HuntnGather

Subversion Repositories:
Compare Path: Rev
With Path: Rev
?path1? @ HEAD  →  ?path2? @ 1
/trunk/HuntnGather/Gather/Gather.c
@@ -5,74 +5,55 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#if !defined ___AmigaOS___
#include <dirent.h>
#include <sys/stat.h>
#include <signal.h>
#endif
 
#include <sys/types.h>
#include <sys/syslimits.h>
#include <sys/stat.h>
 
#if defined ___AmigaOS___
#include <proto/dos.h>
#include <proto/exec.h>
#include <proto/locale.h>
#include <clib/utility_protos.h>
 
#include "StringStack.h"
 
#if defined ___AmigaOS___
#include "getopt.h"
#endif
 
#if defined ___AsyncIO___
#include <asyncio.h>
 
#if !defined TRUE
#define TRUE 1;
#endif
 
#if !defined ___HAVE_GETOPT___
#include "/shared/getopt.h"
#if !defined FALSE
#define FALSE 0;
#endif
 
#include "stack.h"
#include "/shared/utilities.h"
#define MAX_MEM 524288
 
#define PROGRAM_VERSION "1.7.7"
 
#if defined ___AmigaOS___
/*************************************************************************/
/* Version string used for querrying the program version. */
/*************************************************************************/
TEXT version_string[] =
"\0$VER: Gather " PROGRAM_VERSION " "__DATE__" by Wizardry and Steamworks";
#endif
"\0$VER: Gather 1.2 "__DATE__" by Wizardry and Steamworks";
 
int PROGRAM_RUN = TRUE;
int PROGRAM_VERBOSE = TRUE;
int maxmem = MAX_MEM;
typedef struct {
unsigned int dirs;
unsigned int files;
} stats;
 
// Define global locale for string compare.
#if defined ___AmigaOS___
struct Locale *locale;
#endif
int run = TRUE;
int verbose = TRUE;
 
void SignalHandler(int sig) {
/* Toggle the run flag to stop execution. */
PROGRAM_RUN = FALSE;
// Toggle the run flag to stop execution.
run = FALSE;
}
 
/*
*
* Used for sorting database lines.
*/
int QsortCompare(const void *a, const void *b) {
#if defined ___AmigaOS___
return StrnCmp(
locale,
(STRPTR)(*(const char **)a),
(STRPTR)*((const char **)b),
-1,
SC_ASCII
);
#else
return strcmp(*(const char **)a, *(const char **)b);
#endif
int compare(const void *a, const void *b) {
const char **p = (const char **)a;
const char **q = (const char **)b;
return strcmp(*p, *q);
}
 
/*
@@ -79,189 +60,99 @@
*
* Sorts a database file lexicographically.
*/
void SortDatabase(char *dbFile, int lines) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
#else
void SortDatabase(char *dbFile) {
FILE *fp;
#endif
char *name = NULL;
char *path = NULL;
char **database;
dbEntry *entry;
dbLine *line;
char *rem;
int count;
char c;
int i;
int side;
unsigned int line;
 
// Open database file for reading.
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for reading.\n", dbFile);
fprintf(stderr, "Unable to open gather database for reading.\n");
return;
}
 
if((database = malloc(lines * sizeof(*database))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
}
database = (char **) malloc(sizeof(char *));
name = (char *) malloc(sizeof(char));
path = (char *) malloc(sizeof(char));
line = 0;
side = 0;
i = 0;
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Reading lines from file '%s' to array...\n", dbFile);
if(verbose) {
fprintf(stdout, "Sorting database: '%s'\n", dbFile);
}
 
count = 0;
while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) {
while(run && fscanf(fp, "%c", &c) == 1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
run = FALSE;
continue;
}
#endif
if((entry = CreateDatabaseEntry(line)) == NULL) {
fprintf(stderr, "Unable to create database entry.\n");
switch(c) {
case '\n':
// Load up the name and path into the database variable.
database = (char **) realloc(database, (line + 1) * sizeof(char *));
database[line] = (char *) malloc((strlen(name) + strlen(path) + 1 + 1) * sizeof(char));
sprintf(database[line], "%s\t%s", name, path);
++line;
 
free(line->string);
free(line);
line = NULL;
free(name);
name = (char *) malloc(sizeof(char));
--side;
i = 0;
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
break;
case '\t':
free(path);
path = (char *) malloc(sizeof(char));
++side;
i = 0;
break;
default:
switch(side) {
case 0:
name = (char *) realloc(name, (i + 1 + 1) * sizeof(char));
name[i] = c;
name[i + 1] = '\0';
break;
case 1:
path = (char *) realloc(path, (i + 1 + 1) * sizeof(char));
path[i] = c;
path[i + 1] = '\0';
break;
default:
fprintf(stderr, "Database corrupted.\n");
break;
}
++i;
break;
}
 
if((database[count] = malloc((strlen(entry->name) + strlen(entry->path) + 1 + 1) * sizeof(*database[count]))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
 
// Free database entry.
free(entry->name);
free(entry->path);
free(entry);
entry = NULL;
 
// Free the line.
free(line->string);
free(line);
line = NULL;
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
}
 
sprintf(database[count], "%s\t%s", entry->name, entry->path);
++count;
 
// Free the database entry.
free(entry->name);
free(entry->path);
free(entry);
entry = NULL;
 
// Free the line.
free(line->string);
free(line);
line = NULL;
}
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Sorting %d lines in '%s'...\n", count, dbFile);
}
 
// Sort the database.
qsort(database, (unsigned int)count, sizeof(char *), QsortCompare);
qsort(database, line, sizeof(char *), compare);
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Writing %d sorted lines to '%s'...\n", count, dbFile);
}
 
// Write the database lines back to the database.
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "w")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for writing.\n", dbFile);
if((fp = fopen(dbFile, "w+")) == NULL) {
fprintf(stderr, "Unable to open gather database for writing.\n");
return;
}
 
rem = NULL;
for(i = 0; PROGRAM_RUN && i < count; ++i) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
continue;
}
#endif
 
if(rem != NULL) {
#if defined ___AmigaOS___
if(StrnCmp(locale, database[i], rem, -1, SC_ASCII) == 0) {
#else
if(strcmp(database[i], rem) == 0) {
#endif
continue;
}
}
 
#if defined ___AsyncIO___
WriteAsync(fp, database[i], (LONG)strlen(database[i]));
WriteAsync(fp, "\n", 1);
#else
for(i = 0; i < line; ++i) {
fprintf(fp, "%s\n", database[i]);
#endif
 
if(rem != NULL) {
free(rem);
rem = NULL;
}
 
rem = malloc((strlen(database[i]) + 1) * sizeof(*rem));
sprintf(rem, "%s", database[i]);
}
 
if(rem != NULL) {
free(rem);
rem = NULL;
}
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
free(database);
fclose(fp);
#endif
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Disposing %d lines of file '%s'...\n", count, dbFile);
}
 
// Free up database.
for(i = 0; i < count; ++i) {
free(database[i]);
database[i] = NULL;
}
 
free(database);
}
 
/*
@@ -268,280 +159,182 @@
*
* Updates a database file "dbFile".
*/
dbStats *CollectFiles(char *dbFile, VECTOR *paths) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
#else
void UpdateDatabase(char *dbFile, stringStack *dirStack, stats *stats) {
FILE *fp;
#endif
#if defined ___AmigaOS___
struct FileInfoBlock *FIB;
BPTR lock;
#else
DIR *dir;
struct dirent *entry;
#endif
stack *stack;
dbStats *stats = NULL;
int i;
struct dirent *dirEntry;
struct stat dirStat;
unsigned int size;
char *path;
char *sub;
char *subPath;
 
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_APPEND, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "a")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for writing.\n", dbFile);
return NULL;
if((fp = fopen(dbFile, "w+")) == NULL) {
fprintf(stderr, "Unable to open gather database for writing.\n");
return;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Collecting files...\r");
}
 
// Initialize metrics.
if((stats = malloc(sizeof(*stats))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return NULL;
}
 
stats->dirs = 0;
stats->files = 0;
stats->lines = 0;
stats->size = 0;
 
// Push the first path onto the stack.
stack = stackCreate((unsigned int)paths->length);
for(i = 0; PROGRAM_RUN && i < paths->length; ++i) {
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Pushing path '%s'\n", (char *)paths->array[i]);
}
stackPush(stack, paths->array[i], (strlen(paths->array[i]) + 1) * sizeof(char));
}
 
while(PROGRAM_RUN && !stackIsEmpty(stack)) {
while(run && !stringStackIsEmpty(dirStack)) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
continue;
run = FALSE;
}
#endif
if((path = (char *)stackPop(stack)) == NULL) {
break;
if((path = stringStackPop(dirStack)) == NULL) {
return;
}
 
#if defined ___AmigaOS___
if((lock = Lock(path, ACCESS_READ)) == NULL) {
fprintf(stderr, "Could not lock path '%s' for reading.\n", path);
free(path);
path = NULL;
continue;
if((dir = opendir(path)) == NULL) {
return;
}
 
if((FIB = AllocDosObject(DOS_FIB, NULL)) == NULL) {
fprintf(stderr, "File information block for path '%s' could not be allocated.\n", path);
UnLock(lock);
free(path);
path = NULL;
continue;
}
 
if(Examine(lock, FIB) == FALSE) {
fprintf(stderr, "Path '%s' could not be examined.\n", path);
FreeDosObject(DOS_FIB, FIB);
FIB = NULL;
UnLock(lock);
free(path);
path = NULL;
continue;
}
 
while(PROGRAM_RUN && ExNext(lock, FIB)) {
while(run && (dirEntry = readdir(dir)) != NULL) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
continue;
run = FALSE;
}
#else
 
if((dir = opendir(path)) == NULL) {
fprintf(stderr, "Directory '%s' could not be opened.\n", path);
free(path);
path = NULL;
continue;
}
 
while(PROGRAM_RUN && (entry = readdir(dir)) != NULL) {
#endif
size = sizeof(path) + sizeof(dirEntry->d_name) + 1;
switch(path[strlen(path) - 1]) {
case '/':
case ':': // This is a drive path.
#if defined ___AmigaOS___
if((sub = malloc(strlen(path) + strlen(FIB->fib_FileName) + 1)) == NULL) {
#else
if((sub = malloc(strlen(path) + strlen(entry->d_name) + 1)) == NULL) {
#endif
fprintf(stderr, "Memory allocation failure.\n");
#if defined ___AmigaOS___
FreeDosObject(DOS_FIB, FIB);
FIB = NULL;
UnLock(lock);
#else
closedir(dir);
#endif
free(path);
path = NULL;
 
stackDestroy(stack);
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return NULL;
}
#if defined ___AmigaOS___
sprintf(sub, "%s%s", path, FIB->fib_FileName);
#else
sprintf(sub, "%s%s", path, entry->d_name);
#endif
subPath = (char *) malloc(size);
sprintf(subPath, "%s%s", path, dirEntry->d_name);
break;
default:
#if defined ___AmigaOS___
if((sub = malloc(strlen(path) + strlen(FIB->fib_FileName) + 1 + 1)) == NULL) {
#else
if((sub = malloc(strlen(path) + strlen(entry->d_name) + 1 + 1)) == NULL) {
#endif
fprintf(stderr, "Memory allocation failure.\n");
#if defined ___AmigaOS___
FreeDosObject(DOS_FIB, FIB);
FIB = NULL;
UnLock(lock);
#else
closedir(dir);
#endif
free(path);
path = NULL;
 
stackDestroy(stack);
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return NULL;
}
#if defined ___AmigaOS___
sprintf(sub, "%s/%s", path, FIB->fib_FileName);
#else
sprintf(sub, "%s/%s", path, entry->d_name);
#endif
subPath = (char *) malloc(size + 1);
sprintf(subPath, "%s/%s", path, dirEntry->d_name);
break;
}
stat(subPath, &dirStat);
if(S_ISDIR(dirStat.st_mode)) {
stringStackPush(dirStack, subPath);
 
switch(GetFsType(sub)) {
case UNKNOWN:
free(sub);
sub = NULL;
continue;
case REGULAR:
++stats->files;
++stats->dirs;
 
if(PROGRAM_VERBOSE) {
fprintf(stdout,
"Gathered '%d' directories and '%d' files.\r",
if(verbose) {
fprintf(stdout,
"Gathered %d directories and %d files.\r",
stats->dirs,
stats->files);
}
break;
case DIRECTORY:
stackPush(stack, sub, (strlen(sub) + 1) * sizeof(char));
}
 
++stats->dirs;
free(subPath);
continue;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout,
"Gathered '%d' directories and '%d' files.\r",
stats->dirs,
stats->files);
}
// Write to database file.
fprintf(fp, "%s\t%s\n", dirEntry->d_name, subPath);
 
free(sub);
sub = NULL;
continue;
}
++stats->files;
 
#if defined ___NOCASE_FS___
#if defined ___AmigaOS___
StrUpr(FIB->fib_FileName);
#else
StrUpr(entry->d_name);
#endif
#endif
if(verbose) {
fprintf(stdout,
"Gathered %d directories and %d files.\r",
stats->dirs,
stats->files);
}
 
// Write to database file.
#if defined ___AsyncIO___
#if defined ___AmigaOS___
WriteAsync(fp, FIB->fib_FileName, (LONG)strlen(FIB->fib_FileName));
stats->size = stats->size + strlen(FIB->fib_FileName);
#else
WriteAsync(fp, entry->d_name, (LONG)strlen(entry->d_name));
stats->size = stats->size + strlen(entry->d_name);
#endif
WriteAsync(fp, "\t", 1);
++stats->size;
WriteAsync(fp, sub, (LONG)strlen(sub));
stats->size = stats->size + strlen(sub);
WriteAsync(fp, "\n", 1);
++stats->size;
#else
#if defined ___AmigaOS___
fprintf(fp, "%s\t%s\n", FIB->fib_FileName, sub);
stats->size = stats->size + strlen(FIB->fib_FileName) + strlen(sub) + 1 + 1 + 1;
#else
fprintf(fp, "%s\t%s\n", entry->d_name, sub);
stats->size = stats->size + strlen(entry->d_name) + strlen(sub) + 1 + 1 + 1;
#endif
#endif
 
++stats->lines;
 
free(sub);
sub = NULL;
free(subPath);
}
 
#if defined ___AmigaOS___
FreeDosObject(DOS_FIB, FIB);
FIB = NULL;
UnLock(lock);
#else
closedir(dir);
#endif
free(path);
path = NULL;
}
 
if(PROGRAM_VERBOSE) {
if(verbose) {
fprintf(stdout, "\n");
}
 
stackDestroy(stack);
fclose(fp);
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
}
 
/*
*
* Gets the size of a database "dbFle".
*/
int GetDatabaseSize(char *dbFile) {
FILE *fp;
int size;
 
if((fp = fopen(dbFile, "r")) == NULL) {
fprintf(stderr, "Unable to open gather database for reading.\n");
fclose(fp);
return 0;
}
 
fseek(fp, 0L, SEEK_END);
size = ftell(fp);
 
fclose(fp);
return size;
}
 
/*
*
* Counts the lines in a database file "dbFile".
*/
int CountDatabaseLines(char *dbFile) {
FILE *fp;
int lines;
char c;
 
if((fp = fopen(dbFile, "r")) == NULL) {
fprintf(stderr, "Unable to open gather database for reading.\n");
fclose(fp);
return 0;
}
 
lines = 0;
while(fscanf(fp, "%c", &c) == 1) {
switch(c) {
case '\n':
++lines;
break;
}
}
 
fclose(fp);
 
return lines;
}
 
/*
*
* Creates "files" temporary filenames.
*/
char **CreateTempFiles(int files) {
char **tmpNames;
int count;
 
tmpNames = (char **) malloc(files * sizeof(char *));
 
if(verbose) {
fprintf(stdout, "Creating temporary files.\r");
}
 
count = files;
while(--count > -1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
tmpNames[count] = tmpnam(NULL);
 
return stats;
if(verbose) {
fprintf(stdout, "Creating temporary files: %d%%\r", 100 - (int)(((float)count / files) * 100.0));
}
}
 
if(verbose) {
fprintf(stdout, "\n");
}
 
return tmpNames;
}
 
/*
@@ -548,128 +341,137 @@
*
* Writes lines from the database "dbFile" to temporary filenames "tmpNames".
*/
void WriteTemporaryFiles(char *dbFile, VECTOR *tmpNames, int tmpLines, int total) {
#if defined ___AsyncIO___
struct AsyncFile *fp, *tp;
#else
void WriteTempFiles(char *dbFile, char **tmpNames, int tmpFiles, int tmpLines, int total) {
FILE *fp, *tp;
#endif
char c;
int lines;
int write;
int files;
dbLine *line = NULL;
int linesWritten;
 
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for reading.\n", dbFile);
fprintf(stderr, "Unable to open gather database for reading.\n");
return;
}
 
files = tmpNames->length;
#if defined ___AsyncIO___
if((tp = OpenAsync(tmpNames->array[--files], MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((tp = fopen(tmpNames->array[--files], "w")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for writing.\n", (char *)tmpNames->array[files]);
#if defined ___AsyncIO___
CloseAsync(fp);
#else
if((tp = fopen(tmpNames[--tmpFiles], "w+")) == NULL) {
fprintf(stderr, "Unable to open temporary file '%s' for writing.\n", tmpNames[tmpFiles]);
fclose(fp);
#endif
return;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Writing to temporary files...\r");
if(verbose) {
fprintf(stdout, "Writing to temporary files.\r");
}
 
write = 0;
linesWritten = 0;
lines = 0;
 
while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) {
while(run && fscanf(fp, "%c", &c) == 1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
free(line->string);
free(line);
line = NULL;
 
PROGRAM_RUN = FALSE;
continue;
run = FALSE;
}
#endif
switch(c) {
case '\n':
// Increment the total written lines.
++linesWritten;
 
#if defined ___AsyncIO___
WriteAsync(tp, line->string, (LONG)line->length);
WriteAsync(tp, "\n", 1);
#else
fprintf(tp, "%s\n", line->string);
#endif
if(verbose) {
fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)linesWritten / total) * 100.0));
}
 
++write;
// Write the newline character back.
if(fprintf(tp, "%c", c) != 1) {
fprintf(stderr, "Unable to write to temporary file '%s'.\n", tmpNames[tmpFiles]);
fclose(tp);
fclose(fp);
return;
}
// Switch to the next temporary file.
if(++lines >= tmpLines) {
// If there are no temporary files left then run till the end.
if(tmpFiles - 1 < 0) {
break;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)write / total) * 100.0));
// Close the previous temporary file and write to the next temporary file.
fclose(tp);
if((tp = fopen(tmpNames[--tmpFiles], "w+")) == NULL) {
fprintf(stderr, "Unable to open temporary file '%s' for writing.\n", tmpNames[tmpFiles]);
fclose(tp);
fclose(fp);
}
lines = 0;
break;
}
break;
default:
if(fprintf(tp, "%c", c) != 1) {
fprintf(stderr, "Unable to write to temporary file '%s'.\n", tmpNames[tmpFiles]);
fclose(tp);
fclose(fp);
return;
}
break;
}
}
 
// Switch to the next temporary file.
if(++lines >= tmpLines) {
// If there are no temporary files left then run till the end.
if(files - 1 < 0) {
free(line->string);
free(line);
line = NULL;
continue;
}
fprintf(stdout, "\n");
 
// Close the previous temporary file and write to the next temporary file.
#if defined ___AsyncIO___
CloseAsync(tp);
if((tp = OpenAsync(tmpNames->array[--files], MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
fclose(tp);
if((tp = fopen(tmpNames->array[--files], "w")) == NULL) {
#endif
fprintf(stderr, "Could not open '%s' for writing.\n", (char *)tmpNames->array[files]);
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
free(line->string);
free(line);
line = NULL;
return;
}
lines = 0;
}
fclose(tp);
fclose(fp);
}
 
free(line->string);
free(line);
line = NULL;
}
/*
*
* Skips a line in a database file "fp".
*/
void SkipDatabaseLine(FILE *fp) {
char c;
 
if(line != NULL) {
free(line->string);
free(line);
line = NULL;
while(fscanf(fp, "%c", &c) == 1) {
if(c == '\n') {
break;
}
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "\n");
return;
}
 
/*
*
* Reads a line from the database file "fp".
*/
char *ReadDatabaseLine(FILE *fp) {
char c;
char *line;
int chars;
 
line = (char *) malloc(sizeof(char));
 
chars = 0;
while(run && fscanf(fp, "%c", &c) == 1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
switch(c) {
case '\n':
// Rewind the file by the number of read characters.
fseek(fp, -(chars + 1), SEEK_CUR);
return line;
default:
line = (char *) realloc(line, (chars + 1 + 1) * sizeof(char));
line[chars] = c;
line[chars + 1] = '\0';
break;
}
++chars;
}
 
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
return NULL;
}
 
/*
@@ -676,513 +478,159 @@
*
* Merges temporary files "tmpNames" into a database "dbFile".
*/
void MergeTemporaryFiles(char *dbFile, VECTOR *tmpNames, int lines) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
struct AsyncFile **tp;
#else
void MergeDatabase(char *dbFile, char **tmpNames, int files, int lines) {
FILE *fp;
FILE **tp;
#endif
int i;
int j;
dbLine *tmp;
char *rem;
char *min;
char *tmp;
char *tmpMin;
int idxMin;
int count;
 
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "w")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for writing.\n", dbFile);
if((fp = fopen(dbFile, "w+")) == NULL) {
fprintf(stderr, "Unable to open gather database for writing.\n");
return;
}
 
// Allocate as many file pointers as temporary files.
if((tp = malloc(tmpNames->length * sizeof(*tp))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
}
tp = (FILE **) malloc(files * sizeof(FILE *));
 
// Open all temporary files for reading.
for(i = 0; i < tmpNames->length; ++i) {
#if defined ___AsyncIO___
if((tp[i] = OpenAsync(tmpNames->array[i], MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((tp[i] = fopen(tmpNames->array[i], "r")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for reading.\n", (char *)tmpNames->array[i]);
for(i = 0; i < files; ++i) {
if((tp[i] = fopen(tmpNames[i], "r")) == NULL) {
fprintf(stderr, "Unable to open temporary file '%s' for reading.\n", tmpNames[i]);
// Close all temporary files.
while(--i > -1) {
#if defined ___AsyncIO___
CloseAsync(tp[i]);
#else
--i;
while(i >= 0) {
fclose(tp[i]);
#endif
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
}
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Merging all files...\r");
if(verbose) {
fprintf(stdout, "Merging all database lines in temporary files.\r");
}
 
rem = NULL;
count = lines;
j = 0;
while(PROGRAM_RUN && --count > -1) {
idxMin = 0;
while(run && --count > -1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
continue;
run = FALSE;
}
#endif
// Find the smallest line in all temporary files.
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Merging all files: %d%%.\r", 100 - (int)(((float)count / lines) * 100.0));
if(verbose) {
fprintf(stdout, "Merging all database lines in temporary files: %d%%.\r", 100 - (int)(((float)count / lines) * 100.0));
}
 
min = NULL;
for(i = 0; i < tmpNames->length; ++i) {
tmp = PeekLine(tp[i]);
tmpMin = NULL;
for(i = 0; i < files; ++i) {
tmp = ReadDatabaseLine(tp[i]);
if(tmp == NULL) {
free(tmp);
continue;
}
#if defined ___AmigaOS___
if(min == NULL || StrnCmp(locale, tmp->string, min, -1, SC_ASCII) < 0) {
#else
if(min == NULL || strcmp(tmp->string, min) < 0) {
#endif
if(min != NULL) {
// Free previous instance.
free(min);
min = NULL;
}
if((min = malloc((strlen(tmp->string) + 1) * sizeof(*min))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
 
free(tmp->string);
free(tmp);
tmp = NULL;
 
if(min != NULL) {
free(min);
min = NULL;
}
if(rem != NULL) {
free(rem);
rem = NULL;
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
}
sprintf(min, "%s", tmp->string);
if(tmpMin == NULL || strcmp(tmp, tmpMin) < 0) {
tmpMin = (char *) malloc((strlen(tmp) + 1) * sizeof(char));
sprintf(tmpMin, "%s", tmp);
// Remember the index of the file where the smallest entry has been found.
j = i;
idxMin = i;
free(tmp);
continue;
}
free(tmp->string);
free(tmp);
tmp = NULL;
}
 
// Forward the file where the smallest line was found.
SkipLine(tp[j]);
SkipDatabaseLine(tp[idxMin]);
 
// Write the smallest line.
if(min != NULL) {
// If current minimum line is identical to previous minimum line then skip to remove duplicates.
if(rem != NULL) {
#if defined ___AmigaOS___
if(StrnCmp(locale, min, rem, -1, SC_ASCII) == 0) {
#else
if(strcmp(min, rem) == 0) {
#endif
free(min);
min = NULL;
continue;
}
}
 
#if defined ___AsyncIO___
WriteAsync(fp, min, (LONG)strlen(min));
WriteAsync(fp, "\n", 1);
#else
fprintf(fp, "%s\n", min);
#endif
 
if(rem != NULL) {
free(rem);
rem = NULL;
}
 
if((rem = malloc((strlen(min) + 1) * sizeof(*rem))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
 
free(min);
min = NULL;
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return;
}
 
// Remember the last minimal line.
sprintf(rem, "%s", min);
 
free(min);
min = NULL;
if(tmpMin != NULL) {
fprintf(fp, "%s\n", tmpMin);
free(tmpMin);
}
}
 
if(rem != NULL) {
free(rem);
rem = NULL;
}
 
// Write out any remaining contents from the temporary files.
for(i = 0; PROGRAM_RUN && i < tmpNames->length; ++i) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
continue;
}
#endif
tmp = ReadLine(tp[i]);
for(i = 0; i < files; ++i) {
tmp = ReadDatabaseLine(tp[i]);
if(tmp == NULL) {
continue;
}
#if defined ___AsyncIO___
WriteAsync(fp, tmp->string, (LONG)strlen(tmp->string));
WriteAsync(fp, "\n", 1);
#else
fprintf(fp, "%s\n", tmp->string);
#endif
free(tmp->string);
free(tmp);
tmp = NULL;
fprintf(fp, "%s\n", tmp);
}
 
// Close all temporary files.
for(i = 0; i < tmpNames->length; ++i) {
#if defined ___AsyncIO___
CloseAsync(tp[i]);
#else
// Close and delete all temporary files.
for(i = 0; i < files; ++i) {
fclose(tp[i]);
#endif
// Delete temporary file.
remove(tmpNames[i]);
}
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
 
if(PROGRAM_VERBOSE) {
if(verbose) {
fprintf(stdout, "\n");
}
}
 
/*
*
* Filter the paths inside the database with provided paths.
*/
void FilterDatabasePaths(char *dbFile, char *tmpName, VECTOR *paths) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
struct AsyncFile *tp;
#else
FILE *fp;
FILE *tp;
#endif
dbLine *line;
dbEntry *entry;
int lines;
int i;
 
// Open database file for reading.
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for reading.\n", dbFile);
return;
}
 
// Open temporary file for writing.
#if defined ___AsyncIO___
if((tp = OpenAsync(tmpName, MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((tp = fopen(tmpName, "w")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for writing.\n", tmpName);
 
// Close database file.
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
 
return;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Removing lines...\r");
}
 
lines = 0;
while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
PROGRAM_RUN = FALSE;
continue;
}
#endif
 
if((entry = CreateDatabaseEntry(line)) == NULL) {
fprintf(stderr, "Unable to create database entry.\n");
free(line->string);
free(line);
line = NULL;
continue;
}
 
for(i = 0; i < paths->length; ++i) {
if(PathCompare(entry->path, paths->array[i]) == TRUE) {
++lines;
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Removing lines: %d.\r", lines);
}
continue;
}
#if defined ___AsyncIO___
WriteAsync(tp, line->string, (LONG)strlen(line->string));
WriteAsync(tp, "\n", 1);
#else
fprintf(tp, "%s\n", line->string);
#endif
break;
}
 
// Free up database entry.
free(entry->name);
free(entry->path);
free(entry);
entry = NULL;
 
// Free up line.
free(line->string);
free(line);
line = NULL;
}
 
#if defined ___AsyncIO___
CloseAsync(fp);
CloseAsync(tp);
#else
fclose(fp);
fclose(tp);
#endif
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "\n");
}
}
 
/*
*
* Indexes paths and adds to a database file.
* Indexes a "path" by creating a database "dbFile".
*/
void GatherDatabaseFiles(char *dbFile, VECTOR *paths) {
dbStats *stats;
VECTOR *tmpNames;
int tmpFiles;
int tmpLines;
void Gather(char *dbFile, char *path) {
stringStack *stack = stringStackCreate(1);
stats *stats = malloc(sizeof(stats));
char **tmpNames;
int dbSize, dbLines, tmpFiles, tmpLines;
int i;
int line;
int size;
 
// Generate the database file from the supplied paths.
if((stats = CollectFiles(dbFile, paths)) == NULL) {
fprintf(stderr, "Collecting files failed.\n");
return;
}
// Initialize metrics.
stats->dirs = 0;
stats->files = 0;
 
// The size and amount of lines are not necessarily what has been gathered now.
size = GetFileSize(dbFile);
line = CountFileLines(dbFile);
// Push the first path onto the stack.
stringStackPush(stack, path);
 
// Calculate the total number of temporary files required.
tmpFiles = size / maxmem;
// Generate the database file.
UpdateDatabase(dbFile, stack, stats);
 
/* In case no temporary files are required,
* just sort the database and terminate.
*/
if(tmpFiles < 2) {
SortDatabase(dbFile, line);
return;
}
// Get the database metrics.
dbSize = GetDatabaseSize(dbFile);
dbLines = CountDatabaseLines(dbFile);
 
// Calculate the number of lines per temporary file.
tmpLines = ceil(((double)line) / ((double)tmpFiles));
// Compute the amount of temporary files needed.
tmpFiles = dbSize / MAX_MEM;
 
// Create temporary files.
if((tmpNames = CreateTemporaryFiles(tmpFiles)) == NULL) {
fprintf(stderr, "Unable to create temporary files.\n");
// In case no temporary files are required,
// just sort the database and terminate.
if(tmpFiles == 0) {
SortDatabase(dbFile);
return;
}
 
// Write "tmpLines" to temporary files in "tmpNames" from "dbFile".
WriteTemporaryFiles(dbFile, tmpNames, tmpLines, line);
tmpLines = dbLines / tmpFiles;
 
// Sort the temporary files.
for(i = 0; i < tmpNames->length; ++i) {
SortDatabase(tmpNames->array[i], tmpLines);
}
 
// Merge all the temporary files to the database file.
MergeTemporaryFiles(dbFile, tmpNames, line);
 
// Remove all temporary files.
RemoveFiles(tmpNames);
 
// Free temporary file names.
free(tmpNames);
tmpNames = NULL;
 
// Free statistics.
free(stats);
stats = NULL;
}
 
/*
*
* Indexes paths and creates a daabase file.
*/
void CreateDatabaseFiles(char *dbFile, VECTOR *paths) {
dbStats *stats;
VECTOR *tmpNames;
int tmpFiles;
int tmpLines;
int i;
 
// Generate the database file from the supplied paths.
if((stats = CollectFiles(dbFile, paths)) == NULL) {
fprintf(stderr, "Collecting files failed.\n");
return;
}
 
// Calculate the total number of temporary files required.
tmpFiles = stats->size / maxmem;
 
/* In case no temporary files are required,
* just sort the database and terminate.
*/
if(tmpFiles < 2) {
SortDatabase(dbFile, stats->lines);
return;
}
 
// Calculate the number of lines per temporary file.
tmpLines = ceil(((double)stats->lines) / ((double)tmpFiles));
 
// Create temporary files.
if((tmpNames = CreateTemporaryFiles(tmpFiles)) == NULL) {
if((tmpNames = CreateTempFiles(tmpFiles)) == NULL) {
fprintf(stderr, "Unable to create temporary files.\n");
return;
}
 
// Write "tmpLines" to temporary files in "tmpNames" from "dbFile".
WriteTemporaryFiles(dbFile, tmpNames, tmpLines, stats->lines);
// Write "tmpLines" to temporary files in "tmpFiles" from "dbFile".
WriteTempFiles(dbFile, tmpNames, tmpFiles, tmpLines, dbLines);
 
// Sort the temporary files.
for(i = 0; i < tmpNames->length; ++i) {
SortDatabase(tmpNames->array[i], tmpLines);
for(i = 0; i < tmpFiles; ++i) {
SortDatabase(tmpNames[i]);
}
 
// Merge all the temporary files to the database file.
MergeTemporaryFiles(dbFile, tmpNames, stats->lines);
 
// Remove all temporary files.
RemoveFiles(tmpNames);
 
// Free temporary file names.
free(tmpNames);
tmpNames = NULL;
 
// Free statistics.
free(stats);
stats = NULL;
MergeDatabase(dbFile, tmpNames, tmpFiles, dbLines);
}
 
void RemoveDatabaseFiles(char *dbFile, VECTOR *paths) {
char *tmpName;
 
// Create a temporary file to hold the changes.
if((tmpName = CreateTemporaryFile()) == NULL) {
fprintf(stderr, "Unable to create temporary file.\n");
return;
}
 
// Filter the database of the provided paths.
FilterDatabasePaths(dbFile, tmpName, paths);
 
// Overwrite the database file with the filtered paths.
CopyLines(tmpName, dbFile);
 
// Remove temporary file.
if(RemoveFile(tmpName) == FALSE) {
fprintf(stderr, "Temporary file could not be removed.\n");
return;
}
}
 
void usage(char *name) {
fprintf(stdout, "Hunt & Gather - %s, a file index generating tool. \n", name);
fprintf(stdout, "Version: %s \n", PROGRAM_VERSION);
fprintf(stdout, " \n");
fprintf(stdout, "SYNTAX: %s [-q] <-a|-r|-c> <PATH PATH PATH...> \n", name);
fprintf(stdout, " \n");
fprintf(stdout, "Required: \n");
fprintf(stdout, " -a [PATH...] Add files. \n");
fprintf(stdout, " -c [PATH...] Create from scratch. \n");
fprintf(stdout, " -r [PATH...] Remove files. \n");
fprintf(stdout, " \n");
fprintf(stdout, "Optional: \n");
fprintf(stdout, " -d [FIILE] Where to store the database. \n");
fprintf(stdout, " -m BYTES Memory to use (default: %d). \n", maxmem);
fprintf(stdout, " -q Do not print out any messages. \n");
fprintf(stdout, " \n");
fprintf(stdout, "DATABASE is a path to where the indexed results will be \n");
fprintf(stdout, "stored for searching with the Hunt tool. \n");
fprintf(stdout, " \n");
fprintf(stdout, "(c) 2021 Wizardry and Steamworks, MIT. \n");
}
 
/*
*
* Main entry point.
@@ -1189,164 +637,40 @@
*/
int main(int argc, char **argv) {
int option;
int i;
char *dbFile;
char *path;
VECTOR *paths;
OPERATION operation = NONE;
struct stat path;
 
// Bind handler to SIGINT.
#if !defined ___AmigaOS___
signal(SIGINT, SignalHandler);
#endif
 
dbFile = DEFAULT_DATABASE_FILE;
while((option = getopt(argc, argv, "hqdm:arc")) != -1) {
while((option = getopt(argc, argv, "hq")) != -1) {
switch(option) {
case 'a':
operation = GATHER;
break;
case 'r':
operation = REMOVE;
break;
case 'c':
operation = CREATE;
break;
case 'm':
maxmem = strtoul(optarg, NULL, 10);
break;
case 'd':
dbFile = optarg;
break;
case 'q':
PROGRAM_VERBOSE = FALSE;
verbose = FALSE;
break;
case 'h':
usage(argv[0]);
return 0;
fprintf(stdout, "SYNTAX: %s [-q] DIRECTORY", argv[0]);
break;
case '?':
fprintf(stderr, "Invalid option %ct.\n", optopt);
return 5;
fprintf(stdout, "SYNTAX: %s [-q] DIRECTORY\n", argv[0]);
return 1;
}
}
 
if(operation == NONE) {
usage(argv[0]);
return 5;
if(optind > argc) {
fprintf(stdout, "SYNTAX: %s [-q] DIRECTORY\n", argv[0]);
return 1;
}
 
if(optind >= argc) {
usage(argv[0]);
return 5;
}
stat(argv[optind], &path);
 
// Build the path vector.
if((paths = malloc(1 * sizeof(*paths))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
return 20;
if(!S_ISDIR(path.st_mode)) {
fprintf(stdout, "%s is not a directory.\n", argv[optind]);
return 1;
}
 
// Go through all supplied arguments and add paths to search.
if((paths->array = malloc((argc - optind) * sizeof(*paths))) == NULL) {
fprintf(stderr, "Memory allocation failure.\n");
return 20;
}
// Gather.
Gather("S:gather.db", argv[optind]);
 
for(i = optind, paths->length = 0; i < argc; ++i) {
if((path = PathToAbsolute(argv[i])) == NULL) {
fprintf(stderr, "Absolute path for '%s' failed to resolve.\n", argv[optind]);
continue;
}
 
switch(GetFsType(path)) {
case UNKNOWN:
case REGULAR:
fprintf(stderr, "Path '%s' is not a directory.\n", path);
free(path);
path = NULL;
continue;
case DIRECTORY:
break;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Will process path: '%s'\n", path);
}
 
// Add the path to the array of paths.
if((paths->array[paths->length] = malloc((strlen(path) + 1) * sizeof(*paths->array[paths->length]))) == NULL) {
fprintf(stderr, "Memory allocation failure.");
return 20;
}
 
sprintf(paths->array[paths->length], "%s", path);
++paths->length;
 
free(path);
path = NULL;
 
}
 
if(paths->length == 0) {
fprintf(stderr, "No valid paths are available.\n");
free(paths->array);
free(paths);
paths = NULL;
return 5;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Gathering to: '%s'\n", dbFile);
}
 
#if defined ___AmigaOS___
locale = OpenLocale(NULL);
#endif
 
switch(operation) {
case CREATE:
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Removing '%s' and creating a new database.\n", dbFile);
}
RemoveFile(dbFile);
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Gathering files to database...\n");
}
CreateDatabaseFiles(dbFile, paths);
break;
case GATHER:
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Gathering files to database...\n");
}
GatherDatabaseFiles(dbFile, paths);
break;
case REMOVE:
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Removing files from database...\n");
}
RemoveDatabaseFiles(dbFile, paths);
break;
default:
fprintf(stderr, "Unknown operation.\n");
#if defined ___AmigaOS___
CloseLocale(locale);
#endif
 
free(paths->array);
free(paths);
paths = NULL;
return 5;
}
 
#if defined ___AmigaOS___
CloseLocale(locale);
#endif
 
if(paths != NULL) {
free(paths->array);
free(paths);
paths = NULL;
}
 
return 0;
return 0;
}