/trunk/HuntnGather/Gather/Gather.c |
@@ -5,6 +5,7 @@ |
#include <stdio.h> |
#include <stdlib.h> |
#include <string.h> |
#include <math.h> |
#if !defined ___AmigaOS___ |
#include <dirent.h> |
#include <sys/stat.h> |
@@ -86,7 +87,7 @@ |
#endif |
char **database; |
dbEntry *entry; |
char *line = NULL; |
dbLine *line; |
char *rem; |
int count; |
int i; |
@@ -126,7 +127,11 @@ |
#endif |
if((entry = CreateDatabaseEntry(line)) == NULL) { |
fprintf(stderr, "Unable to create database entry.\n"); |
|
free(line->string); |
free(line); |
line = NULL; |
|
#if defined ___AsyncIO___ |
CloseAsync(fp); |
#else |
@@ -137,10 +142,18 @@ |
|
if((database[count] = malloc((strlen(entry->name) + strlen(entry->path) + 1 + 1) * sizeof(*database[count]))) == NULL) { |
fprintf(stderr, "Memory allocation failure.\n"); |
|
// Free database entry. |
free(entry->name); |
free(entry->path); |
free(entry); |
entry = NULL; |
|
// Free the line. |
free(line->string); |
free(line); |
line = NULL; |
|
#if defined ___AsyncIO___ |
CloseAsync(fp); |
#else |
@@ -156,14 +169,14 @@ |
free(entry->name); |
free(entry->path); |
free(entry); |
entry = NULL; |
|
// Free the line. |
free(line->string); |
free(line); |
line = NULL; |
} |
|
if(line != NULL) { |
free(line); |
} |
|
#if defined ___AsyncIO___ |
CloseAsync(fp); |
#else |
@@ -220,6 +233,7 @@ |
|
if(rem != NULL) { |
free(rem); |
rem = NULL; |
} |
|
rem = malloc((strlen(database[i]) + 1) * sizeof(*rem)); |
@@ -228,6 +242,7 @@ |
|
if(rem != NULL) { |
free(rem); |
rem = NULL; |
} |
|
#if defined ___AsyncIO___ |
@@ -243,6 +258,7 @@ |
// Free up database. |
for(i = 0; i < count; ++i) { |
free(database[i]); |
database[i] = NULL; |
} |
|
free(database); |
@@ -252,7 +268,7 @@ |
* |
* Updates a database file "dbFile". |
*/ |
dbStats *CollectFiles(char *dbFile, char **paths, int count) { |
dbStats *CollectFiles(char *dbFile, VECTOR *paths) { |
#if defined ___AsyncIO___ |
struct AsyncFile *fp; |
#else |
@@ -264,7 +280,6 @@ |
#else |
DIR *dir; |
struct dirent *entry; |
struct stat dirStat; |
#endif |
stringStack *stack; |
dbStats *stats = NULL; |
@@ -302,9 +317,9 @@ |
stats->size = 0; |
|
// Push the first path onto the stack. |
stack = stringStackCreate((unsigned int)count); |
for(i = 0; PROGRAM_RUN && i < count; ++i) { |
stringStackPush(stack, paths[i]); |
stack = stringStackCreate((unsigned int)paths->length); |
for(i = 0; PROGRAM_RUN && i < paths->length; ++i) { |
stringStackPush(stack, paths->array[i]); |
} |
|
while(PROGRAM_RUN && !stringStackIsEmpty(stack)) { |
@@ -323,6 +338,7 @@ |
if((lock = Lock(path, ACCESS_READ)) == NULL) { |
fprintf(stderr, "Could not lock path '%s' for reading.\n", path); |
free(path); |
path = NULL; |
continue; |
} |
|
@@ -330,6 +346,7 @@ |
fprintf(stderr, "File information block for path '%s' could not be allocated.\n", path); |
UnLock(lock); |
free(path); |
path = NULL; |
continue; |
} |
|
@@ -336,8 +353,10 @@ |
if(Examine(lock, FIB) == FALSE) { |
fprintf(stderr, "Path '%s' could not be examined.\n", path); |
FreeDosObject(DOS_FIB, FIB); |
FIB = NULL; |
UnLock(lock); |
free(path); |
path = NULL; |
continue; |
} |
|
@@ -352,6 +371,7 @@ |
if((dir = opendir(path)) == NULL) { |
fprintf(stderr, "Directory '%s' could not be opened.\n", path); |
free(path); |
path = NULL; |
continue; |
} |
|
@@ -368,11 +388,15 @@ |
fprintf(stderr, "Memory allocation failure.\n"); |
#if defined ___AmigaOS___ |
FreeDosObject(DOS_FIB, FIB); |
FIB = NULL; |
UnLock(lock); |
#else |
closedir(dir); |
#endif |
|
free(path); |
path = NULL; |
|
stringStackDestroy(stack); |
#if defined ___AsyncIO___ |
CloseAsync(fp); |
@@ -396,11 +420,15 @@ |
fprintf(stderr, "Memory allocation failure.\n"); |
#if defined ___AmigaOS___ |
FreeDosObject(DOS_FIB, FIB); |
FIB = NULL; |
UnLock(lock); |
#else |
closedir(dir); |
#endif |
|
free(path); |
path = NULL; |
|
stringStackDestroy(stack); |
#if defined ___AsyncIO___ |
CloseAsync(fp); |
@@ -420,6 +448,7 @@ |
switch(GetFsType(sub)) { |
case UNKNOWN: |
free(sub); |
sub = NULL; |
continue; |
case REGULAR: |
++stats->files; |
@@ -426,7 +455,7 @@ |
|
if(PROGRAM_VERBOSE) { |
fprintf(stdout, |
"Gathered %d directories and %d files.\r", |
"Gathered '%d' directories and '%d' files.\r", |
stats->dirs, |
stats->files); |
} |
@@ -438,12 +467,13 @@ |
|
if(PROGRAM_VERBOSE) { |
fprintf(stdout, |
"Gathered %d directories and %d files.\r", |
"Gathered '%d' directories and '%d' files.\r", |
stats->dirs, |
stats->files); |
} |
|
free(sub); |
sub = NULL; |
continue; |
} |
|
@@ -483,15 +513,18 @@ |
++stats->lines; |
|
free(sub); |
sub = NULL; |
} |
|
#if defined ___AmigaOS___ |
FreeDosObject(DOS_FIB, FIB); |
FIB = NULL; |
UnLock(lock); |
#else |
closedir(dir); |
#endif |
free(path); |
path = NULL; |
} |
|
if(PROGRAM_VERBOSE) { |
@@ -514,7 +547,7 @@ |
* |
* Writes lines from the database "dbFile" to temporary filenames "tmpNames". |
*/ |
void WriteTemporaryFiles(char *dbFile, char **tmpNames, int tmpFiles, int tmpLines, int total) { |
void WriteTemporaryFiles(char *dbFile, VECTOR *tmpNames, int tmpLines, int total) { |
#if defined ___AsyncIO___ |
struct AsyncFile *fp, *tp; |
LONG c; |
@@ -524,6 +557,7 @@ |
#endif |
int lines; |
int write; |
int files; |
|
#if defined ___AsyncIO___ |
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) { |
@@ -534,12 +568,13 @@ |
return; |
} |
|
files = tmpNames->length; |
#if defined ___AsyncIO___ |
if((tp = OpenAsync(tmpNames[--tmpFiles], MODE_WRITE, ASYNC_BUF)) == NULL) { |
if((tp = OpenAsync(tmpNames->array[--files], MODE_WRITE, ASYNC_BUF)) == NULL) { |
#else |
if((tp = fopen(tmpNames[--tmpFiles], "w")) == NULL) { |
if((tp = fopen(tmpNames->array[--files], "w")) == NULL) { |
#endif |
fprintf(stderr, "Could not open file '%s' for writing.\n", tmpNames[tmpFiles]); |
fprintf(stderr, "Could not open file '%s' for writing.\n", (char *)tmpNames->array[files]); |
#if defined ___AsyncIO___ |
CloseAsync(fp); |
#else |
@@ -581,7 +616,7 @@ |
#else |
if(fprintf(tp, "%c", c) != 1) { |
#endif |
fprintf(stderr, "Unable to write to '%s'.\n", tmpNames[tmpFiles]); |
fprintf(stderr, "Unable to write to '%s'.\n", (char *)tmpNames->array[files]); |
#if defined ___AsyncIO___ |
CloseAsync(tp); |
CloseAsync(fp); |
@@ -594,7 +629,7 @@ |
// Switch to the next temporary file. |
if(++lines >= tmpLines) { |
// If there are no temporary files left then run till the end. |
if(tmpFiles - 1 < 0) { |
if(files - 1 < 0) { |
break; |
} |
|
@@ -601,12 +636,12 @@ |
// Close the previous temporary file and write to the next temporary file. |
#if defined ___AsyncIO___ |
CloseAsync(tp); |
if((tp = OpenAsync(tmpNames[--tmpFiles], MODE_WRITE, ASYNC_BUF)) == NULL) { |
if((tp = OpenAsync(tmpNames->array[--files], MODE_WRITE, ASYNC_BUF)) == NULL) { |
#else |
fclose(tp); |
if((tp = fopen(tmpNames[--tmpFiles], "w")) == NULL) { |
if((tp = fopen(tmpNames->array[--files], "w")) == NULL) { |
#endif |
fprintf(stderr, "Could not open '%s' for writing.\n", tmpNames[tmpFiles]); |
fprintf(stderr, "Could not open '%s' for writing.\n", (char *)tmpNames->array[files]); |
#if defined ___AsyncIO___ |
CloseAsync(fp); |
#else |
@@ -624,7 +659,7 @@ |
#else |
if(fprintf(tp, "%c", c) != 1) { |
#endif |
fprintf(stderr, "Could not write to file '%s'.\n", tmpNames[tmpFiles]); |
fprintf(stderr, "Could not write to file '%s'.\n", (char *)tmpNames->array[files]); |
#if defined ___AsyncIO___ |
CloseAsync(tp); |
CloseAsync(fp); |
@@ -655,7 +690,7 @@ |
* |
* Merges temporary files "tmpNames" into a database "dbFile". |
*/ |
void MergeTemporaryFiles(char *dbFile, char **tmpNames, int files, int lines) { |
void MergeTemporaryFiles(char *dbFile, VECTOR *tmpNames, int lines) { |
#if defined ___AsyncIO___ |
struct AsyncFile *fp; |
struct AsyncFile **tp; |
@@ -665,7 +700,7 @@ |
#endif |
int i; |
int j; |
char *tmp; |
dbLine *tmp; |
char *rem; |
char *min; |
int count; |
@@ -680,7 +715,7 @@ |
} |
|
// Allocate as many file pointers as temporary files. |
if((tp = malloc(files * sizeof(*tp))) == NULL) { |
if((tp = malloc(tmpNames->length * sizeof(*tp))) == NULL) { |
fprintf(stderr, "Memory allocation failure.\n"); |
#if defined ___AsyncIO___ |
CloseAsync(fp); |
@@ -691,13 +726,13 @@ |
} |
|
// Open all temporary files for reading. |
for(i = 0; i < files; ++i) { |
for(i = 0; i < tmpNames->length; ++i) { |
#if defined ___AsyncIO___ |
if((tp[i] = OpenAsync(tmpNames[i], MODE_READ, ASYNC_BUF)) == NULL) { |
if((tp[i] = OpenAsync(tmpNames->array[i], MODE_READ, ASYNC_BUF)) == NULL) { |
#else |
if((tp[i] = fopen(tmpNames[i], "r")) == NULL) { |
if((tp[i] = fopen(tmpNames->array[i], "r")) == NULL) { |
#endif |
fprintf(stderr, "Could not open file '%s' for reading.\n", tmpNames[i]); |
fprintf(stderr, "Could not open file '%s' for reading.\n", tmpNames->array[i]); |
// Close all temporary files. |
while(--i > -1) { |
#if defined ___AsyncIO___ |
@@ -736,28 +771,35 @@ |
} |
|
min = NULL; |
for(i = 0; i < files; ++i) { |
for(i = 0; i < tmpNames->length; ++i) { |
tmp = PeekLine(tp[i]); |
if(tmp == NULL) { |
continue; |
} |
#if defined ___AmigaOS___ |
if(min == NULL || StrnCmp(locale, tmp, min, -1, SC_ASCII) < 0) { |
if(min == NULL || StrnCmp(locale, tmp->string, min, -1, SC_ASCII) < 0) { |
#else |
if(min == NULL || strcmp(tmp, min) < 0) { |
if(min == NULL || strcmp(tmp->string, min) < 0) { |
#endif |
if(min != NULL) { |
// Free previous instance. |
free(min); |
min = NULL; |
} |
if((min = malloc((strlen(tmp) + 1) * sizeof(*min))) == NULL) { |
if((min = malloc((strlen(tmp->string) + 1) * sizeof(*min))) == NULL) { |
fprintf(stderr, "Memory allocation failure.\n"); |
|
free(tmp->string); |
free(tmp); |
tmp = NULL; |
|
if(min != NULL) { |
free(min); |
min = NULL; |
} |
if(rem != NULL) { |
free(rem); |
rem = NULL; |
} |
#if defined ___AsyncIO___ |
CloseAsync(fp); |
@@ -766,11 +808,13 @@ |
#endif |
return; |
} |
sprintf(min, "%s", tmp); |
sprintf(min, "%s", tmp->string); |
// Remember the index of the file where the smallest entry has been found. |
j = i; |
} |
free(tmp->string); |
free(tmp); |
tmp = NULL; |
} |
|
// Forward the file where the smallest line was found. |
@@ -786,6 +830,7 @@ |
if(strcmp(min, rem) == 0) { |
#endif |
free(min); |
min = NULL; |
continue; |
} |
} |
@@ -799,11 +844,15 @@ |
|
if(rem != NULL) { |
free(rem); |
rem = NULL; |
} |
|
if((rem = malloc((strlen(min) + 1) * sizeof(*rem))) == NULL) { |
fprintf(stderr, "Memory allocation failure.\n"); |
|
free(min); |
min = NULL; |
|
#if defined ___AsyncIO___ |
CloseAsync(fp); |
#else |
@@ -812,17 +861,21 @@ |
return; |
} |
|
// Rememb er the last minimal line. |
sprintf(rem, "%s", min); |
|
free(min); |
min = NULL; |
} |
} |
|
if(rem != NULL) { |
free(rem); |
rem = NULL; |
} |
|
// Write out any remaining contents from the temporary files. |
for(i = 0; PROGRAM_RUN && i < files; ++i) { |
for(i = 0; PROGRAM_RUN && i < tmpNames->length; ++i) { |
#if defined ___AmigaOS___ |
// Check if CTRL+C was pressed and abort the program. |
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) { |
@@ -835,16 +888,18 @@ |
continue; |
} |
#if defined ___AsyncIO___ |
WriteAsync(fp, tmp, (LONG)strlen(tmp)); |
WriteAsync(fp, tmp->string, (LONG)strlen(tmp->string)); |
WriteAsync(fp, "\n", 1); |
#else |
fprintf(fp, "%s\n", tmp); |
fprintf(fp, "%s\n", tmp->string); |
#endif |
free(tmp->string); |
free(tmp); |
tmp = NULL; |
} |
|
// Close all temporary files. |
for(i = 0; i < files; ++i) { |
for(i = 0; i < tmpNames->length; ++i) { |
#if defined ___AsyncIO___ |
CloseAsync(tp[i]); |
#else |
@@ -867,7 +922,7 @@ |
* |
* Filter the paths inside the database with provided paths. |
*/ |
void FilterDatabasePaths(char *dbFile, char *tmpName, char **paths, int count) { |
void FilterDatabasePaths(char *dbFile, char *tmpName, VECTOR *paths) { |
#if defined ___AsyncIO___ |
struct AsyncFile *fp; |
struct AsyncFile *tp; |
@@ -875,10 +930,10 @@ |
FILE *fp; |
FILE *tp; |
#endif |
char *line; |
dbLine *line; |
dbEntry *entry; |
int lines; |
int i; |
dbEntry *entry; |
|
// Open database file for reading. |
#if defined ___AsyncIO___ |
@@ -921,14 +976,17 @@ |
continue; |
} |
#endif |
|
if((entry = CreateDatabaseEntry(line)) == NULL) { |
fprintf(stderr, "Unable to create database entry.\n"); |
free(line->string); |
free(line); |
line = NULL; |
continue; |
} |
|
for(i = 0; i < count; ++i) { |
if(PathCompare(entry->path, paths[i]) == TRUE) { |
for(i = 0; i < paths->length; ++i) { |
if(PathCompare(entry->path, paths->array[i]) == TRUE) { |
++lines; |
if(PROGRAM_VERBOSE) { |
fprintf(stdout, "Removing lines: %d.\r", lines); |
@@ -936,20 +994,24 @@ |
continue; |
} |
#if defined ___AsyncIO___ |
WriteAsync(tp, line, (LONG)strlen(line)); |
WriteAsync(tp, line->string, (LONG)strlen(line->string)); |
WriteAsync(tp, "\n", 1); |
#else |
fprintf(tp, "%s\n", line); |
fprintf(tp, "%s\n", line->string); |
#endif |
break; |
} |
|
|
// Free up database entry. |
free(entry->name); |
free(entry->path); |
free(entry); |
entry = NULL; |
|
// Free up line. |
free(line->string); |
free(line); |
line = NULL; |
} |
|
#if defined ___AsyncIO___ |
@@ -969,15 +1031,15 @@ |
* |
* Indexes a "path" by creating a database "dbFile". |
*/ |
void GatherDatabaseFiles(char *dbFile, char **paths, int count) { |
void GatherDatabaseFiles(char *dbFile, VECTOR *paths) { |
dbStats *stats; |
char **tmpNames; |
VECTOR *tmpNames; |
int tmpFiles; |
int tmpLines; |
int i; |
|
// Generate the database file from the supplied paths. |
if((stats = CollectFiles(dbFile, paths, count)) == NULL) { |
if((stats = CollectFiles(dbFile, paths)) == NULL) { |
fprintf(stderr, "Collecting files failed.\n"); |
return; |
} |
@@ -994,7 +1056,7 @@ |
} |
|
// Calculate the number of lines per temporary file. |
tmpLines = stats->lines / tmpFiles; |
tmpLines = ceil(((double)stats->lines) / ((double)tmpFiles)); |
|
// Create temporary files. |
if((tmpNames = CreateTemporaryFiles(tmpFiles)) == NULL) { |
@@ -1003,16 +1065,15 @@ |
} |
|
// Write "tmpLines" to temporary files in "tmpNames" from "dbFile". |
WriteTemporaryFiles(dbFile, tmpNames, tmpFiles, tmpLines, stats->lines); |
WriteTemporaryFiles(dbFile, tmpNames, tmpLines, stats->lines); |
|
// Sort the temporary files. |
for(i = 0; i < tmpFiles; ++i) { |
// Twice the number of computed lines required should cover the last file. |
SortDatabase(tmpNames[i], 2 * tmpLines); |
for(i = 0; i < tmpNames->length; ++i) { |
SortDatabase(tmpNames->array[i], tmpLines); |
} |
|
// Merge all the temporary files to the database file. |
MergeTemporaryFiles(dbFile, tmpNames, tmpFiles, stats->lines); |
MergeTemporaryFiles(dbFile, tmpNames, stats->lines); |
|
// Remove all temporary files. |
RemoveFiles(tmpNames, tmpFiles); |
@@ -1019,12 +1080,14 @@ |
|
// Free temporary file names. |
free(tmpNames); |
tmpNames = NULL; |
|
// Free statistics. |
free(stats); |
stats = NULL; |
} |
|
void RemoveDatabaseFiles(char *dbFile, char **paths, int count) { |
void RemoveDatabaseFiles(char *dbFile, VECTOR *paths) { |
char *tmpName; |
|
// Create a temporary file to hold the changes. |
@@ -1034,7 +1097,7 @@ |
} |
|
// Filter the database of the provided paths. |
FilterDatabasePaths(dbFile, tmpName, paths, count); |
FilterDatabasePaths(dbFile, tmpName, paths); |
|
// Overwrite the database file with the filtered paths. |
CopyFile(tmpName, dbFile); |
@@ -1075,10 +1138,9 @@ |
int main(int argc, char **argv) { |
int option; |
int i; |
int count; |
char *dbFile; |
char *path; |
char **paths; |
VECTOR *paths; |
OPERATION operation = NONE; |
|
// Bind handler to SIGINT. |
@@ -1126,14 +1188,19 @@ |
return 5; |
} |
|
// Build the path vector. |
if((paths = malloc(1 * sizeof(*paths))) == NULL) { |
fprintf(stderr, "Memory allocation failure.\n"); |
return 20; |
} |
|
// Go through all supplied arguments and add paths to search. |
if((paths = malloc((argc - optind) * sizeof(*paths))) == NULL) { |
if((paths->array = malloc((argc - optind) * sizeof(*paths))) == NULL) { |
fprintf(stderr, "Memory allocation failure.\n"); |
return 20; |
} |
|
count = 0; |
for(i = optind, count = 0; i < argc; ++i) { |
for(i = optind, paths->length = 0; i < argc; ++i) { |
if((path = PathToAbsolute(argv[i])) == NULL) { |
fprintf(stderr, "Absolute path for '%s' failed to resolve.\n", argv[optind]); |
continue; |
@@ -1144,8 +1211,8 @@ |
case REGULAR: |
fprintf(stderr, "Path '%s' is not a directory.\n", path); |
free(path); |
path = NULL; |
continue; |
break; |
case DIRECTORY: |
break; |
} |
@@ -1155,21 +1222,24 @@ |
} |
|
// Add the path to the array of paths. |
if((paths[count] = malloc((strlen(path) + 1) * sizeof(*paths[count]))) == NULL) { |
if((paths->array[paths->length] = malloc((strlen(path) + 1) * sizeof(*paths->array[paths->length]))) == NULL) { |
fprintf(stderr, "Memory allocation failure."); |
return 20; |
} |
|
sprintf(paths[count], "%s", path); |
++count; |
sprintf(paths->array[paths->length], "%s", path); |
++paths->length; |
|
free(path); |
path = NULL; |
|
} |
|
if(count == 0) { |
if(paths->length == 0) { |
fprintf(stderr, "No valid paths are available.\n"); |
free(paths->array); |
free(paths); |
paths = NULL; |
return 5; |
} |
|
@@ -1191,13 +1261,13 @@ |
if(PROGRAM_VERBOSE) { |
fprintf(stdout, "Gathering files to database...\n"); |
} |
GatherDatabaseFiles(dbFile, paths, count); |
GatherDatabaseFiles(dbFile, paths); |
break; |
case REMOVE: |
if(PROGRAM_VERBOSE) { |
fprintf(stdout, "Removing files from database...\n"); |
} |
RemoveDatabaseFiles(dbFile, paths, count); |
RemoveDatabaseFiles(dbFile, paths); |
break; |
default: |
fprintf(stderr, "Unknown operation.\n"); |
@@ -1206,6 +1276,7 @@ |
#endif |
|
free(paths); |
paths = NULL; |
return 5; |
} |
|
@@ -1214,6 +1285,6 @@ |
#endif |
|
free(paths); |
|
paths = NULL; |
return 0; |
} |