/trunk/HuntnGather/Gather/Gather.c |
@@ -550,14 +550,13 @@ |
void WriteTemporaryFiles(char *dbFile, VECTOR *tmpNames, int tmpLines, int total) { |
#if defined ___AsyncIO___ |
struct AsyncFile *fp, *tp; |
LONG c; |
#else |
FILE *fp, *tp; |
char c; |
#endif |
int lines; |
int write; |
int files; |
dbLine *line = NULL; |
|
#if defined ___AsyncIO___ |
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) { |
@@ -589,21 +588,27 @@ |
|
write = 0; |
lines = 0; |
#if defined ___AsyncIO___ |
while(PROGRAM_RUN && (c = ReadCharAsync(fp)) != -1) { |
#else |
while(PROGRAM_RUN && fscanf(fp, "%c", &c) == 1) { |
#endif |
|
while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) { |
#if defined ___AmigaOS___ |
// Check if CTRL+C was pressed and abort the program. |
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) { |
free(line->string); |
free(line); |
line = NULL; |
|
PROGRAM_RUN = FALSE; |
continue; |
} |
#endif |
switch(c) { |
case '\n': |
// Increment the total written lines. |
|
#if defined ___AsyncIO___ |
WriteAsync(tp, line->string, (LONG)line->length); |
WriteAsync(tp, "\n", 1); |
#else |
fprintf(tp, "%s\n", line->string); |
#endif |
|
++write; |
|
if(PROGRAM_VERBOSE) { |
@@ -610,27 +615,14 @@ |
fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)write / total) * 100.0)); |
} |
|
// Write the newline character back. |
#if defined ___AsyncIO___ |
if(WriteCharAsync(tp, (UBYTE)c) != 1) { |
#else |
if(fprintf(tp, "%c", c) != 1) { |
#endif |
fprintf(stderr, "Unable to write to '%s'.\n", (char *)tmpNames->array[files]); |
#if defined ___AsyncIO___ |
CloseAsync(tp); |
CloseAsync(fp); |
#else |
fclose(tp); |
fclose(fp); |
#endif |
return; |
} |
// Switch to the next temporary file. |
if(++lines >= tmpLines) { |
// If there are no temporary files left then run till the end. |
if(files - 1 < 0) { |
break; |
free(line->string); |
free(line); |
line = NULL; |
continue; |
} |
|
// Close the previous temporary file and write to the next temporary file. |
@@ -647,31 +639,24 @@ |
#else |
fclose(fp); |
#endif |
free(line->string); |
free(line); |
line = NULL; |
return; |
} |
lines = 0; |
break; |
} |
break; |
default: |
#if defined ___AsyncIO___ |
if(WriteCharAsync(tp, (UBYTE)c) != 1) { |
#else |
if(fprintf(tp, "%c", c) != 1) { |
#endif |
fprintf(stderr, "Could not write to file '%s'.\n", (char *)tmpNames->array[files]); |
#if defined ___AsyncIO___ |
CloseAsync(tp); |
CloseAsync(fp); |
#else |
fclose(tp); |
fclose(fp); |
#endif |
return; |
|
free(line->string); |
free(line); |
line = NULL; |
} |
break; |
|
if(line != NULL) { |
free(line->string); |
free(line); |
line = NULL; |
} |
} |
|
if(PROGRAM_VERBOSE) { |
fprintf(stdout, "\n"); |
@@ -1029,7 +1014,7 @@ |
|
/* |
* |
* Indexes a "path" by creating a database "dbFile". |
* Indexes paths and adds to a database file. |
*/ |
void GatherDatabaseFiles(char *dbFile, VECTOR *paths) { |
dbStats *stats; |
@@ -1037,6 +1022,8 @@ |
int tmpFiles; |
int tmpLines; |
int i; |
int line; |
int size; |
|
// Generate the database file from the supplied paths. |
if((stats = CollectFiles(dbFile, paths)) == NULL) { |
@@ -1044,7 +1031,70 @@ |
return; |
} |
|
size = GetFileSize(dbFile); |
line = CountFileLines(dbFile); |
|
// Calculate the total number of temporary files required. |
tmpFiles = size / maxmem; |
|
/* In case no temporary files are required, |
* just sort the database and terminate. |
*/ |
if(tmpFiles < 2) { |
SortDatabase(dbFile, line); |
return; |
} |
|
// Calculate the number of lines per temporary file. |
tmpLines = ceil(((double)line) / ((double)tmpFiles)); |
|
// Create temporary files. |
if((tmpNames = CreateTemporaryFiles(tmpFiles)) == NULL) { |
fprintf(stderr, "Unable to create temporary files.\n"); |
return; |
} |
|
// Write "tmpLines" to temporary files in "tmpNames" from "dbFile". |
WriteTemporaryFiles(dbFile, tmpNames, tmpLines, line); |
|
// Sort the temporary files. |
for(i = 0; i < tmpNames->length; ++i) { |
SortDatabase(tmpNames->array[i], tmpLines); |
} |
|
// Merge all the temporary files to the database file. |
MergeTemporaryFiles(dbFile, tmpNames, line); |
|
// Remove all temporary files. |
RemoveFiles(tmpNames); |
|
// Free temporary file names. |
free(tmpNames); |
tmpNames = NULL; |
|
// Free statistics. |
free(stats); |
stats = NULL; |
} |
|
/* |
* |
* Indexes paths and creates a daabase file. |
*/ |
void CreateDatabaseFiles(char *dbFile, VECTOR *paths) { |
dbStats *stats; |
VECTOR *tmpNames; |
int tmpFiles; |
int tmpLines; |
int i; |
|
// Generate the database file from the supplied paths. |
if((stats = CollectFiles(dbFile, paths)) == NULL) { |
fprintf(stderr, "Collecting files failed.\n"); |
return; |
} |
|
// Calculate the total number of temporary files required. |
tmpFiles = stats->size / maxmem; |
|
/* In case no temporary files are required, |
@@ -1257,6 +1307,11 @@ |
fprintf(stdout, "Removing '%s' and creating a new database.\n", dbFile); |
} |
RemoveFile(dbFile); |
if(PROGRAM_VERBOSE) { |
fprintf(stdout, "Gathering files to database...\n"); |
} |
CreateDatabaseFiles(dbFile, paths); |
break; |
case GATHER: |
if(PROGRAM_VERBOSE) { |
fprintf(stdout, "Gathering files to database...\n"); |
@@ -1275,6 +1330,7 @@ |
CloseLocale(locale); |
#endif |
|
free(paths->array); |
free(paths); |
paths = NULL; |
return 5; |
@@ -1284,7 +1340,11 @@ |
CloseLocale(locale); |
#endif |
|
if(paths != NULL) { |
free(paths->array); |
free(paths); |
paths = NULL; |
} |
|
return 0; |
} |