HuntnGather

Subversion Repositories:
Compare Path: Rev
With Path: Rev
?path1? @ 40  →  ?path2? @ 41
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
/trunk/HuntnGather/C/Gather
/trunk/HuntnGather/Gather/Gather.c
@@ -550,14 +550,13 @@
void WriteTemporaryFiles(char *dbFile, VECTOR *tmpNames, int tmpLines, int total) {
#if defined ___AsyncIO___
struct AsyncFile *fp, *tp;
LONG c;
#else
FILE *fp, *tp;
char c;
#endif
int lines;
int write;
int files;
dbLine *line = NULL;
 
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
@@ -589,90 +588,76 @@
 
write = 0;
lines = 0;
#if defined ___AsyncIO___
while(PROGRAM_RUN && (c = ReadCharAsync(fp)) != -1) {
#else
while(PROGRAM_RUN && fscanf(fp, "%c", &c) == 1) {
#endif
 
while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
free(line->string);
free(line);
line = NULL;
 
PROGRAM_RUN = FALSE;
continue;
}
#endif
switch(c) {
case '\n':
// Increment the total written lines.
++write;
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)write / total) * 100.0));
}
 
// Write the newline character back.
#if defined ___AsyncIO___
if(WriteCharAsync(tp, (UBYTE)c) != 1) {
WriteAsync(tp, line->string, (LONG)line->length);
WriteAsync(tp, "\n", 1);
#else
if(fprintf(tp, "%c", c) != 1) {
fprintf(tp, "%s\n", line->string);
#endif
fprintf(stderr, "Unable to write to '%s'.\n", (char *)tmpNames->array[files]);
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
return;
}
// Switch to the next temporary file.
if(++lines >= tmpLines) {
// If there are no temporary files left then run till the end.
if(files - 1 < 0) {
break;
}
 
// Close the previous temporary file and write to the next temporary file.
++write;
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)write / total) * 100.0));
}
 
// Switch to the next temporary file.
if(++lines >= tmpLines) {
// If there are no temporary files left then run till the end.
if(files - 1 < 0) {
free(line->string);
free(line);
line = NULL;
continue;
}
 
// Close the previous temporary file and write to the next temporary file.
#if defined ___AsyncIO___
CloseAsync(tp);
if((tp = OpenAsync(tmpNames->array[--files], MODE_WRITE, ASYNC_BUF)) == NULL) {
CloseAsync(tp);
if((tp = OpenAsync(tmpNames->array[--files], MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
fclose(tp);
if((tp = fopen(tmpNames->array[--files], "w")) == NULL) {
fclose(tp);
if((tp = fopen(tmpNames->array[--files], "w")) == NULL) {
#endif
fprintf(stderr, "Could not open '%s' for writing.\n", (char *)tmpNames->array[files]);
fprintf(stderr, "Could not open '%s' for writing.\n", (char *)tmpNames->array[files]);
#if defined ___AsyncIO___
CloseAsync(fp);
CloseAsync(fp);
#else
fclose(fp);
fclose(fp);
#endif
return;
}
lines = 0;
break;
}
break;
default:
#if defined ___AsyncIO___
if(WriteCharAsync(tp, (UBYTE)c) != 1) {
#else
if(fprintf(tp, "%c", c) != 1) {
#endif
fprintf(stderr, "Could not write to file '%s'.\n", (char *)tmpNames->array[files]);
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
return;
}
break;
free(line->string);
free(line);
line = NULL;
return;
}
lines = 0;
}
 
free(line->string);
free(line);
line = NULL;
}
 
if(line != NULL) {
free(line->string);
free(line);
line = NULL;
}
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "\n");
}
@@ -1029,7 +1014,7 @@
 
/*
*
* Indexes a "path" by creating a database "dbFile".
* Indexes paths and adds to a database file.
*/
void GatherDatabaseFiles(char *dbFile, VECTOR *paths) {
dbStats *stats;
@@ -1037,6 +1022,8 @@
int tmpFiles;
int tmpLines;
int i;
int line;
int size;
 
// Generate the database file from the supplied paths.
if((stats = CollectFiles(dbFile, paths)) == NULL) {
@@ -1044,7 +1031,70 @@
return;
}
 
size = GetFileSize(dbFile);
line = CountFileLines(dbFile);
 
// Calculate the total number of temporary files required.
tmpFiles = size / maxmem;
 
/* In case no temporary files are required,
* just sort the database and terminate.
*/
if(tmpFiles < 2) {
SortDatabase(dbFile, line);
return;
}
 
// Calculate the number of lines per temporary file.
tmpLines = ceil(((double)line) / ((double)tmpFiles));
 
// Create temporary files.
if((tmpNames = CreateTemporaryFiles(tmpFiles)) == NULL) {
fprintf(stderr, "Unable to create temporary files.\n");
return;
}
 
// Write "tmpLines" to temporary files in "tmpNames" from "dbFile".
WriteTemporaryFiles(dbFile, tmpNames, tmpLines, line);
 
// Sort the temporary files.
for(i = 0; i < tmpNames->length; ++i) {
SortDatabase(tmpNames->array[i], tmpLines);
}
 
// Merge all the temporary files to the database file.
MergeTemporaryFiles(dbFile, tmpNames, line);
 
// Remove all temporary files.
RemoveFiles(tmpNames);
 
// Free temporary file names.
free(tmpNames);
tmpNames = NULL;
 
// Free statistics.
free(stats);
stats = NULL;
}
 
/*
*
* Indexes paths and creates a daabase file.
*/
void CreateDatabaseFiles(char *dbFile, VECTOR *paths) {
dbStats *stats;
VECTOR *tmpNames;
int tmpFiles;
int tmpLines;
int i;
 
// Generate the database file from the supplied paths.
if((stats = CollectFiles(dbFile, paths)) == NULL) {
fprintf(stderr, "Collecting files failed.\n");
return;
}
 
// Calculate the total number of temporary files required.
tmpFiles = stats->size / maxmem;
 
/* In case no temporary files are required,
@@ -1257,6 +1307,11 @@
fprintf(stdout, "Removing '%s' and creating a new database.\n", dbFile);
}
RemoveFile(dbFile);
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Gathering files to database...\n");
}
CreateDatabaseFiles(dbFile, paths);
break;
case GATHER:
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Gathering files to database...\n");
@@ -1275,6 +1330,7 @@
CloseLocale(locale);
#endif
 
free(paths->array);
free(paths);
paths = NULL;
return 5;
@@ -1284,7 +1340,11 @@
CloseLocale(locale);
#endif
 
free(paths);
paths = NULL;
if(paths != NULL) {
free(paths->array);
free(paths);
paths = NULL;
}
 
return 0;
}
/trunk/HuntnGather/shared/utilities.c
@@ -130,17 +130,54 @@
 
/*
*
* Get the size of a file.
*/
int GetFileSize(char *dbFile) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
#else
FILE *fp;
#endif
int size;
 
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Could not open file '%s' for reading.\n", dbFile);
return -1;
}
 
#if defined ___AsyncIO___
SeekAsync(fp, 0, MODE_END);
size = SeekAsync(fp, 0, MODE_CURRENT);
#else
fseek(fp, 0L, SEEK_END);
size = ftell(fp);
#endif
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
 
return size;
}
 
/*
*
* Counts the lines of a file.
*/
int CountFileLines(char *dbFile) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
LONG c;
#else
FILE *fp;
char c;
#endif
int lines;
dbLine *line = NULL;
 
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
@@ -151,16 +188,12 @@
return -1;
}
 
lines = 0;
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Counting lines in '%s'...\r", dbFile);
}
 
#if defined ___AsyncIO___
while(PROGRAM_RUN && (c = ReadCharAsync(fp)) != -1) {
#else
while(PROGRAM_RUN && fscanf(fp, "%c", &c) == 1) {
#endif
lines = 0;
while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
@@ -168,17 +201,24 @@
continue;
}
#endif
switch(c) {
case '\n':
++lines;
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Counting lines in '%s': %d.\r", dbFile, lines);
}
break;
++lines;
 
if(PROGRAM_VERBOSE) {
fprintf(stdout, "Counting lines in '%s': %d.\r", dbFile, lines);
}
 
free(line->string);
free(line);
line = NULL;
}
 
if(line != NULL) {
free(line->string);
free(line);
line = NULL;
}
 
#if defined ___AsyncIO___
CloseAsync(fp);
#else
/trunk/HuntnGather/shared/utilities.h
@@ -78,6 +78,7 @@
extern void StrUpr(char *);
extern FS_TYPE GetFsType(char *path);
extern int CountFileLines(char *dbFile);
extern int GetFileSize(char *dbFile);
extern char *PathToAbsolute(char *path);
extern char *CreateTemporaryFile(void);
extern VECTOR *CreateTemporaryFiles(int files);