HuntnGather – Rev 22
?pathlinks?
///////////////////////////////////////////////////////////////////////////
// Copyright (C) 2021 Wizardry and Steamworks - License: MIT //
///////////////////////////////////////////////////////////////////////////
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/syslimits.h>
#include <proto/dos.h>
#include <proto/exec.h>
#if defined ___AsyncIO___
#include <asyncio.h>
#endif
#include "StringStack.h"
#if !defined ___HAVE_GETOPT___
#include "getopt.h"
#endif
#define PROGRAM_VERSION "1.7.3"
#if defined ___AmigaOS___
/*************************************************************************/
/* Version string used for querrying the program version. */
/*************************************************************************/
TEXT version_string[] =
"\0$VER: Gather " PROGRAM_VERSION " "__DATE__" by Wizardry and Steamworks";
#endif
#if !defined TRUE
#define TRUE 1;
#endif
#if !defined FALSE
#define FALSE 0;
#endif
#define ASYNC_BUF 8192
#define MAX_MEM 262144
#define NAME_BUF 32
#define PATH_BUF 128
#define LINE_BUF 256
#define DEFAULT_DATABASE_FILE "S:gather.db"
typedef struct {
unsigned int dirs;
unsigned int files;
} stats;
int run = TRUE;
int verbose = TRUE;
void SignalHandler(int sig) {
// Toggle the run flag to stop execution.
run = FALSE;
}
int compare(const void *a, const void *b) {
const char **p = (const char **)a;
const char **q = (const char **)b;
return strncmp(*p, *q, strlen(*p));
}
/*
*
* Sorts a database file lexicographically.
*/
void SortDatabase(char *dbFile) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
LONG c;
#else
FILE *fp;
char c;
#endif
char *name = NULL;
char *path = NULL;
char **database;
int i;
int side;
unsigned int line;
int name_size;
int path_size;
// Open database file for reading.
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Unable to open gather database for reading.\n");
return;
}
database = malloc(sizeof(*database));
name_size = NAME_BUF;
name = malloc(name_size * sizeof(*name));
path_size = PATH_BUF;
path = malloc(path_size * sizeof(*path));
line = 0;
side = 0;
i = 0;
if(verbose) {
fprintf(stdout, "Sorting database: '%s'\n", dbFile);
}
#if defined ___AsyncIO___
while(run && (c = ReadCharAsync(fp)) != -1) {
#else
while(run && fscanf(fp, "%c", &c) == 1) {
#endif
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
continue;
}
#endif
switch(c) {
case '\n':
// Load up the name and path into the database variable.
database = realloc(database, (line + 1) * sizeof(*database));
database[line] = malloc((strlen(name) + strlen(path) + 1 + 1) * sizeof(*database[line]));
sprintf(database[line], "%s\t%s", name, path);
++line;
free(name);
name_size = NAME_BUF;
name = malloc(name_size * sizeof(*name));
--side;
i = 0;
break;
case '\t':
free(path);
path_size = PATH_BUF;
path = malloc(path_size * sizeof(*path));
++side;
i = 0;
break;
default:
switch(side) {
case 0:
if(strlen(name) == name_size) {
name_size = name_size * 1.5;
name = realloc(name, name_size * sizeof(*name));
}
//name = realloc(name, (i + 1 + 1) * sizeof(char));
name[i] = c;
name[i + 1] = '\0';
break;
case 1:
if(strlen(path) == path_size) {
path_size = path_size * 1.5;
path = realloc(path, path_size * sizeof(*path));
}
//path = realloc(path, (i + 1 + 1) * sizeof(char));
path[i] = c;
path[i + 1] = '\0';
break;
default:
fprintf(stderr, "Database corrupted: %d\n", side);
break;
}
++i;
break;
}
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
// Sort the database.
qsort(database, line, sizeof(char *), compare);
// Write the database lines back to the database.
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ|MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "w+")) == NULL) {
#endif
fprintf(stderr, "Unable to open gather database for writing.\n");
return;
}
for(i = 0; i < line; ++i) {
#if defined ___AsyncIO___
WriteAsync(fp, database[i], (LONG)(strlen(database[i]) * sizeof(char)));
WriteAsync(fp, "\n", 1 * sizeof(char));
#else
fprintf(fp, "%s\n", database[i]);
#endif
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
free(database);
}
/*
*
* Updates a database file "dbFile".
*/
void UpdateDatabase(char *dbFile, stringStack *dirStack, stats *stats) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
#else
FILE *fp;
#endif
DIR *dir;
struct dirent *dirEntry;
struct stat dirStat;
unsigned int size;
char *path;
char *subPath;
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ|MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "w+")) == NULL) {
#endif
fprintf(stderr, "Unable to open gather database for writing.\n");
return;
}
while(run && !stringStackIsEmpty(dirStack)) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
if((path = stringStackPop(dirStack)) == NULL) {
return;
}
if((dir = opendir(path)) == NULL) {
return;
}
while(run && (dirEntry = readdir(dir)) != NULL) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
size = sizeof(path) + sizeof(dirEntry->d_name) + 1;
switch(path[strlen(path) - 1]) {
case '/':
case ':': // This is a drive path.
subPath = malloc(size);
sprintf(subPath, "%s%s", path, dirEntry->d_name);
break;
default:
subPath = malloc(size + 1);
sprintf(subPath, "%s/%s", path, dirEntry->d_name);
break;
}
stat(subPath, &dirStat);
if(S_ISDIR(dirStat.st_mode)) {
stringStackPush(dirStack, subPath);
++stats->dirs;
if(verbose) {
fprintf(stdout,
"Gathered %d directories and %d files.\r",
stats->dirs,
stats->files);
}
free(subPath);
continue;
}
// Write to database file.
#if defined ___NOCASE_FS___
strupr(dirEntry->d_name);
#endif
#if defined ___AsyncIO___
WriteAsync(fp, dirEntry->d_name, (LONG)(strlen(dirEntry->d_name) * sizeof(char)));
WriteAsync(fp, "\t", 1 * sizeof(char));
WriteAsync(fp, subPath, (LONG)(strlen(subPath) * sizeof(char)));
WriteAsync(fp, "\n", 1 * sizeof(char));
#else
fprintf(fp, "%s\t%s\n", dirEntry->d_name, subPath);
#endif
++stats->files;
if(verbose) {
fprintf(stdout,
"Gathered %d directories and %d files.\r",
stats->dirs,
stats->files);
}
free(subPath);
}
closedir(dir);
free(path);
}
if(verbose) {
fprintf(stdout, "\n");
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
}
/*
*
* Gets the size of a database "dbFle".
*/
int GetDatabaseSize(char *dbFile) {
FILE *fp;
int size;
if((fp = fopen(dbFile, "r")) == NULL) {
fprintf(stderr, "Unable to open gather database for reading.\n");
fclose(fp);
return 0;
}
fseek(fp, 0L, SEEK_END);
size = ftell(fp);
fclose(fp);
return size;
}
/*
*
* Counts the lines in a database file "dbFile".
*/
int CountDatabaseLines(char *dbFile) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
LONG c;
#else
FILE *fp;
char c;
#endif
int lines;
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Unable to open gather database for reading.\n");
return 0;
}
lines = 0;
#if defined ___AsyncIO___
while((c = ReadCharAsync(fp)) != -1) {
#else
while(fscanf(fp, "%c", &c) == 1) {
#endif
switch(c) {
case '\n':
++lines;
break;
}
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
return lines;
}
/*
*
* Creates "files" temporary filenames.
*/
char **CreateTempFiles(int files) {
char **tmpNames;
int count;
tmpNames = malloc(files * sizeof(char *));
if(verbose) {
fprintf(stdout, "Creating temporary files.\r");
}
count = files;
while(--count > -1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
tmpNames[count] = tmpnam(NULL);
if(verbose) {
fprintf(stdout, "Creating temporary files: %d%%\r", 100 - (int)(((float)count / files) * 100.0));
}
}
if(verbose) {
fprintf(stdout, "\n");
}
return tmpNames;
}
/*
*
* Writes lines from the database "dbFile" to temporary filenames "tmpNames".
*/
void WriteTempFiles(char *dbFile, char **tmpNames, int tmpFiles, int tmpLines, int total) {
#if defined ___AsyncIO___
struct AsyncFile *fp, *tp;
LONG c;
#else
FILE *fp, *tp;
char c;
#endif
int lines;
int linesWritten;
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "r")) == NULL) {
#endif
fprintf(stderr, "Unable to open gather database for reading.\n");
return;
}
#if defined ___AsyncIO___
if((tp = OpenAsync(tmpNames[--tmpFiles], MODE_READ|MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((tp = fopen(tmpNames[--tmpFiles], "w+")) == NULL) {
#endif
fprintf(stderr, "Unable to open temporary file '%s' for writing.\n", tmpNames[tmpFiles]);
return;
}
if(verbose) {
fprintf(stdout, "Writing to temporary files.\r");
}
linesWritten = 0;
lines = 0;
#if defined ___AsyncIO___
while(run && (c = ReadCharAsync(fp)) != -1) {
#else
while(run && fscanf(fp, "%c", &c) == 1) {
#endif
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
switch(c) {
case '\n':
// Increment the total written lines.
++linesWritten;
if(verbose) {
fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)linesWritten / total) * 100.0));
}
// Write the newline character back.
#if defined ___AsyncIO___
if(WriteCharAsync(tp, (UBYTE)c) != 1) {
#else
if(fprintf(tp, "%c", c) != 1) {
#endif
fprintf(stderr, "Unable to write to temporary file '%s'.\n", tmpNames[tmpFiles]);
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
return;
}
// Switch to the next temporary file.
if(++lines >= tmpLines) {
// If there are no temporary files left then run till the end.
if(tmpFiles - 1 < 0) {
break;
}
// Close the previous temporary file and write to the next temporary file.
#if defined ___AsyncIO___
CloseAsync(tp);
if((tp = OpenAsync(tmpNames[--tmpFiles], MODE_READ|MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
fclose(tp);
if((tp = fopen(tmpNames[--tmpFiles], "w+")) == NULL) {
#endif
fprintf(stderr, "Unable to open temporary file '%s' for writing.\n", tmpNames[tmpFiles]);
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
}
lines = 0;
break;
}
break;
default:
#if defined ___AsyncIO___
if(WriteCharAsync(tp, (UBYTE)c) != 1) {
#else
if(fprintf(tp, "%c", c) != 1) {
#endif
fprintf(stderr, "Unable to write to temporary file '%s'.\n", tmpNames[tmpFiles]);
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
return;
}
break;
}
}
fprintf(stdout, "\n");
#if defined ___AsyncIO___
CloseAsync(tp);
CloseAsync(fp);
#else
fclose(tp);
fclose(fp);
#endif
}
/*
*
* Skips a line in a database file "fp".
*/
#if defined ___AsyncIO___
void SkipDatabaseLine(struct AsyncFile *fp) {
LONG c;
while((c = ReadCharAsync(fp)) != -1) {
#else
void SkipDatabaseLine(FILE *fp) {
char c;
while(fscanf(fp, "%c", &c) == 1) {
#endif
switch(c) {
case '\n':
return;
}
}
}
/*
*
* Reads a line from the database file "fp".
*/
#if defined ___AsyncIO___
char *ReadDatabaseLine(struct AsyncFile *fp) {
LONG c;
#else
char *ReadDatabaseLine(FILE *fp) {
char c;
#endif
char *line;
int line_size;
int i;
line_size = LINE_BUF;
line = malloc(line_size * sizeof(*line));
i = 0;
#if defined ___AsyncIO___
while(run && (c = ReadCharAsync(fp)) != -1) {
#else
while(run && fscanf(fp, "%c", &c) == 1) {
#endif
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
switch(c) {
case '\n':
// Rewind the file by the number of read characters.
#if defined ___AsyncIO___
if(SeekAsync(fp, -(i + 1), MODE_CURRENT) == -1) {
fprintf(stderr, "Could not seek in file.\n");
return NULL;
}
#else
fseek(fp, -(i + 1), SEEK_CUR);
#endif
return line;
default:
if(strlen(line) == line_size) {
line_size = line_size * 1.5;
line = realloc(line, line_size * sizeof(*line));
}
//line = realloc(line, (chars + 1 + 1) * sizeof(char));
line[i] = c;
line[i + 1] = '\0';
break;
}
++i;
}
return NULL;
}
/*
*
* Merges temporary files "tmpNames" into a database "dbFile".
*/
void MergeDatabase(char *dbFile, char **tmpNames, int files, int lines) {
#if defined ___AsyncIO___
struct AsyncFile *fp;
struct AsyncFile **tp;
#else
FILE *fp;
FILE **tp;
#endif
int i;
int j;
char *tmp;
char *min;
int count;
#if defined ___AsyncIO___
if((fp = OpenAsync(dbFile, MODE_READ|MODE_WRITE, ASYNC_BUF)) == NULL) {
#else
if((fp = fopen(dbFile, "w+")) == NULL) {
#endif
fprintf(stderr, "Unable to open gather database for writing.\n");
return;
}
// Allocate as many file pointers as temporary files.
tp = malloc(files * sizeof(*tp));
// Open all temporary files for reading.
for(i = 0; i < files; ++i) {
#if defined ___AsyncIO___
if((tp[i] = OpenAsync(tmpNames[i], MODE_READ, ASYNC_BUF)) == NULL) {
#else
if((tp[i] = fopen(tmpNames[i], "r")) == NULL) {
#endif
fprintf(stderr, "Unable to open temporary file '%s' for reading.\n", tmpNames[i]);
// Close all temporary files.
--i;
while(i >= 0) {
#if defined ___AsyncIO___
CloseAsync(tp[i]);
#else
fclose(tp[i]);
#endif
}
return;
}
}
if(verbose) {
fprintf(stdout, "Merging all database lines in temporary files.\r");
}
count = lines;
j = 0;
while(run && --count > -1) {
#if defined ___AmigaOS___
// Check if CTRL+C was pressed and abort the program.
if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) {
run = FALSE;
}
#endif
// Find the smallest line in all temporary files.
if(verbose) {
fprintf(stdout, "Merging all database lines in temporary files: %d%%.\r", 100 - (int)(((float)count / lines) * 100.0));
}
min = NULL;
for(i = 0; i < files; ++i) {
tmp = ReadDatabaseLine(tp[i]);
if(tmp == NULL) {
continue;
}
if(min == NULL || strncmp(tmp, min, strlen(tmp)) < 0) {
if(min != NULL) {
// Free previous instance.
free(min);
}
min = malloc((strlen(tmp) + 1) * sizeof(*min));
sprintf(min, "%s", tmp);
// Remember the index of the file where the smallest entry has been found.
j = i;
free(tmp);
continue;
}
free(tmp);
}
// Forward the file where the smallest line was found.
SkipDatabaseLine(tp[j]);
// Write the smallest line.
if(min != NULL) {
#if defined ___AsyncIO___
WriteAsync(fp, min, (LONG)(strlen(min) * sizeof(char)));
WriteAsync(fp, "\n", 1 * sizeof(char));
#else
fprintf(fp, "%s\n", min);
#endif
free(min);
}
}
// Write out any remaining contents from the temporary files.
for(i = 0; i < files; ++i) {
tmp = ReadDatabaseLine(tp[i]);
if(tmp == NULL) {
continue;
}
#if defined ___AsyncIO___
WriteAsync(fp, tmp, (LONG)(strlen(tmp) * sizeof(char)));
WriteAsync(fp, "\n", 1 * sizeof(char));
#else
fprintf(fp, "%s\n", tmp);
#endif
free(tmp);
}
// Close and delete all temporary files.
for(i = 0; i < files; ++i) {
#if defined ___AsyncIO___
CloseAsync(tp[i]);
#else
fclose(tp[i]);
#endif
// Delete temporary file.
remove(tmpNames[i]);
}
if(verbose) {
fprintf(stdout, "\n");
}
#if defined ___AsyncIO___
CloseAsync(fp);
#else
fclose(fp);
#endif
}
/*
*
* Indexes a "path" by creating a database "dbFile".
*/
void Gather(char *dbFile, char *path) {
stringStack *stack = stringStackCreate(1);
stats *stats = malloc(sizeof(stats));
char **tmpNames;
int dbSize, dbLines, tmpFiles, tmpLines;
int i;
// Initialize metrics.
stats->dirs = 0;
stats->files = 0;
// Push the first path onto the stack.
stringStackPush(stack, path);
// Generate the database file.
UpdateDatabase(dbFile, stack, stats);
// Get the database metrics.
dbSize = GetDatabaseSize(dbFile);
dbLines = CountDatabaseLines(dbFile);
// Compute the amount of temporary files needed.
tmpFiles = dbSize / MAX_MEM;
// In case no temporary files are required,
// just sort the database and terminate.
if(tmpFiles <= 1) {
SortDatabase(dbFile);
return;
}
tmpLines = dbLines / tmpFiles;
// Create temporary files.
if((tmpNames = CreateTempFiles(tmpFiles)) == NULL) {
fprintf(stderr, "Unable to create temporary files.\n");
return;
}
// Write "tmpLines" to temporary files in "tmpFiles" from "dbFile".
WriteTempFiles(dbFile, tmpNames, tmpFiles, tmpLines, dbLines);
// Sort the temporary files.
for(i = 0; i < tmpFiles; ++i) {
SortDatabase(tmpNames[i]);
}
MergeDatabase(dbFile, tmpNames, tmpFiles, dbLines);
}
void usage(char *name) {
fprintf(stdout, "Hunt & Gather - %s, a file index generating tool. \n", name);
fprintf(stdout, "Version: %s \n", PROGRAM_VERSION);
fprintf(stdout, " \n");
fprintf(stdout, "SYNTAX: %s [-q] DATABASE \n", name);
fprintf(stdout, " \n");
fprintf(stdout, " -q Do not print out any messages. \n");
fprintf(stdout, " \n");
fprintf(stdout, "DATABASE is a path to where the indexed results will be \n");
fprintf(stdout, "stored for searching with the Hunt tool. \n");
fprintf(stdout, " \n");
fprintf(stdout, "(c) 2021 Wizardry and Steamworks, MIT. \n");
}
/*
*
* Main entry point.
*/
int main(int argc, char **argv) {
int option;
char *dbFile;
char *path;
struct stat dirStat;
#if defined ___AmigaOS___
BPTR lock;
#endif
// Bind handler to SIGINT.
signal(SIGINT, SignalHandler);
dbFile = DEFAULT_DATABASE_FILE;
while((option = getopt(argc, argv, "hqd:")) != -1) {
switch(option) {
case 'd':
dbFile = optarg;
break;
case 'q':
verbose = FALSE;
break;
case 'h':
usage(argv[0]);
return 0;
case '?':
fprintf(stderr, "Invalid option %ct.\n", optopt);
return 1;
}
}
if(optind >= argc) {
usage(argv[0]);
return 1;
}
#if defined ___AmigaOS___
path = malloc(PATH_MAX * sizeof(*path));
lock = Lock(argv[optind], SHARED_LOCK);
NameFromLock(lock, path, PATH_MAX);
UnLock(lock);
#else
path = realpath(argv[optind], NULL);
#endif
stat(path, &dirStat);
if(!S_ISDIR(dirStat.st_mode)) {
fprintf(stderr, "Path '%s' is not a directory.\n", argv[optind]);
return 1;
}
if(verbose) {
fprintf(stdout, "Gathering to database file: %s\n", dbFile);
}
// Gather.
Gather(dbFile, path);
free(path);
return 0;
}