HuntnGather – Diff between revs 39 and 41
?pathlinks?
Rev 39 | Rev 41 | |||
---|---|---|---|---|
Line 548... | Line 548... | |||
548 | * Writes lines from the database "dbFile" to temporary filenames "tmpNames". |
548 | * Writes lines from the database "dbFile" to temporary filenames "tmpNames". |
|
549 | */ |
549 | */ |
|
550 | void WriteTemporaryFiles(char *dbFile, VECTOR *tmpNames, int tmpLines, int total) { |
550 | void WriteTemporaryFiles(char *dbFile, VECTOR *tmpNames, int tmpLines, int total) { |
|
551 | #if defined ___AsyncIO___ |
551 | #if defined ___AsyncIO___ |
|
552 | struct AsyncFile *fp, *tp; |
552 | struct AsyncFile *fp, *tp; |
|
553 | LONG c; |
- | ||
554 | #else |
553 | #else |
|
555 | FILE *fp, *tp; |
554 | FILE *fp, *tp; |
|
556 | char c; |
- | ||
557 | #endif |
555 | #endif |
|
558 | int lines; |
556 | int lines; |
|
559 | int write; |
557 | int write; |
|
560 | int files; |
558 | int files; |
|
- | 559 | dbLine *line = NULL; |
||
Line 561... | Line 560... | |||
561 | |
560 | |
|
562 | #if defined ___AsyncIO___ |
561 | #if defined ___AsyncIO___ |
|
563 | if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) { |
562 | if((fp = OpenAsync(dbFile, MODE_READ, ASYNC_BUF)) == NULL) { |
|
564 | #else |
563 | #else |
|
Line 587... | Line 586... | |||
587 | fprintf(stdout, "Writing to temporary files...\r"); |
586 | fprintf(stdout, "Writing to temporary files...\r"); |
|
588 | } |
587 | } |
|
Line 589... | Line 588... | |||
589 | |
588 | |
|
590 | write = 0; |
589 | write = 0; |
|
591 | lines = 0; |
- | ||
592 | #if defined ___AsyncIO___ |
- | ||
593 | while(PROGRAM_RUN && (c = ReadCharAsync(fp)) != -1) { |
590 | lines = 0; |
|
594 | #else |
591 | |
|
595 | while(PROGRAM_RUN && fscanf(fp, "%c", &c) == 1) { |
- | ||
596 | #endif |
592 | while(PROGRAM_RUN && (line = ReadLine(fp)) != NULL) { |
|
597 | #if defined ___AmigaOS___ |
593 | #if defined ___AmigaOS___ |
|
598 | // Check if CTRL+C was pressed and abort the program. |
594 | // Check if CTRL+C was pressed and abort the program. |
|
- | 595 | if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) { |
||
- | 596 | free(line->string); |
||
- | 597 | free(line); |
||
- | 598 | line = NULL; |
||
599 | if(SetSignal(0L, SIGBREAKF_CTRL_C) & SIGBREAKF_CTRL_C) { |
599 | |
|
600 | PROGRAM_RUN = FALSE; |
600 | PROGRAM_RUN = FALSE; |
|
601 | continue; |
601 | continue; |
|
602 | } |
602 | } |
|
- | 603 | #endif |
||
- | 604 | |
||
- | 605 | #if defined ___AsyncIO___ |
||
603 | #endif |
606 | WriteAsync(tp, line->string, (LONG)line->length); |
|
604 | switch(c) { |
607 | WriteAsync(tp, "\n", 1); |
|
605 | case '\n': |
608 | #else |
|
- | 609 | fprintf(tp, "%s\n", line->string); |
||
- | 610 | #endif |
||
606 | // Increment the total written lines. |
611 | |
|
Line 607... | Line 612... | |||
607 | ++write; |
612 | ++write; |
|
608 | |
613 | |
|
609 | if(PROGRAM_VERBOSE) { |
614 | if(PROGRAM_VERBOSE) { |
|
Line 610... | Line -... | |||
610 | fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)write / total) * 100.0)); |
- | ||
611 | } |
- | ||
612 | |
- | ||
613 | // Write the newline character back. |
- | ||
614 | #if defined ___AsyncIO___ |
- | ||
615 | if(WriteCharAsync(tp, (UBYTE)c) != 1) { |
- | ||
616 | #else |
- | ||
617 | if(fprintf(tp, "%c", c) != 1) { |
- | ||
618 | #endif |
- | ||
619 | fprintf(stderr, "Unable to write to '%s'.\n", (char *)tmpNames->array[files]); |
- | ||
620 | #if defined ___AsyncIO___ |
- | ||
621 | CloseAsync(tp); |
- | ||
622 | CloseAsync(fp); |
- | ||
623 | #else |
- | ||
624 | fclose(tp); |
- | ||
625 | fclose(fp); |
- | ||
626 | #endif |
615 | fprintf(stdout, "Writing to temporary files: %d%%.\r", (int)(((float)write / total) * 100.0)); |
|
627 | return; |
616 | } |
|
628 | } |
617 | |
|
629 | // Switch to the next temporary file. |
618 | // Switch to the next temporary file. |
|
- | 619 | if(++lines >= tmpLines) { |
||
- | 620 | // If there are no temporary files left then run till the end. |
||
- | 621 | if(files - 1 < 0) { |
||
630 | if(++lines >= tmpLines) { |
622 | free(line->string); |
|
631 | // If there are no temporary files left then run till the end. |
623 | free(line); |
|
Line 632... | Line 624... | |||
632 | if(files - 1 < 0) { |
624 | line = NULL; |
|
633 | break; |
625 | continue; |
|
634 | } |
626 | } |
|
Line 645... | Line 637... | |||
645 | #if defined ___AsyncIO___ |
637 | #if defined ___AsyncIO___ |
|
646 | CloseAsync(fp); |
638 | CloseAsync(fp); |
|
647 | #else |
639 | #else |
|
648 | fclose(fp); |
640 | fclose(fp); |
|
649 | #endif |
641 | #endif |
|
- | 642 | free(line->string); |
||
- | 643 | free(line); |
||
- | 644 | line = NULL; |
||
650 | return; |
645 | return; |
|
651 | } |
646 | } |
|
652 | lines = 0; |
647 | lines = 0; |
|
653 | break; |
- | ||
654 | } |
- | ||
655 | break; |
- | ||
656 | default: |
- | ||
657 | #if defined ___AsyncIO___ |
- | ||
658 | if(WriteCharAsync(tp, (UBYTE)c) != 1) { |
- | ||
659 | #else |
- | ||
660 | if(fprintf(tp, "%c", c) != 1) { |
- | ||
661 | #endif |
- | ||
662 | fprintf(stderr, "Could not write to file '%s'.\n", (char *)tmpNames->array[files]); |
- | ||
663 | #if defined ___AsyncIO___ |
- | ||
664 | CloseAsync(tp); |
- | ||
665 | CloseAsync(fp); |
- | ||
666 | #else |
- | ||
667 | fclose(tp); |
- | ||
668 | fclose(fp); |
- | ||
669 | #endif |
- | ||
670 | return; |
- | ||
671 | } |
648 | } |
|
- | 649 | |
||
- | 650 | free(line->string); |
||
672 | break; |
651 | free(line); |
|
- | 652 | line = NULL; |
||
673 | } |
653 | } |
|
- | 654 | |
||
- | 655 | if(line != NULL) { |
||
- | 656 | free(line->string); |
||
- | 657 | free(line); |
||
- | 658 | line = NULL; |
||
674 | } |
659 | } |
|
Line 675... | Line 660... | |||
675 | |
660 | |
|
676 | if(PROGRAM_VERBOSE) { |
661 | if(PROGRAM_VERBOSE) { |
|
677 | fprintf(stdout, "\n"); |
662 | fprintf(stdout, "\n"); |
|
Line 1027... | Line 1012... | |||
1027 | } |
1012 | } |
|
1028 | } |
1013 | } |
|
Line 1029... | Line 1014... | |||
1029 | |
1014 | |
|
1030 | /* |
1015 | /* |
|
1031 | * |
1016 | * |
|
1032 | * Indexes a "path" by creating a database "dbFile". |
1017 | * Indexes paths and adds to a database file. |
|
1033 | */ |
1018 | */ |
|
1034 | void GatherDatabaseFiles(char *dbFile, VECTOR *paths) { |
1019 | void GatherDatabaseFiles(char *dbFile, VECTOR *paths) { |
|
1035 | dbStats *stats; |
1020 | dbStats *stats; |
|
1036 | VECTOR *tmpNames; |
1021 | VECTOR *tmpNames; |
|
1037 | int tmpFiles; |
1022 | int tmpFiles; |
|
1038 | int tmpLines; |
1023 | int tmpLines; |
|
- | 1024 | int i; |
||
- | 1025 | int line; |
||
- | 1026 | int size; |
||
- | 1027 | |
||
- | 1028 | // Generate the database file from the supplied paths. |
||
- | 1029 | if((stats = CollectFiles(dbFile, paths)) == NULL) { |
||
- | 1030 | fprintf(stderr, "Collecting files failed.\n"); |
||
- | 1031 | return; |
||
- | 1032 | } |
||
- | 1033 | |
||
- | 1034 | size = GetFileSize(dbFile); |
||
- | 1035 | line = CountFileLines(dbFile); |
||
- | 1036 | |
||
- | 1037 | // Calculate the total number of temporary files required. |
||
- | 1038 | tmpFiles = size / maxmem; |
||
- | 1039 | |
||
- | 1040 | /* In case no temporary files are required, |
||
- | 1041 | * just sort the database and terminate. |
||
- | 1042 | */ |
||
- | 1043 | if(tmpFiles < 2) { |
||
- | 1044 | SortDatabase(dbFile, line); |
||
- | 1045 | return; |
||
- | 1046 | } |
||
- | 1047 | |
||
- | 1048 | // Calculate the number of lines per temporary file. |
||
- | 1049 | tmpLines = ceil(((double)line) / ((double)tmpFiles)); |
||
- | 1050 | |
||
- | 1051 | // Create temporary files. |
||
- | 1052 | if((tmpNames = CreateTemporaryFiles(tmpFiles)) == NULL) { |
||
- | 1053 | fprintf(stderr, "Unable to create temporary files.\n"); |
||
- | 1054 | return; |
||
- | 1055 | } |
||
- | 1056 | |
||
- | 1057 | // Write "tmpLines" to temporary files in "tmpNames" from "dbFile". |
||
- | 1058 | WriteTemporaryFiles(dbFile, tmpNames, tmpLines, line); |
||
- | 1059 | |
||
- | 1060 | // Sort the temporary files. |
||
- | 1061 | for(i = 0; i < tmpNames->length; ++i) { |
||
- | 1062 | SortDatabase(tmpNames->array[i], tmpLines); |
||
- | 1063 | } |
||
- | 1064 | |
||
- | 1065 | // Merge all the temporary files to the database file. |
||
- | 1066 | MergeTemporaryFiles(dbFile, tmpNames, line); |
||
- | 1067 | |
||
- | 1068 | // Remove all temporary files. |
||
- | 1069 | RemoveFiles(tmpNames); |
||
- | 1070 | |
||
- | 1071 | // Free temporary file names. |
||
- | 1072 | free(tmpNames); |
||
- | 1073 | tmpNames = NULL; |
||
- | 1074 | |
||
- | 1075 | // Free statistics. |
||
- | 1076 | free(stats); |
||
- | 1077 | stats = NULL; |
||
- | 1078 | } |
||
- | 1079 | |
||
- | 1080 | /* |
||
- | 1081 | * |
||
- | 1082 | * Indexes paths and creates a daabase file. |
||
- | 1083 | */ |
||
- | 1084 | void CreateDatabaseFiles(char *dbFile, VECTOR *paths) { |
||
- | 1085 | dbStats *stats; |
||
- | 1086 | VECTOR *tmpNames; |
||
- | 1087 | int tmpFiles; |
||
- | 1088 | int tmpLines; |
||
Line 1039... | Line 1089... | |||
1039 | int i; |
1089 | int i; |
|
1040 | |
1090 | |
|
1041 | // Generate the database file from the supplied paths. |
1091 | // Generate the database file from the supplied paths. |
|
1042 | if((stats = CollectFiles(dbFile, paths)) == NULL) { |
1092 | if((stats = CollectFiles(dbFile, paths)) == NULL) { |
|
Line 1255... | Line 1305... | |||
1255 | case CREATE: |
1305 | case CREATE: |
|
1256 | if(PROGRAM_VERBOSE) { |
1306 | if(PROGRAM_VERBOSE) { |
|
1257 | fprintf(stdout, "Removing '%s' and creating a new database.\n", dbFile); |
1307 | fprintf(stdout, "Removing '%s' and creating a new database.\n", dbFile); |
|
1258 | } |
1308 | } |
|
1259 | RemoveFile(dbFile); |
1309 | RemoveFile(dbFile); |
|
- | 1310 | if(PROGRAM_VERBOSE) { |
||
- | 1311 | fprintf(stdout, "Gathering files to database...\n"); |
||
- | 1312 | } |
||
- | 1313 | CreateDatabaseFiles(dbFile, paths); |
||
- | 1314 | break; |
||
1260 | case GATHER: |
1315 | case GATHER: |
|
1261 | if(PROGRAM_VERBOSE) { |
1316 | if(PROGRAM_VERBOSE) { |
|
1262 | fprintf(stdout, "Gathering files to database...\n"); |
1317 | fprintf(stdout, "Gathering files to database...\n"); |
|
1263 | } |
1318 | } |
|
1264 | GatherDatabaseFiles(dbFile, paths); |
1319 | GatherDatabaseFiles(dbFile, paths); |
|
Line 1273... | Line 1328... | |||
1273 | fprintf(stderr, "Unknown operation.\n"); |
1328 | fprintf(stderr, "Unknown operation.\n"); |
|
1274 | #if defined ___AmigaOS___ |
1329 | #if defined ___AmigaOS___ |
|
1275 | CloseLocale(locale); |
1330 | CloseLocale(locale); |
|
1276 | #endif |
1331 | #endif |
|
Line -... | Line 1332... | |||
- | 1332 | |
||
1277 | |
1333 | free(paths->array); |
|
1278 | free(paths); |
1334 | free(paths); |
|
1279 | paths = NULL; |
1335 | paths = NULL; |
|
1280 | return 5; |
1336 | return 5; |
|
Line 1281... | Line 1337... | |||
1281 | } |
1337 | } |
|
1282 | |
1338 | |
|
1283 | #if defined ___AmigaOS___ |
1339 | #if defined ___AmigaOS___ |
|
Line -... | Line 1340... | |||
- | 1340 | CloseLocale(locale); |
||
- | 1341 | #endif |
||
1284 | CloseLocale(locale); |
1342 | |
|
1285 | #endif |
1343 | if(paths != NULL) { |
|
- | 1344 | free(paths->array); |
||
- | 1345 | free(paths); |
||
1286 | |
1346 | paths = NULL; |
|
1287 | free(paths); |
1347 | } |