constrain parser is not reading the pages file in correct order while processing the pages file

[Marco Tusa]

In the Main class when the directory is open and the files are read
The pages files are NOT read from the readdir function in sorted order.

As such the resulting file has the pages not following the correct sorted order, and this can be an issue in general and is not optimal when reloading the data on the MySQL server.

The following patch replace part of the code and allow to have them sorted correctly.

**********************************
Replace

/*
  while(NULL != (de = readdir(src_dir))){
   if(!strncmp(de->d_name, ".", sizeof(de->d_name))) continue;
   if(!strncmp(de->d_name, "..", sizeof(de->d_name))) continue;
            snprintf(src_file, sizeof(src_file), "%s/%s", src, de->d_name);
   if(debug) { fprintf(stderr, "Processing %s\n", src_file); }
   fprintf(stderr, "Processing %s\n", src_file);
   if(0 == (fn = open_ibfile(src_file))){
    fprintf(stderr, "Can't open %s\n", src_file);
    perror("open_ibfile");
    exit(-1);
    }
   process_ibfile(fn);
   close(fn);
   }
*/

WITH
=======================================

*/
/*The pages are NOT read from the readdir function in sorted order
 * To have them sorted we have to process them first
 *
 * Below we first identify the number of files to parse
 *
 * then we create an array with all the names and we sort it
 *
 * Then we process it.
 *
 * Process can be optimize, but at least is not presenting the pages in sparse order in the result file.
 */

/*
 * Start Patch \/
 */
  while(NULL != (de = readdir(src_dir))){
   if(!strncmp(de->d_name, ".", sizeof(de->d_name))) continue;
   if(!strncmp(de->d_name, "..", sizeof(de->d_name))) continue;
   file_count++;

  }

  char result[file_count][255];
  int indexres = 0;

  src_dir = opendir(src);
  while(NULL != (de = readdir(src_dir))){
   if(!strncmp(de->d_name, ".", sizeof(de->d_name))) continue;
   if(!strncmp(de->d_name, "..", sizeof(de->d_name))) continue;
            snprintf(src_file, sizeof(src_file), "%s/%s", src, de->d_name);

            strcpy(result[indexres++], src_file) ;
   if(debug){ fprintf(stderr, "Processing %s\n", src_file); }
  }

  closedir(src_dir);

  int i,j;
  char t[file_count];

  for(i=1;i<file_count;i++){
      for(j=1;j<file_count;j++)
          {
       //fprintf(stderr, "value %d\n", strcmp(result[j-1],result[j])>0);
          if(strcmp(result[j-1],result[j])>0)
              {
              strcpy(t,result[j-1]);
              strcpy(result[j-1],result[j]);
              strcpy(result[j],t);
              }
          }
      }

   indexres = 0;
   fprintf(stderr, "Processing, \nTotal pages to process %d %s",file_count, ":");
   while(indexres<file_count){
    fprintf(stderr, "%s", ".");
    if(0 == (fn = open_ibfile(result[indexres]))){
     fprintf(stderr, "Can't open %s\n", result[indexres]);
     perror("open_ibfile");
     exit(-1);
     }
    process_ibfile(fn);
    indexres++;
    close(fn);
    }
   fprintf(stderr, "%s\n", ".");
/*
 * END Patch /\
 */

[Marco Tusa (marcotusa)]

Here the final patch with also the option to see the process when in debug mode, I had forgot to remove them in the previous
-------------------------------------------------------------------------

Index: constraints_parser.c
===================================================================
--- constraints_parser.c (revision 4)
+++ constraints_parser.c (working copy)
@@ -657,6 +657,7 @@
  struct stat st;
  char src[256];

+
  char buffer[16*1024];
         setvbuf(stdout, buffer, _IOFBF, sizeof(buffer));

@@ -727,11 +728,16 @@
   char src_file[256];
   struct dirent *de;
   src_dir = opendir(src);
+ int file_count = 0;
+
+
+/*
   while(NULL != (de = readdir(src_dir))){
    if(!strncmp(de->d_name, ".", sizeof(de->d_name))) continue;
    if(!strncmp(de->d_name, "..", sizeof(de->d_name))) continue;
             snprintf(src_file, sizeof(src_file), "%s/%s", src, de->d_name);
    if(debug) { fprintf(stderr, "Processing %s\n", src_file); }
+ fprintf(stderr, "Processing %s\n", src_file);
    if(0 == (fn = open_ibfile(src_file))){
     fprintf(stderr, "Can't open %s\n", src_file);
     perror("open_ibfile");
@@ -740,7 +746,82 @@
    process_ibfile(fn);
    close(fn);
    }
+*/
+/*The pages are NOT read from the readdir function in sorted order
+ * To have them sorted we have to process them first
+ *
+ * Below we first identify the number of files to parse
+ *
+ * then we create an array with all the names and we sort it
+ *
+ * Then we process it.
+ *
+ * Process can be optimize, but at least is not presenting the pages in sparse order in the result file.
+ *
+ * Use debug option to enable the progress view
+ */
+
+/*
+ * Start Patch \/
+ */
+ while(NULL != (de = readdir(src_dir))){
+ if(!strncmp(de->d_name, ".", sizeof(de->d_name))) continue;
+ if(!strncmp(de->d_name, "..", sizeof(de->d_name))) continue;
+ file_count++;
+
+ }
+
+ char result[file_count][255];
+ int indexres = 0;
+
+ src_dir = opendir(src);
+ while(NULL != (de = readdir(src_dir))){
+ if(!strncmp(de->d_name, ".", sizeof(de->d_name))) continue;
+ if(!strncmp(de->d_name, "..", sizeof(de->d_name))) continue;
+ snprintf(src_file, sizeof(src_file), "%s/%s", src, de->d_name);
+
+ strcpy(result[indexres++], src_file) ;
+ if(debug){ fprintf(stderr, "Processing %s\n", src_file); }
+ }
+
   closedir(src_dir);
+
+ int i,j;
+ char t[file_count];
+
+ for(i=1;i<file_count;i++){
+ for(j=1;j<file_count;j++)
+ {
+ //fprintf(stderr, "value %d\n", strcmp(result[j-1],result[j])>0);
+ if(strcmp(result[j-1],result[j])>0)
+ {
+ strcpy(t,result[j-1]);
+ strcpy(result[j-1],result[j]);
+ strcpy(result[j],t);
+ }
+ }
+ }
+
+
+ indexres = 0;
+ if(debug){fprintf(stderr, "Processing, \nTotal pages to process %d %s",file_count, ":");}
+ while(indexres<file_count){
+ if(debug){fprintf(stderr, "%s", ".");}
+ if(0 == (fn = open_ibfile(result[indexres]))){
+ fprintf(stderr, "Can't open %s\n", result[indexres]);
+ perror("open_ibfile");
+ exit(-1);
+ }
+ process_ibfile(fn);
+ indexres++;
+ close(fn);
+ }
+ if(debug){fp...