/***************************************************************************
* csv2dat.c
*
* H.G.Berns (berns@phys.washington.edu), UW Seattle, 28 April 1999
*
* last revision: 17 Mar 2001
*
* This little code just converts the comma-separated entries in the *.csv
* logdata file of the KEK watersystem monitor to space-separated entries
* for gnuplot.
***************************************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#define  FALSE  0
#define  TRUE   1
#define  ERRORFILE "wsys.error"    /* for logging error entries in logdata */

main(int argc, char **argv)
{
  int       year=0, mon=0, day=0, hour=0, min=0, sec=0;
  int       vsort[36], histo[45][2], numscan;
  float     valo[36], pre_valo[36], fsec;
  int       i,j,k,l, e, e0;
  int       DATA_NONZERO = 0;
  time_t    t;
  struct tm tt;
  FILE     *in, *out, *e_out;
  char      str[1024], date_str[1024], time_str[1024], mon_str[1024];
  char     *p;

  if (argc!=3)
  {
    fprintf(stderr,"syntax %s <inputfile> <outputfile>\n", argv[0]);
    exit(1);
  }
  if ((in=fopen(argv[1],"rb"))==NULL)
  {
    fprintf(stderr,"cannot read %s ... I quit.\n", argv[1]);
    exit(1);
  }
  if ((out=fopen(argv[2],"w"))==NULL)
  {
    fprintf(stderr,"cannot write to %s ... I quit.\n", argv[2]);
    exit(1);
  }
  if ((e_out=fopen(ERRORFILE,"w"))==NULL)
  {
    fprintf(stderr,"cannot write to %s ... I quit.\n", ERRORFILE);
    exit(1);
  }

  /* write header of output data */

  for (k=0; k<45; k++) histo[k][0] = 0;
  for (k=0; k<45; k++) histo[k][1] = 0;
  i = j = k = e = e0 = 0;
  t = time(NULL);

  fprintf(out,"#  watersystem log as of %s#\n# columns:\n", ctime(&t));
  fprintf(out,"#  1. timestamp\n");
  fprintf(out,"#  2. Signet Flow Product                     0-100   GPM\n");
  fprintf(out,"#  3. Signet Flow Waste                       0-100   GPM\n");
  fprintf(out,"#  4. Signet Flow to Detector                 0-100   GPM\n");
  fprintf(out,"#  5. Signet Flow to Fast Recirculation       0-100   GPM\n");
  fprintf(out,"#  6. Pressure after Anion Exchange           0-100   PSI\n");
  fprintf(out,"#  7. Pressure after Mixbed Filter            0-100   PSI\n");
  fprintf(out,"#  8. Pressure after Ultra Violet Sterilizer  0-100   PSI\n");
  fprintf(out,"#  9. Pressure of City Water                  0-100   PSI\n");
  fprintf(out,"# 10. Pressure from detector pumps            0-100   PSI\n");
  fprintf(out,"# 11. Pressure of detector (Water Height)     0-1100  cm\n");
  fprintf(out,"# 12. Pressure of detector (low setting)      0-1100  cm\n");
  fprintf(out,"# 13. Pressure after Ultra Filter             0-100   PSI\n");
  fprintf(out,"# 14. Pressure after Depth Filter             0-100   PSI\n");
  fprintf(out,"# 15. Pressure before Ultra Pre-Filter        0-100   PSI\n");
  fprintf(out,"# 16. Pressure to detector                    0-100   PSI\n");
  fprintf(out,"# 17. Water Quality after Mixbed Filter       0-20    MegOhms\n");
  fprintf(out,"# 18. Water Quality to the Detector           0-20    MegOhms\n");
  fprintf(out,"# 19. Water Temperature before Chiller        0-100   Degrees F\n");
  fprintf(out,"# 20. Temperature of Detector Return Water    0-100   Degrees F\n");
  fprintf(out,"# 21. Water Temperature to Detector           0-100   Degrees F\n");
  fprintf(out,"# 22. Temperature in Tent                     0-100   Degrees F\n");
  fprintf(out,"# 23. Water Quality from Detector             0-20    MegOhms\n");
  fprintf(out,"# 24. Water Quality out of UF                 0-20    MegOhms\n");
  fprintf(out,"# 25. Water Quality after UV                  0-20    MegOhms\n");
  fprintf(out,"# 26. Signet Flow Total                       0-100   GPM\n");
  fprintf(out,"# 27. UV Lamps Output Power                   0-100   %%\n");
  fprintf(out,"# 28. Pressure after Repressurization Pumps   0-100   PSI\n");
  fprintf(out,"# 29. Pressure after Fast Recirc Filters      0-100   PSI\n");
  fprintf(out,"# 30. Pressure after Carbon                   0-100   PSI\n");
  fprintf(out,"# 31. Pressure after Pre-UV                   0-100   PSI\n");
  fprintf(out,"# 32. Pressure after RO Pre-filter            0-100   PSI\n");
  fprintf(out,"# 33. Pressure after Softener                 0-100   PSI\n");
  fprintf(out,"# 34. Temperature of City Water               0-100   Degrees F\n");
  fprintf(out,"# 35. Signet Flow UF Waste                    0-100   GPM\n");
  fprintf(out,"# 36. Signet Flow UF Bypass                   0-100   GPM\n");
  fprintf(out,"#\n#YYY-MM-DD-hh:mm:ss");
  for (k=0;k<35;k++) fprintf(out,"\t%d",k+2);
  fprintf(out,"\n");
  fprintf(out,"#===================");
  for (k=0;k<35;k++) fprintf(out,"========");
  fprintf(out,"\n");

  /* read input data file line-by-line */

  for (k=0; k<36; k++) pre_valo[k] = 0.0;

  while(fgets(str,1000,in) && !feof(in) && !ferror(in))
  {
    i++;
    for (k=0; k<36; k++) valo[k] = 0.0;

    /* quick error check: use only those lines with >23 valid entries each */

    if (strstr(str,"timestamp"))
    {
      i--;    /* don't count header line */
      for (k=0; k<36; k++) vsort[k] = 35;
      k = 0;
      p = strtok(str,",");
      while(p!=NULL && k<35)
      {
        l = 35;
        p = strtok('\0',",");
        if (p)
        {
          if (strstr(p,"SIGNET_FLOW_PRODUCT"))  l = 0;     /*  1 */
          if (strstr(p,"SIGNET_FLOW_RO_WASTE")) l = 1;     /*  2 */
          if (strstr(p,"SIGNET_FLOW_TO_DETEC")) l = 2;     /*  3 */
          if (strstr(p,"SIGNET_FLOW_TO_FAST_")) l = 3;     /* 11 */
          if (strstr(p,"PRESSURE_AFTER_ANION")) l = 4;     /* 23 */
          if (strstr(p,"PRESSURE_AFTER_MIXBE")) l = 5;
          if (strstr(p,"PRESSURE_AFTER_UV"))    l = 6;     /* 21 */
          if (strstr(p,"PRESSURE_CITY_WATER"))  l = 7;     /*  4 */
          if (strstr(p,"PRESSURE_DETECTOR_PU")) l = 8;     /* 25 */
          if (strstr(p,"PRESSURE_OF_DETECTOR")) l = 9;     /*  5 */
          if (strstr(p,"PRESSURE_OF_DETE_009")) l = 10;    /*  6 */
          if (strstr(p,"PSI_AFTER_UF"))         l = 11;    /* 20 */
          if (strstr(p,"PSI_AFT_DEPTH_FILTER")) l = 12;    /*  7 */
          if (strstr(p,"PSI_BEFORE_UF_PREFIL")) l = 13;    /* 22 */
          if (strstr(p,"PSI_TO_DETECTOR"))      l = 14;    /* 24 */
          if (strstr(p,"QUALITY_AFTER_MIXBED")) l = 15;    /* 13 */
          if (strstr(p,"QUALITY_TO_THE_DETEC")) l = 16;
          if (strstr(p,"TEMPERATURE_BEFORE_C")) l = 17;    /*  8 */
          if (strstr(p,"TEMPERATURE_OF_DETEC")) l = 18;    /*  9 */
          if (strstr(p,"TEMPERATURE_TO_DETEC")) l = 19;    /* 10 */
          if (strstr(p,"TEMP_TENT"))            l = 20;    /* 12 */
          if (strstr(p,"QUALITY_FROM_DETECTO")) l = 21;    /* 14 */
          if (strstr(p,"QUALITY_OUTOF_UF"))     l = 22;    /* 15 */
          if (strstr(p,"QUALITY_AFTER_UV"))     l = 23;    /* 18 */
          if (strstr(p,"SIGNET_FLOW_TOTAL"))    l = 24;    /* 16 */
          if (strstr(p,"UV_LAMPS"))             l = 25;    /* 17 */
          if (strstr(p,"PRESSURE_AFTER_REPRE")) l = 26;    /* 19 */
          if (strstr(p,"PRESSURE_AFTER_FAST"))  l = 27;    /* 26 */
          if (strstr(p,"PRESSURE_AFT_CARBON"))  l = 28;    /* 27 */
          if (strstr(p,"PRESSURE_AFT_PRE_UV"))  l = 29;    /* 28 */
          if (strstr(p,"PRESSURE_AFT_RO_PREF")) l = 30;    /* 29 */
          if (strstr(p,"PRESSURE_AFT_SOFTENE")) l = 31;    /* 30 */
          if (strstr(p,"TEMPERATURE_CITY_WAT")) l = 32;    /* 30 */
          if (strstr(p,"SIGNET_FLOW_UF_WASTE")) l = 33;    /* 30 */
          if (strstr(p,"SIGNET_FLOW_UF_BYPAS")) l = 34;    /* 30 */
          vsort[l] = k;
          if (l == k)
          {
            fprintf(stderr,"%2d:               %s\n", k, p);
          }
          else if (l<35)
          {
            fprintf(stderr,"%2d swapped to %2d: %s\n", k, l, p);
          }
          else
          {
            fprintf(stderr,"%2d NO REFERENCE!: %s\n", k, p);
          }
        }
        k++;
      }
    }
    else if ((numscan=sscanf(str, \
        "%d-%3s-%d %d:%d:%g,%g,%g,%g,%g,%g" \
        ",%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g" \
        ",%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g,%g", \
        &day, mon_str, &year, &hour, &min, &fsec, \
        &valo[0],  &valo[1],  &valo[2],  &valo[3],  &valo[4],  &valo[5], \
        &valo[6],  &valo[7],  &valo[8],  &valo[9],  &valo[10], &valo[11],\
        &valo[12], &valo[13], &valo[14], &valo[15], &valo[16], &valo[17],\
        &valo[18], &valo[19], &valo[20], &valo[21], &valo[22], &valo[23],\
        &valo[24], &valo[25], &valo[26], &valo[27], &valo[28], &valo[29],\
        &valo[30], &valo[31], &valo[32], &valo[33], &valo[34]))>25)
    {
      histo[numscan][0]++;
      j++;
      if (strstr(mon_str,"JAN")) mon=1;
      if (strstr(mon_str,"FEB")) mon=2;
      if (strstr(mon_str,"MAR")) mon=3;
      if (strstr(mon_str,"APR")) mon=4;
      if (strstr(mon_str,"MAY")) mon=5;
      if (strstr(mon_str,"JUN")) mon=6;
      if (strstr(mon_str,"JUL")) mon=7;
      if (strstr(mon_str,"AUG")) mon=8;
      if (strstr(mon_str,"SEP")) mon=9;
      if (strstr(mon_str,"OCT")) mon=10;
      if (strstr(mon_str,"NOV")) mon=11;
      if (strstr(mon_str,"DEC")) mon=12;

      DATA_NONZERO = 0;
      for (k=0;k<35;k++)
      {
        if (valo[vsort[k]]!=0) DATA_NONZERO++;
      }
      if (DATA_NONZERO>=10)  /* minimum 10 non-zero datapoints required, otherwise ignored */
      {
        /* PC clock was accidentally set from AM to PM on 3/17/2001 between
           01:50 and 04:30 => correcting by 12 hours for that time period */

        if (year==2001 && mon==3 && day==17 && (int)fsec==6) hour -= 12;

        fprintf(out,"%04d-%02d-%02d-%02d:%02d:%02d",\
                year, mon, day, hour, min, (int)fsec);
        for (k=0;k<35;k++)
        {
          /* temperature sensor (pos. 17-20) overflow (showing 0F)? => set to 100F */
          /* if (k>=17 && k<=20 && valo[vsort[k]]==0) valo[vsort[k]]=100;  */

          /* UV lamps sensor (pos. 25) overflow (showing 0%)? => set to 100% */
          if (k==25 && valo[vsort[k]]==0) valo[vsort[k]]=100;

          /* Quality sensors (pos. 15, 16, 21-23) 0? => use previous value */
          if (valo[vsort[k]]==0 && (k==15 || k==16 || k==21 || k==22 || k==23))
          {
            if (pre_valo[k]>0) 
            {
              valo[vsort[k]] = pre_valo[k];
              fprintf(e_out,"%04d-%02d-%02d-%02d:%02d:%02d sensor %d => gap filled with %g\n", \
                      year, mon, day, hour, min, (int)fsec, k, pre_valo[k]);
              pre_valo[k] = 0;
            }
          }
          else
          {
            pre_valo[k] = valo[vsort[k]];
          }
          fprintf(out,"\t%g",valo[vsort[k]]);
        }
        fprintf(out,"\n");
      }
      else
      {
        /* log rejected lines in separate error file */

        e0++;
        fprintf(e_out,"%s", str);
      }
      histo[DATA_NONZERO][1]++;
    }
    else
    {
      /* log rejected lines in separate error file */

      e++;
      fprintf(e_out,"%s", str);
    }
  }
  fclose(in);
  fclose(out);
  fclose(e_out);

  fprintf(stderr,"%s: %d lines (%d valid, %d invalid, %d no data)\n", argv[1],i,j,e,e0);

  fprintf(stderr,"histogram:\n");
  for (k=0; k<45; k++) fprintf(stderr,"%2d: %6d %6d\n", k, histo[k][0], histo[k][1]);
}