#!/bin/sh # # shell script for automated plotting of KEK watersystem monitor logdata # H.G.Berns (berns@phys.washington.edu), UW Seattle, 2 Nov. 1999 # updated 1/7/2004 start_time=`date +"%a %Y/%m/%d %Z %T"` stime=`date +%T` homedir=/u/hans workdir=$homedir/watersys tempdir=$workdir/temp #ftpdir=$workdir/ftpdir logdir=$workdir/log webdir=$homedir/public_html/watersys newweb=neutrino:public_html/watersys olddata=$webdir/wsys.last gnuscript=$workdir/wsys.gnu htmlfile=$webdir/index.html htmltemp=$webdir/wsys.html logfile=$logdir/plotwsys.log sumlog=$logdir/plotwsys.sh.log infile=wsys.last ftpwarnfile=$workdir/ftp_alarm_time.txt ftpwarnlog=$logdir/ftp_alarm.log ftpwarnnew="no" plot_status="-" cd $tempdir # echo "`date`: check for latest data file via wget" > $logfile echo "`date`: check for latest data file via pftp" > $logfile wgetlog=$tempdir/dl.log ftptempdir=$tempdir/.listing ftpdir=$tempdir/ftpdir # wsyspath=130.87.218.221/e:/cimplicity/hmi/projects/uci_k2k/arc # wsyspath=superk:route41@130.87.223.30/e:/cimplicity/hmi/projects/uci_k2k/arc # wsyspath=kekwater:fanuc@128.200.48.187/c:/cimplicity/hmi/projects/uci_k2k/arc # wsyspath=kekwater:fanuc@128.200.48.159/c:/cimplicity/hmi/projects/uci_k2k/arc # options="-nr -nH -P $tempdir" # wget $options -o $wgetlog ftp://$wsyspath/ wsysaddress=128.200.48.159 (/usr/bin/pftp $wsysaddress < $ftptempdir dir bye EOF1 if [ -f $ftptempdir ] then mv $ftptempdir $ftpdir else echo "`date`: no data! ftp to watersystem PC failed!" > $ftpdir fi cp $ftpdir $webdir/ftpdir.last # scp -o Protocol=1 $ftpdir $newweb/ftpdir.last case `cat $ftpdir` in *CSV*) echo "`date`: scanning for latest data file in ftp listing" >> $logfile ## ####### 00/04/14: new PC generates different file listing than old one. ####### old: filesize= 1st column, filename= 4th column ####### new: filesize= 5th column, filename= 9th column ## ## files9=`grep WATERASCII9 $ftpdir | grep .CSV | awk '{print $4}' | awk -F. '{print $1}' | sort -n` ## files0=`grep WATERASCII0 $ftpdir | grep .CSV | awk '{print $4}' | awk -F. '{print $1}' | sort -n` ## files1=`grep WATERASCII1 $ftpdir | grep .CSV | awk '{print $4}' | awk -F. '{print $1}' | sort -n` ## files2=`grep WATERASCII2 $ftpdir | grep .CSV | awk '{print $4}' | awk -F. '{print $1}' | sort -n` files9=`grep WATERASCII9 $ftpdir | grep .CSV | awk '{print $9}' | awk -F. '{print $1}' | sort -n` files0=`grep WATERASCII0 $ftpdir | grep .CSV | awk '{print $9}' | awk -F. '{print $1}' | sort -n` files1=`grep WATERASCII1 $ftpdir | grep .CSV | awk '{print $9}' | awk -F. '{print $1}' | sort -n` files2=`grep WATERASCII2 $ftpdir | grep .CSV | awk '{print $9}' | awk -F. '{print $1}' | sort -n` allfiles="$files9 $files0 $files1 $files2" if [ -f $ftpdir.temp ] then rm $ftpdir.temp fi if [ -f $ftpwarnfile ] then rm $ftpwarnfile echo "`date`: network recovered" >> $ftpwarnlog fi for filename in $allfiles do lastfile="$filename.CSV" echo "$lastfile" > $ftpdir.temp done ## filesize=`grep $lastfile $ftpdir | tail -1 | awk '{print $1}'` filesize=`grep $lastfile $ftpdir | tail -1 | awk '{print $5}'` echo "$lastfile $filesize" > $workdir/ftpdir.new case `/usr/bin/diff $workdir/ftpdir.new $workdir/ftpdir.last` in *'>'* | *'<'*) echo "`date`: get new file $lastfile ($filesize bytes)" >> $logfile # wget $options -a $wgetlog ftp://$wsyspath/$lastfile;type=a # mv $lastfile $infile (/usr/bin/pftp $wsysaddress <> $logfile 2>&1 get $lastfile $infile bye EOF2 echo "$lastfile $filesize" > $workdir/ftpdir.last ;; *) echo "`date`: datafile ($lastfile) unchanged" >> $logfile infile=$olddata ;; esac echo "`date` 0" > $workdir/wsys.ftp_last ;; *) echo " ==> ERROR!! No data file found on watersystem PC!" >> $logfile cat $ftpdir >> $logfile if [ -f $ftpwarnfile ] then echo "`date`: $ftpwarnfile already exists" >> $logfile else echo "`date`: creating $ftpwarnfile" >> $logfile echo "`date`" > $ftpwarnfile echo "----------------------------------------" >> $ftpwarnlog echo "`date`: fdwat2 not responding" >> $ftpwarnlog ftpwarnnew="yes" fi lastfile="Network ERROR! Watersystem PC connection failed at `cat $ftpwarnfile`" infile=$olddata ftpalarm=$workdir/ftp_alarm.txt $workdir/wsyswarn > $ftpalarm case `cat $ftpalarm` in *Alert*) m_cc="hsobel@uci.edu,berns@phys.washington.edu,Yoshinari.Hayato@kek.jp,itow@suketto.icrr.u-tokyo.ac.jp,jzal@neutrino.kek.jp" m_to="mine@neutrino.kek.jp" cat $ftpalarm | mail -s "ftp problems at KEK watersystem PC" -c $m_cc $m_to echo "`date`: email sent to inform about ftp problem" >> $logfile ;; esac plot_status="ftp failed" ;; esac case `ls -a $tempdir` in *listing*) echo "`date`: ... remove old ftp listing files ..." >> $logfile rm .listing* ;; esac case `ls $tempdir` in *index.htm*) echo "`date`: ... remove index.htm* files from temp dir ..." >> $logfile rm $tempdir/index.htm* ;; esac # check if new datafile is different from the one retrieved last time # (execute the gnuplot scripts only if new data is different from old) case `/usr/bin/diff $infile $olddata` in *'>'* | *'<'*) echo "`date`: convert ascii data to gnuplot readable data file" >> $logfile # first, read each value and rewrite it with %g format. This way, if an entry # is blank, then it'll be converted to 0 and the data can be read out correctly # by the csv2dat process. head -1 $infile > $infile.temp tail +2 $infile | grep -v ",,,," | awk -F, '{ printf "%s", $1 for (i = 2 ; i <= 35 ; i++) { printf ",%g",$i } printf "\n" }' >> $infile.temp # cp $infile.temp $infile.temp.save lasttime=`tail -1 $infile | awk -F. '{print $1}'` echo "`date`: last datalog from $lasttime" >> $logfile # extract last week's data and errors to website head -1 $infile > $webdir/wsys.last.week tail -1010 $infile | grep -v "timestamp" >> $webdir/wsys.last.week grep ",," $infile > $webdir/all/wsys.errors tail -200 $webdir/all/wsys.errors > $webdir/all/wsys.errors.last # scp -o Protocol=1 $webdir/wsys.last.week $newweb/ # scp -o Protocol=1 $webdir/all/wsys.error* $newweb/all/ echo "`date`: save original file to website" >> $logfile mv $infile $webdir # scp -o Protocol=1 $webdir/$infile $newweb/ echo "`date`: run csvdat" >> $logfile mv $infile.temp $infile ( $workdir/csv2dat $infile $tempdir/wsys.temp ) > $logdir/csv2dat.lastlog 2>&1 echo "`date`: run mergedat" >> $logfile $workdir/mergedat $tempdir/wsys.save $tempdir/wsys.temp echo "`date`: check for shift warnings" >> $logfile alertmsg=$tempdir/wsys_warning.email case `grep ! $tempdir/wsys_warning.data` in *!*) echo "`date`: shift warning found -> send email" >> $logfile # m_cc=hgberns@comcast.net # m_to=hgb@u.washington.edu m_cc="hsobel@uci.edu,berns@phys.washington.edu,Yoshinari.Hayato@kek.jp,itow@suketto.icrr.u-tokyo.ac.jp,jzal@neutrino.kek.jp,ishii@neutrino.kek.jp,miuram@suketto.icrr.u-tokyo.ac.jp,kameda@neutrino.kek.jp" m_to="mine@neutrino.kek.jp" echo "This is an automatic message generated by a script" > $alertmsg echo "" >> $alertmsg echo "WARNING!" >> $alertmsg echo "Abnormal data are detected at 1kt water system:" >> $alertmsg echo "" >> $alertmsg grep ! $tempdir/wsys_warning.data >> $alertmsg echo "" >> $alertmsg echo "Please phone the experts (try from 1 to 3) right now!" >> $alertmsg echo "Follow his/her suggestion." >> $alertmsg echo "" >> $alertmsg echo "Experts:" >> $alertmsg echo " 1. Shun'ichi Mine: 0011-949-824-9928 (USA, UCI office)," >> $alertmsg echo " 0011-949-278-7129 (USA cell phone)," >> $alertmsg echo " 090-4136-8379 (Japan cell phone)" >> $alertmsg echo " 2. Asia Zalipska: 029-864-5220 (KEK, K2K US office)," >> $alertmsg echo " 090-8591-5822 (cell phone)" >> $alertmsg echo " 3. Makoto Miura: 0578-5-9608 (Super-K office)," >> $alertmsg echo " 090-2453-0842 (cell phone)" >> $alertmsg echo "" >> $alertmsg echo "" >> $alertmsg echo "URL: http://neutrino.kek.jp/shift/monitor/watersys/shift/" >> $alertmsg echo "`date`: --------------------------" >> $alertmsg.sent grep ! $tempdir/wsys_warning.data >> $alertmsg.sent cat $alertmsg | mail -s "sensor warning at K2K 1kt watersystem" -c $m_cc $m_to ;; esac # sorting not needed anymore (I hope). source files seem to be # correctly sorted now. therefore, following section disabled. # echo "`date`: sort data file by date/time" >> $logfile # cat $tempdir/wsys.temp | grep "#" > $tempdir/wsys.data # merge with old data before 28 April 1999 (removed 5/12/99) # cat $tempdir/wsys.data.old | grep -v "#" >> $tempdir/wsys.data # cat $tempdir/wsys.temp | grep -v "#" | sort -n >> $tempdir/wsys.data echo "`date`: copy 24h & 7-day data to website" >> $logfile rm $tempdir/wsys.temp echo "# watersystem log as of `date`" > $tempdir/wsys.data cat $tempdir/wsys.save >> $tempdir/wsys.data # echo "`date`: extract last 7 days and last 24 hours" >> $logfile # cat $tempdir/wsys.data | grep "#" > $tempdir/wsys_7day.data # cat $tempdir/wsys.data | grep "#" > $tempdir/wsys_24h.data # tail -505 $tempdir/wsys.data >> $tempdir/wsys_7day.data # tail -73 $tempdir/wsys.data >> $tempdir/wsys_24h.data cp $tempdir/wsys.data $webdir/all/ echo "# watersystem log of last 7 days as of `date`" > $webdir/week/wsys_7day.data cat $tempdir/wsys_7day.data >> $webdir/week/wsys_7day.data echo "# watersystem log of last 24 hours as of `date`" > $webdir/day/wsys_24h.data cat $tempdir/wsys_24h.data >> $webdir/day/wsys_24h.data # scp -o Protocol=1 $tempdir/wsys.data $newweb/all/ # scp -o Protocol=1 $webdir/week/wsys_7day.data $newweb/week/ # scp -o Protocol=1 $webdir/day/wsys_24h.data $newweb/day/ # echo "`date`: extract data of year 1999" >> $logfile # cat $tempdir/wsys.data | grep "#" > $tempdir/wsys_1999.data # grep "^1999-" $tempdir/wsys.data >> $tempdir/wsys_1999.data # cp $tempdir/wsys_1999.data $webdir/year1999/ # echo "`date`: extract data of year 2000" >> $logfile # grep "#" $tempdir/wsys.data > $tempdir/wsys_2000.data # grep "^2000-" $tempdir/wsys.data >> $tempdir/wsys_2000.data # cp $tempdir/wsys_2000.data $webdir/year2000/ # echo "`date`: extract data of year 2001" >> $logfile # grep "#" $tempdir/wsys.data > $tempdir/wsys_2001.data # grep "^2001-" $tempdir/wsys.data >> $tempdir/wsys_2001.data # cp $tempdir/wsys_2001.data $webdir/year2001/ # echo "`date`: extract data of year 2002" >> $logfile # grep "#" $tempdir/wsys.data > $tempdir/wsys_2002.data # grep "^2002-" $tempdir/wsys.data >> $tempdir/wsys_2002.data # cp $tempdir/wsys_2002.data $webdir/year2002/ # echo "`date`: extract data of year 2003" >> $logfile # grep "#" $tempdir/wsys.data > $tempdir/wsys_2003.data # grep "^2003-" $tempdir/wsys.data >> $tempdir/wsys_2003.data # cp $tempdir/wsys_2003.data $webdir/year2003/ # scp -o Protocol=1 $tempdir/wsys_2003.data $newweb/year2003/ echo "`date`: extract data of year 2004" >> $logfile grep "#" $tempdir/wsys.data > $tempdir/wsys_year.data grep "^2004-" $tempdir/wsys.data >> $tempdir/wsys_year.data cp $tempdir/wsys_year.data $webdir/year2004/ # echo "`date`: run plot_all.sh script" >> $logfile # $workdir/plot_all.sh echo "`date`: run plot_24h.sh script" >> $logfile $workdir/plot_24h.sh echo "`date`: run plot_7day.sh script" >> $logfile $workdir/plot_7day.sh echo "`date`: run plot_shift.sh script" >> $logfile $workdir/plot_shift.sh case $stime in '00:0'* | '00:1'* | '00:2'* | '09:1'* | '12:0'* | '12:1'* | '12:2'*) # echo "`date`: run plot_1999.sh script" >> $logfile # $workdir/plot_1999.sh # echo "`date`: run plot_2000.sh script" >> $logfile # $workdir/plot_2000.sh # echo "`date`: run plot_2001.sh script" >> $logfile # $workdir/plot_2001.sh # echo "`date`: run plot_2002.sh script" >> $logfile # $workdir/plot_2002.sh # echo "`date`: run plot_2003.sh script" >> $logfile # $workdir/plot_2003.sh echo "`date`: run plot_2004.sh script" >> $logfile $workdir/plot_2004.sh plot_status="new plots ($lasttime) + 2004 plots" ;; *) plot_status="new plots ($lasttime)" ;; esac echo "`date`: deleting temp files" >> $logfile if [ -f $tempdir/wsys.data ] then rm $tempdir/wsys.data fi if [ -f $tempdir/wsys_24h.data ] then rm $tempdir/wsys_24h.data fi if [ -f $tempdir/wsys_7day.data ] then rm $tempdir/wsys_7day.data fi if [ -f $tempdir/wsys_1999.data ] then rm $tempdir/wsys_1999.data fi if [ -f $tempdir/wsys_2000.data ] then rm $tempdir/wsys_2000.data fi if [ -f $tempdir/wsys_2001.data ] then rm $tempdir/wsys_2001.data fi if [ -f $tempdir/wsys_2002.data ] then rm $tempdir/wsys_2002.data fi if [ -f $tempdir/wsys_2003.data ] then rm $tempdir/wsys_2003.data fi if [ -f $tempdir/wsys_year.data ] then rm $tempdir/wsys_year.data fi ;; *) echo "`date`: datafile same as before --> plots not updated" >> $logfile case $ftpwarnnew in *yes*) echo "`date`: run plot_shift.sh script" >> $logfile cp $webdir/week/wsys_7day.data $tempdir $workdir/plot_shift.sh rm $tempdir/wsys_7day.data ;; esac ;; esac echo "`date`: create html page" >> $logfile echo "" > $htmltemp echo "K2K 1KT Watersystem Logs" >> $htmltemp echo "

K2K " >> $htmltemp echo "1kt Watersystem Logs Overview

" >> $htmltemp echo "
" >> $htmltemp echo "Please make sure the time stamps below are up to date!
" >> $htmltemp echo "If they are more than 2 hours behind current time, try a " >> $htmltemp echo "''Shift-Reload'' of the page (push and hold Shift key on the" >> $htmltemp echo " keyboard and then push the Reload button in the Netscape " >> $htmltemp echo "toolbar with the mouse).
" >> $htmltemp echo "

" >> $htmltemp echo "last automatic ftp/scan/plot " >> $htmltemp echo "update: `date`

" >> $htmltemp if [ -f $workdir/shift.note ] then cat $workdir/shift.note >> $htmltemp fi echo "

" >> $htmltemp # echo "`date`: (testpoint 3)" >> $logfile echo "

Current sensor values and 24-hour min./average/max.

" >> $htmltemp $workdir/wsysstat >> $logfile cat $webdir/wsys_24h.html >> $htmltemp # echo "`date`: (testpoint 4)" >> $logfile echo "
" >> $htmltemp echo "This page is automatically generated hourly by a shell " >> $htmltemp echo "script plotwsys.sh" >> $htmltemp echo "which does the following:
    " >> $htmltemp echo "
  • ftp to watersystem PC and retrieve *.csv data file" >> $htmltemp echo "" >> $htmltemp echo "
  • convert *.csv data file to gnuplot readable format" >> $htmltemp echo "via csv2dat.c program" >> $htmltemp echo "
  • sort by date [note: original data seems to be" >> $htmltemp echo "out of order in blocks...]" >> $htmltemp echo "
  • extract last 7 days and 24 hours in separat files" >> $htmltemp echo "
  • run separate sub scripts:
    " >> $htmltemp echo "a) plot_XXXX.sh (XXXX = current year):
      " >> $htmltemp echo "
    • create plots (PBM format) for data set of the current year" >> $htmltemp echo "with gnuplot macro wsys_XXXX.gnu" >> $htmltemp echo "
    • convert plots from PBM to GIF via ppmtogif" >> $htmltemp echo "
    • create HTML file
    " >> $htmltemp echo "
  • b) plot_7day.sh:
      " >> $htmltemp echo "
    • create plots (PBM format) for 7-day data set with" >> $htmltemp echo "gnuplot macro wsys_7day.gnu" >> $htmltemp echo "
    • convert plots from PBM to GIF via ppmtogif" >> $htmltemp echo "
    • create HTML file
    " >> $htmltemp echo "
  • c) plot_24h.sh:
      " >> $htmltemp echo "
    • create plots (PBM format) for 24-hour data set with" >> $htmltemp echo "gnuplot macro wsys_24h.gnu" >> $htmltemp echo "
    • convert plots from PBM to GIF via ppmtogif" >> $htmltemp echo "
    • create HTML file
    " >> $htmltemp echo "
  • run wsysstat.c to create " >> $htmltemp echo "overview table of current sensor values and last 24-hour " >> $htmltemp echo "statistics" >> $htmltemp echo "
  • create this HTML file" >> $htmltemp echo "
" >> $htmltemp echo "
questions? please send comments to " >> $htmltemp echo "HGB " >> $htmltemp echo "@ UW, Seattle.

" >> $htmltemp echo "updated `date` (recent" >> $htmltemp echo "update summary)

" >> $htmltemp echo "`date`: move html page to website" >> $logfile mv $htmltemp $htmlfile # scp -o Protocol=1 $htmlfile $newweb/index.html echo "`date`: done plotting watersys data" >> $logfile cp $logfile $webdir # scp -o Protocol=1 $logfile $newweb/ echo "$start_time ... `date +%T`: $plot_status" tail -150 $sumlog > $webdir/plotwsys.sumlog.last150 # scp -o Protocol=1 $webdir/plotwsys.sumlog.last150 $newweb/plotwsys.sumlog.last150 scplog=$logfile.scp echo "`date`: starting scp transfer to neutrino" > $scplog ( scp -prvBC -o Protocol=1 $webdir/* $newweb/ ) >> $scplog 2>&1 echo "`date`: done with scp transfer" >> $scplog echo "$start_time ... `date +%T`: $plot_status" >> $logfile.summary