#!/usr/local/bin/ingperl
#$Id: fld_crunch.p,v 1.1 2007/09/25 23:18:51 kaehler Exp $
#$Log: fld_crunch.p,v $
#Revision 1.1  2007/09/25 23:18:51  kaehler
#Initial revision
#
#Revision 1.2  1998/10/08  19:02:58  pittag
# commented out II_SYSTEM.
#
#Revision 1.1  1998/06/17  23:54:47  pittag
#Initial revision
#
#########################################################################
#
# program:        fld_crunch.p
#
# programmer:     jennifer mead
#
# creation date:  march 1997
#
# purpose:        to crunch validated data from files that fld_val.p created
#                 for loading to the database.
#
# usage:          fld_crunch.p
#
# files read:
#
#                 $LOADDIR/TAG_COM.VAL.<filename>
#                 format: file name, line number, note record
#
#                 $LOADDIR/TAG_FLAG.VAL.<filename>
#                 format: tag id, file name, flag code
#
#                 $LOADDIR/TAG_DATA.VAL.<filename>
#                 format: migr yr, file name, seq no, tag id, chk sum, length,
#                         weight, species, run, rear type, brd yr, var num,
#                         flags, text comments
#
#                 $LOADDIR/TAG_VAR.VAL.<filename>
#                 format:  file name, var number, var date
#
#                 $LOADDIR/TAG_HDR.VAL.<filename>
#                 format: project message, file name, tag date, tagger,
#                         hatchery, stock, brood yr, migr yr, tag site,
#                         raceway, capt meth, tag temp, post temp, rel temp,
#                         tag meth, org, coord id, rel date, rel site, rkm,
#                         close date, reach
#
#                 $LOADDIR/MORT_COM.VAL.<filename>
#                 format: file name, line number, note record
#
#                 $LOADDIR/MORT_HDR.VAL.<filename>
#                 format: file name, file date, coll site,
#                         coll river km, capt meth, org, coord id,
#                         close date, proj msg
#
#                 $LOADDIR/MORT_DATA.VAL.<filename>
#                 format: file name, seq no, tag id, chk sum, length, weight,
#                         mort date, flags, text comments
#
#                 $LOADDIR/MORT_FLAG.VAL.<filename>
#                 format: tag id, file name, flag code
#
# files written:
#
#                 /tmp/$pname.$filename
#                 log file, in case of sql error this file is e-mailed to
#                 $admin
#
#
#                 $CRUNCHDIR/TAG_DATA.CR.<filename>
#                 format: migr yr, file name, seq no, tag id, chk sum, length,
#                         weight, species, run, rear type, brd yr, var num,
#                         flags, text comments, var date
#
#                 $CRUNCHDIR/TAG_HDR.CR.<filename>
#                 format: project message, file name, tag date, tagger,
#                         hatchery, stock, brood yr, migr yr, tag site,
#                         raceway, capt meth, tag temp, post temp, rel temp,
#                         tag meth, org, coord id, rel date, rel site, rkm,
#                         close date, reach, nfish, species, run, rear type
#                         rel_file, trans dur, trans type, assoc mark
#
#                 $CRUNCHDIR/TAG_COM.CR.<filename>
#                 format: file name, line number, note record
#
#                 $CRUNCHDIR/TAG_DOT_OUT.CR.<filename>
#                 format: migr_yr, tag_file, seq_no, species, run, rear type,
#                         brd_yr, wt, length, flags, remark
#
#                 $CRUNCHDIR/TAG_DUP.CR.<filename>
#                 format: migr yr, file name, tag id, seq no, species, run,
#                         rear type, brd yr, wt, length, chk sum, flags,
#                         remark
#
#                 $CRUNCHDIR/TAG_VAR.CR.<filename>
#                 format:  file name, var number, var date
#
#                 $CRUNCHDIR/TAG_FLAG.CR.<filename>
#                 format: tag id, file name, flag code
#
#                 $CRUNCHDIR/RECAP_DATA.CR.<filename>
#                 format: tag id, re file name, tag file, re migr yr,
#                         re chk sum, re seq num, re len type, re len,
#                         re weight, re species, re run, re rear type,
#                         re brood yr, re rel var, re rel v date, re flags,
#                         re text comments
#
#                 $CRUNCHDIR/RECAP_HDR.CR.<filename>
#                 format: re migr yr, re file name, re site, re date,
#                         re num of fish, re mort num, re species, re run,
#                         re rear type, re hatchery, re stock, re brood yr,
#                         re raceway, re capt meth, re tag temp, re tag meth,
#                         re org, re coord id, re tagger, re rel file,
#                         re rel date, re rel site, re rel temp, re rkm,
#                         re epa reach, re transp_dur, re transp_type,
#                         re post tag temp, re close date, re session 
#
#                 $CRUNCHDIR/RECAP_FLAG.CR.<filename>
#                 format: tag id, file name, flag code
#
#                 $CRUNCHDIR/RECAP_COM.CR.<filename>
#                 format: file name, line number, note record
#
#                 $CRUNCHDIR/MORT_DATA.CR.<filename>
#                 format: mort file, tag file, tag id, chksm, migr yr, seq no
#                         tag len, tag wt, mort len, mort wt, org, coord id,
#                         tag rel site, tag rel date, mort date, species, run,
#                         rear type, flags, mort remark
#
#                 $CRUNCHDIR/MORT_HDR.CR.<filename>
#                 format: file name, mort date, site, rkm, capt meth, org,
#                         coord id, close date, proj msg
#
#                 $CRUNCHDIR/MORT_FLAG.CR.<filename>
#                 format: tag id, file name, flag code
#
#                 $CRUNCHDIR/MORT_COM.CR.<filename>
#                 format: file name, line number, note record
#
#                 $CRUNCHDIR/COUNT_SRRT.CR.<filename>
#                 format: load migr yr, file name, species, run, rear type,
#                         number of fish
#
#                 $CRUNCHDIR/filestoload.julian_date.pid.vernbr
#                 format: file name
#
# files moved from $LOADDIR to $CRUNCHDIR:
#
#                 TAG_COM.VAL.<filename>
#                 TAG_FLAG.VAL.<filename>
#                 TAG_DATA.VAL.<filename>
#                 TAG_VAR.VAL.<filename>
#                 TAG_HDR.VAL.<filename>
#                 MORT_HDR.VAL.<filename>
#                 MORT_DATA.VAL.<filename>
#                 MORT_FLAG.VAL.<filename>
#
# files removed:
#
#                 $CRUNCHDIR/TAG_COM.VAL.<filename>
#                 $CRUNCHDIR/TAG_FLAG.VAL.<filename>
#                 $CRUNCHDIR/TAG_DATA.VAL.<filename>
#                 $CRUNCHDIR/TAG_VAR.VAL.<filename>
#                 $CRUNCHDIR/TAG_HDR.VAL.<filename>
#                 $CRUNCHDIR/MORT_HDR.VAL.<filename>
#                 $CRUNCHDIR/MORT_DATA.VAL.<filename>
#                 $CRUNCHDIR/MORT_FLAG.VAL.<filename>
#
# tables updated:
#                 fld_load_summary
#                 tag_data
#                 mort_data
#
# programs called:
#
# modifications:
#
#  mar 06 98    jen   added gary to mailing list
#
#  Jan 23 98    jen   added code to commit after all updates in
#                     chkMortUpdate and chkTagUpdate
#
#  Oct 17 97    jen   deleted out code that removes mort data from
#                     tagging records.
#
#  Oct 13 97    jen   dbname load env. var.
#
#  Aug 26 97    jen   fix row crunch counter, deleted all references to
#                     performance report writer
#
#  Aug 25 97    jen   commented out performance report mailed to admin
#
#  Aug 21 97    jen   added function insert_fld_load_summary_tbl to insert
#                     crunch time into fld_load_summary table.
#
#  Jul 08 97    jen   fixed bug created by adding while loops to all
#                     selects (Jun 20 97)
#
#  Jun 25 97    jen   cleaned out main (put code into functions) and
#                     added code for clean exit if ~pittag/etc/FDL_SHUTDOWN
#                     file exists
#
#  Jun 20 97    jen   made all select statements into while loops
#                     removed commit statements within loops
#
#  May 21 97    jen   append mort_flag to tag_data records
#
#########################################################################
#
#
require "ingperl.pl";
require "/usr/local/lib/perl4/timelocal.pl";

##################################
#
#  Subroutines
#
# 18 lines
##################################
#
# 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#
##########
sub template
{ 

} # 
#
##################################
#
# insert_fld_load_summary_tbl 
#
#   calls:
#
#   receives:
#             _[0]: total process time per one file
#
#   returns:
#
#   called by:
#             main
#
##########
sub insert_fld_load_summary_tbl
{ 
   # get total number of rows crunched
   $rows  = 0;
   $rows += $#TAG_COM + 1;
   $rows += $#TAG_FLAG + 1;
   $rows += $#TAG_DATA + 1;
   if ($TAG_HDR) { $rows++; }
   $rows += $#TAG_VAR + 1;
   $rows += $#COUNT_SRRT + 1;
   $rows += $#RECAP_DATA + 1;;
   if ($RECAP_HDR) { $rows++; }
   $rows += $#RECAP_FLAG + 1;;
   $rows += $#RECAP_COM + 1;;
   if ($MORT_HDR) { $rows++; }
   $rows += $#MORT_DATA + 1;
   $rows += $#MORT_COM + 1;
   $rows += $#MORT_FLAG + 1;

   $query  = "insert into fld_load_summary (\n";
   $query .= "file_name,upload_date,row_num,crunch_time )\n";
   $query .= "values (\'$filename\',date(\'NOW\'),$rows,\'$_[0]\')\n";
   &sql_exec("$query");
   if ($sql_error) {
     print F "\n$0:***SQL ERROR*** $sql_error\n";
     print F "$query";
     $sqlerrs++;
   }
   &sql_exec("commit");
   &sql("disconnect");
} # insert_fld_load_summary_tbl
#
##################################
#
# getMortFileNames 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             Main
#
##########
sub getMortFileNames
{ 
  #
  # clean-out for re-use (used by tagging file names)
  #
  @allfiles = ();
  #
  # 
  # read-in list of filestocrunch.$jdate.$$.$vernbr files.  these files
  # hold the file names that have been validated and are ready to be
  # crunched.
  #
  opendir(DIR,$mortholddir);
  @mortfilestocrunch = grep(/^filestocrunch\.*/,readdir(DIR));
  closedir(DIR);
  foreach $mftc (@mortfilestocrunch) {
     system "mv $mortholddir/$mftc $mortworkdir";
     #
     # open and read in file names from mortfilestocrunch.vernbr
     #
     open(MFTC,"$mortworkdir/$mftc") ||
        die "\n\ncould not open $mortworkdir/$mftc\n\n";
     while (<MFTC>)
     {
        chop;
        push(@allfiles,$_);
     }
     close(MFTL);
     system "mv $mortworkdir/$mftc /tmp";
  } # for each mortfilestocrunch file
  #
  #                   MORT FILES
  # if a user uploads a file twice before the cruncher is run then the
  # same file name could be in two or more different filestocrunch files
  # so i splice out duplicate file names found in different filestocrunch
  # files below
  #
  @allfiles = sort(@allfiles);
  for($i=0;$i<=$#allfiles;$i++) {
     if ($i != 0) {
        $id2 = $id;
        $id = $allfiles[$i];
        if ($id2 eq $id) {
          splice(@allfiles,$i,1);
          $i--;
        }
     } else { $id = $allfiles[$i]; }
  }
} # getMortFileNames
#
##################################
#
# getTagFileNames  
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             Main
#
##########
sub getTagFileNames
{ 
  #
  # read-in list of filestocrunch.$jdate.$$.$vernbr files.  these files
  # hold the file names that have been validated and are ready to be
  # crunched.
  #
  opendir(DIR,$tagholddir);
  @tagfilestocrunch = grep(/^filestocrunch\.*/,readdir(DIR));
  closedir(DIR);

  foreach $tftc (@tagfilestocrunch) {
     system "mv $tagholddir/$tftc $tagworkdir";
     #
     # open and read in file names from tagfilestocrunch.vernbr
     #
     open(TFTC,"$tagworkdir/$tftc") ||
        die "\n\ncould not open $tagworkdir/$tftc\n\n";
     while (<TFTC>)
     {
        chop;
        push(@allfiles,$_);
     }
     close(TFTC);
     system "mv $tagworkdir/$tftc /tmp";
  } # for each tagfilestocrunch file
  #
  # if a user uploads a file twice before the cruncher is run then the
  # same file name could be in two or more different filestocrunch files
  # so i splice out duplicate file names found in different filestocrunch
  # files below
  #
  @allfiles = sort(@allfiles);
  for($i=0;$i<=$#allfiles;$i++) {
     if ($i != 0) {
        $id2 = $id;
        $id = $allfiles[$i];
        if ($id2 eq $id) {
          splice(@allfiles,$i,1);
          $i--;
        }
     } else { $id = $allfiles[$i]; }
  }
} # getTagFileNames
# 
##################################
#
# gracefulExit - called when FDL_SHUTDOWN file is detected
#
#   calls:
#             doEndStuff
#
#   receives:
#             $_[0]: if 0 = tagging, else 1 = mortality
#
#   returns:
#
#   called by:
#             Main
#
##########
sub gracefulExit
{ 
  $shuttime = &getTime;
  $sbj="shutdown fld_crunch.p @ $shuttime";
  #
  # tagging
  #
  if ($_[0] eq "0") {
     opendir(DIR,"$tagholddir");
     @ftocrunch = grep(/^filestocrunch\.*/, readdir(DIR));
     closedir(DIR);
     @vernbr=split('\.',$ftocrunch[$#ftocrunch]);
     if($vernbr[$#vernbr-2] eq $jdate) {
       $v=$vernbr[$#vernbr]+1;
       open(FTC,">$tagholddir/filestocrunch.$jdate.$$.$v") ||
         die "\n\ncould not open $tagholddir/filestocrunch.$jdate.$$.$v\n\n";
     }
     else { 
       open(FTC,">$tagholddir/filestocrunch.$jdate.$$.1") ||
         die "\n\ncould not open $tagholddir/filestocrunch.$jdate.$$.1\n\n";
     }
     opendir(DIR,"$tagholddir");
     @f = grep(!/^\./, readdir(DIR));
     closedir(DIR);
     for($i=0;$i<=$#f;$i++) {
        if (!($f[$i] =~ "filestocrunch")) {
           print FTC "$f[$i]\n";
        }
     }
     close(FTC);
  }
  #
  # mortality
  #
  else {
     opendir(DIR,"$mortholddir");
     @ftocrunch = grep(/^filestocrunch\.*/, readdir(DIR));
     closedir(DIR);
     @vernbr=split('\.',$ftocrunch[$#ftocrunch]);
     if($vernbr[$#vernbr-2] eq $jdate) {
       $v=$vernbr[$#vernbr]+1;
       open(FTC,">$mortholddir/filestocrunch.$jdate.$$.$v") ||
         die "\n\ncould not open $mortholddir/filestocrunch.$jdate.$$.$v\n\n";
     }
     else { 
       open(FTC,">$mortholddir/filestocrunch.$jdate.$$.1") ||
         die "\n\ncould not open $mortholddir/filestocrunch.$jdate.$$.1\n\n";
     }
     opendir(DIR,"$mortholddir");
     @f = grep(!/^\./, readdir(DIR));
     closedir(DIR);
     for($i=0;$i<=$#f;$i++) {
        if (!($f[$i] =~ "filestocrunch")) {
           print FTC "$f[$i]\n";
        }
     }
     close(FTC);
  }
  #
  # write the filestoload file
  #
  &doEndStuff;
  system "mailx -s \"$sbj\" $admin < /dev/null";
  system "echo \"$sbj\" >>$job_log";
  exit(0);
} # gracefulExit
#
##################################
#
# undoSecs 
#
#   calls:
#
#   receives:
#             $_[0]: variable in seconds
#
#   returns:
#             incoming seconds to date string (hours:minutes:secs)
#
#   called by:
#             Main
#
##########
sub undoSecs
{ 
  $seconds=$_[0];
  $minutes=0;
  $hours=0;
  if($seconds >= 60) {
    $minutes = ($seconds / 60);
    $seconds = ($seconds % 60);
    if($minutes >= 60) {
       $hours = ($minutes / 60);
       $minutes = ($minutes % 60);
    }
  }
  $seconds = sprintf("%02d",$seconds);
  $minutes = sprintf("%02d",$minutes);
  $hours = sprintf("%02d",$hours);
  return("$hours:$minutes:$seconds");
} # undoSecs
#
##################################
#
# doSecs 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#            Main
#
##########
sub doSecs
{ 
   $mon--;
   $dateInSeconds=&timelocal($sec,$min,$hour,$mday,$mon,$year);
} # doSecs
#
##################################
#
# chkTagUpdate - update mort flag in tag data table
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#            finishMort
#
##########
sub chkTagUpdate
{ 
   $nowtime = &getTime;
   print F "$0: chkTagUpdate: $filename begin $nowtime\n";
   # mort_data= mort file, tag file, tag id, chksm, migr yr, seq no
   #            tag len, tag wt, mort len, mort wt, org, coord id,
   #            tag rel site, tag rel date, mort date, species, run,
   #            rear type, flags, mort remark
   for($ii=0;$ii<=$#MORT_DATA;$ii++) {
      ($a,$b,$c,$d,$e,$f,$g,$h,$i) = split('\^',$MORT_DATA[$ii]);
      $query  = "select tag_id from tag_data\n";
      $query .= "where tag_id = \'$c\'";
      &sql($query) || die "\n$sql_error\n";
      $found = 0;
      while (@row = &sql_fetch) {
        if ($sql_rowcount > 0) { $found = 1; }
        if($sql_error) {
          print F "\n$0:***SQL ERROR*** $sql_error\n";
          print F "$query\n";
          $sqlerrs++;
        }
      }
      &sql_exec("commit");
      if($sql_error) {
         print F "\n$0:***SQL ERROR*** on commit: $sql_error\n";
         $sqlerrs++;
      } # if sql error
      if ($found > 0) {
         $query  = "update tag_data\n";
         $query .= "set mort_flag = \'M\'\n";
         $query .= "where tag_id = \'$c\'";
         &sql_exec("$query");
         if ($sql_error) {
            print F "\n$0:***SQL ERROR*** $sql_error\n";
            print F "$query\n";
            $sqlerrs++;
         }
         print F "$0: chkTagUpdate: update on tag_data performed\n";
         &sql_exec("commit");
         if($sql_error) {
            print F "\n$0:***SQL ERROR*** on commit: $sql_error\n";
            $sqlerrs++;
         } # if sql error
      } # if tag id found in tag_data table
   }
   $nowtime = &getTime;
   print F "$0: chkTagUpdate: $filename end $nowtime\n";
} # chkTagUpdate
#
##################################
#
# chkMortUpdate - update mort_data table with incoming tag data
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#            finishTag
#
##########
sub chkMortUpdate
{ 
   $nowtime = &getTime;
   print F "$0: chkMortUpdate: $filename begin $nowtime\n";
   # tag hdr: project message, file name, tag date, tagger,
   #          hatchery, stock, brood yr, migr yr, tag site,
   #          raceway, capt meth, tag temp, post temp, rel temp,
   #          tag meth, org, coord id, rel date, rel site, rkm,
   #          close date, reach, nfish, species, run, rear type
   #          rel_file, trans dur, trans type, assoc mark
   # tag data: migr yr, file name, seq no, tag id, chk sum, length,
   #           weight, species, run, rear type, brd yr, var num,
   #           flags, text comments, var date, mort flag
   $rdate="";
   ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p,$q,$r,$s)
      = split('\^',$TAG_HDR);
   $tfile = $b;
   $myr = $h;
   $org = $p;
   $cid = $q;
   $rsite = $s;
   $rdate = $r;
   for($jj=0;$jj<=$#TAG_DATA;$jj++) {
      ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) =
         split('\^',$TAG_DATA[$jj]);
      $query  = "select m_tagid from mort_data\n";
      $query .= "where m_tagid = \'$d\'\n";
      &sql($query) || die "\n$sql_error\n";
      $found = 0;
      while (@row = &sql_fetch) {
        if ($sql_rowcount > 0) { $found = 1; }
        if ($sql_error) {
          print F "\n$0:***SQL ERROR*** $sql_error\n";
          print F "$query\n";
          $sqlerrs++;
        } # if sql error
      } # while fetch 
      &sql_exec("commit");
      if($sql_error) {
         print F "\n$0:***SQL ERROR*** on commit: $sql_error\n";
         $sqlerrs++;
      } # if sql error
      if ($found > 0) {
         if($g eq "") { $g = 0; }
         if($f eq "") { $f = 0; }
         $query  = "update mort_data\n";
         $query .= "set tag_file = \'$tfile\'\n";
         $query .= ", migr_yr = \'$myr\'\n";
         $query .= ", tag_lgth = $f\n";
         $query .= ", tag_wt = $g\n";
         $query .= ", org = \'$org\'\n";
         $query .= ", coord_id = \'$cid\'\n";
         $query .= ", tag_rel_site = \'$rsite\'\n";
         $query .= ", tag_rel_date = \'$rdate\'\n";
         $query .= "where m_tagid = \'$d\'\n";
         &sql_exec($query);
         if ($sql_error) {
            print F "\n$0:***SQL ERROR*** $sql_error\n";
            print F "$query\n";
            $sqlerrs++;
         } # if sql error
         print F "$0: chkMortUpdate: update on mort_data performed\n";
         $TAG_DATA[$jj]="$a^$b^$c^$d^$e^$f^$g^$h^$i^$j^$k^$l^$m^$n^$o^M";
         &sql_exec("commit");
         if($sql_error) {
            print F "\n$0:***SQL ERROR*** on commit: $sql_error\n";
            $sqlerrs++;
         } # if sql error
      } # if pit tag id found in mort_data table
   } # for length of tag data
   $nowtime = &getTime;
   print F "$0: chkMortUpdate: $filename end $nowtime\n";
} # chkMortUpdate
#
##################################
#
# chkRelUpdate - update tag data with release info from database
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#            finishTag
#
##########
sub chkRelUpdate
{ 
  # TAG_HDR=  a)project message, b)file name, c)tag date, d)tagger,
  #           e)hatchery, f)stock, g)brood yr, h)migr yr, i)tag site,
  #           j)raceway, k)capt meth, l)tag temp, m)post temp, n)rel temp,
  #           o)tag meth, p)org, q)coord id, r)rel date, s)rel site, t)rkm,
  #           u)close date, v)reach, w)nfish, x)species, y)run, z)rear type
  $rdate="";
  ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p,$q,$r,$s,$t)
     = split('\^',$TAG_HDR);
  $rdate = $r;
  if($rdate eq "") {
    # TAG_DATA= a)migr_yr, b)file name, c)seq no, d)tag id, e)chk sum,
    #           f)length, g)weight, h)species, i)run, j)rear type, k)brd yr,
    #           l)var num, m)flags, n)text comments, o)vardate, p)mort flag
    ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p)=split('\^',$TAG_DATA[0]);
    if($o eq "") {
      $query  = "select rel_date, rel_site, rel_file, river_km\n";
      $query .= ", rel_batch, trans_dur, trans_type, assoc_mark\n",
      $query .= " from rel_hdr where file_id = \'$b\'\n";
      &sql("$query") || die "\n$sql_error\n";
      $row0=$row1=$row2=$row3=$row4=$row5=$row6=$row7="";
      $found=0;
      while (@row = &sql_fetch) {
       if($sql_rowcount > 0) {
        $found = 1;
        $row[0] =~ s/\ *$//g; # rel_date
        $row0 = $row[0];
        $row[1] =~ s/\ *$//g; # rel_site
        $row1 = $row[1];
        $row[2] =~ s/\ *$//g; # rel_file
        $row2 = $row[2];
        $row[3] =~ s/\ *$//g; # river_km
        $row3 = $row[3];
        # yo, substr is here because ingperl returns the number of bytes
        # used to store the binary reporesentation of the value, 
        $rtemp = substr($row[4],0,4); # rel_batch
        $row[5] =~ s/\ *$//g; # trans_dur
        $row5 = $row[5];
        $row[6] =~ s/\ *$//g; # trans_type
        $row6 = $row[6];
        $row[7] =~ s/\ *$//g; # assoc_mark
        $row7 = $row[7];
       }
        if ($sql_error) {
          print F "\n$0:***SQL ERROR*** $sql_error\n";
          print F "$query\n";
         $sqlerrs++;
        }
      }
      if ($found > 0) {
         for($jj=0;$jj<=$#TAG_DATA;$jj++) {
            ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) = 
               split('\^',$TAG_DATA[$jj]);
            # rel_v_time
            $o = $row0;
            $TAG_DATA[$jj]="$a^$b^$c^$d^$e^$f^$g^$h^$i^$j^$k^$l^$m^$n^$o^$p";
         } # for length of tag data records
         # TAG_HDR=  a)project message, b)file name, c)tag date, d)tagger,
         #           e)hatchery, f)stock, g)brood yr, h)migr yr, i)tag site,
         #           j)raceway, k)capt meth, l)tag temp, m)post temp,
         #           n)rel temp, o)tag meth, p)org, q)coord id, r)rel date,
         #           s)rel site, t)rkm, u)close date, v)reach, w)nfish,
         #           x)species, y)run, z)rear type
($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p,$q,$r,$s,$t,$u,$v,$w,$x,$y,$z)
        = split('\^',$TAG_HDR);
         # TAG_HDR=  project message, file name, tag date, tagger,
         #           hatchery, stock, brood yr, migr yr, tag site,
         #           raceway, capt meth, tag temp, post temp,
         #           rel temp, tag meth, org, coord id, rel date,
         #           rel site, rkm, close date, reach, nfish
         #           species, run, rear type, rel file, trans dur
         #           trans type, assoc mark
         $TAG_HDR  = "$a^$b^$c^$d^$e^$f^$g^$h^$i^$j^$k^$l^$m^$rtemp^$o^$p";
         $TAG_HDR .= "^$q^$row0^$row1^$row3^$u^$v^$w^$x^$y^$z^$row2";
         $TAG_HDR .= "^$row5^$row6^$row7";
       } else { $TAG_HDR .= "^^^^"; }
    } else { $TAG_HDR .= "^^^^"; }
  } else {
   # TAG_DATA= a)migr_yr, b)file name, c)seq no, d)tag id, e)chk sum,
   #           f)length, g)weight, h)species, i)run, j)rear type, k)brd yr,
   #           l)var num, m)flags, n)text comments, o)vardate, p)mort flag
   #
    ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p)=split('\^',$TAG_DATA[0]);
    for($jj=0;$jj<=$#TAG_DATA;$jj++) {
       ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) = 
          split('\^',$TAG_DATA[$jj]);
   # Next if added to update rel_v_time with release time from header if
   # this is not a variable release time.
       if ($o eq "") {
       $TAG_DATA[$jj]="$a^$b^$c^$d^$e^$f^$g^$h^$i^$j^$k^$l^$m^$n^$rdate^$p";
       }
    }
   # adding: rel_file, trans dur, trans type, assoc mark to tag_hdr
    $TAG_HDR .= "^^^^";
  } # got release info from tag hdr
} # chkRelUpdate
#
##################################
#
# doCountSrrt - do the count srrt thing...count species, run and rear types
#               per file.  load into an array in the same format as the
#               count_srrt table in ptagis.
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub doCountSrrt
{ 
  # TAG_DATA= migr_yr, file name, seq no, tag id, chk sum, length, weight,
  #           species, run, rear type, brd yr, var num, flags, text comments,
  #           vardate, mort date

  # strip out pertinent data from tag data for count srrt table
  for($jj=0;$jj<=$#TAG_DATA;$jj++) {
     ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) = 
        split('\^',$TAG_DATA[$jj]);
     # load migr yr, file name, species, run, rear type and one fish
     push(@SRRT,"$a^$b^$h^$i^$j"); 
  }
  # count up codes
  for($jj=0;$jj<=$#SRRT;$jj++) { push(@COUNT,1) }
  for($jj=0;$jj<=$#SRRT;$jj++) {
     ($a,$b,$c,$d,$e) = split('\^',$SRRT[$jj]);
     for($ii=0;$ii<=$#SRRT;$ii++) {
        ($f,$g,$h,$i,$j) = split('\^',$SRRT[$ii]);
        if($jj != $ii) {
           if($c eq $h) {
              if($d eq $i) {
                 if($e eq $j) { 
                    $COUNT[$jj]++;
                    splice(@COUNT,$ii,1);
                    splice(@SRRT,$ii,1);
                    $ii--;
                 }
              }
           }
        }
     }
  }
  # load count srrt array
  for($jj=0;$jj<=$#SRRT;$jj++) {
     push(@COUNT_SRRT, "$SRRT[$jj]^$COUNT[$jj]"); }
} # doCountSrrt
#
##################################
# dumpArray
#
#   calls:
#
#   receives:
#             $_[0]: name of array (MUST BE PASSED BY REFERENCE)
#             $_[1]: size of array
#             $_[2]: name of file to write to
#
#   returns:
#
#   called by:
#             finishTag
#             finishMort
#
##########
sub dumpArray
{
   (*name,$size,$fn)=@_;
   open(DAT,">$working_dir/$fn") ||
      die "\n\ncould not open $fn for writing\n\n";
   for($i=0;$i<=$size;$i++) {
      print DAT "$name[$i]\n"; }
   close(DAT);
} # dumpArray
#
##################################
#
#  cleanTagArrays
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             finishTag
#
##########
sub cleanTagArrays
{ 
   #
   # commented out this whole area.  if the fish is a mort on a tagging
   # event then these records need to be included, if it was morted 
   # during a recapture event then the next chunk of code will get
   # it.
   #
   # clean-out mort data from tag data arrays
   #
   # data
   #for($ii=0;$ii<=$#MORTS;$ii++) {
   #    ($x,$y) = split('\^',$MORTS[$ii]);
   #    for($jj=0;$jj<=$#TAG_DATA;$jj++) {
   #       ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) =
   #            split('\^',$TAG_DATA[$jj]);
   #       if($y eq $d) { splice(@TAG_DATA,$jj,1); $jj--; }
   #    }
   #}
   # flags
   #for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
   #   ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
   #   for($jj=0;$jj<=$#MORTS;$jj++) {
   #      ($x,$y) = split('\^',$MORTS[$jj]);
   #      if($y eq $a) { splice(@TAG_FLAG,$ii,1); $ii--; }
   #   }
   #}
   # set mort flag in tag data
   for($ii=0;$ii<=$#MORTS;$ii++) {
       ($x,$y) = split('\^',$MORTS[$ii]);
       for($jj=0;$jj<=$#TAG_DATA;$jj++) {
          ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) =
               split('\^',$TAG_DATA[$jj]);
          if($y eq $d) {
             $TAG_DATA[$jj]="$a^$b^$c^$d^$e^$f^$g^$h^$i^$j^$k^$l^$m^$n^$o^M";
          }
       }
   }

   #
   # clean-out recap data from tag data arrays
   #
   # data
   for($ii=0;$ii<=$#RECAPS;$ii++) {
      ($x,$y) = split('\^',$RECAPS[$ii]);
      for($jj=0;$jj<=$#TAG_DATA;$jj++) {
         ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) =
            split('\^',$TAG_DATA[$jj]);
         if($y eq $d) { splice(@TAG_DATA,$jj,1); $jj--; }
      }
   }
   # flags
   for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
      ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
      for($jj=0;$jj<=$#RECAPS;$jj++) {
         ($x,$y) = split('\^',$RECAPS[$jj]);
         if($y eq $a) { splice(@TAG_FLAG,$ii,1); $ii--; }
      }
   }
   #
   # append nfish, species, run, rear type to tag hdr
   #
   # TAG_HDR=  project message, file name, tag date, tagger,
   #           hatchery, stock, brood yr, migr yr, tag site,
   #           raceway, capt meth, tag temp, post temp, rel temp,
   #           tag meth, org, coord id, rel date, rel site, rkm,
   #           close date, reach, nfish, species, run, rear type 
   #           nfish
   $nfish = $#TAG_DATA+1;
   $TAG_HDR.="^$nfish^NA^NA^NA";
} # cleanTagArrays
#
##################################
#
#  findTagDups
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             finishTag
#
##########
sub findTagDups
{ 
    # tag_data
    # format: migr_yr, file name, seq no, tag id, chk sum, length, weight,
    #         species, run, rear type, brd yr, var num, flags, text comments
    #         var date, mort flag
   for($jj=0;$jj<=$#TAG_DATA;$jj++) {
      ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) =
         split('\^',$TAG_DATA[$jj]);
      $query  = "select tag_id from tag_data\n";
      $query .= "where tag_file = \'ORPHAN\'\n";
      $query .= "and tag_id = \'$d\'";
      &sql("$query") || die "\n$sql_error\n$query";
      $found = 0;
      while (@row = &sql_fetch) {
        if($sql_rowcount > 0) { $found = 1; }
        if($sql_error) {
          print F "\n$0:***SQL ERROR*** $sql_error\n";
          print F "$query\n";
          $sqlerrs++;
        }
      }
      if ($found > 0) {
         #
         #  an orphan is not a dup... so get it out of there
         #
         $query  = "delete from tag_data\n";
         $query .= "where tag_file = \'ORPHAN\'\n";
         $query .= "and tag_id = \'$d\'";
         &sql_exec("$query");
         if ($sql_error) {
            print F "\n$0:***SQL ERROR*** $sql_error\n";
            print F "$query\n";
            $sqlerrs++;
         }
      }
      $query  = "select tag_id from tag_data\n";
      $query .= "where tag_id = \'$d\'";
      &sql("$query") || die "\n$sql_error\n$query";
      $row0="";
      while (@row = &sql_fetch) {
        if($sql_rowcount > 0) { $row[0] =~ s/\ *$//g; $row0 = $row[0]; }
        if ($sql_error) {
          print F "\n$0:***SQL ERROR*** $sql_error\n";
          print F "$query\n";
          $sqlerrs++;
        }
      }
      if($row0 ne "") {
         # tag dup format: migr yr, file name, tag id, seq no, species, run,
         #                 rear type, brd yr, wt, length, chk sum, flags,
         #                 remark
         push(@TAG_DUP,"$a^$b^$d^$c^$h^$i^$j^$k^$g^$f^$e^$m^$n");
         splice(@TAG_DATA,$jj,1);
         $jj--;
      }
   }
} # findTagDups 
#
##################################
#
# updateLogs 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#
##########
sub updateLogs
{ 
   #
   # update user upload log table
   #
   $MSG  = "update user_upload_log\n";
   $MSG .= "set status = 'FILE READY TO LOAD'\n";
   $MSG .= ",msg = 'FLD_CRUNCH.P'\n";
   $MSG .= ",process_date = \'$processdate\'\n";
   $MSG .= "where file_name = \'$filename\'\n";
   &sql_exec("$MSG");
   if ($sql_error) {
     print F "\n$0:***SQL ERROR*** $sql_error\n";
     print F "$MSG";
     $sqlerrs++;
   }

   #
   # select important info from user upload log table for insert into
   # user upload archive table
   #
   $query  = "select type,username,upload_date from user_upload_log\n";
   $query .= "where file_name = \'$filename\'";
   &sql("$query") || die "\n$sql_error\n$query";
   $row0=$row1=$row2="";
   while (@row = &sql_fetch) {
     if ($sql_rowcount > 0) {
        $row[0] =~ s/\ *$//g; # type
        $row0 = $row[0];
        $row[1] =~ s/\ *$//g; # username
        $row1 = $row[1];
        $row[2] =~ s/\ *$//g; # upload_date
        $row2 = $row[2];
     }
     if ($sql_error) {
       print F "\n$0:***SQL ERROR*** $sql_error\n";
       print F "$query\n";
       $sqlerrs++;
     }
   }

   #
   # update user upload archive table
   #
   $MSG  = "insert into user_upload_archive(type,file_name,username\n";
   $MSG .= ",process_date,upload_date,status,msg)\n";
   $MSG .= "values (\'$row0\',\'$filename\',\'$row1\',\'$processdate\'\n";
   $MSG .= ",\'$row2\',\'FILE READY TO LOAD\'\n";
   $MSG .= ",\'FLD_CRUNCH.P\')\n";
   &sql_exec("$MSG");
   if ($sql_error) {
     print F "\n$0:***SQL ERROR*** $sql_error\n";
     print F "$MSG";
     $sqlerrs++;
   }
} # updateLogs
# 
##################################
#
# dumpHdr 
#
#   calls:
#
#   receives:
#            $_[0]: string name (hdr data)
#            $_[1]: file name to write to
#
#   returns:
#
#   called by:
#            finishTag
#            finishMort
#
##########
sub dumpHdr
{
   open(DAT,">$working_dir/$_[1]") ||
      die "\n\ncould not open $working_dir/$_[1] for writing\n\n";
   if($_[0] ne "") { print DAT "$_[0]\n"; }
   close(DAT)
} # dumpHdr
#
##################################
#
#  finishTag - clear-out tag arrays if all data was found to be recaptures and
#              mortalities.  if tag data still remains then count the species
#              run and rear types for the count srrt table.
#
#   calls:
#             cleanTagArrays
#             findTagDups
#             doCountSrrt
#             chkRelUpdate
#             chkMortUpdate
#             dumpArray
#             dumpHdr
#             updateLogs
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub finishTag
{ 
   &cleanTagArrays;
   &findTagDups;
   if($#TAG_DATA < 0) {
      $TAG_HDR="";
      @TAG_COM=(); }
   else { &doCountSrrt; &chkRelUpdate; &chkMortUpdate; }
   &dumpArray("TAG_COM",$#TAG_COM,"TAG_COM.CR.$filename");
   &dumpArray("TAG_VAR",$#TAG_VAR,"TAG_VAR.CR.$filename");
   &dumpArray("TAG_FLAG",$#TAG_FLAG,"TAG_FLAG.CR.$filename");
   &dumpArray("TAG_DATA",$#TAG_DATA,"TAG_DATA.CR.$filename");
   &dumpHdr($TAG_HDR,"TAG_HDR.CR.$filename");
   &dumpArray("TAG_DOT_OUT",$#TAG_DOT_OUT,"TAG_DOT_OUT.CR.$filename");
   &dumpArray("TAG_DUP",$#TAG_DUP,"TAG_DUP.CR.$filename");
   &dumpArray("RECAP_FLAG",$#RECAP_FLAG,"RECAP_FLAG.CR.$filename");
   &dumpArray("RECAP_DATA",$#RECAP_DATA,"RECAP_DATA.CR.$filename");
   &dumpHdr($RECAP_HDR,"RECAP_HDR.CR.$filename");
   &dumpArray("RECAP_COM",$#RECAP_COM,"RECAP_COM.CR.$filename");
   &dumpArray("MORT_FLAG",$#MORT_FLAG,"MORT_FLAG.CR.$filename");
   &dumpArray("MORT_DATA",$#MORT_DATA,"MORT_DATA.CR.$filename");
   &dumpHdr($MORT_HDR,"MORT_HDR.CR.$filename");
   &dumpArray("MORT_COM",$#MORT_COM,"MORT_COM.CR.$filename");
   &dumpArray("COUNT_SRRT",$#COUNT_SRRT,"COUNT_SRRT.CR.$filename");
   &updateLogs;
   close(F);
   if($sqlerrs != 0) {
      &sql_exec("rollback");
      if($sql_error) {
         print F "\n$0:***SQL ERROR*** on rollback: $sql_error\n"; }
      system "mailx -s \"$adminmsg\" $admin </tmp/$pname.$filename"; }
   else {
      &sql_exec("commit");
      if($sql_error) {
         print F "\n$0:***SQL ERROR*** on commit: $sql_error\n"; }
   }
   system "mv -f $tagworkdir/$filename $tagcrdir";
} # finishTag
#
##################################
#
#  finishMort 
#
#   calls:
#             chkTagUpdate
#             dumpArray
#             dumpHdr
#             updateLogs
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub finishMort
{ 
   &chkTagUpdate;
   &dumpArray("MORT_FLAG",$#MORT_FLAG,"MORT_FLAG.CR.$filename");
   &dumpArray("MORT_DATA",$#MORT_DATA,"MORT_DATA.CR.$filename");
   &dumpHdr($MORT_HDR,"MORT_HDR.CR.$filename");
   &dumpArray("MORT_COM",$#MORT_COM,"MORT_COM.CR.$filename");
   &updateLogs;
   close(F);
   if($sqlerrs != 0) {
      &sql_exec("rollback");
      if($sql_error) {
         print F "\n$0:***SQL ERROR*** on rollback: $sql_error\n"; }
      system "mailx -s \"$adminmsg\" $admin </tmp/$pname.$filename"; }
   else {
      &sql_exec("commit");
      if($sql_error) {
         print F "\n$0:***SQL ERROR*** on commit: $sql_error\n"; }
   }
   system "mv -f $mortworkdir/$filename $mortcrdir";
} # finishMort
#
##################################
#
# loadTagMorts - find morts if any exist and strip them out of the tag data
#             arrays.
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub loadTagMorts
{ 
  # TAG_HDR=  project message, file name, tag date, tagger,
  #           hatchery, stock, brood yr, migr yr, tag site,
  #           raceway, capt meth, tag temp, post temp, rel temp,
  #           tag meth, org, coord id, rel date, rel site, rkm,
  #           close date, reach
  # TAG_DATA= migr_yr, file name, seq no, tag id, chk sum, length, weight,
  #           species, run, rear type, brd yr, var num, flags, text comments,
  #           vardate, mort flag
  # tag flag format: tag id, file name, flag code
  # recaps: file name, tag id
  # morts: file name, tag id
  # mort com format: file name, line number, note record
  # get tag ids with the L, M, MB, MK, MS, SM flag and save in array
  for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
    ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
    # if it is a recap then previous tagging info exists else
    # take data from header records
    $recap=0;
    for($jj=0;$jj<=$#RECAPS;$jj++) {
       ($d,$e) = split('\^',$RECAPS[$jj]);
       if($a eq $e) { $recap = 1; }
    }
    if($c eq "L") { push(@MORTS,"$b^$a^$recap"); }
    elsif($c eq "M") { push(@MORTS,"$b^$a^$recap"); }
    elsif($c eq "MB") { push(@MORTS,"$b^$a^$recap"); }
    elsif($c eq "MK") { push(@MORTS,"$b^$a^$recap"); }
    elsif($c eq "MS") { push(@MORTS,"$b^$a^$recap"); }
    elsif($c eq "SM") { push(@MORTS,"$b^$a^$recap"); }
  } 
  # get mort hdr from tag hdr
  # morts: file name, tag id
  ($x,$y) = split('\^',$MORTS[0]);
  # TAG_HDR=  project message, file name, tag date, tagger,
  #           hatchery, stock, brood yr, migr yr, tag site,
  #           raceway, capt meth, tag temp, post temp, rel temp,
  #           tag meth, org, coord id, rel date, rel site, rkm,
  #           close date, reach
  $rdate="";
  ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p,$q,$r,$s,$t,$u,$v) =
     split('\^',$TAG_HDR);
  $rdate=$r;
  # find rkm for tagging site i.e. mort site
  # rkm_tbl format: site, loc_name, river_km, reach, fixed_rel_site_y_n
  if($x ne "") {
     for($ii=0;$ii<=$#RKM_TBL;$ii++) {
        ($site,$loc,$rkm,$rch,$fix) = split('\^',$RKM_TBL[$ii]);
        if ($i eq $site) { $ii = $#RKM_TBL + 1; }
     }
  }
  if($x eq $b) {
     # mort_hdr= file name, mort date, site, rkm, capt meth, org, coord id,
     #           close date, proj msg
     $MORT_HDR = "$b^$c^$i^$rkm^$k^$p^$q^$u^$a";
     # nab below for mort_data info
     $rsite = $s;
     $cid = $q;
     $org = $p;
     $tdate = $c;
  }
  # TAG_DATA= migr_yr, file name, seq no, tag id, chk sum, length, weight,
  #           species, run, rear type, brd yr, var num, flags, text comments,
  #           vardate, mort flag
  # use array created above to strip out mort data from tag data
  for($ii=0;$ii<=$#MORTS;$ii++) {
    ($x,$y,$z) = split('\^',$MORTS[$ii]);
    for($jj=0;$jj<=$#TAG_DATA;$jj++) {
       ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) = 
          split('\^',$TAG_DATA[$jj]);
       $tfile = $filename;
       $twt = $g;
       $tlen = $f;
       if($y eq $d) {
          # z is the recap flag
          if($z eq "1") {
            $rdate="";
            #
            # load query and get the tag file name, tag wt and tag len
            #
            $query  = "select tag_file, wt, length, rel_v_time ";
            $query .= "from tag_data ";
            $query .= "where tag_id = \'$d\'";
            &sql("$query") || die "\n$sql_error\n$query";
            while (@row = &sql_fetch) {
             if($sql_rowcount > 0) {
              $row[0] =~ s/\ *$//g; # tag file
              $tfile = $row[0];
              $row[1] =~ s/\ *$//g; # wt
              # yo, substr is here because ingperl returns the number of bytes
              # used to store the binary reporesentation of the value, 
              # trust me, it's way too much.
              $twt = substr($row[1],0,4); 
              $row[2] =~ s/\ *$//g; # length
              $tlen = $row[2];
              $row[3] =~ s/\ *$//g; # rel_v_time
              $rdate = $row[3];
             }
             if ($sql_error) {
               print F "\n$0:***SQL ERROR*** $sql_error\n";
               print F "$query\n";
               $sqlerrs++;
             }
            }
            #
            # load query and get the tag rel site
            #
            $query  = "select rel_site from tag_hdr ";
            $query .= "where file_id = \'$tfile\'";
            &sql("$query") || die "\n$sql_error\n$query";
            while (@row = &sql_fetch) {
             if($sql_rowcount > 0) {
              $row[0] =~ s/\ *$//g;
              $rsite = $row[0];
             }
              if ($sql_error) {
                print F "\n$0:***SQL ERROR*** $sql_error\n";
                print F "$query\n";
                $sqlerrs++;
              }
            }
          }
          # mort_data= mort file, tag file, tag id, chksm, migr yr, seq no
          #            tag len, tag wt, mort len, mort wt, org, coord id,
          #            tag rel site, tag rel date, mort date, species, run,
          #            rear type, flags, mort remark
          if($f eq "") { $f = "0"; }
          if($g eq "") { $g = "0"; }
          $str  = "$b\^$tfile\^$d\^$e\^$a\^$c\^$tlen\^$twt\^$f\^$g\^$org";
          $str .= "\^$cid\^$rsite\^$rdate\^$tdate\^$h\^$i\^$j\^$m\^$n";
          push(@MORT_DATA,$str);
       }
    }
  }
  # load mort comments from tag comments
  # tag com format: file name, line number, note record
  for($ii=0;$ii<=$#TAG_COM;$ii++) {
     ($a,$b,$c) = split('\^',$TAG_COM[$ii]);
     if($x eq $a) { push(@MORT_COM,$TAG_COM[$ii]); }
  }
  # get mort flags and remove tagging flags
  # tag flag format: tag id, file name, flag code
  for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
     ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
     for($jj=0;$jj<=$#MORTS;$jj++) {
       ($x,$y) = split('\^',$MORTS[$jj]);
       if($y eq $a) { push(@MORT_FLAG,$TAG_FLAG[$ii]); }
     }
  }
} # loadTagMorts
#
##################################
#
# loadRecaps - find recap data if it exists and strip out of the tag data.
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub loadRecaps
{ 
  # TAG_HDR=  project message, file name, tag date, tagger,
  #           hatchery, stock, brood yr, migr yr, tag site,
  #           raceway, capt meth, tag temp, post temp, rel temp,
  #           tag meth, org, coord id, rel date, rel site, rkm,
  #           close date, reach
  # TAG_DATA= migr_yr, file name, seq no, tag id, chk sum, length, weight,
  #           species, run, rear type, brd yr, var num, flags, text comments,
  #           vardate, mort flag
  # tag flag format: tag id, file name, flag code
  # recaps: file name, tag id
  # tag com format: file name, line number, note record

  # get tag ids with the re flag and save in array
  for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
    ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
    if($c eq "RE") { push(@RECAPS,"$b^$a"); }
  } 

  # get recap hdr from tag hdr
  # recaps: file name, tag id
  ($x,$y) = split('\^',$RECAPS[0]);
  # TAG_HDR=  project message, file name, tag date, tagger,
  #           hatchery, stock, brood yr, migr yr, tag site,
  #           raceway, capt meth, tag temp, post temp, rel temp,
  #           tag meth, org, coord id, rel date, rel site, rkm,
  #           close date, reach
  $r="";
  ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p,$q,$r,$s,$t,$u,$v) =
     split('\^',$TAG_HDR);
  if($x eq $b) { 
     # fix number of fish
     $w = $#RECAPS+1;
     # load rel file name if tagging file was the rel file
     $rel = "";
     $rdate = "";
     if($r ne "") { $rel = $b; $rdate = $r; }
     # recap_hdr= re migr yr, re file name, re site, re date, re num of fish,
     #            re mort num, re species, re run, re rear type, re hatchery,
     #            re stock, re brood yr, re raceway, re capt meth, re tag temp,
     #            re tag meth, re org, re coord id, re tagger, re rel file,
     #            re rel date, re rel site, re rel temp, re rkm, re epa reach,
     #            re transp_dur, re transp_type, re post tag temp,
     #            re close date, re session (proj msg)
     $str  = "$h\^$b\^$i\^$c\^$w\^\^\^\^\^$e\^$f\^$g\^$j\^$k\^$l\^$o";
     $str .= "\^$p\^$q\^$d\^$rel\^$r\^$s\^$n\^$t\^$v\^\^\^$m\^$u\^$a";
     $RECAP_HDR = $str;
  }

  # use array created above to strip out recap data from tag data
  # TAG_DATA= migr_yr, file name, seq no, tag id, chk sum, length, weight,
  #           species, run, rear type, brd yr, var num, flags, text comments,
  #           vardate, mort flag
  for($ii=0;$ii<=$#RECAPS;$ii++) {
    ($x,$y) = split('\^',$RECAPS[$ii]);
    for($jj=0;$jj<=$#TAG_DATA;$jj++) {
       $o="";
       ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p) = 
          split('\^',$TAG_DATA[$jj]);
       if($y eq $d) {
          #
          # load query and get the tag file name
          #
          $query  = "select tag_file from tag_data ";
          $query .= "where tag_id = \'$d\'";
          &sql("$query") || "\n$sql_error\n$query";
          $row0="";
          while (@row = &sql_fetch) {
            if($sql_rowcount > 0) { $row[0] =~ s/\ *$//g; $row0 = $row[0]; } # tag file
            if ($sql_error) {
              print F "\n$0:***SQL ERROR*** $sql_error\n";
              print F "$query\n";
              $sqlerrs++;
            }
          }
          # recap_data= tag id, re file name, tag file, re migr yr, re chk sum,
          #             re seq num, re len type, re len, re weight, re species,
          #             re run, re rear type, re brood yr, re rel var,
          #             re rel v date, re flags, re text comments
          # Added $l, re_rel_var to $str below.
          if ($o eq "") { $o = $rdate; }
          $str  = "$d\^$b\^$row0\^$a\^$e\^$c\^\^$f\^$g\^$h\^$i\^$j\^$k\^$l";
          $str .= "\^$o\^$m\^$n";
          push(@RECAP_DATA,$str);
       }
    }
  }

  # load recap comments from tag comments
  # tag com format: file name, line number, note record
  for($ii=0;$ii<=$#TAG_COM;$ii++) {
     ($a,$b,$c) = split('\^',$TAG_COM[$ii]);
     if($x eq $a) { push(@RECAP_COM,$TAG_COM[$ii]); }
  }
  # get recap flags and remove tagging flags
  # tag flag format: tag id, file name, flag code
  for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
     ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
     for($jj=0;$jj<=$#RECAPS;$jj++) {
       ($x,$y) = split('\^',$RECAPS[$jj]);

       if($y eq $a) { push(@RECAP_FLAG,$TAG_FLAG[$ii]); }
     }
  }

} # loadRecaps
#
##################################
#
# doDotOut - find dotted out tag id records, add them to dot out array
#            and remove them from the tag data array.
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub doDotOut
{ 
  #tag data format: migr yr, file name, seq no, tag id, chk sum, length,
  #                 weight, species, run, rear type, brd yr, var num,
  #                 flags, text comments, var date, mort flag
  # tag flag format: tag id, file name, flag code
  for($ii=0;$ii<=$#TAG_DATA;$ii++) {
    ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p)=split('\^',$TAG_DATA[$ii]);
    if($d =~ /\.\.\.\.\.\.\.\./) {
       #tag dot out: migr_yr, tag_file, seq_no, species, run, rear type,
       #             brd_yr, wt, length, flags, remark
       $g =~ s/\-//g;
       $f =~ s/\-//g;
       push(@TAG_DOT_OUT,"$a^$b^$c^$h^$i^$j^$k^$g^$f^$m^$n");
       splice(@TAG_DATA,$ii,1);
       $ii--; 
    }
  } # for tag data
  for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
    ($a,$b,$c) = split('\^',$TAG_FLAG[$ii]);
    if($a =~ /\.\.\.\.\.\.\.\./) {
       splice(@TAG_FLAG,$ii,1);
       $ii--;
    }
  } 
} # doDotOut
#
##################################
#
# loadVarDates - match up tag var numbers on individual fish records with
#                the tag var dates in the note records of the file.
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub loadVarDates
{ 
  # @TD_var="$tg^$vn"
  # @TAG_VAR="$fn^$varnm^$vardate"
  # TAG_HDR=  project message, file name, tag date, tagger,
  #           hatchery, stock, brood yr, migr yr, tag site,
  #           raceway, capt meth, tag temp, post temp, rel temp,
  #           tag meth, org, coord id, rel date, rel site, rkm,
  #           close date, reach
  # TAG_DATA= migr_yr, file name, seq no, tag id, chk sum, length, weight,
  #           species, run, rear type, brd yr, var num, flags, text comments,
  #           vardate, mort flag (vardate gets added here i.e. rel_v_time)
  #           (mort data gets added here also)

  $vardate="";
  $varnm="";
  $fn="";
  $tg="";
  $vn="";
  #
  # if var nums match then plug in release time in tag data
  #
  open(DEBUG,">/tmp/fldcrunch.dbug");
  for($i=0;$i<=$#TAG_VAR;$i++) {
     print DEBUG "TAG_VAR for $i = $TAG_VAR[$i]\n";
     ($fn,$varnm,$vardate)=split('\^',$TAG_VAR[$i]);
     print DEBUG "vardate is $vardate\n";
     for($j=0;$j<=$#TD_var;$j++) {
	print DEBUG "TD_var for $j = $TD_var[$j]\n";
        ($tg,$vn)=split('\^',$TD_var[$j]);
        print DEBUG "varnm is $varnm; vn is $vn\n";
        if($varnm eq $vn) {
            $TAG_DATA[$j] .= "^$vardate^"; 
	    print DEBUG "Match - TAG_DATA = $TAG_DATA[$j]\n";
        }
     } # for number of fish detail records
  } # for number of var dates
  #
  # put last var date in tag header
  #
  if($vardate ne "") {
     ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p,$q,$r,$s,$t,$u,$v) =
        split('\^',$TAG_HDR);
     $str="$a\^$b\^$c\^$d\^$e\^$f\^$g\^$h\^$i\^$j\^$k\^$l\^$m\^$n\^$o";
     $str.="\^$p\^$q\^$vardate\^$s\^$t\^$u\^$v";
     $TAG_HDR=$str;
  }
  #
  # if rel_v_time blank then fill with var date
  #
  for($ii=0;$ii<=$#TAG_DATA;$ii++) {
    ($a,$b,$c,$d,$e,$f,$g,$h,$i,$j,$k,$l,$m,$n,$o,$p)=split('\^',$TAG_DATA[$ii]);
    print DEBUG "TAG_DATA[$ii] is $TAG_DATA[$ii]\n";
    if($o eq "") {
       print DEBUG "o is blank for $b\n";
       $o = $vardate;
       $TAG_DATA[$ii]="$a^$b^$c^$d^$e^$f^$g^$h^$i^$j^$k^$l^$m^$n^$o^$p"; }
  } # for tag data
} # loadVarDates
#
##################################
#
# removeRepeatingFlags - find repeating flags in array and remove them. 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             loadTagArrays
#
##########
sub removeRepeatingFlags
{
   @TAG_FLAG = sort(@TAG_FLAG);
   for($ii=0;$ii<=$#TAG_FLAG;$ii++) {
     if ($ii != 0) {
        $str2 = $str1;
        $str1 = $TAG_FLAG[$ii];
        if ($str1 eq $str2) {
           splice(@TAG_FLAG,$ii,1);
           $ii--;
        }
     } else { $str1 = $TAG_FLAG[$ii]; }
   }
} # removeRepeatingFlags
#
##################################
# 
# loadTagArrays - load arrays from files that fld_val.p wrote for tagging data
#                 that require some crunching
#
#   calls:
#             removeRepeatingFlags
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub loadTagArrays
{
   open(TC,"$working_dir/TAG_COM.VAL.$filename") ||
      die "could not open $working_dir/TAG_COM.VAL.$filename";
   while(<TC>) {
      # ( format: file name, line number, note record )
      chop;
      push(@TAG_COM,$_);
   }
   close(TC);
   #system "rm $working_dir/TAG_COM.VAL.$filename";
   system "mv $working_dir/TAG_COM.VAL.$filename /tmp";
   open(TF,"$working_dir/TAG_FLAG.VAL.$filename") ||
      die "could not open $working_dir/TAG_FLAG.VAL.$filename";
   while(<TF>) {
      # ( format: tag id, file name, flag code )
      chop;
      push(@TAG_FLAG,$_);
   }
   close(TF);
   &removeRepeatingFlags;
   #system "rm $working_dir/TAG_FLAG.VAL.$filename";
   system "mv $working_dir/TAG_FLAG.VAL.$filename /tmp";
   open(TD,"$working_dir/TAG_DATA.VAL.$filename") ||
      die "could not open $working_dir/TAG_DATA.VAL.$filename";
   while(<TD>) {
      # format: migr_yr, file name, seq no, tag id, chk sum, length, weight,
      #         species, run, rear type, brd yr, var num, flags, text comments
      chop;
      ($my,$fn,$seq,$tg,$chk,$ln,$wt,$sp,$rn,$rt,$by,$vn,$flg,$txt)=split('\^');
      push(@TAG_DATA,$_);
      push(@TD_var,"$tg^$vn");
   }
   close(TD);
   #system "rm $working_dir/TAG_DATA.VAL.$filename";
   system "mv $working_dir/TAG_DATA.VAL.$filename /tmp";
   open(TH,"$working_dir/TAG_HDR.VAL.$filename") ||
      die "could not open $working_dir/TAG_HDR.VAL.$filename";
   while(<TH>) {
      # format: project message, file name, tag date, tagger,
      #         hatchery, stock, brood yr, migr yr, tag site,
      #         raceway, capt meth, tag temp, post temp, rel temp,
      #         tag meth, org, coord id, rel date, rel site, rkm,
      #         close date, reach
      chop;
      $TAG_HDR=$_;
   }
   close(TH);
   #system "rm $working_dir/TAG_HDR.VAL.$filename";
   system "mv $working_dir/TAG_HDR.VAL.$filename /tmp";
   open(TV,"$working_dir/TAG_VAR.VAL.$filename") ||
      die "could not open $working_dir/TAG_VAR.VAL.$filename";
   while(<TV>) {
      # ( format:  file name, var number, var date)
      chop;
      push(@TAG_VAR,$_);
   }
   close(TV);
   #system "rm $working_dir/TAG_VAR.VAL.$filename";
   system "mv $working_dir/TAG_VAR.VAL.$filename /tmp";
} # loadTagArrays
#
##################################
# 
# loadMortArrays - load arrays from files that fld_val.p wrote for mort data
#                  that require some crunching
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub loadMortArrays
{
   open(MH,"$working_dir/MORT_HDR.VAL.$filename") ||
      die "could not open $working_dir/MORT_HDR.VAL.$filename";
   while(<MH>) {
      # ( format: project message, file name, file date, coll site,
      #           coll river km, capt meth, org, coord id,
      #           close date )
      chop;
      $MORT_HDR=$_;
   }
   close(MH);
   #system "rm $working_dir/MORT_HDR.VAL.$filename";
   system "mv $working_dir/MORT_HDR.VAL.$filename /tmp";

   open(MD,"$working_dir/MORT_DATA.VAL.$filename") ||
      die "could not open $working_dir/MORT_DATA.VAL.$filename";
   while(<MD>) {
       # ( format: file name, seq no, tag id, length, weight,
       #           mort date, flags, text comments)
      chop;
      push(@MORT_DATA,$_);
   }
   close(MD);
   #system "rm $working_dir/MORT_DATA.VAL.$filename";
   system "mv $working_dir/MORT_DATA.VAL.$filename /tmp";

   open(MF,"$working_dir/MORT_COM.VAL.$filename") ||
      die "could not open $working_dir/MORT_COM.VAL.$filename";
   while(<MC>) {
      # ( format: tag id, file name, flag code )
      chop;
      push(@MORT_COM,$_);
   }
   close(MF);
   #system "rm $working_dir/MORT_COM.VAL.$filename";
   system "mv $working_dir/MORT_COM.VAL.$filename /tmp";

   open(MF,"$working_dir/MORT_FLAG.VAL.$filename") ||
      die "could not open $working_dir/MORT_FLAG.VAL.$filename";
   while(<MF>) {
      # ( format: tag id, file name, flag code )
      chop;
      push(@MORT_FLAG,$_);
   }
   close(MF);
   #system "rm $working_dir/MORT_FLAG.VAL.$filename";
   system "mv $working_dir/MORT_FLAG.VAL.$filename /tmp";

} # loadMortArrays
#
##################################
#
# processMorts  
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#
##########
sub processMorts
{ 
   # MORT_HDR format: project message, file name, file date, coll site,
   #                  coll river km, capt meth, org, coord id,
   #                  close date 
   #
   # MORT_DATA format: file name, seq no, tag id, chk sum, length, weight,
   #                   mort date, flags, text comments
   #
   # MORT_FLAG format: tag id, file name, flag code
   #
   # update mort data
   #
   for($ii=0;$ii<=$#MORT_DATA;$ii++) {
      ($a,$b,$c,$d,$e,$f,$g,$h,$i) = split('\^',$MORT_DATA[$ii]);
      $query  = "select tag_file, wt, length, rel_v_time, t_species, \n";
      $query .= "t_run, t_rear_type from tag_data where tag_id = \'$c\'";
      &sql("$query") || die "\n$sql_error\n$query";
      while (@row = &sql_fetch) {
       if($sql_rowcount > 0) {
        $row[0] =~ s/\ *$//g; # tag file
        $tfile = $row[0];
        $row[1] =~ s/\ *$//g; # wt
        # yo, substr is here because ingperl returns the number of bytes
        # used to store the binary reporesentation of the value, 
        # trust me, it's way too much.
        $twt=substr($row[1],0,4);
        $row[2] =~ s/\ *$//g; # length
        $tlen = $row[2];
        $row[3] =~ s/\ *$//g; # rel_v_time
        $rtime = $row[3];
        $row[4] =~ s/\ *$//g; # t_species
        $sp = $row[4];
        $row[5] =~ s/\ *$//g; # t_run
        $run = $row[5];
        $row[6] =~ s/\ *$//g; # t_rear_type
        $rt = $row[6];
       }
        if ($sql_error) {
           print F "\n$0:***SQL ERROR*** $sql_error\n";
           print F "$query\n";
           $sqlerrs++;
        }
      }
      if($tfile ne "") {
         $query  = "select rel_site, coord_id, org, migr_yr from tag_hdr";
         $query .= " where file_id = \'$tfile\'";
         &sql("$query") || die "\n$sql_error\n$query";
         $row0=$row1=$row2=$row3=0;
         while (@row = &sql_fetch) {
          if($sql_rowcount > 0) {
           $row[0] =~ s/\ *$//g; # rel site
           $row0 = $row[0];
           $row[1] =~ s/\ *$//g; # coord id
           $row1 = $row[1];
           $row[2] =~ s/\ *$//g; # org
           $row2 = $row[2];
           $row[3] =~ s/\ *$//g; # migr yr
           $row3 = $row[3];
          }
           if ($sql_error) {
             print F "\n$0:***SQL ERROR*** $sql_error\n";
             print F "$query\n";
             $sqlerrs++;
           }
         }
      }
      # MORTDATA = m file, tag file, m tag id, m chksum, migr yr, m seq no,
      #            tag len, tag wt, m len, m wt, org, coord id, tag rel site,
      #            tag rel date, mort date, species, run, rear type, flags,
      #            m remark
      if($e eq "") { $e = "0"; }
      if($f eq "") { $f = "0"; }
      $str  = "$a\^$tfile\^$c\^$d\^$row3\^$b\^$tlen\^$twt\^$e\^$f\^$row2";
      $str .= "\^$row1\^$row0\^$rtime\^$g\^$sp\^$run\^$rt\^$h\^$i";
      $MORT_DATA[$ii]=$str;
   }
} # processMorts
#
##################################
#
# getTime
#
#   calls:
#            localtime
#
#   receives:
#
#   returns:
#            current date and time (mm/dd/yy hh:mm) 
#
#   called by:
#            fileInit
#
##########
sub getTime
{
  ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$lndst) = localtime(time);
  $mon++;
  $zmon = sprintf("%02d",$mon);
  $zhour = sprintf("%02d",$hour);
  $zmday = sprintf("%02d",$mday);
  $zmin = sprintf("%02d",$min);
  $zsec = sprintf("%02d",$sec);
  return("$zmon\/$zmday\/$year $zhour:$zmin:$zsec");
} # getTime
#
##################################
#
# fileInit - initial file variables
#
#   calls:
#            getTime
#
#   receives:
#
#   returns:
#
#   called by:
#            Main
#
##########
sub fileInit
{ 
  push(@filestoload,$filename);
  open(F,">/tmp/$pname.$filename") || "cound not open /tmp/$pname.$filename";
  $processdate=&getTime;
  print F "$0: starting to process $filename at $processdate...\n";
  print F "moving $hold_dir/*.$filename to $working_dir...\n";
  system "mv $hold_dir/*.$filename $working_dir";
  print F "$0: connecting to $db with role = field_data_upload...\n";
  $sql_showerrors = 1; #  ingperl debugging error messages turned on
  &sql("connect $db -Rfield_data_upload");
  &sql_exec("set lockmode session where readlock=nolock");
  &sql_exec("set lockmode on tag_data where maxlocks=50");
  &sql_exec("set lockmode on tag_flag_codes where maxlocks=30");
  $adminmsg="SQL ERROR $0: $filename";
  $sqlerrs=0;
  @SRRT=();
  @COUNT=();
  @COUNT_SRRT=();
  @MORT_FLAG=();
  @MORT_DATA=();
  @MORT_COM=();
  $MORT_HDR="";
  @MORTS=();
  @RECAP_FLAG=();
  @RECAP_DATA=();
  @RECAP_COM=();
  $RECAP_HDR="";
  @RECAPS=();
  @TAG_DUP=();
  @TAG_COM=();
  @TAG_FLAG=();
  @TAG_DOT_OUT=();
  @TAG_DATA=();
  @TD_var=();
  @TAG_VAR=();
  $TAG_HDR="";
  # release vars
  $rdate="";
  $rsite = "";
  $rtemp = "";
  $rtime = "";
  # tagging vars
  $vardate="";
  $tdate="";
  $tfile = "";
  $twt = "";
  $tlen = "";
  $myr = "";
  $org = "";
  $cid = "";
  $sp = "";
  $run = "";
  $rt = "";
} # fileInit
#
##################################
#
# loadRkmTbl 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             main
#
##########
sub loadRkmTbl
{ 
  # $rkm_tbl="/home/jen/etc/rkm.ptagis3";
  $rkm_tbl="/usr/pit/pittag/etc/rkm.ptagis3";
  open(RKM,$rkm_tbl) || die "\n\ncould not open $rkm_tbl\n\n";
  while (<RKM>)
  {
     chop;
     push(@RKM_TBL,$_);
  }
  close(RKM);
} # loadRkmTbl
#
##################################
#
# progInit - initial program variables
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#
##########
sub progInit
{
#
#
# PRODUCTION
######################################################
  # $db="ptagis3";
  $db=$ENV{'DBNAME'};
  $admin="pittag";
  $mortworkdir=$ENV{'MORTWORK'};
  $tagworkdir=$ENV{'TAGWORK'};
  $tagholddir=$ENV{'TAGHOLD'};
  $mortholddir=$ENV{'MORTHOLD'};
  $working_dir=$ENV{'CRUNCHDIR'};
  $hold_dir=$ENV{'LOADDIR'};
  $mortloadeddir=$ENV{'MORTLOADED'};
  $tagloadeddir=$ENV{'TAGLOADED'};
  $mortcrdir=$ENV{'MORTCRUNCHED'};
  $tagcrdir=$ENV{'TAGCRUNCHED'};
  $logdir=$ENV{'LOGDIR'};
  $fdlshut=$ENV{'FDL_SHUTDOWN'};
  $job_log=$ENV{'JOB_LOG'};
  $username="pittag";
#
#
# DEVELOPMENT
######################################################
#  $db="p3_dev";
#  $admin="carters";
#  $mortworkdir="/home/jen/data/mort/working";
#  $tagworkdir="/home/jen/data/tagging/working";
#  $tagholddir="/home/jen/data/tagging/hold";
#  $mortholddir="/home/jen/data/mort/hold";
#  $working_dir="/home/jen/data/crunching";
#  $hold_dir="/home/jen/data/dat";
#  $mortloadeddir="/home/jen/data/mort";
#  $tagloadeddir="/home/jen/data/tagging";
#  $tagcrdir="/home/jen/data/tagging/crunched";
#  $mortcrdir="/home/jen/data/mort/crunched";
#  $logdir="/home/jen/log";
#  $fdlshut="/home/jen/etc/FDL_SHUTDOWN";
#  $job_log="/home/jen/log/job_log";
#  $username="ptagdev";
######################################################
#  $ENV{"II_SYSTEM"}="/usr/op_ing/ing12";
  @progname = split('\/',$0);
  $pname = $progname[$#progname];
  $jdate=$ENV{'JULIAN_DATE'};
  $done = 0;
} #progInit
#
##################################
#
# doEndStuff 
#
#   calls:
#
#   receives:
#
#   returns:
#
#   called by:
#             Main
#             gracefulExit
#
##########
sub doEndStuff
{ 
  #
  # load file names for fld_load.sh to read in
  # (keep track of julian dates and version numbers)
  #
  if(@filestoload) {
    opendir(DIR,"$working_dir");
    @ftoload=grep(/^filestoload\.*/,readdir(DIR));
    closedir(DIR);
    @vernbr=split('\.',$ftoload[$#ftoload]);
    if($vernbr[$#vernbr-2] eq $jdate) {
      $v=$vernbr[$#vernbr]+1;
      open(DAT,">>$working_dir/filestoload.$jdate.$$.$v") ||
       die "\n\ncould not open $working_dir/filestoload.$jdate.$$.$v\n\n";
      for($i=0;$i<=$#filestoload;$i++) {
         print DAT "$filestoload[$i]\n"; }
      close(DAT);
    } else {
      open(DAT,">>$working_dir/filestoload.$jdate.$$.1") ||
       die "\n\ncould not open $working_dir/filestoload.$jdate.$$.1\n\n";
      for($i=0;$i<=$#filestoload;$i++) {
         print DAT "$filestoload[$i]\n"; }
      close(DAT);
    }
  }
  $filenum = ($#filestoload + 1);
  $done = 1;

} # doEndStuff
##################################
#
# Main Body
#
#   calls:
#             progInit
#             loadRkmTbl
#             fileInit
#             loadTagArrays
#             loadVarDates
#             doDotOut
#             loadRecaps
#             loadTagMorts
#             finishTag
#             loadMortArrays
#             processMorts
#             finishMort
#             doEndStuff
#             gracefulExit
#             getTagFileNames
#             getMortFileNames
#
#   receives:
#             @ARGV[0]: username
#             @ARGV[1]: filetype
#             @ARGV[2]: filename
#
#   returns:
#
#   called by:
#
#
##################################
#
# check for existence of $FDL_SHUTDOWN and exit if found
#
stat("$fdlshut");
if (-e _) {
   $shuttime = &getTime;
   $sbj="shutdown fld_crunch.p @ $shuttime";
   system "mailx -s \"$sbj\" $admin < /dev/null";
   system "echo \"$sbj\" >>$job_log";
   exit(0);
}
#
# set up program variables and files
#
&progInit;
&loadRkmTbl;
#
# process tagging files
#
&getTagFileNames;
foreach $filename (@allfiles) {
   #
   # check for existence of $FDL_SHUTDOWN and call gracefulExit
   # if found else continue processing data file
   #
   stat("$fdlshut");
   if (-e _) { &gracefulExit(0); }
   else {
     system "mv -f $tagholddir/$filename $tagworkdir";
     $starttime = &getTime;
     $starttimesecs = &doSecs;
     &fileInit;
     &loadTagArrays;
     &loadVarDates;
     &doDotOut;
     &loadRecaps;
     &loadTagMorts;
     &finishTag;
     $endtimesecs = &doSecs;
     $tottimesecs = ($endtimesecs - $starttimesecs);
     $tottime = &undoSecs($tottimesecs);
     &insert_fld_load_summary_tbl($tottime);
   } # else no fld_shutdown file
}
#
# process mortality files
#
&getMortFileNames;
foreach $filename (@allfiles) {
   #
   # check for existence of $FDL_SHUTDOWN and call gracefulExit
   # if found else continue processing data file
   #
   stat("$fdlshut");
   if (-e _) { &gracefulExit(1); }
   else {
     system "mv -f $mortholddir/$filename $mortworkdir";
     $starttime = &getTime;
     $starttimesecs = &doSecs;
     &fileInit;
     &loadMortArrays;
     &processMorts;
     &finishMort;
     $endtimesecs = &doSecs;
     $tottimesecs = ($endtimesecs - $starttimesecs);
     $tottime = &undoSecs($tottimesecs);
     &insert_fld_load_summary_tbl($tottime);
   } # else no fld_shutdown file
}
if (!($done)) { &doEndStuff; }