#!/usr/bin/perl

use Getopt::Long;
use File::Copy;
use Cwd;

use lib "/mt/apps/gpms/DateTrans/lib";
use DateTrans qw(monName monNumber timeFormat AmPmFormat dayNumbeFormat);

use lib "/mt/apps/gpms/DateTrans/lib";
use  xsdxsltcheck qw(xsdcheck xsltcheck FGDC2MRFcheck xsl4mrf2fgdccheck);


use File::Basename;

$correct=substr($0,2);



sub usage() {
    print "\nUsage: perl $0 [-listfile=<listfile>][-directory=<directory>][-product=<product>]\n";
    print "Example:\n";
    print "perl $0 -listfile=list.txt -directory=/dir/subdir -product=TLFE2007\n";
    print"where list.txt is the filename of the list file containg the names of the files to validated,\n";
       print "/dir/subdir is the directory containing the files to be validated\n";
       print "and TLFE2007 is the product.\n";
   print "Please also take note of the following shortcuts for the directory variable:\n";
   print "1. putting mtdata for the directory name will yield the /mtdata/geo/gpms/input/mif directory.\n";
   print "2. putting mtdataTest for the directory name will yield the /mtdata/geo/gpms/input/mif/cpb_files_20110401 directory.\n";
   print "3. putting mtdataDir/subdirectoryname  directory name will yield the subdirectory under the /mtdata/geo/gpms/input/mif directory.\n";
   print "For example, lets say all the mif files are in the cpb_test_files_20110224 directory, which is a subdirectory of /mtdata/geo/gpms/input/mif directory.\n";
   print "Inserting mtdataDir/cpb_test_files_20110224  for the directory variable would yield the mtdata/geo/gpms/input/mif/cpb_test_files_20110224 directory.\n\n";

}
%gOptions;
&GetOptions(\%gOptions, qw(listfile=s directory=s  product=s));
unless ($gOptions{listfile} and $gOptions{directory} and $gOptions{product}) {
        usage();
        exit(1);
};
#---------------------------------------------change log-------------------------------------------------------------------------------------------------
=pod
1/5/2011 Commented out the Here 6!!!!!!!!!! diagnostic and the Here 2 diagnostic

1/5/2011 commented out print LOG "$FGDCdir/$newdir/$FGDCname   in the second chmod\n\n";

1/5/2011 changed the if condition on the list of files that passed validation from if( $pass_counter>0){ to if( $pass_counter>0 and $badAFFfile2_counter2 == 0){
  this was done so that only the mif files that passed validation and had no problems with the mif translation were shown  

1/25/2011 Put in a second test to make sure that errir files with a size of 0 really do not exist

2/17/2011 Created the ReRun Array Decision Structure/loop to process files that do exist, have errors in them but are not producing error files. This is due to 
problems with the <Map_Projection_Name gid1="1"> element. These errors include including a "(" in the text, not including the state or misspelling the state
name. The rules for this element have to be strict becuase it used to create a an element name in the MRF file by the xslt sheets that convert MIF to MRF.
This decision structure/loop uses two subroutines: ProjectionName, which prints to the error the actual name of the projection element, and 
 ErrorFileSub, which creates the error files.

2/18/2011 "Subroutined" the ReRun Array Decision Structure/loop for putting the rerun files whose error has been determined to be a problem with the 
<Map_Projection_Name gid1="1"> element. This was done through creating the badMifSub subroutine.

2/24/2011 inserted the mtdataTest keyword for mif files located in the /mtdata/geo/gpms/input/mif/cpb_test_files_20110224 directory

3/2/2011 modified the second check for error files =0 to check that the xml declaration(<?xml version="1.0" encoding="ISO-8859-1"?>) is in the mif file.
 If the declaration is not in the file, than an error message stating this fact is printed to the error file. This declaration is need in order for non-English
characters like the funny n to transform successfully from the mif file to the MRF file.

3/2/2011 modified the last line of the error messages in the ReRun Array Decision Structure/loop to read "Transformation failed: A Run-time error was reported".

3/3/2011 modified the else clause to an elsif clause of ReRun Array Decision Structure/loop to put any mif files that exist do not have error with their XML declaration or projection but do not have an error file greater than 0 to put this file in the 2nd rereun array.

3/3/2011 inserted the  mtdataDir keyword to make filling subdirectories under /mtdata/geo/gpms/input/mif on the command line easier

1/13/2012 inserted an if decision structure that automatically deletes the newly created folders under the output/FGDC, output/MRF and output/AFF if none of the files being tested pass validation. This was
done to clear up server space and make the directories less confusing  
=cut
#--------------------------------------------file mangement section----------------------------------------------------------------------------------------
$Infile = $gOptions{listfile};
chomp($Branch = $gOptions{product});
chomp($indir= $gOptions{directory});

$indirA=$indir;
#print "\$indirA = $indirA\n";
$indirB=substr($indirA,0,9);
#print "\$indirB = $indirB\n";

#for the special shortcut directories
#print "\$indirA = $indirA\n";
$mtdataDirYesNo=index($indirA,"mtdataDir");
#print "\$mtdataDirYesNo = $mtdataDirYesNo\n";
$mtdataTestYesNo=index($indirA,"mtdataTest");
#print "\$mtdataTestYesNo = $mtdataTestYesNo\n";


print "\$indir = $indir\n";
if ($indir eq "mtdata"){
$indir = "/mtdata/geo/gpms/input/mif";

#}elsif ($indir eq "mtdataTest"){
#/mtdata/geo/gpms/input/mif/cpb_files_20110401
#$indir = "/mtdata/geo/gpms/input/mif/cpb_test_files_20110224";
#$indir = "/mtdata/geo/gpms/input/mif/cpb_files_20110401";
}elsif ($mtdataTestYesNo >=0){
$slashLoc=index($indir,"/");
$subDir=substr($indir,$slashLoc);
$indir= "/mtdata/geo/gpms/input/mif/cpb_files_20110401" . $subDir; 
}elsif ($mtdataDirYesNo >=0){
#print "in the mtData if \n"; 
$slashLoc=index($indir,"/");
$subDir=substr($indir,$slashLoc);
$indir= "/mtdata/geo/gpms/input/mif" . $subDir;
}elsif ($indir eq "nfs"){
$indir ="/nfs/gpms/input/mif";
}elsif ($indir eq "input"){
$indir ="/home/mccre004/input_files";
};

$root="/mt/apps";
$rootout="/mtdata/geo";
$inlist="$rootout/gpms/input/list";
$outputdir ="$rootout/gpms/output/AFF";
$errlogdir ="$rootout/gpms/log";
$combolist = $inlist . "/" . $Infile;
$stylesheet ="$root/gpms/MIF2AFF/MIF2MRF.xsl";
$FGDCdir="$rootout/gpms/output/fgdc";
$MRFdir ="$rootout/gpms/output/mrf";

#------------------------------------------------------error trap--------------------------------------------------------------------------------------------

if (-e $indir){
print "Please Wait, Now processing the files ----------------------------------------------\n\n";
}else{
print "The directory $indir  could not be found! \n";
print "Please make sure the pathname, $indir, was spelled correctly and that it exists.\n";
exit(1);
};

if (-s $indir){
}else{
print "The directory $indir does not contain any files to be validated! \n";
exit(1);
};

if (-e $combolist){
}else{
print "The file $combolist could not be found! \n";
print "Please make sure the file, $Infile,  was spelled correctly and that the file is in the $inlist directory.\n";
exit(1);
}; 

#making sure the file list contains something
if (-s $combolist){
#comment
}else{
print "The file $combolist does not contain the list of files to be validated! \n";
print "Please make sure the file, $Infile,  contains a list of files to be validated.\n";
exit(1);
};

#------------------------------------getting the date---------------------------------------------------------------------------------------------------------
my $now=`date`;
$mon = substr($now,4,3);
$day=substr($now,8,2);
$year =substr($now,24,4);
$hour = substr($now,11,2);
$min = substr($now,14,2);
my $daynum= dayNumbeFormat($day);

#converting the date to a number
my $monnum = monName($mon);
my $monnum =monNumber($mon);
my $Month_Number = monNumber($mon);

$dateString =$year . $monnum . $daynum . $hour . $min;

$time = $hour . ":" . $min;
$oldhour = $hour;
my $newhour =timeFormat($hour);
my $ampm =AmPmFormat($oldhour);


$newtime = $newhour . ":" . $min . " $ampm";

#-----------------------------------------------------making the directories---------------------------------------------------------------------------
$newdir=$Branch . $dateString;
system "mkdir $outputdir/$newdir";
$newerrdir ="$errlogdir/$newdir";
system "mkdir $newerrdir";
chmod 0777, "$outputdir/$newdir";
chmod 0777, "$newerrdir";

$FGDCResult = system "mkdir $FGDCdir/$newdir";

chmod 0777, "$FGDCdir/$newdir";
$MRFreslut= system "mkdir $MRFdir/$newdir";

if ($reslut >0 ){
print "can not make mrf directory\n";
};
chmod 0777, "$MRFdir/$newdir";

#-----------------------------------------create the failed files---------------------------------------------------------------------------------
#open FAILEDFILES, ">$newerrdir/FailedFiles.txt";
#---------------------------------------------------------------------Counter variable Management section------------------------------------------
$badfile_counter2=0;
$goodfile_counter=0;
$countere=0;
$flag_array_counter=0;
$badAFFfile2_counter2=0;
$sizebadMif=0;
#----------------------------------------------------------Saxon-----------------------------------------------------------------------------------------
#$ENV{'PATH'} = "/opt/saxon_lib/jdk1.5.0_11/bin:/mt/apps/gpms/output/mrf$newerrdir:$ENV{'PATH'}";#goodone
#$cp = "CLASSPATH";#goodone
#$ENV{$cp} = "/opt/saxon_lib/jdk1.5.0_11/lib/:/opt/saxon_lib/saxon9/saxon9sa.jar:/opt/saxon_lib/saxon9/saxon9-jdom.jar:/opt/saxon_lib/saxon9/:/opt/saxon_lib/saxon9/saxon9.jar:$ENV{$cp}";

$ENV{'PATH'} = "/apps/saxon/jdk1.5.0_24/bin::/apps/saxon/saxon9ee.jar::/apps/saxon:$rootout/mrf/$newerrdir:$ENV{'PATH'}";
$cp = "CLASSPATH";#goodone
$ENV{$cp} = "/apps/saxon/jdk1.5.0_24/lib/:/apps/saxon/saxon9ee.jar:/apps/saxon/saxon9ee.jar $CLASSPATH:/usr/share/java/saxon.jar";
#---------------------------------------redirecting standard output section------------------------------------------------------------------------------


#redirecting STDOUT because the Saxon program automatically sends all its output to STDOUT
# Saving a copy of STDOUT as SAVEOUT so that items can still be printed to the screen
open (SAVEOUT,">&STDOUT");
#open DIAG, ">diagnos.err";
#-----------------------------------------Filename array creation section------------------------------------------------------------------------------
#open a filehamdle for the file containing the filenames

#print "Prereadingxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n\n";
open INPUTFILE, "$combolist";
while(<INPUTFILE>){

$slashpos= rindex($_, "/");

if ($slashpos < 0){
#print "Slash < 0\n";
$flagpos=index($_,"mif");
$flagless=substr($_, 0,$flagpos+3);
$flag=substr($_,$flagpos+4);
$ppos=index($flag,"p");

#print "\$flag = $flag\n";
#print "\$ppos = $ppos\n";
$filelen=length($_);

if ($ppos>=0){
$filename[$first_counter]="$indir/$flagless";
$flagarray[$first_counter]="$flag";
$flaghash{$filename}="$flag";
$first_counter++;
} elsif($filelen>1){
$noflagarray[$flag_array_counter]=$_;
$flag_array_counter++;
};


}elsif($slashpos > 0){


my $dirnameA = dirname $_;

$slash_directory_name[$slash_directory_name_counter]=$dirnameA;
$slash_directory_name_counter++;

$infile=$_;
$dotloc=rindex($infile,".")+4;
$slashfilename = substr($infile,0,$dotloc);
$flagpos=index($infile,"xml");
$ppos=index($flag,"p");
$flag2=substr($flag,0,1);
$filelen=length($infile);

if ($ppos>=0 and $filelen>1){
$filename[$first_counter]=$slashfilename;
print SAVEOUT "Here: \$filename = $filename\n";
$flagarray[$first_counter]="$flag";
$flaghash{$slashfilename}="$flag";
$first_counter++;
} elsif($ppos=0 and $filelen>1){
$noflagarray[$flag_array_counter]=$slashfilename;
$flag_array_counter++;
};

};
};
#-------------------------------------------validating the incoming MIF File--------------------------------------------------------------------------------------
$valcounter=0;
$filecounter =1;

foreach $filename (@filename){
print "now reading $filename \n";
$ISOind="no";
chomp($filename);
#setting up the variables necessary to make the the finalfilename variable, which is needed to make the MRF file
$slashpos= rindex($filename, "/") +1;
$newfilename = substr($filename,$slashpos);
$dotlocation = index($newfilename, ".");
$finalfilename=substr($newfilename, 0, $dotlocation);

if ($valcounter >0){
close STDERR;
#system "rm $TMP";
};

$MRF = $finalfilename . ".mrf";
$TMP = "TMP" . "-" . $newfilename;
$valcounter++;


open STDOUT,">$TMP";

#making the error file and redirecting the STDERR to the error file
#have a new $dotlocation variable because the $filename and $newfilename variables are different lengths
$errmain= substr($newfilename, 0, $dotlocation);
$errfile = $errmain .".err";

open STDERR, ">$errfile";

#starts schema aware transformation. This will cause the transformation to fail if the document is invalid
# the -vw option indicates that validation errors found when validating the result tree should be treated as warnings only.
#This option is available only with the Saxon-SA version of the command, com.saxonica.Transform.
printf SAVEOUT "$filecounter: Now validating $filename\n";
$filecounter++;
my @csax = "java com.saxonica.Transform -vw $filename $root/gpms/xsl/MIF2MRF.xsl";
my $mif = system(@csax);
#print SAVEOUT "\$mif = $mif\n";

#opening fielhandles so that *.xsd will be replaced by Repository.xsd
#basically replicating the sed command 
open MRFOUTPUT, ">$MRF";
open TMPFILE, "<$TMP";

close STDERR; #new one
if ($mif > 0) {
$badMIFfile[$badMIFfile_counter] = $errfile;
$badMIFfile_counter++;
#print SAVEOUT "In the mif greater than 0 \n";
$sizeTMP= -s $TMP;
#print SAVEOUT"\$sizeTMP = $sizeTMP\n";

if ($sizeTMP >0 ){
print SAVEOUT "moving $TMP  to $newerrdir\n";
system "mv $TMP $newerrdir";
}else{
#system "rm $TMP";
}; 

$sizeMRF = -s $MRF;

if ($sizeMRF >0){
system "mv $MRF $newerrdir";
}else{
system "rm $MRF";
};

}else{
unlink $errfile;

$goodmif[$goodMif_counter]=$filename;
$goodMif_counter++;

#system "rm $errfile";
$goodfile[$goodfile_counter]= $MRF;


$goodflaghash{$MRF}=$flaghash{$filename};
$npos=index($flaghash{$filename},"n");
if ($npos==0){
$nopassarray[$no_pass_counter]=$goodfile[$goodfile_counter];
$no_pass_counter++;
#print SAVEOUT "in the posting array\n";
}elsif ($npos<0){
$passarray[$pass_counter]=$goodfile[$goodfile_counter];
$pass_counter++;

};



$goodfile_counter++;
unlink $errfile;
 while(<TMPFILE>){
#print SAVEOUT " reading the TMP file\n";
  if($_ =~ /xsd/){
   #this happens if the the line has xsd in it
   $xsdlocation =index($_, "/xsd");
   $originalstring= substr($_,0,$xsdlocation);
   $Repository = "xsd/Repository.xsd\">\n";
   printf MRFOUTPUT ("$originalstring/$Repository");
  }elsif ($_ =~ /<\/MRF>/){
   printf  MRFOUTPUT ("$_");
   printf  MRFOUTPUT "\n";
   }elsif($_=~ /%/){
        chomp($_);
        $perloc=index($_, "%");
        $firsthalf =substr($_,0,$perloc);
        $secondhalf =substr($_, $perloc+1);
        $perloc2=index($secondhalf, "%");
                if ($perloc2 < 0){
                        printf  MRFOUTPUT"$firsthalf" . "%%" . "$secondhalf";
                }elsif($perloc2 >0){
                        $thirdhalf =substr($secondhalf,0,$perloc2);
                        $fourthhalf =substr($secondhalf, $perloc2+1);
                        $newstring = "$firsthalf" . "%%" . "$thirdhalf" . "%%" . "$fourthhalf\n";
if ($newstring=~/&#147;/){
$amploc2 =index($newstring, "&#148;");
                           $amp1string = substr($newstring,0,$amploc1);
                           $midstringlength= $amploc2 - $amploc1-6;
                           $amp2string = substr($newstring,$amploc1+6,$midstringlength);
                           $amp3string = substr($newstring,$amploc2+6);
                           printf  MRFOUTPUT $amp1string . "\"" . $amp2string . "\"" . $amp3string;
                         }else{
                        printf  MRFOUTPUT"$firsthalf" . "%%" . "$thirdhalf" . "%%" . "$fourthhalf\n";
                         };;#if ($newstring=~/&#147;/){
                         };#if ($perloc2 <  0){
 }elsif($_=~/&#147;/){
$amploc1=index($_, "&#147;");
 $amploc2 =index($_, "&#148;");
 $amp1string = substr($_,0,$amploc1);
 $midstringlength= $amploc2 - $amploc1-6;
 $amp2string = substr($_,$amploc1+6,$midstringlength);
 $amp3string = substr($_,$amploc2+6);
 printf  MRFOUTPUT $amp1string . "\"" . $amp2string . "\"" . $amp3string;

}elsif($_=~/&#146;/){
$aposloc1 = index($_,"&#146;");
$aposstring1 = substr($_,0,$aposloc1);
$aposstring2 = substr($_,$aposloc1+6);
printf  MRFOUTPUT $aposstring1 . "'" . $aposstring2;


  }else{
   printf  MRFOUTPUT "$_";
  };#if xsd
 };#while<TMPFILE>
close MRFOUTPUT;
close TMPFILE;

open TMPFILE, "<$TMP";


};#if mif>0
unlink $TMP
};#foreach

printf SAVEOUT "\n";
close STDOUT;
#---------------------
#-------------------------------error handling for invalid MIF files-------------------------------------------------------------------------------------------------
#setting the $badMIFfile2_counter variable 
if ($badMIFfile_counter >0){
print SAVEOUT "Now creating the Error Files----------------------------------------------------------------------------------------------\n";
$badMIFfile2_counter=0;
$badMIFfile3_counter=1;#for the screen
foreach $badMIFfile (@badMIFfile){
print SAVEOUT "$badMIFfile3_counter: Now working on  $badMIFfile\n";
$badMIFfile3_counter++;
open BADFILEH, "$badMIFfile";
$bad="no";
$slashpos= rindex($badMIFfile, "/")+1;
$wheredot2 = index($badMIFfile, ".");


if ($slashpos >0){
#print SAVEOUT "Here 1!!!!!!!!!!!!\n";
$AFF_Main_name = substr($badMIFfile,$slashpos,$wheredot2);
}else{
#print SAVEOUT "Here 2!!!!!!!!!!!!\n";
$AFF_Main_name = substr($badMIFfile,0,$wheredot2);
};

$name = "$AFF_Main_name.xml";
$mifErrFile ="$AFF_Main_name.err";
$mifMrfFile ="$AFF_Main_name.mrf";
$bad ="no";
$sizebadMif = -s $badMif;
if (-s $badMIFfile == 0){
#print SAVEOUT "Here 3!!!!!!!!!!!!\n";
$MIfDotLoc=index($badMIFfile,".");
$MifPreDot=substr($badMIFfile,0,$MIfDotLoc);
$FinalMif = $MifPreDot . ".xml";
$sizeFile ="$indir/$FinalMif";
if (-e $sizeFile){
#print SAVEOUT "Here 4!!!!!!!!!!!!\n";
$ReRunList[$ReRunListCounter] = "$name";
$ReRunListCounter ++;
}else{
#print SAVEOUT "Here 5!!!!!!!!!!!!\n";
#system "mv $badMIFfile $newerrdir";  
$remove=shift(@badMif);
$not_exist[$countere]= "$name";
$countere++;
$bad ="yes";
#unlink $badMIFfile;#this one
};
}else{


while(<BADFILEH>){
#print SAVEOUT "Now reading the error file\n";
if ($_=~ /does not exist/){
&DoesNotExist;
$bad ="yes";
print SAVEOUT "$badMIFfile2 does not exist";
#unlink $badMIFfile;

#print SAVEOUT "\$badMIFfile = $badMIFfile\n";
#print SAVEOUT "\$badMIFfilePrev = $badMIFfilePrev\n";
if ($badMIFfile ne  $badMIFfilePrev){
print SAVEOUT "$badMIFfile File does not exist\n";
};

$badMIFfilePrev = $badMIFfile;
}elsif($badMiffile2[$badMif_counter] ne $name and $bad ne "yes"){
#print SAVEOUT "Here 6!!!!!!!!!!!!\n";
if ($badMIFfile ne  $badMIFfilePrev){
};
$badMIFfilePrev = $badMIFfile;

if ($name ne $MifPrev and $MifPrev ne "0"){
$MifPrev= chomp($name);
#print SAVEOUT "Here 7!!!!!!!!!!!!\n";
$badMIFfile2[$badMIFfile2_counter] = "$name";

$bad ="no";
$outmif ="$indir/$name";
copy ($outmif, $newerrdir) or warn "Can't copy $name to $newerrdir: $!\n";
system "mv $name $newerrdir";
system "mv $mifErrFile $newerrdir";
system "mv $badMIFfile $newerrdir";
system "chmod 777  $newerrdir";
$badMIFfile2_counter++;
$MifPrev = $name;
};


unlink $badMif
}elsif ($bad ne "no" and  $MifPrev ne $name){
#$NOErrorFile[$NOErrorFileCounter]=$badMIFfile;
#$NOErrorFileCounter;
#print SAVEOUT "3. default ($badMIFfile)\n";

};#if
}#s if
};#while
close BADFILEH
};#foreach
};#$badMIFfile_counter if statement
print SAVEOUT "\n\n";

#foreach $NOErrorFile (@NOErrorFile){
#print SAVEOUT"now removing the error files with size\n";
#unlink $NOErrorFile;
#};

#------------------------------------------Transforming MIF to AFF-----------------------------------------------------------------------------------------------------
if ($goodMif_counter>0){
print SAVEOUT "Now transforming the MIF files into AFF files ---------------------------------------------------------------------\n";
};

$valcounter=0;
$filecounter =1;

foreach $goodmif (@goodmif){
chomp($goodmif);
#print SAVEOUT "In the loop\n";
#print SAVEOUT "MIF2AFF: $goodmif\n";
#setting up the variables necessary to make the the finalfilename variable, which is needed to make the MRF file
$slashpos= rindex($goodmif, "/") +1;
$newfilename = substr($goodmif,$slashpos);
$dotlocation = rindex($newfilename, ".");
$finalfilename=substr($newfilename, 0, $dotlocation);
#setting up the TMP and MRF filenames

#if (-e $TMP){
if ($valcounter >0){
close STDERR;
#system "rm $AFFOutFile";
};

$AFFOutFile="AFF" . "-" . $finalfilename . ".xml" ;

open STDOUT,">$AFFOutFile";

#making the error file and redirecting the STDERR to the error file
#have a new $dotlocation variable because the $goodmif and $newfilename variables are different lengths
$errmain= substr($newfilename, 0, $dotlocation);
$errfile ="AFF_" . $errmain . ".err";

open STDERR, ">$errfile";

#starts schema aware transformation. This will cause the transformation to fail if the document is invalid
# the -vw option indicates that validation errors found when validating the result tree should be treated as warnings only.
#This option is available only with the Saxon-SA version of the command, com.saxonica.Transform.
printf SAVEOUT "$filecounter: Now transforming $goodmif from the MIF format to the American Fact Finder Format(AFF)\n";
$filecounter++;
my @csax = "java com.saxonica.Transform -vw $goodmif $root/gpms/MIF2AFF/MIF2AFF.xsl";
my $affResult  = system(@csax);
#opening fielhandles so that *.xsd will be replaced by Repository.xsd
#basically replicating the sed command 
close STDERR;
 
if ($affResult > 0) {
#print SAVEOUT "bad Mif\n";
$badAFFfile[$badAFFfile_counter] = $errfile;
$badAFFfile_counter++;
system "cp $errfile $errlogdir/$newdir";
if (-e  $TMP){
unlink $TMP;
};

 
}else{
#print SAVEOUT "removing $errfile\n";
unlink $errfile;
#determine if the files are to be posted. If the position of n, as indicated by $npos is -1, then the file is not to be posted
$goodflaghash{$AFFOutFile}=$flaghash{$goodmif};#was com
$npos=index($flaghash{$goodmif},"n");#was com
$AFFFileLength=length($AFFOutFile);

if ($AFFFileLength >4){
#system "cp $AFFOutFile  $outputdir/$newdir";
system "mv $AFFOutFile  $outputdir/$newdir";
};

};#if ($affResult > 0) {
};#foreach
 

 
 
printf SAVEOUT "\n";


#------------------------------error filehandling section for bad AFF files--------------------------------------------------------------------------------------------

sub DoesNotExist{
$not_exist[$countere]= "$name";
$countere++;
};

foreach $badAFFfile (@badAFFfile){
open BADFILEH, "$badAFFfile";
$bad="no";
$slashpos= rindex($badAFFfile, "/")+1;
$wheredot2 = index($badAFFfile, ".");
$AFF_Main_name = substr($badAFFfile,$slashpos,$wheredot2);
$name = "$AFF_Main_name.xml";
$bad ="no";
if (-s $badAFFfile == 0){
#print SAVEOUT "In the size = 0 section(bad AFF).\n";
$not_exist[$countere]="$name";

#removing the element in the array so that the file does not show up twice: as a bad mif and missing file
$countere++;
$bad ="yes";

}else{

while(<BADFILEH>){
if ($_=~ /does not exist/){
&DoesNotExist;
$bad ="yes";
#print SAVEOUT "badd AFF does not exist \n";
}elsif (-s $badAFFfile == 0 and $badfile2[$badfile_counter2-1] ne $name and $bad ne "yes"){
&DoesNotExist;
printf  MRFOUTPUT "Here!";
unlink "$badfile";
$countere++;
$bad ="yes";
}elsif($badAFFfile[$badAFFfile2_counter2-1] ne $name and $bad ne "yes"){

if ($name ne $PrevAFF){
$badAFFfile2[$badAFFfile2_counter2] = "$name";
$badAFFfile2_counter2++;
$bad ="no";
system "cp $badfile $newerrdir/$name";
system "chmod 777 $newerrdir/$name";
unlink $badfile;
$PrevAFF=$name;
};


}else{
#print SAVEOUT"3. Default\n";
};#if
}#s if
};#while
close BADFILEH
};#foreach
#--------------------------------------final variables counting section--------------------------------------------------------------------------------
$total = $badfile_counter2 + $pass_counter+$no_pass_counter ;
$newtotal=$total+$countere+$flag_array_counter+$badMIFfile2_counter;
if ($pass_counter >0){
$pergood = (($pass_counter )/$newtotal) *100;
}else{
$pergood =0;
$pass_counter  =0;
};

#if ($badfile_counter2){
if ($badMIFfile2_counter>0){
$perbad = (($badMIFfile2_counter)/$newtotal) *100;
}else{
$perbad =0;
$badfile_counter2=0;
};

#-------------------------------working with the log file----------------------------------------------------------------------------------------------
#This is the introductary paragraph.
open LOG, ">$errlogdir/$newdir/$newdir.log";
system "chmod 777 $errlogdir/$newdir/$newdir.log";
printf LOG "This log details the transformation and validation results of the $newtotal mif files contained in \n the  $combolist list file.\n";

if ($slash_directory_name_counter >0){
printf LOG "\nThese files can be found in the following directories:\n";
printf LOG "1. $indir (This is the directory given at the command line)\n";
$slash_directory_nameb =2;
@slash_directory_name=sort(@slash_directory_name);

foreach $slash_directory_name (@slash_directory_name){

$slash_directory_nameca =$slash_directory_namec-1;

if ($slash_directory_name[$slash_directory_namec] ne $slash_directory_name[$slash_directory_nameca] and $slash_directory_name[$slash_directory_namec] ne $indir){
printf LOG "$slash_directory_nameb: $slash_directory_name\n";
$slash_directory_nameb++;
$slash_directory_namec++;
}elsif($slash_directory_namec == 0){
printf LOG "$slash_directory_nameb: $slash_directory_name\n";
$slash_directory_nameb++;
$slash_directory_namec++;
};#if ($slash_directory_name[$slash_directory_namec] ne $slash_directory_name[$slash_directory_nameca] and $slash_directory_name[$slash_directory_namec] ne $indir){
};#foreach $slash_directory_name (@slash_directory_name){

printf LOG "\n$0 was run on $monnum/$daynum/$year at $time ($newtime) for the $Branch product.\n\n";
}else{
printf LOG "\nThese files can be found in the $indir directory.\n";
printf LOG "It was run on $monnum/$daynum/$year at $time ($newtime) for the $Branch product.\n\n";
#print LOG "Here 0\n";
};#if ($slash_directory_name_counter >0){
printf LOG "\n Of these $newtotal files, $pass_counter, or $pergood%, passsed validation and $badMIFfile2_counter, or $perbad%, failed validation.  ";


#if ($countere >0 or $flag_array_counter>0) {
if ($countere >0 or $flag_array_counter>0) {
printf LOG "Also, ";

if ($countere >0){

if ($countere ==1){
$notFoundFile = "file";
}else{
$notFoundFile = "files";
};

printf LOG "$countere $notFoundFile could not be found";
}

if ($countere >0 and $flag_array_counter>0){
printf LOG " and ";
}

if ($flag_array_counter>0){

if ($flag_array_counter ==1){
$FlagFile = "file";
$FlagVerb = "does";
}else{
$FlagFile = "files";
$FlagVerb = "do";
};


printf LOG "$flag_array_counter $FlagFile $FlagVerb not have a FGDC posting flag";
}

printf LOG ".\n\n";  
}
#--------------------------grammar---------------------------------------------------------------------------------------------------------------------
if($badMIFfile2_counter ==1){
$badMIFfile_File="file";
$badMIFfile_ThisThese="This";
$badMIFfile_Verb="WAS";
$badMIFfile_VerbPres="is an";
$badMIFfile_an =" an "
}else{
$badMIFfile_File="files";
$badMIFfile_ThisThese="These";
$badMIFfile_Verb="WERE";
$badMIFfile_VerbPres="are";
$badMIFfile_an=" "
};

#--------------------------Processing the ReRun Array--------------------------------------------------------------------------------------------------
sub ProjectionName{
$rightBracPos=index($mapProj,">")+1;
$leftBracPos=rindex($mapProj,"<");#was -1;
$mapPojectionNameLength=$leftBracPos-$rightBracPos;
$mapPojectionName=substr($mapProj,$rightBracPos,$mapPojectionNameLength);
};
sub ErrorFileSub{
$dotlocation = index($ReRunList, ".");
$errmain= substr($ReRunList, 0, $dotlocation);
$errfile = $errmain . ".err";
$errfileB = $newerrdir . "/" . $errfile;
};

sub badMifSub{
$badMIFfile2[$badMIFfile2_counter]=$ReRunList;
$badMIFfile2_counter++;
};

if ($ReRunListCounter >0){

$flagfile="ReRunList.txt";

$ReRunListCounterB =1;
foreach $ReRunList (@ReRunList){

$ReRunListCounterB ++;
$xmlDecl="no";
$xmlDeclErrRes="no";
$XmldecCounter=0;
$ReRunFile =$indir . "/" . $ReRunList;
open REM, "<$ReRunFile";

while (<REM>){
$line =$_;
$ISOLoc=index($line,"encoding=\"ISO-8859-1\"");
if ($ISOLoc>0){
$xmlDecl="yes";
}elsif($_ =~ /<Map_Projection_Name gid1="1">/ and $xmlDecl eq "yes" and $xmlDeclErrRes eq "no"){
$mapProj = $_;
$paraPos=index($mapProj,"(");
$AlbersLength =length($mapProj);
$wrongAlbers1 = index($mapProj,"Albers Conical Equal Area for");

#-b--------------------------Beginning of projection if-----------------------------------------------------------------------------------------------------------
if ($paraPos>0){
$errfileB =&ErrorFileSub;
open STDERRB, ">$errfileB";
system "rm $errfile";
print STDERRB "Error: The element <Map_Projection_Name> has a parenthesis in it. This is not allowed. Please remove the '('.\n";
print STDERRB "Transformation failed: A Run-time error was reported\n";
&badMifSub;
}elsif($mapProj =~ /Albers Equal Area Conic/){
$errfileB =&ErrorFileSub;
open STDERRB, ">$errfileB";
system "rm $errfile";
print STDERRB "Error: Albers Equal Area Conic is not a recognized projection name. Try Albers Conical Equal Area.\n";
print STDERRB "Transformation failed: A Run-time error was reported\n";
&badMifSub;
}elsif($mapProj =~ /Albers Conical Equal Area for/ and $AlbersLength < 84){
$errfileB =&ErrorFileSub;
open STDERRB, ">$errfileB";
system "rm $errfile";
print STDERRB "Error: 'Albers Conical Equal Area for' is not a recognized projection name. Try Albers Conical Equal Area with the state name.\n";
print STDERRB "Transformation failed: A Run-time error was reported\n";
&badMifSub;
}elsif($mapProj =~ /Albers Conical Equal Area for/ and $AlbersLength > 85){
$errfileBA =&ErrorFileSub;
open STDERRB, ">$errfileBA";
system "rm $errfile";
$mapPojectionNameB=&ProjectionName;
print STDERRB "Error: '$mapPojectionNameB' is not a recognized projection name. The state is spelled wrong.\n";
print STDERRB "Transformation failed: A Run-time error was reported\n";
&badMifSub;
}else{
$ReRunListFinal[$ReRunListFinalCounter]=$ReRunList;
$ReRunListFinalCounter++;
#print SAVEOUT "\$AlbersLength = $AlbersLength\n";
};#if ($paraPos>0){
#----end of projection if---------------------------------------------------------------------------------------------------------------------------------------

#-----------------------beginning of xml declaration test------------------------------------------------------------------------------------------------------
}elsif($xmlDecl eq "no" ){
$errfileBA =&ErrorFileSub;
open STDERRB, ">$errfileBA";
$xmlDecl = "yes";
$xmlDeclErrRes="yes";
system "rm $errfile";
print STDERRB " UTF error. The XML declaration is missing. This is needed for non-English characters."; 
print STDERRB " Please insert \<?xml version=\"1.0\" encoding= \"ISO-8859-1\"?\>\n";
print STDERRB " at the top of the file. This line should go above the MIF element.\n";
print STDERRB "Transformation failed: A Run-time error was reported\n";
&badMifSub;
#-----------------------ending of xml declaration test--------------------------------------------------------------------------------------------------------
#</MIF>
}elsif($_ =~ /<\/MIF>/ and $xmlDecl eq "no" ){
$ReRunListFinal[$ReRunListFinalCounter]=$ReRunList;
$ReRunListFinalCounter++;
#print SAVEOUT "here $ReRunList\n";

};#if($_ =~ /<Map_Projection_Name gid1="1">/){
};#while (<REM>){
};  #foreach $ReRunList (@ReRunList){
printf SAVEOUT "\n";
printf LOG "\n";
};#if ($ReRunListCounter >0){

#--------------------------indicator section-----------------------------------------------------------------------------------------------------------
if ($ReRunListFinalCounter >0){
$ReRunListFinalCounterB=1;
printf SAVEOUT "The following file(s) need to be rerun:\n";
printf LOG "The following file(s) need to be rerun:\n";
open RERUN, ">>$errlogdir/$newdir/$flagfile";
system "chmod 777 $errlogdir/$newdir/$flagfile";

printf SAVEOUT "Please see the the file $flagfile at $errlogdir/$newdir\n";
printf LOG "Please see the the file $flagfile at $errlogdir/$newdir\n";

foreach $ReRunListFinal (@ReRunListFinal){
print SAVEOUT "$ReRunListFinalCounterB: $ReRunListFinal\n";
$ReRunListFinalCounterB++;

};
printf SAVEOUT "\n";
printf LOG "\n"; 
};

#print SAVEOUT"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n\n";


if ($countere == 1) {
	printf SAVEOUT "The following file could not be found!\n";
	printf LOG "The following file could not be found!\n";
         #printf LOG "Here 1!!!!!!!!!!\n";
		foreach $not_exist (@not_exist){

			printf SAVEOUT "$countere: $not_exist\n";
			printf LOG "$countere: $not_exist\n"; 
		};
	printf SAVEOUT"\n";
	printf LOG "\n";		
}elsif ($countere >1) {
	printf SAVEOUT "The following files could not be found!\n";
	printf LOG "The following files could not be found!\n";
        
	foreach $not_exist (@not_exist){
		$notexistcounter++;
                $dotLocation=index($not_exist,".");
                $mainPart=substr($not_exist,0,$dotLocation);
                #print SAVEOUT"\$mainPart = $mainPart\n";
                $noErr=$mainPart . ".err";
                unlink $noErr;
		printf SAVEOUT "$notexistcounter: $not_exist\n";
		printf LOG "$notexistcounter: $not_exist\n";
	};
	printf SAVEOUT"\n";
	printf LOG"\n";
};

$counter=1;

if ($flag_array_counter ==1){
	$flagfile="FailedFiles";
	open FLAGLIST, ">>$errlogdir/$newdir/$flagfile";
        system "chmod 777 $errlogdir/$newdir/$flagfile";
#to the screen
	printf SAVEOUT "The following file does not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post) after the filename. A list containg this file can be found in $flagfile in the \n";

	printf SAVEOUT "$errlogdir/$newdir directory.\n";
	#to the logfile
	printf LOG  "The following file does not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
after the filename.\n";
	printf LOG  "A file called $flagfile has a list with this file in it for you to flag.\n";
        #printf LOG "Here 2!!!!!!!!!!\n";	
	foreach $noflagarray (@noflagarray){
		printf SAVEOUT "$counter: $noflagarray\n";
		printf LOG "$counter: $noflagarray\n"; 
		printf FLAGLIST "$noflagarray";
	};
close FLAGLIST;	
}elsif ($flag_array_counter >1) {
	$flagfile="FailedFiles";
	 open FLAGLIST, ">$errlogdir/$newdir/$flagfile";
#	#to the screen
	 printf SAVEOUT "The following files do not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
#after the filename. A list of these files can be found in $flagfile in the \n";
	 printf SAVEOUT "$errlogdir/$newdir directory.\n";
	 #to the logfile
	 printf LOG  "The following files do not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
after the filename. A list of these files can be found in $flagfile in the $errlogdir/$newdir directory.\n";
	foreach $noflagarray (@noflagarray){ 
			printf SAVEOUT "$counter: $noflagarray";
			printf LOG "$counter: $noflagarray";
			printf FLAGLIST "$noflagarray";
			$counter++;
		};
	close FLAGLIST;
	printf SAVEOUT"\n";
	printf LOG"\n";
};


if ($badMIFfile2_counter >0 ){
#printf LOG "Here 3!!!!!!!!!!\n";
$flagfile="FailedFiles";
        open FLAGLIST, ">>$errlogdir/$newdir/$flagfile";
        system "chmod 777 $errlogdir/$newdir/$flagfile";

$badMif_counter_counter=1;
print  SAVEOUT "$badMIFfile2_counter $badMIFfile_File $badMIFfile_VerbPres invalid MIF $badMIFfile_File. $badMIFfile_ThisThese $badMIFfile_File  $badMIFfile_Verb NOT TRANSFORMED into $badMIFfile_an AFF $badMIFfile_File. Please see the error $badMIFfile_File at $errlogdir/$newdir\n";
print LOG "$badMIFfile2_counter $badMIFfile_File $badMIFfile_VerbPres invalid MIF $badMIFfile_File. $badMIFfile_ThisThese $badMIFfile_File  $badMIFfile_Verb NOT TRANSFORMED into $badMIFfile_an AFF $badMIFfile_File. Please see the error $badMIFfile_File at $errlogdir/$newdir\n";
foreach $badMIFfile2 (@badMIFfile2){ 

if ($badMIFfile2 ne $badMIFfile2Prev){
print  SAVEOUT "$badMif_counter_counter: $badMIFfile2\n";
print LOG "$badMif_counter_counter: $badMIFfile2\n";
print FLAGLIST "$badMIFfile2\n";
$badMif_counter_counter++;
$badMIFfile2Prev = $badMIFfile2;
};
};
print SAVEOUT"\n";
print LOG "\n";
close FLAGLIST;
};

#printf LOG "Here 3Aaaaaaaaa!!!!!!!!!!\n";
if ($badAFFfile2_counter2 >0){
#printf LOG "Here 4!!!!!!!!!!\n";
$flagfile="FailedFiles";
        open FLAGLIST, ">>$errlogdir/$newdir/$flagfile";
        system "chmod 777 $errlogdir/$newdir/$flagfile";
$badfile2_Post_counter=1;
print  SAVEOUT "$badAFFfile2_counter2 files have errors in them that occured in the transformation from the MIF format to the AFF format\n";
print  SAVEOUT "Please see the error files at  $errlogdir/$newdir\n";
print LOG "$badAFFfile2_counter2 files have errors in them that occured in the transformation from the MIF format to the AFF format\n";
print LOG "Please see the error files at $errlogdir/$newdir\n";
 
foreach $badAFFfile2 (@badAFFfile2){
print SAVEOUT "$badfile2_Post_counter: $badAFFfile2\n";
print LOG "$badfile2_Post_counter: $badAFFfile2\n";
print FLAGLIST " $badAFFfile2\n";
$badfile2_Post_counter++;

};
print  SAVEOUT "\n";
print LOG "\n";
close FLAGLIST;
};
#printf LOG "Here 3A!!!!!!!!!!\n";
if ($no_pass_counter >0){
printf LOG "Here 5!!!!!!!!!!\n";
$AFFNoPostCounter=1;
print  SAVEOUT "$no_pass_counter MIF files that WERE NOT POSTED to the GOS server were transformed into AFF files.  \n";
print  SAVEOUT "Please see them at $outputdir/$newdir\n";
print  SAVEOUT "Also, see the MRF files at $MRFdir/$newdir.\n";
print LOG "$no_pass_counter MIF files that WERE NOT POSTED to the GOS server were transformed into AFF files.  \n";
print LOG "Please see them at $outputdir/$newdir\n";
print LOG "Also, see the MRF files at $MRFdir/$newdir. \n";

foreach $nopassarray(@nopassarray){
print LOG "Here 1";
print SAVEOUT "$AFFNoPostCounter: $nopassarray\n";
print LOG  "$AFFNoPostCounter: $nopassarray\n"; 
$AFFNoPostCounter++;
$dotLoc=index($nopassarray,".");
$preGood=substr($nopassarray,0,$dotLoc);
$goodfile=$preGood ."_Good.xml";

$nopassDotLoc=index($nopassarray,0,".")-3;#was 2
$dotLessnopassarray=substr($nopassarray, 0,$nopassDotLoc);

$goodfile= "AFF-"  . $dotLessnopassarray . "_Good.xml";
$nonGoodloc = index($goodfile, "_Good.xml");#was -1
$nonGood= substr($goodfile,0,$nonGoodloc);
$nonGoodFinal = $nonGood . ".xml"; 

$AFFGoodCounter++;
#system "cp $nopassarray  $MRFdir/$newdir";
system "mv $nopassarray  $MRFdir/$newdir";

unlink $nopassarray;
if (-e "$goodfile"){
system "chmod 777 $goodfile";
};
printf LOG "Here 3B!!!!!!!!!!\n";

};
print SAVEOUT"\n";
print LOG "\n";
};

#print SAVEOUT "\$badAFFfile2_counter2 = $badAFFfile2_counter2\n";
if( $pass_counter>0 and $badAFFfile2_counter2 == 0){
#print LOG "Here 2";
print  SAVEOUT "$pass_counter MIF files that WERE POSTED to the GOS server were transformed into AFF files.  \n";
print  SAVEOUT "Please see them at $outputdir/$newdir.\n";
print  SAVEOUT "Also, see the MRF files at $MRFdir/$newdir and the FGDC files at $FGDCdir/$newdir.\n"; 
print LOG "$pass_counter MIF files that WERE POSTED to the GOS server  were transformed into AFF files.  \n";
print LOG  "Please see them at $outputdir/$newdir\n";
print LOG "Also, see the MRF files at $MRFdir/$newdir and the FGDC files at $FGDCdir/$newdir.\n";
$AFFGoodCounterA=1;
$AFFGoodCounter=0;

foreach $passarray (@passarray){
#printf LOG "Here 6!!!!!!!!!!\n";
print SAVEOUT "$AFFGoodCounterA: $passarray\n";
print LOG "$AFFGoodCounterA: $passarray\n";

#handling the AFF files
$dotLoc=index($passAFFarray[$AFFGoodCounter],".");
$preGood=substr($passAFFarray[$AFFGoodCounter],0,$dotLoc);
$goodfile=$preGood . ".xml"; ; # ."_Good.xml";
$AFFGoodCounterA++;
$nonGoodloc = index($passAFFarray[$AFFGoodCounter], "_Good.xml");
$nonGood= substr($passAFFarray[$AFFGoodCounter],0,$nonGoodloc);
$nonGoodFinal = $nonGood . ".xml";
if (-e "$outputdir/$newdir/$nonGoodFinal"){
#print LOG "chmod -1";
system "chmod 777 $outputdir/$newdir/$nonGoodFinal"; 
};
$AFFGoodCounter++;

#working with the FGDC file
$fgdcdotloc = index($passarray, ".");
$mainpart=substr($passarray,0,$fgdcdotloc);
$FGDCname = "$mainpart.xml";


open STDOUT,">$FGDCdir/$newdir/$FGDCname";

my @csax = "java com.saxonica.Transform -vw $passarray $root/gpms/xsl/xsl4mrf2fgdc/MRF2FGDC.xsl";
my $mif = system(@csax);
#print SAVEOUT "\$passarray = $passarray\n";
#print SAVEOUT "\$mif =$mif\n";
if (-e "$FGDCdir/$newdir/$FGDCname"){
#print LOG "\n $FGDCdir/$newdir/$FGDCname   in the first chmod\n\n";
system "chmod 777 $FGDCdir/$newdir/$FGDCname";
};

#handling the MRF files
if (-e $passarray){
#};
#print LOG "$FGDCdir/$newdir/$FGDCname   in the second chmod\n\n";
#system "cp $passarray  $MRFdir/$newdir";
system "mv $passarray  $MRFdir/$newdir";
if (-e $goodfile){
system "chmod 777 $goodfile";
unlink $passarray;
};
};


};
print SAVEOUT "\n";
print LOG  "\n";
};


print SAVEOUT "Please see the log file at $newerrdir.\n";
print LOG "Please see the log file at $newerrdir.\n";

#print SAVEOUT "\$pass_counter = $pass_counter\n";

if ($pass_counter == 0){
#print SAVEOUT " IN the deleting if \n";
#$FGDCdir/$newdir
#print SAVEOUT "$FGDCdir/$newdir";
#print SAVEOUT"\n----------------------------------\n";
#print SAVEOUT "$FGDCname\n";
#print SAVEOUT"The End\n";
$removeFGDCDir ="$FGDCdir/$newdir";
#$MRFdir
# $outputdir/$newdir
$removeAffDir = "$outputdir/$newdir";
$removeMRFDir ="$MRFdir/$newdir";
chomp($removeFGDCDir);
chomp($removeMRFDir);
chomp($removeAffDir);
system "rmdir $removeFGDCDir";
system "rmdir $removeMRFDir";
system "rmdir $removeAffDir";
};

if ($badfile_counter2>0 or $badMIFfile2_counter>0 or $flag_array_counter>0){
exit(2);
}else{
exit(0);
};

print SAVEOUT "\$pass_counter = $pass_counter\n";

my $Presdirecory =cwd;
system "chmod 777 $Presdirecory";