#!/usr/bin/perl

use Getopt::Long;
use File::Copy;
use Cwd;

use lib "/mt/apps/gpms/DateTrans/lib";
use DateTrans qw(monName monNumber timeFormat AmPmFormat dayNumbeFormat);

use lib "/mt/apps/gpms/DateTrans/lib";
use  xsdxsltcheck qw(xsdcheck xsltcheck FGDC2MRFcheck xsl4mrf2fgdccheck);


use File::Basename;

$correct=substr($0,2);



sub usage() {
    print "Usage: perl $0 [-listfile=<listfile>][-directory=<directory>][-product=<product>]\n";
    print "Example:\n";
    print "perl $0 -listfile=list.txt -directory=/dir/subdir -product=TLFE2007\n";
    print"where list.txt is the filename of the list file containg the names of the files to validated,\n";
       print "/dir/subdir is the directory containing the files to be validated\n";
       print "and TLFE2007 is the product.\n";
   print "Please also take note of the following shortcut for the directory variable: putting mtdata for the directory name\n";
  print "will yield the /mtdata/geo/gpms/input/mif directory.\n"; 
}
%gOptions;
&GetOptions(\%gOptions, qw(listfile=s directory=s  product=s));
unless ($gOptions{listfile} and $gOptions{directory} and $gOptions{product}) {
        usage();
        exit(1);
};
#---------------------------------------------change log-------------------------------------------------------------------------------------------------
#1/5/2011 Commented out the Here 6!!!!!!!!!! diagnostic and the Here 2 diagnostic
#1/5/2011 commented out print LOG "$FGDCdir/$newdir/$FGDCname   in the second chmod\n\n";
#1/5/2011 changed the if condition on the list of files that passed validation from if( $pass_counter>0){ to if( $pass_counter>0 and $badAFFfile2_counter2 == 0){
#  this was done so that only the mif files that passed validation and had no problems with the mif translation were shown  
#1/25/2011 Put in a second test to make sure that files with a size of 0 really do not exist
#---------------------------------------------file mangement section----------------------------------------------------------------------------------------
$Infile = $gOptions{listfile};
chomp($Branch = $gOptions{product});
chomp($indir= $gOptions{directory});

if ($indir eq "mtdata"){
$indir = "/mtdata/geo/gpms/input/mif";
}elsif ($indir eq "nfs"){
$indir ="/nfs/gpms/input/mif";
}elsif ($indir eq "input"){
$indir ="/home/mccre004/input_files";
};

$root="/mt/apps";
$rootout="/mtdata/geo";
$inlist="$rootout/gpms/input/list";
$outputdir ="$rootout/gpms/output/AFF";
$errlogdir ="$rootout/gpms/log";
$combolist = $inlist . "/" . $Infile;
$stylesheet ="$root/gpms/MIF2AFF/MIF2MRF.xsl";
$FGDCdir="$rootout/gpms/output/fgdc";
$MRFdir ="$rootout/gpms/output/mrf";

#------------------------------------------------------error trap--------------------------------------------------------------------------------------------

if (-e $indir){
print "Please Wait, Now processing the files ----------------------------------------------\n\n";
}else{
print "The directory $indir  could not be found! \n";
print "Please make sure the pathname, $indir, was spelled correctly and that it exists.\n";
exit(1);
};

if (-s $indir){
}else{
print "The directory $indir does not contain any files to be validated! \n";
exit(1);
};

if (-e $combolist){
}else{
print "The file $combolist could not be found! \n";
print "Please make sure the file, $Infile,  was spelled correctly and that the file is in the $inlist directory.\n";
exit(1);
}; 

#making sure the file list contains something
if (-s $combolist){
#comment
}else{
print "The file $combolist does not contain the list of files to be validated! \n";
print "Please make sure the file, $Infile,  contains a list of files to be validated.\n";
exit(1);
};

#------------------------------------getting the date---------------------------------------------------------------------------------------------------------
my $now=`date`;
$mon = substr($now,4,3);
$day=substr($now,8,2);
$year =substr($now,24,4);
$hour = substr($now,11,2);
$min = substr($now,14,2);
my $daynum= dayNumbeFormat($day);

#converting the date to a number
my $monnum = monName($mon);
my $monnum =monNumber($mon);
my $Month_Number = monNumber($mon);

$dateString =$year . $monnum . $daynum . $hour . $min;

$time = $hour . ":" . $min;
$oldhour = $hour;
my $newhour =timeFormat($hour);
my $ampm =AmPmFormat($oldhour);


$newtime = $newhour . ":" . $min . " $ampm";

#-----------------------------------------------------making the directories---------------------------------------------------------------------------
$newdir=$Branch . $dateString;
system "mkdir $outputdir/$newdir";
$newerrdir ="$errlogdir/$newdir";
system "mkdir $newerrdir";
chmod 0777, "$outputdir/$newdir";
chmod 0777, "$newerrdir";

$FGDCResult = system "mkdir $FGDCdir/$newdir";

chmod 0777, "$FGDCdir/$newdir";
$MRFreslut= system "mkdir $MRFdir/$newdir";

if ($reslut >0 ){
print "can not make mrf directory\n";
};
chmod 0777, "$MRFdir/$newdir";

#-----------------------------------------create the failed files---------------------------------------------------------------------------------
#open FAILEDFILES, ">$newerrdir/FailedFiles.txt";
#---------------------------------------------------------------------Counter variable Management section------------------------------------------
$badfile_counter2=0;
$goodfile_counter=0;
$countere=0;
$flag_array_counter=0;
$badAFFfile2_counter2=0;
$sizebadMif=0;
#----------------------------------------------------------Saxon-----------------------------------------------------------------------------------------
$ENV{'PATH'} = "/opt/saxon_lib/jdk1.5.0_11/bin:/mt/apps/gpms/output/mrf$newerrdir:$ENV{'PATH'}";#goodone
$cp = "CLASSPATH";#goodone
$ENV{$cp} = "/opt/saxon_lib/jdk1.5.0_11/lib/:/opt/saxon_lib/saxon9/saxon9sa.jar:/opt/saxon_lib/saxon9/saxon9-jdom.jar:/opt/saxon_lib/saxon9/:/opt/saxon_lib/saxon9/saxon9.jar:$ENV{$cp}";
#---------------------------------------redirecting standard output section------------------------------------------------------------------------------


#redirecting STDOUT because the Saxon program automatically sends all its output to STDOUT
# Saving a copy of STDOUT as SAVEOUT so that items can still be printed to the screen
open (SAVEOUT,">&STDOUT");

#-----------------------------------------Filename array creation section------------------------------------------------------------------------------
#open a filehamdle for the file containing the filenames
open INPUTFILE, "$combolist";
while(<INPUTFILE>){
$slashpos= rindex($_, "/");
if ($slashpos < 0){
$flagpos=index($_,"xml");
$flagless=substr($_, 0,$flagpos+3);
$flag=substr($_,$flagpos+4);
$ppos=index($flag,"p");
$filename="$indir/$flagless";
$filelen=length($_);

if ($ppos>=0){
$filename[$first_counter]="$indir/$flagless";
$flagarray[$first_counter]="$flag";
$flaghash{$filename}="$flag";
$first_counter++;
} elsif($filelen>1){
$noflagarray[$flag_array_counter]=$_;
$flag_array_counter++;
};


}elsif($slashpos > 0){


my $dirnameA = dirname $_;

$slash_directory_name[$slash_directory_name_counter]=$dirnameA;
$slash_directory_name_counter++;

$infile=$_;
$dotloc=rindex($infile,".")+4;
$slashfilename = substr($infile,0,$dotloc);
$flagpos=index($infile,"xml");
$ppos=index($flag,"p");
$flag2=substr($flag,0,1);
$filelen=length($infile);

if ($ppos>=0 and $filelen>1){
$filename[$first_counter]=$slashfilename;
$flagarray[$first_counter]="$flag";
$flaghash{$slashfilename}="$flag";
$first_counter++;
} elsif($ppos=0 and $filelen>1){
$noflagarray[$flag_array_counter]=$slashfilename;
$flag_array_counter++;
};

};
};
#-------------------------------------------validating the incoming MIF File--------------------------------------------------------------------------------------
$valcounter=0;
$filecounter =1;

foreach $filename (@filename){
$ISOind="no";
chomp($filename);
#setting up the variables necessary to make the the finalfilename variable, which is needed to make the MRF file
$slashpos= rindex($filename, "/") +1;
$newfilename = substr($filename,$slashpos);
$dotlocation = index($newfilename, ".");
$finalfilename=substr($newfilename, 0, $dotlocation);

if ($valcounter >0){
close STDERR;
system "rm $TMP";
};

$MRF = $finalfilename . ".mrf";
$TMP = "TMP" . "-" . $newfilename;
$valcounter++;


open STDOUT,">$TMP";

#making the error file and redirecting the STDERR to the error file
#have a new $dotlocation variable because the $filename and $newfilename variables are different lengths
$errmain= substr($newfilename, 0, $dotlocation);
$errfile = $errmain .".err";

open STDERR, ">$errfile";

#starts schema aware transformation. This will cause the transformation to fail if the document is invalid
# the -vw option indicates that validation errors found when validating the result tree should be treated as warnings only.
#This option is available only with the Saxon-SA version of the command, com.saxonica.Transform.
printf SAVEOUT "$filecounter: Now validating $filename\n";
$filecounter++;
my @csax = "java com.saxonica.Transform -vw $filename $root/gpms/xsl/MIF2MRF.xsl";
my $mif = system(@csax);
#printf SAVEOUT "\$mif = $mif\n";

#opening fielhandles so that *.xsd will be replaced by Repository.xsd
#basically replicating the sed command 
open MRFOUTPUT, ">$MRF";
open TMPFILE, "<$TMP";

close STDERR; #new one
if ($mif > 0) {
$badMIFfile[$badMIFfile_counter] = $errfile;
$badMIFfile_counter++;

$sizeTMP= -s $TMP;

if ($sizeTMP >0 ){
system "mv $TMP $newerrdir";
}else{
system "rm $TMP";
}; 

$sizeMRF = -s $MRF;
#print SAVEOUT "\$sizeMRF = $sizeMRF\n";

if ($sizeMRF >0){
system "mv $MRF $newerrdir";
}else{
system "rm $MRF";
};

#printf SAVEOUT "\$badMIFfile[$badMIFfile_counter] = $errfile\n";
}else{
unlink $errfile;

$goodmif[$goodMif_counter]=$filename;
$goodMif_counter++;

system "rm $errfile";
$goodfile[$goodfile_counter]= $MRF;


$goodflaghash{$MRF}=$flaghash{$filename};
$npos=index($flaghash{$filename},"n");
if ($npos==0){
$nopassarray[$no_pass_counter]=$goodfile[$goodfile_counter];
$no_pass_counter++;
print SAVEOUT "in the posting array\n";
}elsif ($npos<0){
$passarray[$pass_counter]=$goodfile[$goodfile_counter];
$pass_counter++;

};



$goodfile_counter++;
unlink $errfile;
 while(<TMPFILE>){
#print SAVEOUT " reading the TMP file\n";
  if($_ =~ /xsd/){
   #this happens if the the line has xsd in it
   $xsdlocation =index($_, "/xsd");
   $originalstring= substr($_,0,$xsdlocation);
   $Repository = "xsd/Repository.xsd\">\n";
   printf MRFOUTPUT ("$originalstring/$Repository");
  }elsif ($_ =~ /<\/MRF>/){
   printf  MRFOUTPUT ("$_");
   printf  MRFOUTPUT "\n";
   }elsif($_=~ /%/){
        chomp($_);
        $perloc=index($_, "%");
        $firsthalf =substr($_,0,$perloc);
        $secondhalf =substr($_, $perloc+1);
        $perloc2=index($secondhalf, "%");
                if ($perloc2 < 0){
                        printf  MRFOUTPUT"$firsthalf" . "%%" . "$secondhalf";
                }elsif($perloc2 >0){
                        $thirdhalf =substr($secondhalf,0,$perloc2);
                        $fourthhalf =substr($secondhalf, $perloc2+1);
                        $newstring = "$firsthalf" . "%%" . "$thirdhalf" . "%%" . "$fourthhalf\n";
if ($newstring=~/&#147;/){
$amploc2 =index($newstring, "&#148;");
                           $amp1string = substr($newstring,0,$amploc1);
                           $midstringlength= $amploc2 - $amploc1-6;
                           $amp2string = substr($newstring,$amploc1+6,$midstringlength);
                           $amp3string = substr($newstring,$amploc2+6);
                           printf  MRFOUTPUT $amp1string . "\"" . $amp2string . "\"" . $amp3string;
                         }else{
                        printf  MRFOUTPUT"$firsthalf" . "%%" . "$thirdhalf" . "%%" . "$fourthhalf\n";
                         };;#if ($newstring=~/&#147;/){
                         };#if ($perloc2 <  0){
 }elsif($_=~/&#147;/){
$amploc1=index($_, "&#147;");
 $amploc2 =index($_, "&#148;");
 $amp1string = substr($_,0,$amploc1);
 $midstringlength= $amploc2 - $amploc1-6;
 $amp2string = substr($_,$amploc1+6,$midstringlength);
 $amp3string = substr($_,$amploc2+6);
 printf  MRFOUTPUT $amp1string . "\"" . $amp2string . "\"" . $amp3string;

}elsif($_=~/&#146;/){
$aposloc1 = index($_,"&#146;");
$aposstring1 = substr($_,0,$aposloc1);
$aposstring2 = substr($_,$aposloc1+6);
printf  MRFOUTPUT $aposstring1 . "'" . $aposstring2;


  }else{
   printf  MRFOUTPUT "$_";
  };#if xsd
 };#while<TMPFILE>
close MRFOUTPUT;
close TMPFILE;

open TMPFILE, "<$TMP";


};#if mif>0
unlink $TMP
};#foreach

printf SAVEOUT "\n";
close STDOUT;
#---------------------
#-------------------------------error handling for invalid MIF files-------------------------------------------------------------------------------------------------
#setting the $badMIFfile2_counter variable 
print SAVEOUT "Now creating the Error Files----------------------------------------------------------------------------------------------\n";
$badMIFfile2_counter=0;
$badMIFfile3_counter=1;#for the screen
foreach $badMIFfile (@badMIFfile){
print SAVEOUT "$badMIFfile3_counter: Now working on  $badMIFfile\n";
$badMIFfile3_counter++;
open BADFILEH, "$badMIFfile";
$bad="no";
$slashpos= rindex($badMIFfile, "/")+1;
$wheredot2 = index($badMIFfile, ".");


if ($slashpos >0){
$AFF_Main_name = substr($badMIFfile,$slashpos,$wheredot2);
}else{
$AFF_Main_name = substr($badMIFfile,0,$wheredot2);
};

$name = "$AFF_Main_name.xml";
$mifErrFile ="$AFF_Main_name.err";
$mifMrfFile ="$AFF_Main_name.mrf";
$bad ="no";
$sizebadMif = -s $badMif;
#print SAVEOUT  "\$sizebadMif = $sizebadMif\n";
if (-s $badMIFfile == 0){
#print SAVEOUT "In the size =0 zero section\n";
$MIfDotLoc=index($badMIFfile,".");
$MifPreDot=substr($badMIFfile,0,$MIfDotLoc);
$FinalMif = $MifPreDot . ".xml";
#print SAVEOUT "\$FinalMif = $FinalMif\n";
#$sizeFile ="$indir/$badMIFfile";
$sizeFile ="$indir/$FinalMif";
#print SAVEOUT "\$sizeFile = $sizeFile\n";
#if (-e $badMIFfile){
if (-e $sizeFile){
#print SAVEOUT "the $badMIFfile File Exists!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n";
$ReRunList[$ReRunListCounter] = "$name";
$ReRunListCounter ++;
#unlink $badMIFfile;
}else{
$remove=shift(@badMif);
$not_exist[$countere]= "$name";
$countere++;
$bad ="yes";
unlink $badMIFfile;
#print SAVEOUT " File DOES NOT Exist!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n";
};
#print SAVEOUT "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n\n";
#print SAVEOUT "In the size =0 zero section\n";
}else{


while(<BADFILEH>){
if ($_=~ /does not exist/){
$not_exist[$countere]= "$name";
$countere++;
$bad ="yes";
unlink $badMIFfile;

if ($badMIFfile ne  $badMIFfilePrev){
print SAVEOUT "$badMIFfile File does not exist\n";
};

#$badMIFfile = $badMIFfilePrev;
$badMIFfilePrev = $badMIFfile;
}elsif($badMiffile2[$badMif_counter] ne $name and $bad ne "yes"){

if ($badMIFfile ne  $badMIFfilePrev){
#print SAVEOUT "$badMIFfile File does exist\n";
};
$badMIFfilePrev = $badMIFfile;

if ($name ne $MifPrev and $MifPrev ne "0"){
$MifPrev= chomp($name);

$badMIFfile2[$badMIFfile2_counter] = "$name";

$bad ="no";
$outmif ="$indir/$name";
copy ($outmif, $newerrdir) or warn "Can't copy $name to $newerrdir: $!\n";
system "mv $name $newerrdir";
system "mv $mifErrFile $newerrdir";
system "mv $badMIFfile $newerrdir";
system "chmod 777  $newerrdir";
$badMIFfile2_counter++;
$MifPrev = $name;
};


unlink $badMif
}elsif ($bad ne "no" and  $MifPrev ne $name){

print SAVEOUT "3. default ($badMIFfile)\n";

};#if
}#s if
};#while
close BADFILEH
};#foreach
print SAVEOUT "\n\n";

#------------------------------------------Transforming MIF to AFF-----------------------------------------------------------------------------------------------------
if ($goodMif_counter){
print SAVEOUT "Now transforming the MIF files into AFF files ---------------------------------------------------------------------\n";
};

$valcounter=0;
$filecounter =1;

foreach $goodmif (@goodmif){
chomp($goodmif);
#setting up the variables necessary to make the the finalfilename variable, which is needed to make the MRF file
$slashpos= rindex($goodmif, "/") +1;
$newfilename = substr($goodmif,$slashpos);
$dotlocation = rindex($newfilename, ".");
$finalfilename=substr($newfilename, 0, $dotlocation);
#setting up the TMP and MRF filenames

#if (-e $TMP){
if ($valcounter >0){
close STDERR;
system "rm $AFFOutFile";
};

$AFFOutFile="AFF" . "-" . $finalfilename . ".xml" ;

open STDOUT,">$AFFOutFile";

#making the error file and redirecting the STDERR to the error file
#have a new $dotlocation variable because the $goodmif and $newfilename variables are different lengths
$errmain= substr($newfilename, 0, $dotlocation);
$errfile ="AFF_" . $errmain . ".err";

open STDERR, ">$errfile";

#starts schema aware transformation. This will cause the transformation to fail if the document is invalid
# the -vw option indicates that validation errors found when validating the result tree should be treated as warnings only.
#This option is available only with the Saxon-SA version of the command, com.saxonica.Transform.
printf SAVEOUT "$filecounter: Now transforming $goodmif from the MIF format to the American Fact Finder Format(AFF)\n";
$filecounter++;
my @csax = "java com.saxonica.Transform -vw $goodmif $root/gpms/MIF2AFF/MIF2AFF.xsl";
my $affResult  = system(@csax);
#opening fielhandles so that *.xsd will be replaced by Repository.xsd
#basically replicating the sed command 
close STDERR;
 
if ($affResult > 0) {
#print SAVEOUT "bad Mif\n";
$badAFFfile[$badAFFfile_counter] = $errfile;
$badAFFfile_counter++;
system "cp $errfile $errlogdir/$newdir";
if (-e  $TMP){
unlink $TMP;
};

#print SAVEOUT "-------------------------------------\n\n";
 
}else{
#print SAVEOUT "removing $errfile\n";
unlink $errfile;
#determine if the files are to be posted. If the position of n, as indicated by $npos is -1, then the file is not to be posted
$goodflaghash{$AFFOutFile}=$flaghash{$goodmif};#was com
$npos=index($flaghash{$goodmif},"n");#was com
$AFFFileLength=length($AFFOutFile);
#print SAVEOUT "\$AFFFileLength = $AFFFileLength\n";

if ($AFFFileLength >4){
#system "cp $AFFOutFile  $outputdir/$newdir";
system "mv $AFFOutFile  $outputdir/$newdir";
};

};#if ($affResult > 0) {
};#foreach
 
 
 
printf SAVEOUT "\n";


#------------------------------error filehandling section for bad AFF files--------------------------------------------------------------------------------------------

foreach $badAFFfile (@badAFFfile){
open BADFILEH, "$badAFFfile";
$bad="no";
$slashpos= rindex($badAFFfile, "/")+1;
$wheredot2 = index($badAFFfile, ".");
$AFF_Main_name = substr($badAFFfile,$slashpos,$wheredot2);
$name = "$AFF_Main_name.xml";
$bad ="no";
if (-s $badAFFfile == 0){
print SAVEOUT "In the size = 0 section(bad AFF).\n";
$not_exist[$countere]="$name";

#removing the element in the array so that the file does not show up twice: as a bad mif and missing file
$countere++;
$bad ="yes";

}else{

while(<BADFILEH>){
if ($_=~ /does not exist/){
$not_exist[$countere]= "$name";
$countere++;
$bad ="yes";
print SAVEOUT "badd AFF does not exist \n";
}elsif (-s $badAFFfile == 0 and $badfile2[$badfile_counter2-1] ne $name and $bad ne "yes"){
$not_exist[$countere]= "$name";
printf  MRFOUTPUT "Here!";
unlink "$badfile";
$countere++;
$bad ="yes";
}elsif($badAFFfile[$badAFFfile2_counter2-1] ne $name and $bad ne "yes"){

if ($name ne $PrevAFF){
$badAFFfile2[$badAFFfile2_counter2] = "$name";
$badAFFfile2_counter2++;
$bad ="no";
system "cp $badfile $newerrdir/$name";
system "chmod 777 $newerrdir/$name";
unlink $badfile;
$PrevAFF=$name;
};


}else{
print SAVEOUT"3. Default\n";
};#if
}#s if
};#while
close BADFILEH
};#foreach
#print SAVEOUT "end of loop\n";
#--------------------------------------final variables counting section--------------------------------------------------------------------------------
#print SAVEOUT  "in the final variables counting section\n";
$total = $badfile_counter2 + $pass_counter+$no_pass_counter ;
$newtotal=$total+$countere+$flag_array_counter+$badMIFfile2_counter;
if ($pass_counter >0){
$pergood = (($pass_counter )/$newtotal) *100;
}else{
$pergood =0;
$pass_counter  =0;
};

#if ($badfile_counter2){
if ($badMIFfile2_counter>0){
$perbad = (($badMIFfile2_counter)/$newtotal) *100;
}else{
$perbad =0;
$badfile_counter2=0;
};

#print SAVEOUT  "out of final variables counting section\n";
#-------------------------------working with the log file----------------------------------------------------------------------------------------------
#This is the introductary paragraph.
open LOG, ">$errlogdir/$newdir/$newdir.log";
system "chmod 777 $errlogdir/$newdir/$newdir.log";
printf LOG "This log details the transformation and validation results of the $newtotal mif files contained in \n the  $combolist list file.\n";

if ($slash_directory_name_counter >0){
printf LOG "\nThese files can be found in the following directories:\n";
printf LOG "1. $indir (This is the directory given at the command line)\n";
$slash_directory_nameb =2;
@slash_directory_name=sort(@slash_directory_name);

foreach $slash_directory_name (@slash_directory_name){

$slash_directory_nameca =$slash_directory_namec-1;

if ($slash_directory_name[$slash_directory_namec] ne $slash_directory_name[$slash_directory_nameca] and $slash_directory_name[$slash_directory_namec] ne $indir){
printf LOG "$slash_directory_nameb: $slash_directory_name\n";
$slash_directory_nameb++;
$slash_directory_namec++;
}elsif($slash_directory_namec == 0){
printf LOG "$slash_directory_nameb: $slash_directory_name\n";
$slash_directory_nameb++;
$slash_directory_namec++;
};#if ($slash_directory_name[$slash_directory_namec] ne $slash_directory_name[$slash_directory_nameca] and $slash_directory_name[$slash_directory_namec] ne $indir){
};#foreach $slash_directory_name (@slash_directory_name){

printf LOG "\n$0 was run on $monnum/$daynum/$year at $time ($newtime) for the $Branch product.\n\n";
}else{
printf LOG "\nThese files can be found in the $indir directory.\n";
printf LOG "It was run on $monnum/$daynum/$year at $time ($newtime) for the $Branch product.\n\n";
#print LOG "Here 0\n";
};#if ($slash_directory_name_counter >0){
printf LOG "\n Of these $newtotal files, $pass_counter, or $pergood%, passsed validation and $badMIFfile2_counter, or $perbad%, failed validation.  ";


#if ($countere >0 or $flag_array_counter>0) {
if ($countere >0 or $flag_array_counter>0) {
printf LOG "Also, ";

if ($countere >0){

if ($countere ==1){
$notFoundFile = "file";
}else{
$notFoundFile = "files";
};

printf LOG "$countere $notFoundFile could not be found";
}

if ($countere >0 and $flag_array_counter>0){
printf LOG " and ";
}

if ($flag_array_counter>0){

if ($flag_array_counter ==1){
$FlagFile = "file";
$FlagVerb = "does";
}else{
$FlagFile = "files";
$FlagVerb = "do";
};


printf LOG "$flag_array_counter $FlagFile $FlagVerb not have a FGDC posting flag";
}

printf LOG ".\n\n";  
}
#--------------------------grammar---------------------------------------------------------------------------------------------------------------------
if($badMIFfile2_counter ==1){
$badMIFfile_File="file";
$badMIFfile_ThisThese="This";
$badMIFfile_Verb="WAS";
$badMIFfile_VerbPres="is an";
$badMIFfile_an =" an "
}else{
$badMIFfile_File="files";
$badMIFfile_ThisThese="These";
$badMIFfile_Verb="WERE";
$badMIFfile_VerbPres="are";
$badMIFfile_an=" "
};

#--------------------------indicator section-----------------------------------------------------------------------------------------------------------
if ($ReRunListCounter >0){
#printf SAVEOUT "The following file(s) need to be rerun:\n";
#printf LOG "The following file(s) need to be rerun:\n";

$flagfile="ReRunList.txt";
#open RERUN, ">>$errlogdir/$newdir/$flagfile";
#system "chmod 777 $errlogdir/$newdir/$flagfile";

#printf SAVEOUT "Please see the the file $flagfile at $errlogdir/$newdir\n";
#printf LOG "Please see the the file $flagfile at $errlogdir/$newdir\n";

$ReRunListCounterB =1;
foreach $ReRunList (@ReRunList){
#printf SAVEOUT "In the loop\n";
#printf SAVEOUT "$ReRunListCounterB:  $ReRunList\n";
#printf LOG "$ReRunListCounterB:  $ReRunList\n";
#printf RERUN " $ReRunList \n";
$ReRunListCounterB ++;

$ReRunFile =$indir . "/" . $ReRunList;
#print SAVEOUT "\$ReRunFile = $ReRunFile\n";
open REM, "<$ReRunFile";

while (<REM>){
if($_ =~ /<Map_Projection_Name gid1="1">/){
$mapProj = $_;
$paraPos=index($mapProj,"(");
$AlbersLength =length($mapProj);
#$wrongAlbers1 = index($mapProj,"Albers Conical Equal Area");
$wrongAlbers1 = index($mapProj,"Albers Conical Equal Area for");
#$mapProjName=
#print SAVEOUT "\$mapProj = $mapProj\n";
#print SAVEOUT "\$wrongAlbers1 = $wrongAlbers1\n";
#print SAVEOUT "\$AlbersLength = $AlbersLength\n";
#$AlberLength=length(
if ($paraPos>0){
$dotlocation = index($ReRunList, ".");
$errmain= substr($ReRunList, 0, $dotlocation);
#print SAVEOUT "\$errmain = $errmain\n";
$errfile = $errmain . ".err";
#$errlogdir
$errfileB = $newerrdir . "/" . $errfile;
#print SAVEOUT "\$errfileB = $errfileB\n";
open STDERRB, ">$errfileB";
system "rm $errfile";
print STDERRB "Error: The element <Map_Projection_Name> has a parenthesis in it. This is not allowed. Please remove the '('.\n";
print STDERRB "1 Error found\n";
$badMIFfile2[$badMIFfile2_counter]=$ReRunList;
$badMIFfile2_counter++;
}elsif($mapProj =~ /Albers Equal Area Conic/){
$dotlocation = index($ReRunList, ".");
$errmain= substr($ReRunList, 0, $dotlocation);
#print SAVEOUT "In the Albers Conical Equal Area Section \n";
#print SAVEOUT "\$errmain = $errmain\n";
$errfile = $errmain . ".err";
#$errlogdir
$errfileB = $newerrdir . "/" . $errfile;
#print SAVEOUT "\$errfileB = $errfileB\n";
open STDERRB, ">$errfileB";
system "rm $errfile";
print STDERRB "Error: Albers Equal Area Conic is not a recognized projection name. Try Albers Conical Equal Area.\n";
print STDERRB "1 Error found\n";

$badMIFfile2[$badMIFfile2_counter]=$ReRunList;
$badMIFfile2_counter++;
}elsif($mapProj =~ /Albers Conical Equal Area for/ and $AlbersLength < 84){
#print SAVEOUT "In the Albers length section \n";
$dotlocation = index($ReRunList, ".");
$errmain= substr($ReRunList, 0, $dotlocation);
#print SAVEOUT "In the Albers Conical Equal Area Section \n";
#print SAVEOUT "\$errmain = $errmain\n";
$errfile = $errmain . ".err";
#$errlogdir
$errfileB = $newerrdir . "/" . $errfile;
#print SAVEOUT "\$errfileB = $errfileB\n";
open STDERRB, ">$errfileB";
system "rm $errfile";
print STDERRB "Error: 'Albers Conical Equal Area for' is not a recognized projection name. Try Albers Conical Equal Area with the state name.\n";
print STDERRB "1 Error found\n";
$badMIFfile2[$badMIFfile2_counter]=$ReRunList;
$badMIFfile2_counter++;

#}elsif(

#}if($mapProj =~ /
}else{
$ReRunListFinal[$ReRunListFinalCounter]=$ReRunList;
$ReRunListFinalCounter++;
};#if ($paraPos>0){

#rint SAVEOUT "\$mapProj = $mapProj\n";
};#if($_ =~ /<Map_Projection_Name gid1="1">/){
};#while (<REM>){
};  #foreach $ReRunList (@ReRunList){
printf SAVEOUT "\n";
printf LOG "\n";
};#if ($ReRunListCounter >0){

#print SAVEOUT "\$ReRunListFinalCounter = $ReRunListFinalCounter\n";
#print SAVEOUT"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n";
if ($ReRunListFinalCounter >0){
$ReRunListFinalCounterB=1;
printf SAVEOUT "The following file(s) need to be rerun:\n";
printf LOG "The following file(s) need to be rerun:\n";
open RERUN, ">>$errlogdir/$newdir/$flagfile";
system "chmod 777 $errlogdir/$newdir/$flagfile";

printf SAVEOUT "Please see the the file $flagfile at $errlogdir/$newdir\n";
printf LOG "Please see the the file $flagfile at $errlogdir/$newdir\n";

foreach $ReRunListFinal (@ReRunListFinal){
print SAVEOUT "$ReRunListFinalCounterB: $ReRunListFinal\n";
$ReRunListFinalCounterB++;

};
printf SAVEOUT "\n";
printf LOG "\n"; 
};

#print SAVEOUT"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n\n";

if ($countere == 1) {
	printf SAVEOUT "The following file could not be found!\n";
	printf LOG "The following file could not be found!\n";
         #printf LOG "Here 1!!!!!!!!!!\n";
		foreach $not_exist (@not_exist){

			printf SAVEOUT "$countere: $not_exist\n";
			printf LOG "$countere: $not_exist\n"; 
		};
	printf SAVEOUT"\n";
	printf LOG "\n";		
}elsif ($countere >1) {
	printf SAVEOUT "The following files could not be found!\n";
	printf LOG "The following files could not be found!\n";
	foreach $not_exist (@not_exist){
		$notexistcounter++;
		printf SAVEOUT "$notexistcounter: $not_exist\n";
		printf LOG "$notexistcounter: $not_exist\n";
	};
	printf SAVEOUT"\n";
	printf LOG"\n";
};

$counter=1;

if ($flag_array_counter ==1){
	$flagfile="FailedFiles";
	open FLAGLIST, ">>$errlogdir/$newdir/$flagfile";
        system "chmod 777 $errlogdir/$newdir/$flagfile";
#to the screen
	printf SAVEOUT "The following file does not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post) after the filename. A list containg this file can be found in $flagfile in the \n";

	printf SAVEOUT "$errlogdir/$newdir directory.\n";
	#to the logfile
	printf LOG  "The following file does not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
after the filename.\n";
	printf LOG  "A file called $flagfile has a list with this file in it for you to flag.\n";
        #printf LOG "Here 2!!!!!!!!!!\n";	
	foreach $noflagarray (@noflagarray){
		printf SAVEOUT "$counter: $noflagarray\n";
		printf LOG "$counter: $noflagarray\n"; 
		printf FLAGLIST "$noflagarray";
	};
close FLAGLIST;	
}elsif ($flag_array_counter >1) {
	$flagfile="FailedFiles";
	 open FLAGLIST, ">$errlogdir/$newdir/$flagfile";
#	#to the screen
	 printf SAVEOUT "The following files do not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
#after the filename. A list of these files can be found in $flagfile in the \n";
	 printf SAVEOUT "$errlogdir/$newdir directory.\n";
	 #to the logfile
	 printf LOG  "The following files do not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
after the filename. A list of these files can be found in $flagfile in the $errlogdir/$newdir directory.\n";
	foreach $noflagarray (@noflagarray){ 
			printf SAVEOUT "$counter: $noflagarray";
			printf LOG "$counter: $noflagarray";
			printf FLAGLIST "$noflagarray";
			$counter++;
		};
	close FLAGLIST;
	printf SAVEOUT"\n";
	printf LOG"\n";
};


if ($badMIFfile2_counter >0 ){
#printf LOG "Here 3!!!!!!!!!!\n";
$flagfile="FailedFiles";
        open FLAGLIST, ">>$errlogdir/$newdir/$flagfile";
        system "chmod 777 $errlogdir/$newdir/$flagfile";

$badMif_counter_counter=1;
print  SAVEOUT "$badMIFfile2_counter $badMIFfile_File $badMIFfile_VerbPres invalid MIF $badMIFfile_File. $badMIFfile_ThisThese $badMIFfile_File  $badMIFfile_Verb NOT TRANSFORMED into $badMIFfile_an AFF $badMIFfile_File. Please see the error $badMIFfile_File at $errlogdir/$newdir\n";
print LOG "$badMIFfile2_counter $badMIFfile_File $badMIFfile_VerbPres invalid MIF $badMIFfile_File. $badMIFfile_ThisThese $badMIFfile_File  $badMIFfile_Verb NOT TRANSFORMED into $badMIFfile_an AFF $badMIFfile_File. Please see the error $badMIFfile_File at $errlogdir/$newdir\n";
foreach $badMIFfile2 (@badMIFfile2){ 

if ($badMIFfile2 ne $badMIFfile2Prev){
print  SAVEOUT "$badMif_counter_counter: $badMIFfile2\n";
print LOG "$badMif_counter_counter: $badMIFfile2\n";
print FLAGLIST "$badMIFfile2\n";
$badMif_counter_counter++;
$badMIFfile2Prev = $badMIFfile2;
};
};
print SAVEOUT"\n";
print LOG "\n";
close FLAGLIST;
};

#printf LOG "Here 3Aaaaaaaaa!!!!!!!!!!\n";
if ($badAFFfile2_counter2 >0){
#printf LOG "Here 4!!!!!!!!!!\n";
$flagfile="FailedFiles";
        open FLAGLIST, ">>$errlogdir/$newdir/$flagfile";
        system "chmod 777 $errlogdir/$newdir/$flagfile";
$badfile2_Post_counter=1;
print  SAVEOUT "$badAFFfile2_counter2 files have errors in them that occured in the transformation from the MIF format to the AFF format\n";
print  SAVEOUT "Please see the error files at  $errlogdir/$newdir\n";
print LOG "$badAFFfile2_counter2 files have errors in them that occured in the transformation from the MIF format to the AFF format\n";
print LOG "Please see the error files at $errlogdir/$newdir\n";
 
foreach $badAFFfile2 (@badAFFfile2){
print SAVEOUT "$badfile2_Post_counter: $badAFFfile2\n";
print LOG "$badfile2_Post_counter: $badAFFfile2\n";
print FLAGLIST " $badAFFfile2\n";
$badfile2_Post_counter++;

};
print  SAVEOUT "\n";
print LOG "\n";
close FLAGLIST;
};
#printf LOG "Here 3A!!!!!!!!!!\n";
if ($no_pass_counter >0){
printf LOG "Here 5!!!!!!!!!!\n";
$AFFNoPostCounter=1;
print  SAVEOUT "$no_pass_counter MIF files that WERE NOT POSTED to the GOS server were transformed into AFF files.  \n";
print  SAVEOUT "Please see them at $outputdir/$newdir\n";
print  SAVEOUT "Also, see the MRF files at $MRFdir/$newdir.\n";
print LOG "$no_pass_counter MIF files that WERE NOT POSTED to the GOS server were transformed into AFF files.  \n";
print LOG "Please see them at $outputdir/$newdir\n";
print LOG "Also, see the MRF files at $MRFdir/$newdir. \n";

foreach $nopassarray(@nopassarray){
print LOG "Here 1";
print SAVEOUT "$AFFNoPostCounter: $nopassarray\n";
print LOG  "$AFFNoPostCounter: $nopassarray\n"; 
$AFFNoPostCounter++;
$dotLoc=index($nopassarray,".");
$preGood=substr($nopassarray,0,$dotLoc);
$goodfile=$preGood ."_Good.xml";

$nopassDotLoc=index($nopassarray,0,".")-3;#was 2
$dotLessnopassarray=substr($nopassarray, 0,$nopassDotLoc);

$goodfile= "AFF-"  . $dotLessnopassarray . "_Good.xml";
$nonGoodloc = index($goodfile, "_Good.xml");#was -1
$nonGood= substr($goodfile,0,$nonGoodloc);
$nonGoodFinal = $nonGood . ".xml"; 

$AFFGoodCounter++;
#system "cp $nopassarray  $MRFdir/$newdir";
system "mv $nopassarray  $MRFdir/$newdir";

unlink $nopassarray;
if (-e "$goodfile"){
system "chmod 777 $goodfile";
};
printf LOG "Here 3B!!!!!!!!!!\n";

};
print SAVEOUT"\n";
print LOG "\n";
};

#print SAVEOUT "\$badAFFfile2_counter2 = $badAFFfile2_counter2\n";
if( $pass_counter>0 and $badAFFfile2_counter2 == 0){
#print LOG "Here 2";
print  SAVEOUT "$pass_counter MIF files that WERE POSTED to the GOS server were transformed into AFF files.  \n";
print  SAVEOUT "Please see them at $outputdir/$newdir.\n";
print  SAVEOUT "Also, see the MRF files at $MRFdir/$newdir and the FGDC files at $FGDCdir/$newdir.\n"; 
print LOG "$pass_counter MIF files that WERE POSTED to the GOS server  were transformed into AFF files.  \n";
print LOG  "Please see them at $outputdir/$newdir\n";
print LOG "Also, see the MRF files at $MRFdir/$newdir and the FGDC files at $FGDCdir/$newdir.\n";
$AFFGoodCounterA=1;
$AFFGoodCounter=0;

foreach $passarray (@passarray){
#printf LOG "Here 6!!!!!!!!!!\n";
print SAVEOUT "$AFFGoodCounterA: $passarray\n";
print LOG "$AFFGoodCounterA: $passarray\n";

#handling the AFF files
$dotLoc=index($passAFFarray[$AFFGoodCounter],".");
$preGood=substr($passAFFarray[$AFFGoodCounter],0,$dotLoc);
$goodfile=$preGood . ".xml"; ; # ."_Good.xml";
$AFFGoodCounterA++;
$nonGoodloc = index($passAFFarray[$AFFGoodCounter], "_Good.xml");
$nonGood= substr($passAFFarray[$AFFGoodCounter],0,$nonGoodloc);
$nonGoodFinal = $nonGood . ".xml";
if (-e "$outputdir/$newdir/$nonGoodFinal"){
#print LOG "chmod -1";
system "chmod 777 $outputdir/$newdir/$nonGoodFinal"; 
};
$AFFGoodCounter++;

#working with the FGDC file
$fgdcdotloc = index($passarray, ".");
$mainpart=substr($passarray,0,$fgdcdotloc);
$FGDCname = "$mainpart.xml";


open STDOUT,">$FGDCdir/$newdir/$FGDCname";

my @csax = "java com.saxonica.Transform -vw $passarray $root/gpms/xsl/xsl4mrf2fgdc/MRF2FGDC.xsl";
my $mif = system(@csax);

if (-e "$FGDCdir/$newdir/$FGDCname"){
#print LOG "\n $FGDCdir/$newdir/$FGDCname   in the first chmod\n\n";
system "chmod 777 $FGDCdir/$newdir/$FGDCname";
};

#handling the MRF files
if (-e $passarray){
#};
#print LOG "$FGDCdir/$newdir/$FGDCname   in the second chmod\n\n";
#system "cp $passarray  $MRFdir/$newdir";
system "mv $passarray  $MRFdir/$newdir";
if (-e $goodfile){
system "chmod 777 $goodfile";
unlink $passarray;
};
};


};
print SAVEOUT "\n";
print LOG  "\n";
};


print SAVEOUT "Please see the log file at $newerrdir.\n";
print LOG "Please see the log file at $newerrdir.\n";

if ($badfile_counter2>0 or $badMIFfile2_counter>0 or $flag_array_counter>0){
exit(2);
}else{
exit(0);
};

my $Presdirecory =cwd;
system "chmod 777 $Presdirecory";