#!/usr/bin/perl

use Getopt::Long;
use File::Copy;

use lib "/mt/apps/gpms/DateTrans/lib";
use DateTrans qw(monName monNumber timeFormat AmPmFormat dayNumbeFormat);

use lib "/mt/apps/gpms/DateTrans/lib";
use  xsdxsltcheck qw(xsdcheck xsltcheck FGDC2MRFcheck xsl4mrf2fgdccheck);


use File::Basename;

$correct=substr($0,2);



sub usage() {
    print "Usage: perl $correct [-listfile=<listfile>][-directory=<directory>][-product=<product>]\n";
    print "Example:\n";
    print "perl $correct -listfile=list.txt -directory=/dir/subdir -product=TLFE2007\n";
    print"where list.txt is the filename of the list file containg the names of the files to validated,\n";
       print "/dir/subdir is the directory containing the files to be validated\n";
       print "and TLFE2007 is the product.\n";
   print "Please also take note of the following shortcut for the directory variable: putting mtdata for the directory name\n";
  print "will yield the /mtdata/geo/gpms/input/mif directory.\n"; 
}
%gOptions;
&GetOptions(\%gOptions, qw(listfile=s directory=s  product=s));
unless ($gOptions{listfile} and $gOptions{directory} and $gOptions{product}) {
        usage();
        exit(1);
};
#---------------------------------------------file mangement section----------------------------------------------------------------------------------------
$Infile = $gOptions{listfile};
chomp($Branch = $gOptions{product});
chomp($indir= $gOptions{directory});

if ($indir eq "mtdata"){
$indir = "/mtdata/geo/gpms/input/mif";
}elsif ($indir eq "nfs"){
$indir ="/nfs/gpms/input/mif";
}elsif ($indir eq "input"){
$indir ="/home/mccre004/input_files";
};

$root="/mt/apps";
$rootout="/mtdata/geo";
#$indir="$rootout/gpms/input/mif";
$inlist="$rootout/gpms/input/list";
$outputdir ="$rootout/gpms/output/AFF";
$errlogdir ="$rootout/gpms/log";
$combolist = $inlist . "/" . $Infile;
$stylesheet ="$root/gpms/MIF2AFF/MIF2MRF.xsl";
$FGDCdir="$rootout/gpms/output/fgdc";
$MRFdir ="$rootout/gpms/output/mrf";

#------------------------------------------------------error trap--------------------------------------------------------------------------------------------

if (-e $indir){
print "Please Wait, Now processing the files ----------------------------------------------\n\n";
#print "for the $indir directory\n";
}else{
print "The directory $indir  could not be found! \n";
print "Please make sure the pathname, $indir, was spelled correctly and that it exists.\n";
exit(1);
};

if (-s $indir){
}else{
print "The directory $indir does not contain any files to be validated! \n";
exit(1);
};

if (-e $combolist){
#print "Please Wait, Now processing -----------------------------------------------\n";
}else{
print "The file $combolist could not be found! \n";
print "Please make sure the file, $Infile,  was spelled correctly and that the file is in the $inlist directory.\n";
exit(1);
}; 

#making sure the file list contains something
if (-s $combolist){
#comment
}else{
print "The file $combolist does not contain the list of files to be validated! \n";
print "Please make sure the file, $Infile,  contains a list of files to be validated.\n";
exit(1);
};

#------------------------------------getting the date---------------------------------------------------------------------------------------------------------
my $now=`date`;
$mon = substr($now,4,3);
$day=substr($now,8,2);
$year =substr($now,24,4);
#$time= substr($now,11,5);
$hour = substr($now,11,2);
$min = substr($now,14,2);
my $daynum= dayNumbeFormat($day);

#converting the date to a number
my $monnum = monName($mon);
my $monnum =monNumber($mon);
my $Month_Number = monNumber($mon);

$dateString =$year . $monnum . $daynum . $hour . $min;

$time = $hour . ":" . $min;
$oldhour = $hour;
my $newhour =timeFormat($hour);
my $ampm =AmPmFormat($oldhour);


$newtime = $newhour . ":" . $min . " $ampm";

#-----------------------------------------------------making the directories---------------------------------------------------------------------------
#$mrf ="MRF_$Branch";
$newdir=$Branch . $dateString;
system "mkdir $outputdir/$newdir";
$newerrdir ="$errlogdir/$newdir";
system "mkdir $newerrdir";
chmod 0777, "$outputdir/$newdir";
chmod 0777, "$newerrdir";

$FGDCResult = system "mkdir $FGDCdir/$newdir";
#print "\$FGDCResult = $FGDCResult\n";

chmod 0777, "$FGDCdir/$newdir";
$MRFreslut= system "mkdir $MRFdir/$newdir";

#print "\$MRFreslut = $MRFreslut\n";
if ($reslut >0 ){
print "can not make mrf directory\n";
};
chmod 0777, "$MRFdir/$newdir";

#---------------------------------------------------------------------Counter variable Management section------------------------------------------
$badfile_counter2=0;
$goodfile_counter=0;
$countere=0;
$flag_array_counter=0;
#----------------------------------------------------------Saxon-----------------------------------------------------------------------------------------
$ENV{'PATH'} = "/opt/saxon_lib/jdk1.5.0_11/bin:/mt/apps/gpms/output/mrf$newerrdir:$ENV{'PATH'}";#goodone
$cp = "CLASSPATH";#goodone
$ENV{$cp} = "/opt/saxon_lib/jdk1.5.0_11/lib/:/opt/saxon_lib/saxon9/saxon9sa.jar:/opt/saxon_lib/saxon9/saxon9-jdom.jar:/opt/saxon_lib/saxon9/:/opt/saxon_lib/saxon9/saxon9.jar:$ENV{$cp}";
#---------------------------------------redirecting standard output section------------------------------------------------------------------------------


#redirecting STDOUT because the Saxon program automatically sends all its output to STDOUT
# Saving a copy of STDOUT as SAVEOUT so that items can still be printed to the screen
open (SAVEOUT,">&STDOUT");

#-----------------------------------------Filename array creation section------------------------------------------------------------------------------
#open a filehamdle for the file containing the filenames
open INPUTFILE, "$combolist";
while(<INPUTFILE>){
$slashpos= rindex($_, "/");
# print "\$slashpos = $slashpos\n";
if ($slashpos < 0){
$flagpos=index($_,"xml");
$flagless=substr($_, 0,$flagpos+3);
$flag=substr($_,$flagpos+4);
$ppos=index($flag,"p");
$filename="$indir/$flagless";

if ($ppos>=0){
$filename[$first_counter]="$indir/$flagless";
$flagarray[$first_counter]="$flag";
#putting the posting flag in a hash array
$flaghash{$filename}="$flag";
$first_counter++;
} else{
$noflagarray[$flag_array_counter]=$_;
$flag_array_counter++;
};


}elsif($slashpos > 0){


my $dirnameA = dirname $_;

$slash_directory_name[$slash_directory_name_counter]=$dirnameA;
$slash_directory_name_counter++;

$infile=$_;
$dotloc=rindex($infile,".")+4;
$slashfilename = substr($infile,0,$dotloc);
$flagpos=index($infile,"xml");
$flag=substr($infile,$flagpos+4);
$flaglen =length($flag);
$ppos=index($flag,"p");
$flag2=substr($flag,0,1);

if ($ppos>=0){
$filename[$first_counter]=$slashfilename;
$flagarray[$first_counter]="$flag";
$flaghash{$slashfilename}="$flag";
$first_counter++;
} else{
$noflagarray[$flag_array_counter]=$slashfilename;
$flag_array_counter++;
};

};
};
#-------------------------------------------validating the incoming MIF File--------------------------------------------------------------------------------------
$valcounter=0;
$filecounter =1;

foreach $filename (@filename){
chomp($filename);
#setting up the variables necessary to make the the finalfilename variable, which is needed to make the MRF file
$slashpos= rindex($filename, "/") +1;
$newfilename = substr($filename,$slashpos);
$dotlocation = index($newfilename, ".");
$finalfilename=substr($newfilename, 0, $dotlocation);
#setting up the TMP and MRF filenames

#if (-e $TMP){
if ($valcounter >0){
close STDERR;
system "rm $TMP";
};

$MRF = $finalfilename . ".mrf";
$TMP = "TMP" . "-" . $newfilename;
$valcounter++;


open STDOUT,">$TMP";

#making the error file and redirecting the STDERR to the error file
#have a new $dotlocation variable because the $filename and $newfilename variables are different lengths
$errmain= substr($newfilename, 0, $dotlocation);
$errfile = $errmain .".err";

open STDERR, ">$errfile";

#starts schema aware transformation. This will cause the transformation to fail if the document is invalid
# the -vw option indicates that validation errors found when validating the result tree should be treated as warnings only.
#This option is available only with the Saxon-SA version of the command, com.saxonica.Transform.
printf SAVEOUT "$filecounter: Now validating $filename\n";
$filecounter++;
my @csax = "java com.saxonica.Transform -vw $filename $root/gpms/xsl/MIF2MRF.xsl";
my $mif = system(@csax);
#print SAVEOUT "\$mif = $mif\n";
#opening fielhandles so that *.xsd will be replaced by Repository.xsd
#basically replicating the sed command 
#close STDERR;
#close STDERR;
open MRFOUTPUT, ">$MRF";
open TMPFILE, "<$TMP";
#printf SAVEOUT "\$mif = $mif\n";

if ($mif > 0) {
print"error\n";
$badfile[$badfile_counter] = $errfile;
$badfile_counter++;
#copy(
unlink $MRF;
unlink $TMP;
}else{
unlink $errfile;
$goodfile[$goodfile_counter]= $MRF;


system "chmod 777 $MRF";
$goodflaghash{$MRF}=$flaghash{$filename};
$npos=index($flaghash{$filename},"n");
#printf SAVEOUT "\$npos = $npos\n";
if ($npos==0){
$nopassarray[$no_pass_counter]=$goodfile[$goodfile_counter];
$no_pass_counter++;
}elsif ($npos<0){
$passarray[$pass_counter]=$goodfile[$goodfile_counter];
$pass_counter++;
#printf SAVEOUT"\$pass_counter = $pass_counter\n";
#printf SAVEOUT $passarray[$pass_counter];
};



$goodfile_counter++;
chmod 0777, "$outputdir/$newdir/$MRF";
unlink $errfile;
 while(<TMPFILE>){
  if($_ =~ /xsd/){
   #this happens if the the line has xsd in it
   $xsdlocation =index($_, "/xsd");
   $originalstring= substr($_,0,$xsdlocation);
   $Repository = "xsd/Repository.xsd\">\n";
   printf MRFOUTPUT ("$originalstring/$Repository");
  }elsif ($_ =~ /<\/MRF>/){
   printf  MRFOUTPUT ("$_");
   printf  MRFOUTPUT "\n";
   }elsif($_=~ /%/){
        chomp($_);
        $perloc=index($_, "%");
        $firsthalf =substr($_,0,$perloc);
        $secondhalf =substr($_, $perloc+1);
        $perloc2=index($secondhalf, "%");
                if ($perloc2 < 0){
                        printf  MRFOUTPUT"$firsthalf" . "%%" . "$secondhalf";
                }elsif($perloc2 >0){
                        $thirdhalf =substr($secondhalf,0,$perloc2);
                        $fourthhalf =substr($secondhalf, $perloc2+1);
                        $newstring = "$firsthalf" . "%%" . "$thirdhalf" . "%%" . "$fourthhalf\n";
if ($newstring=~/&#147;/){
$amploc2 =index($newstring, "&#148;");
                           $amp1string = substr($newstring,0,$amploc1);
                           $midstringlength= $amploc2 - $amploc1-6;
                           $amp2string = substr($newstring,$amploc1+6,$midstringlength);
                           $amp3string = substr($newstring,$amploc2+6);
                           printf  MRFOUTPUT $amp1string . "\"" . $amp2string . "\"" . $amp3string;
                         }else{
                        printf  MRFOUTPUT"$firsthalf" . "%%" . "$thirdhalf" . "%%" . "$fourthhalf\n";
                         };;#if ($newstring=~/&#147;/){
                         };#if ($perloc2 <  0){
 }elsif($_=~/&#147;/){
$amploc1=index($_, "&#147;");
 $amploc2 =index($_, "&#148;");
 $amp1string = substr($_,0,$amploc1);
 $midstringlength= $amploc2 - $amploc1-6;
 $amp2string = substr($_,$amploc1+6,$midstringlength);
 $amp3string = substr($_,$amploc2+6);
 printf  MRFOUTPUT $amp1string . "\"" . $amp2string . "\"" . $amp3string;

}elsif($_=~/&#146;/){
$aposloc1 = index($_,"&#146;");
$aposstring1 = substr($_,0,$aposloc1);
$aposstring2 = substr($_,$aposloc1+6);
#print SAVEOUT "\$aposstring1 = $aposstring1\n";
#print SAVEOUT "\$aposstring2 = $aposstring2\n";
#printf SAVEOUT $aposstring1 . "'" . $aposstring2;
printf  MRFOUTPUT $aposstring1 . "'" . $aposstring2;


  }else{
   printf  MRFOUTPUT "$_";
  };#if xsd
  };#while<TMPFILE>
close MRFOUTPUT;
close TMPFILE;
#close STDERR;
system "rm $TMP";
};#if mif>0
unlink $TMP
};#foreach

#unlink $TMP;
printf SAVEOUT "\n";
#---------------------
#-------------------------------error handling for invalid MIF files-------------------------------------------------------------------------------------------------
#print SAVEOUT "-----------------------------error diagnonsis----------------------------\n\n";
foreach $badMif (@badMif){
print SAVEOUT "\$badMif = $badMif\n";
open BADFILEH, "$badMif";
$bad="no";
$slashpos= rindex($badMif, "/")+1;
$wheredot2 = index($badMif, ".");

$AFF_Main_name = substr($badMif,$slashpos,$wheredot2);
$name = "$AFF_Main_name.xml";
$bad ="no";
$sizebadMif = -s $badMif;
print "\$sizebadMif = $sizebadMif\n";
if (-s $badMif == 0){
#rint SAVEOUT "\$badMif = $badMif\n";
print SAVEOUT "In the size = 0 section.(Invalid MIF)\n";
$remove=shift(@badMif);
$not_exist[$countere]= "$name";
print SAVEOUT"$countere non-existant files\n";
#rintf  MRFOUTPUT "Here!";
#print SAVEOUT "$name removed";
$countere++;
$bad ="yes";
}else{

print SAVEOUT "\nIn the size > 0 Section(MIF Error)\n\n"; 

while(<BADFILEH>){
#print SAVEOUT "reading $badMif\n";
if ($_=~ /does not exist/){
$not_exist[$countere]= "$name";
printf  MRFOUTPUT "not found!\n";
print SAVEOUT "not found!\n";
$countere++;
$bad ="yes";
#file not existing not being reported to the error file
}elsif (-s $badMif == 0 and $badMif2[$badMif_counter2-1] ne $name and $bad ne "yes"){
$not_exist[$countere]= "$name";
#printf  MRFOUTPUT "Here!1";



print SAVEOUT "Here!(-s $badMif == 0)";
unlink "$badMif";

$countere++;
$bad ="yes";
printf SAVEOUT "Emptyfile";
}elsif($badMif2[$badMif_counter2-1] ne $name and $bad ne "yes"){
#printf  MRFOUTPUT "Here!2";
print SAVEOUT "Here!(-s ne \name)";
$name = $badMif;
$badMif2[$badMif_counter2] = "$name";
$badMif_counter2++;
$bad ="no";
system "cp $badMif $newerrdir/$name";

unlink $badMif
};#if
}#s if
};#while
close BADFILEH
};#foreach
print SAVEOUT "\n\n";


#------------------------------------------Transforming MIF to AFF-----------------------------------------------------------------------------------------------------
 print SAVEOUT "Now transforming the MIF files into AFF files ---------------------------------------------------------------------\n";
$valcounter=0;
$filecounter =1;

foreach $goodfile (@goodfile){
chomp($goodfile);
#system "chmod 777 $goodfile";
#setting up the variables necessary to make the the finalfilename variable, which is needed to make the MRF file
 $slashpos= rindex($goodfile, "/") +1;
 $newfilename = substr($goodfile,$slashpos);
$dotlocation = rindex($newfilename, ".");
$finalfilename=substr($newfilename, 0, $dotlocation);
#setting up the TMP and MRF filenames

#if (-e $TMP){
if ($valcounter >0){
close STDERR;
system "rm $AFFOutFile";
};

$AFFOutFile="AFF" . "-" . $finalfilename . ".xml" ;

open STDOUT,">$AFFOutFile";

#making the error file and redirecting the STDERR to the error file
#have a new $dotlocation variable because the $goodmif and $newfilename variables are different lengths
$errmain= substr($newfilename, 0, $dotlocation);
$errfile ="AFF" . $errmain . ".err";

open STDERR, ">$errfile";

#starts schema aware transformation. This will cause the transformation to fail if the document is invalid
# the -vw option indicates that validation errors found when validating the result tree should be treated as warnings only.
#This option is available only with the Saxon-SA version of the command, com.saxonica.Transform.
printf SAVEOUT "$filecounter: Now transforming $goodfile from the MIF format to the American Fact Finder Format(AFF)\n";
$filecounter++; 
my @csax = "java com.saxonica.Transform -vw $goodfile $root/gpms/MIF2AFF/MIF2AFF.xsl";
my $affResult  = system(@csax);
#opening fielhandles so that *.xsd will be replaced by Repository.xsd
#basically replicating the sed command 
close STDERR;

#print SAVEOUT "\$affResult = $affResult\n";
if ($affResult > 0) {
$badfile[$badfile_counter] = $errfile;
$badfile_counter++;
print SAVEOUT "bad AFF to Mif Transformation:  $errfile\n"; 
#copy(
unlink $AFFOutFile;
if (-e $TMP){
unlink $TMP;
print SAVEOUT "removing $TMP-----------------------\n\n";
};

}else{
unlink $errfile;
$goodAFFfile[$goodAFFfile_counter]= $AFFOutFile;
#print SAVEOUT "\$goodAFFfile[$goodAFFfile_counter]= $goodAFFfile[$goodAFFfile_counter]\n";
$goodAFFfile_counter++;
#determine if the files are to be posted. If the position of n, as indicated by $npos is -1, then the file is not to be posted
$goodflaghash{$AFFOutFile}=$flaghash{$goodfile};
#print SAVEOUT "\$goodflaghash{$AFFOutFile} = $goodflaghash{$AFFOutFile}\n";
$npos=index($flaghash{$goodfile},"n");

$xmlDotPos=rindex($AFFOutFile, ".");
$CleanAFFOutFileParta= substr($AFFOutFile,0,$xmlDotPos);
$CleanAFFOutFile = $CleanAFFOutFileParta . "_Good.xml";
#print SAVEOUT "\$CleanAFFOutFile = $CleanAFFOutFile\n";

open CLEANAFF, "> $CleanAFFOutFile";
open INAFF, "<$AFFOutFile";

while (<INAFF>){
#rint SAVEOUT "now reading $AFFOutFile\n ";
if ($_ =~ /Publication_Date/ ){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$PubyearLength=$leftBracPos-$rightBracPos;
$Pubyear= substr($_,$rightBracPos,$PubyearLength);
print CLEANAFF "$Pubyear";
}elsif($_ =~ /Metadata_File_Identifier/){
#print SAVEOUT "\$_ = $_\n";
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$File_Id_Length=$leftBracPos-$rightBracPos;
$File_Id=substr($_,$rightBracPos,$File_Id_Length);
print CLEANAFF "$File_Id";
}elsif($_ =~ /<document/){
$HyPos =index($_, "-");


#using the position of the - to differentiate the document and document-type-ref elements
if ($HyPos >12){
$xmlns="xmlns=\"http://factfinder.census.gov/2009/metadata\"";
$xmlSchema="xsi:schemaLocation=\"http://factfinder.census.gov/2009/metadata\"";
print CLEANAFF $_  . $xmlns . "\n" . $xmlSchema
};

}elsif($_ =~ /Title/){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$Title_Length=$leftBracPos-$rightBracPos;

$Title=substr($_,$rightBracPos,$Title_Length);
print CLEANAFF "$Title";
}elsif($_ =~ /Abstract/){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$Abstract_Length=$leftBracPos-$rightBracPos;
$Abstract=substr($_,$rightBracPos,$Abstract_Length);
print CLEANAFF "$Abstract";
}elsif($_ =~ /Ordering_Instructions/){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$Order_Length=$leftBracPos-$rightBracPos;
$Order=substr($_,$rightBracPos,$Order_Length);
print CLEANAFF "$Order";
}elsif($_ =~ /Network_Resource_Name/){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$Net_Length=$leftBracPos-$rightBracPos;
$Net=substr($_,$rightBracPos,$Net_Length);
print CLEANAFF "$Net";
}elsif($_ =~ /Metadata_Date/){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$Met_Date_Length=$leftBracPos-$rightBracPos;
$Met_Date=substr($_,$rightBracPos,$Met_Date_Length);
print CLEANAFF "$Met_Date";
}elsif($_ =~ /Theme_Keyword/){
}elsif($_ =~ /\/Theme_Keyword/){
}elsif($_ =~ /Transfer_Size/){
$rightBracPos=index($_,">")+1;
$leftBracPos=rindex($_,"<");
$TransSizeLength=$leftBracPos-$rightBracPos;
$Trans=substr($_,$rightBracPos,$TransSizeLength);
print CLEANAFF "$Trans";
}else{
print CLEANAFF "$_";
};
};

#print SAVEOUT "\$npos = $npos\n";
if ($npos==0){
print SAVEOUT  "-----------------------------------------------------------------------\n";
$nopassAFFarray[$no_passAFF_counter]=$CleanAFFOutFile;

#print SAVEOUT "\$nopassAFFarray[$no_passAFF_counter] = $nopassAFFarray[$no_passAFF_counter]\n";
$no_passAFF_counter++;
#print SAVEOUT "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n";
}elsif ($npos<0){
#print SAVEOUT "npos < 0 : \$AFFOutFile = $AFFOutFile\n";
$passAFFarray[$passAFF_counter]=$CleanAFFOutFile;
#$passAFF_counter++;
#printf SAVEOUT"\$passAFF_counter = $passAFF_counter\n";
#printf SAVEOUT "\$passAFFarray[$passAFF_counter]=$passAFFarray[$passAFF_counter]\n" ;
$passAFF_counter++;
system "cp $CleanAFFOutFile  AFFfiles";
};#if ($npos==0){
};#if ($affResult > 0) {
};#foreach



printf SAVEOUT "\n";


#------------------------------error filehandling section for bad AFF files--------------------------------------------------------------------------------------------

foreach $badfile (@badfile){
#print SAVEOUT "in the badfile loop: \$badfile = $badfile\n";
open BADFILEH, "$badfile";
$bad="no";
$slashpos= rindex($badfile, "/")+1;
$wheredot2 = index($badfile, ".");
$AFF_Main_name = substr($badfile,$slashpos,$wheredot2);
$name = "$AFF_Main_name.xml";
#print SAVEOUT "\$name = $name\n";
$bad ="no";
if (-s $badfile == 0){
#print SAVEOUT "In the size = 0 section(bad AFF).\n";
#$not_exist[$countere]= "$name";
$not_exist[$countere]="$name";

#removing the element in the array so that the file does not show up twice: as a bad mif and missing file
#$remove=shift (@badfile);
#print SAVEOUT "removing $badfile\n";
#printf  MRFOUTPUT "Here!";
#unlink "$badfile";
$countere++;
$bad ="yes";

}else{
#print SAVEOUT "\nIn the size > 0 Section\n\n"; 

while(<BADFILEH>){
if ($_=~ /does not exist/){
$not_exist[$countere]= "$name";
#printf  MRFOUTPUT "not found!\n";
#unlink "$badfile";
$countere++;
$bad ="yes";
#file not existing not being reported to the error file
}elsif (-s $badfile == 0 and $badfile2[$badfile_counter2-1] ne $name and $bad ne "yes"){
$not_exist[$countere]= "$name";
printf  MRFOUTPUT "Here!";
unlink "$badfile";
$countere++;
$bad ="yes";
printf SAVEOUT "Emptyfile";
}elsif($badfile2[$badfile_counter2-1] ne $name and $bad ne "yes"){
#print SAVEOUT "\nIn the ne=0  Section\n\n";

$name = $badfile;
$badfile2[$badfile_counter2] = "$name";
$badfile_counter2++;
$bad ="no";
#print SAVEOUT "Error file being copied!\n";
system "cp $badfile $newerrdir/$name";
unlink $badfile
};#if
}#s if
};#while
close BADFILEH
};#foreach

#--------------------------------------final variables counting section--------------------------------------------------------------------------------
$total = $badfile_counter2 + $pass_counter+$no_pass_counter ;
$newtotal=$total+$countere+$flag_array_counter;
$pergood = (($pass_counter )/$total) *100;
$perbad = (($badfile_counter2)/$total) *100;
#-------------------------------working with the log file----------------------------------------------------------------------------------------------
#This is the introductary paragraph.
open LOG, ">$errlogdir/$newdir/$newdir.log";
printf LOG "This log details the transformation and validation results of the $newtotal mif files contained in \n the  $combolist list file.\n";

if ($slash_directory_name_counter >0){
printf LOG "\nThese files can be found in the following directories:\n";
printf LOG "1. $indir (This is the directory given at the command line)\n";
$slash_directory_nameb =2;
@slash_directory_name=sort(@slash_directory_name);

foreach $slash_directory_name (@slash_directory_name){

$slash_directory_nameca =$slash_directory_namec-1;

if ($slash_directory_name[$slash_directory_namec] ne $slash_directory_name[$slash_directory_nameca] and $slash_directory_name[$slash_directory_namec] ne $indir){
printf LOG "$slash_directory_nameb: $slash_directory_name\n";
$slash_directory_nameb++;
$slash_directory_namec++;
}elsif($slash_directory_namec == 0){
printf LOG "$slash_directory_nameb: $slash_directory_name\n";
$slash_directory_nameb++;
$slash_directory_namec++;
};#if ($slash_directory_name[$slash_directory_namec] ne $slash_directory_name[$slash_directory_nameca] and $slash_directory_name[$slash_directory_namec] ne $indir){
};#foreach $slash_directory_name (@slash_directory_name){

printf LOG "\n$0 was run on $monnum/$daynum/$year at $time ($newtime) for the $Branch product.\n\n";
}else{
printf LOG "\nThese files can be found in the $indir directory.\n";
printf LOG "It was run on $monnum/$daynum/$year at $time ($newtime) for the $Branch product.\n\n";

};#if ($slash_directory_name_counter >0){

#--------------------------indicator section-----------------------------------------------------------------------------------------------------------
if ($countere == 1) {
	printf SAVEOUT "The following file could not be found!\n";
	printf LOG "The following file could not be found!\n";
		foreach $not_exist (@not_exist){

			printf SAVEOUT "$countere: $not_exist\n";
			printf LOG "$countere: $not_exist\n"; 
		};
	printf SAVEOUT"\n";
	printf LOG "\n";		
}elsif ($countere >1) {
	printf SAVEOUT "The following files could not be found!\n";
	printf LOG "The following files could not be found!\n";
	foreach $not_exist (@not_exist){
		$notexistcounter++;
		printf SAVEOUT "$notexistcounter: $not_exist\n";
		printf LOG "$notexistcounter: $not_exist\n";
	};
	printf SAVEOUT"\n";
	printf LOG"\n";
};

$counter=1;

if ($flag_array_counter ==1){
	$flagfile="FailedFiles";
	#printf SAVEOUT "\$flagfile = $flagfile";
	open FLAGLIST, ">$errlogdir/$newdir/$flagfile";
        #to the screen
	printf SAVEOUT "The following file does not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post) after the filename. A list containg this file can be found in $flagfile in the \n";

	printf SAVEOUT "$errlogdir/$newdir directory.\n";
	#to the logfile
	printf LOG  "The following file does not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
after the filename.\n";
	printf LOG  "A file called $flagfile has a list with this file in it for you to flag.\n";	
	foreach $noflagarray (@noflagarray){
		printf SAVEOUT "$counter: $noflagarray\n";
		printf LOG "$counter: $noflagarray\n"; 
		printf FLAGLIST "$noflagarray";
	};
close FLAGLIST;	
}elsif ($flag_array_counter >1) {
	$flagfile="FailedFiles";
	 #printf SAVEOUT "\$flagfile = $flagfile\n";
	 open FLAGLIST, ">$errlogdir/$newdir/$flagfile";
#	#to the screen
	 printf SAVEOUT "The following files do not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
#after the filename. A list of these files can be found in $flagfile in the \n";
	 printf SAVEOUT "$errlogdir/$newdir directory.\n";
	 #to the logfile
	 printf LOG  "The following files do not have a FGDC posting flag and therefore can not be processed. Please put a space and a p (for post) or np (for not post)
after the filename. A list of these files can be found in $flagfile in the $errlogdir/$newdir directory.\n";
	foreach $noflagarray (@noflagarray){ 
			printf SAVEOUT "$counter: $noflagarray";
			printf LOG "$counter: $noflagarray";
			printf FLAGLIST "$noflagarray";
			$counter++;
		};
	close FLAGLIST;
	printf SAVEOUT"\n";
	printf LOG"\n";
};


if ($badMif_counter >0 ){
$badMif_counter_counter=1;
print  SAVEOUT "$badMif_counter files are invalid MIF files. These files WERE NOT TRANSFORMED into AFF files. Please see the error files at $errlogdir/$newdir\n";
print LOG "$badMif_counter files are invalid MIF files. These files WERE NOT TRANSFORMED into AFF files. Please see the error files at $errlogdir/$newdir\n";
foreach $badMif (@badMif){ 
print  SAVEOUT "$badMif_counter_counter: $badMif\n";
print LOG "$badMif_counter_counter: $badMif\n";
$badMif_counter_counter++;
};
print SAVEOUT"\n";
print LOG "\n";
};


if ($badfile_counter2 >0){
$badfile2_Post_counter=1;
print  SAVEOUT "$badfile_counter2 files have errors in them that occured in the transformation from the MIF format to the AFF format\n";
print  SAVEOUT "Please see the error files at  $errlogdir/$newdir\n";
print LOG "$badfile_counter2 files have errors in them that occured in the transformation from the MIF format to the AFF format\n";
print LOG "Please see the error files at $errlogdir/$newdir\n";
 
foreach $badfile2 (@badfile2){
print SAVEOUT "$badfile2_Post_counter: $badfile2\n";
print LOG "$badfile2_Post_counter: $badfile2\n";
$badfile2_Post_counter++;

};
print  SAVEOUT "\n";
print LOG "\n";
};

if ($no_pass_counter >0){
$AFFNoPostCounter=1;
print  SAVEOUT "$no_pass_counter MIF files that WERE NOT POSTED to the GOS server were transformed into AFF files.  \n";
print  SAVEOUT "Please see them at $outputdir/$newdir\n";
print  SAVEOUT "Also, see the MRF files ate $MRFdir/$newdir.\n";
print LOG "$no_pass_counter MIF files that WERE NOT POSTED to the GOS server were transformed into AFF files.  \n";
print LOG "Please see them at $outputdir/$newdir\n";
print LOG "Also, see the MRF files ate $MRFdir/$newdir. \n";

foreach $nopassarray(@nopassarray){
print SAVEOUT "$AFFNoPostCounter: $nopassarray\n";
print LOG  "$AFFNoPostCounter: $nopassarray\n"; 
$AFFNoPostCounter++;
$dotLoc=index($nopassarray,".");
$preGood=substr($nopassarray,0,$dotLoc);
$goodfile=$preGood ."_Good.xml";
#$nopassAFFarray[$no_passAFF_counter]

#$dotLoc=index($nopassAFFarray[$AFFGoodCounter],".");
#$preGood=substr($nopassAFFarray[$AFFGoodCounter],0,$dotLoc);
#$goodfile=$preGood . ".xml"; ; # ."_Good.xml";
$nopassDotLoc=index($nopassarray,0,".")-3;#was 2
$dotLessnopassarray=substr($nopassarray, 0,$nopassDotLoc);

#print SAVEOUT "---------------------------------------\n";
$goodfile= "AFF-"  . $dotLessnopassarray . "_Good.xml";
#print SAVEOUT "\$goodfile = $goodfile\n";
$nonGoodloc = index($goodfile, "_Good.xml");#was -1
$nonGood= substr($goodfile,0,$nonGoodloc);
$nonGoodFinal = $nonGood . ".xml"; 
#print SAVEOUT "\$nonGoodFinal = $nonGoodFinal\n";
#print SAVEOUT "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n";

$AFFGoodCounter++;
system "cp $goodfile $outputdir/$newdir/$nonGoodFinal";
system "cp $nopassarray  $MRFdir/$newdir";

#system "cp $goodfile $nopassarray";
#system "cp $nopassarray $MRFdir/$newdir";

#ystem "cp $nopassarray  $outputdir/$newdir";
unlink $nopassarray;
system "chmod 777 $goodfile";
};
print SAVEOUT"\n";
print LOG "\n";
};

if( $pass_counter>0){

print  SAVEOUT "$pass_counter MIF files that WERE POSTED to the GOS server were transformed into AFF files.  \n";
print  SAVEOUT "Please see them at $outputdir/$newdir.\n";
print  SAVEOUT "Also, see the MRF files ate $MRFdir/$newdir and the FGDC files at $FGDCdir/$newdir.\n"; 
print LOG "$pass_counter MIF files that WERE POSTED to the GOS server  were transformed into AFF files.  \n";
print LOG  "Please see them at $outputdir/$newdir\n";
print LOG "Also, see the MRF files ate $MRFdir/$newdir and the FGDC files at $FGDCdir/$newdir.\n";
$AFFGoodCounterA=1;
$AFFGoodCounter=0;

foreach $passarray (@passarray){
print SAVEOUT "$AFFGoodCounterA: $passarray\n";
print LOG "$AFFGoodCounterA: $passarray\n";

#handling the AFF files
$dotLoc=index($passAFFarray[$AFFGoodCounter],".");
$preGood=substr($passAFFarray[$AFFGoodCounter],0,$dotLoc);
$goodfile=$preGood . ".xml"; ; # ."_Good.xml";
$AFFGoodCounterA++;
$nonGoodloc = index($passAFFarray[$AFFGoodCounter], "_Good.xml");
$nonGood= substr($passAFFarray[$AFFGoodCounter],0,$nonGoodloc);
$nonGoodFinal = $nonGood . ".xml";
system "cp $passarray $outputdir/$newdir/$nonGoodFinal";
system "chmod 777 $outputdir/$newdir/$nonGoodFinal"; 
$AFFGoodCounter++;

#working with the FGDC file
$fgdcdotloc = index($passarray, ".");
$mainpart=substr($passarray,0,$fgdcdotloc);
$FGDCname = "$mainpart.xml";

#print SAVEOUT "---------------------------------------\n";
#print SAVEOUT "\$passarray = $passarray\n";
#print SAVEOUT "\$mainpart = $mainpart\n";
#print SAVEOUT "\$FGDCname = $FGDCname\n";
#print SAVEOUT "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n";

open STDOUT,">$FGDCdir/$newdir/$FGDCname";

my @csax = "java com.saxonica.Transform -vw $passarray $root/gpms/xsl/xsl4mrf2fgdc/MRF2FGDC.xsl";
my $mif = system(@csax);
#print SAVEOUT "\$FGDCname =$FGDCname\n";
#print SAVEOUT "\$mif = $mif  ($FGDCdir\/$newdir\/$FGDCname)\n";
system "chmod 777 $FGDCdir/$newdir/$FGDCname";

#handling the MRF files
#system "chmod 777 $passarray";
system "cp $passarray  $MRFdir/$newdir";
#system "chmod 777 $MRFdir/$newdir";
system "chmod 777 $goodfile";
unlink $passarray;


};
print SAVEOUT "\n";
print LOG  "\n";
};


system "rm AFF-*";
#system "rm TMP-*";

print SAVEOUT "Please see the log file at $newerrdir.\n";
print LOG "Please see the log file at $newerrdir.\n";

if ($badfile_counter2>0 or $flag_array_counter>0){
exit(2);
}else{
exit(0);
};