UNIX for Dummies Questions & Answers

#!/usr/bin/sh
#
cd /var/datastage/FRPDEVL/work/source/landing/dspf
for fname in pf_*.csv;do
#Check for files existence in the corresponding directory and perform validation
if [ -f "$fname" ]
then
echo "Expected file(s) found, Performing Validations for file: "$fname
filename=`basename $fname`
fdate=`echo $filename|tr -dc '[:digit:]'`
echo $filename","$fdate

  #Validation 1: Sum of all the columns from D to O (numeric data type) respectively should be equal
  #to the value present in trailer row against the respective column.
  if [ $filename = 'pf_'$fdate'.csv' ]
  then
  echo "------------------------------------------------------------------------------------"
  echo "Checking Specific Validations 2 for File: $filename"
  echo "------------------------------------------------------------------------------------"
  #Trim Header and Trailer record and create temporary file temp1_$fdate.tmp
  sed '1d;$d' $filename >temp1_$fdate.tmp
  
  #Trim the trailer record only from original file and create another temporary file temp_original_$fdate.tmp
  #which will be used for comparison after finding sum from D to O column
  tail -1 $filename|cut -d "," -f 4->temp_original_$fdate.tmp
  
  #Perform sum from column D to O on temporary file temp1_$fdate.tmp and create another temporary file temp_sum_$fdate.tmp
  awk -F, -v OFS="," -v OFMT="%.2E" '{s1+=$4;s2+=$5;s3+=$6;s4+=$7;s5+=$8;s6+=$9;s7+=$10;s8+=$11;s9+=$12;s10+=$13;s11+=$14;s12+=$15}END{print s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,s12}' temp1_$fdate.tmp>temp_sum_$fdate.tmp
  #awk -F, -v OFS="," '{s1+=$4;s2+=$5;s3+=$6;s4+=$7;s5+=$8;s6+=$9;s7+=$10;s8+=$11;s9+=$12;s10+=$13;s11+=$14;s12+=$15}END{print s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,s12}' temp1_$fdate.tmp>temp_sum_$fdate.tmp
  
  #Now compare the sum that is present in trailer record of original file with that of the sum taken from Column D to O,
  #If both the values match in the two files, then the matching record will be printed and a count will be taken which will be 
  #always one. If both the data does not match then the count will be 0
  val=`awk 'NR==FNR{a[$0];next}$0 in a{print $0}' temp_original_$fdate.tmp temp_sum_$fdate.tmp|wc -l`
  
  #If $val is =0, which means the sum is not matching with Trailer record sum, hence kill the job
  if [ "$val" -eq "0" ]
  then
  echo "The sum of either or all columns is not matching with last row sum value of corresponding column. Hence exiting the Job"
  
  #If the validation fails remove all the temporary files before exiting from further processing
  #rm -f temp1_$fdate.tmp
  #rm -f temp_original_$fdate.tmp
  #rm -f temp_sum_$fdate.tmp
  
  #Exit with code 16, If the sums are not matching
  exit 16  
  else
  echo "Sums are matching"  
  fi  
  echo "------------------------------------------------------------------------------------"
  echo "Specific Validations check for File: $filename completed"
  echo "------------------------------------------------------------------------------------"  
  fi
  
  #Remove all temp files if all the validations pass
  #rm -f temp_$fdate.tmp
  #rm -f temp1_$fdate.tmp
  #rm -f temp_original_$fdate.tmp
  #rm -f temp_sum_$fdate.tmp

#If files are not there in landing directory, will not perform validations and exit with normal status  
else
	echo "Expected files not found, Hence not performing any validations"
	exit 0
fi

#End of main For loop
done

awk 'NR>1 {for (i=4; i<=NF; i++) SUM[i]+=$i} END {for (i=4; i<=NF; i++) {SUM[i]-=2*$i; if (SUM[i]) printf "Column %c: %.0f\n", i+64, SUM[i]}}' FS="," /tmp/pf_20150127.csv 
Column D: -71415152

...
  val=`
awk '(NR==FNR){a[FNR]=$1+0;next} ($1+0!=a[FNR]){printf "col %.f: %.f != %.f\n",FNR,$1+0,a[FNR]}' RS=, temp_original_20150127.tmp temp_sum_20150127.tmp
`
  #If $val is not empty, the sum is not matching with Trailer record sum, hence kill the job
  if [ -n "$val" ]
  then
  echo "The sum of either or all columns is not matching with last row sum value of corresponding column. Hence exiting the Job"
  echo "$val"
...

Expected file(s) found, Performing Validations for file: pf_20150325.csv
pf_20150325.csv,20150325
------------------------------------------------------------------------------------
Checking Specific Validations 2 for File: pf_20150325.csv
------------------------------------------------------------------------------------
The sum of either or all columns is not matching with last row sum value of corresponding column. Hence exiting the Job
Errors: col 1: 173000000000 != 172928624441

cat temp_original_20150325.tmp

172928624441.00,334431290.00,346417133.00,354231936.00,443777494.00,526288959.00,769941370.00,918420217.00,1274200675.00,1067695005.00,1122762029.00,1181290201.00

cat temp_sum_20150325.tmp

1.73E+11,334431290,346417133,354231936,443777494,526288959,769941370,918420217,1274200675,1067695005,1122762029,1181290201