S1()
{
stream=S1
path=/sadique/S1
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*.RTM | head -3`
	do

##Read 1st column as key of all the lines match the key with key inside the function and pass the 2nd column as argument during run time
#!/bin/sh
DATE=`date +"%Y-%m-%d %H:%M:%S"`
S1()
{
key=S1
counter=0
while [ $counter -le 100 ]
do
	for  files in `printf "%s\n" $path/*.RTM | head -3`
	do
		if [[ -f $files ]];
		then
		TT_FIRST=`awk -F ',' 'NR==1{print $1}' $files`
		TT_LAST=`awk -F ',' 'END{print $1}' $files`
		FILENAME=`ls $files | cut -d '/' -f6`
		TIMESTAMP=$( date -r $files +'%s')
		FILE_CR_TIME=$( date +'%Y-%m-%d %H:%M:%S' -r $files)
        TRANS_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +'%s')
		TRANS_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +'%s')
		RECORD_TIME_FIRST=$(date -d @$(printf '%.0f\n' $TT_FIRST) +"%Y-%m-%d %H:%M:%S")
		RECORD_TIME_LAST=$(date -d @$(printf '%.0f\n' $TT_LAST) +"%Y-%m-%d %H:%M:%S")
		TIME_LAG_FIRST=$(expr $TIMESTAMP - $TRANS_TIME_FIRST)
		TIME_LAG_LAST=$(expr $TIMESTAMP - $TRANS_TIME_LAST)
		DELAY_PROCESS=$((`expr $(date -u +"%s") - $TIMESTAMP`))
		TRANS_DIFFTIME=$(expr $TRANS_TIME_LAST - $TRANS_TIME_FIRST)
		counter=100
		else
			echo "file doesn't exist" >/dev/null
		fi	
		if [ "${FILENAME}" != "" ]; 
		then
			echo "${DATE} ${stream} ${FILENAME} ${FILE_CR_TIME} ${RECORD_TIME_FIRST} ${TIME_LAG_FIRST} ${RECORD_TIME_LAST} ${TIME_LAG_LAST} ${DELAY_PROCESS} ${TRANS_DIFFTIME}" | hdfs dfs -appendToFile - /bigpfstest/DPI_INVESTIG/AUDIT/FINAL.csv
		else
			echo "Filename is empty" >/dev/null 
		fi 
	done
((counter++))
sleep 5s
done
}


S1 $path &