Shell Programming and Scripting

 1 #!/bin/sh
  2 echo ======= LogManageri start ==========
  3
  4 #This directory is getting the raw data from remote server
  5 Raw_data=/opt/ftplogs
  6
  7 # This directory is ready for process the data
  8 Processing_dir=/opt/processing_dir
  9
 10 # This directory is prcoessed files and taking backup
 11 Processed_dir=/opt/processed_dir
 12
 13 # This directory spliting the files like access and error logs
 14 split_dir=/opt/split_dir
 15
 16 # Copying Raw data to Processing directory
 17 echo starting copying files from $Raw_Data to $Processing_dir
 18 cp -p $Raw_data/*.gz $Processing_dir/
 19 echo done copying raw files.
 20
 21 # Decompress .gz files from $Process_dir
 22 echo starting unziping files in $Process_dir
 23 gunzip $Processing_dir/*.gz
 24 echo done unziping files in $Process_dir
 25 #path=Processing_dir/*.log
 26 #name=$(basename "$Processing_dir/" .log)
 27 #echo $name
 28
 29 # This for loops gives year,month and day from file name
 30
 31 for file in $Processing_dir/*.log
 32 do
 33         name=$(basename "$file" .log)
 34         echo this $name
 35         year=${name:9:4}
 36         echo $year
 37         month=${name:13:2}
 38         echo $month
 39         day=${name:15:2}
 40         echo $day
 41
 42 # This while loop reading the file each and every line and spliting the Respective $cname and $code
 43 echo start reading files
 44 while read -r line
 45 do
 46         # cname is reading every line from all files
 47
 48         cname=$(echo ${line} | awk '{split($11,c,"/"); print c[3]}')
 49
 50         echo $cname
 51
 52         # scode is reading every line from all files
 53
 54         scode=$(echo ${line} | awk -F"[ ]" '{print $13}')
 55
 56         echo $scode
 57
 58         [[ ! -d "$split_dir/$cname/$year/$month/$day" ]] && mkdir -p "$split_dir/$cname/$year/$month/$day"
 59
 60         if [egrep -v '^(#|$)' $file]
 61         then
 62         echo ${line} >> $split_dir/$cname/$month/$day/_line_started_with#.log
 63         elif [ ${scode}-le399 ]
 64
 65         then
 66
 67         echo ${line} >> $split_dir/$cname/$year/$month/$day/${name}_${cname}_access.log
 68
 69         else
 70 #       [[ ( "${scode}"-ge"400" )]]
 71
 72         echo ${line} >> $split_dir/$cname/$year/$month/$day/${name}_${cname}_error.log
 73
 74         fi
 75
 76 done < $file
 77
 78         done