Shell Programming and Scripting

#!/bin/sh

echo ======= LogManageri start ==========

#This directory is getting the raw data from remote server

Raw_data=/opt/ftplogs

# This directory is ready for process the data

Processing_dir=/opt/processing_dir

# This directory is prcoessed files and taking backup

Processed_dir=/opt/processed_dir

# This directory spliting the files like access and error logs

split_dir=/opt/split_dir

# Copying Raw data to Processing directory

echo starting copying files from $Raw_Data to $Processing_dir

cp -p $Raw_data/*.gz $Processing_dir/

echo done copying raw files.

# Decompress .gz files from $Process_dir

echo starting unziping files in $Process_dir

gunzip $Processing_dir/*.gz

echo done unziping files in $Process_dir

# This for loops gives year,month and day from file name

for file in $Processing_dir/*.log
do
        #Name of the input file

        name=$(basename "$file" .log)
        echo this $name

        #getting year from file name

        year=${name:9:4}

        echo $year

        #getting month from file name

        month=${name:13:2}

        echo $month

        #getting day from file name

        day=${name:15:2}

        echo $day

# This while loop reading the file each and every line and spliting the Respective $cname and $code

echo start reading files

while read -r line
do
        # cname is reading every line from all files

        cname=$(echo ${line} | awk '{split($11,c,"/"); print c[3]}')

        echo $cname

        # scode is reading every line from all files

        scode=$(echo ${line} | awk -F"[ ]" '{print $13}')

        echo $scode

        # creating directory if not exists

        [[ ! -d "$split_dir/$cname/$year/$month/$day" ]] && mkdir -p "$split_dir/$cname/$year/$month/$day"
        # this loop not processing starting with # line

        case $line in

        (\#*) continue

        esac

        # If condition scode less than 399 logs goes to access log
        if [ $scode -le 399 ]

        then

        echo ${line} >> $split_dir/$cname/$year/$month/$day/${name}_${cname}_access.log

        # If condition scode greater than 400 logs goes to error log

        else
#       [[ ( "${scode}"-ge"400" )]]

        echo ${line} >> $split_dir/$cname/$year/$month/$day/${name}_${cname}_error.log

        fi

done < $file

        done

echo files reading done

# after successfull splitting the the logs gzip .log file

echo starting zip logs

gzip $Processing_dir/*.log

echo compression done

# gzip file moves to $Processed directory

mv $Processing_dir/*.gz  $Processed_dir

echo moved file to $Processed_dir