Shell Programming and Scripting

#!/bin/bash
set -x

FINALDIR=/data/media/LOGS/archive/_cdrc2

DATE=`date +%Y%m%d`
#echo "`date +'%Y/%m/%d %H:%M:%S'` Starting getting CDR's"

mkdir /data/media/LOGS/archive/logs2/$DATE/
/bin/chmod 750 /data/media/LOGS/archive/logs2/$DATE
/usr/bin/setfacl -m g:ITdep:rx /data/media/LOGS/archive/logs2/$DATE
scp httplogs@192.168.70.123:/data/backup/cdr/hotcdr/MEGAFLOW*$DATE* /data/media/LOGS/current/logs2/

cd /data/media/LOGS/current/logs2

for i in `ls *.gz`
do
        gunzip $i
        a=`echo $i | cut -d'.' -f1-2`
        awk -F '\t' '{ print $2"\t"$4"\t"$6"\t"$16"\t"$17"\t"$18"\t"$22"\t"$37"\t"$43"\t"$44 }' $a >> LOGS_Extracted_CDR_$DATE    #extract selected fields from the CDR and append them to new cdr file
        7za a -bd -mfb=255 $a.7z $a
        rm -f $a
        mv $a.7z /data/media/LOGS/archive/logs2/$DATE/
done

7za a -bd -mfb=150 LOGS_Extracted_CDR_$DATE.7z LOGS_Extracted_CDR_$DATE   #compress with 7za the new cdr file
rm LOGS_Extracted_CDR_$DATE -f
mv LOGS_Extracted_CDR_$DATE.7z $FINALDIR

process_file firstfile
process_file secondfile
process_file thirdfile
... etc.

num_jobs=0
num_CPUs=<enter number of cores here>

(list-files-to-process) | while read FILE ; do
     if [ $num_jobs -lt num_CPUs ] ; then
          process_file "$FILE" &
     fi
     num_jobs=$(jobs | wc -l)
     sleep 5
done

 
real 116m57.289s
user 198m15.878s
sys 5m46.687s