compare & split files


 
Thread Tools Search this Thread
Top Forums Shell Programming and Scripting compare & split files
# 8  
Old 08-12-2009
The last line of the awk code:

Code:
' file CC29072009_CELPCU01.dnr CC04082009_CELPCU04.dnr CC25072009_CELPCU07.dnr

should be, assuming the merged file is in the current directory (otherwise specify the full path):

Code:
' nid_lec_rej_20090804_merged /arbor/FX/data/remote/cpm/output/WORK_TEMP/toDINER/CC29072009_CELPCU01.dnr /arbor/FX/data/remote/cpm/output/WORK_TEMP/toDINER/CC04082009_CELPCU04.dnr /arbor/FX/data/remote/cpm/output/WORK_TEMP/toDINER/CC25072009_CELPCU07.dnr

# 9  
Old 08-12-2009
Thanks a lot for your inputs, its working absolutely fine.

And my mistake i didnt read file

Quote:
Code:
' file CC29072009_CELPCU01.dnr CC04082009_CELPCU04.dnr CC25072009_CELPCU07.dnr
Thanks & Regards Smilie
# 10  
Old 08-12-2009
Quote:
Originally Posted by ss_ss
Thanks a lot for your inputs, its working absolutely fine.

And my mistake i didnt read file

Thanks & Regards Smilie
Your welcome, glad to hear you get it work now!

Regards
# 11  
Old 08-12-2009
Troubling you again but i generalized it and after generalization the output files are getting created but only with the header record i.e. detail n trailer records are not coming Smilie

Code:
#! /usr/bin/ksh

program_name=$0
program_name=`echo $program_name | sed -e 's/.*\///'`

function usage
{
    echo
    echo $*

    cat << EOF

$program_name  [-options value]

Valid options:

              [-b Bank Name]      run for all banks  if not specified

EOF
exit 1
}

# get arguments from command line
while [ $# -gt 0 ]
do
    case $1 in
      -b)
           [ "$2" = "" ] && usage "no value for option $1"
           read_bank=$2
           shift 2
           ;;
       *)
           echo "no such option $1"
           usage
    esac
done

# Main CPM directories
cpm_base=/arbor/FX/data/remote/cpm
cpm_out=${cpm_base}/output
cpm_in=${cpm_base}/input
cpm_bak=${cpm_base}/BACKUP
cpm_work_in=${cpm_in}/WORK_TEMP
cpm_work_out=${cpm_out}/WORK_TEMP
ctrl_dir=${cpm_work_out}/CTRL
log=/SYSTEM/custom/data/log/CPM/cpm_merge_log.`date '+%Y%m%d%H%M%S'`
echo "logs created in $log"

# Function to check success
function check_status
{
 if [ $? -ne 0 ] ; then
    echo "Check directory permissions, files not able to copied or deleted; exiting main program ......... "
    exit 1
 fi
}

# Function to add leading zeroes to numbers
function leading_zeroes
{
  sum=$1
  ln=`echo $sum|awk '{print length}'`
  nr=$2
  zero=`expr $nr - $ln`
  i=1

  while [ $i -le $zero ]
   do
   sum="x${sum}"
   i=`expr $i + 1`
  done

  echo $sum |sed 's/x/0/g'
}

# Function to add trailing blanks to trailer
function trailing_blanks
{
  blank=$1
  i=1
  sum=""
    while [ $i -le $blank ]
   do
   sum="x${sum}"
   i=`expr $i + 1`
  done

  echo $sum |sed 's/x/ /g'
}

function split_files
{
  ch_name=$1
  exp=`echo $ch_name|tr 'A-Z' 'a-z' `
  #ctrl_file=${ctrl_dir}/ctrl_${ch_name}

  cd ${cpm_work_in}/fr${ch_name}
  file_id=`ls -trC1|grep -v .gz|awk /$exp[[:digit:]]{12}.*\.sd/|tail -1`

  cd ${cpm_work_out}/to${ch_name}
  if [ "$exp" = "cob" -o  "$exp" = "amx" ] ; then
     file_id3=`ls -trC1|awk /$exp[[:digit:]]{12}\.03\./|tail -1`
     file_id6=`ls -trC1|awk /$exp[[:digit:]]{12}\.06\./|tail -1`
     file_id9=`ls -trC1|awk /$exp[[:digit:]]{12}\.09\./|tail -1`
  else
     file_id3=`ls -trC1|awk /CC[[:digit:]]{8}_CELPCU01\..../|tail -1`
     echo $file_id3
     file_id6=`ls -trC1|awk /CC[[:digit:]]{8}_CELPCU04\..../|tail -1`
     echo $file_id6
     file_id9=`ls -trC1|awk /CC[[:digit:]]{8}_CELPCU07\..../|tail -1`
     echo $file_id9
  fi

ready_dir=${cpm_in}/fr${ch_name}/ready

awk -F" |_" 'NR==FNR && /^2/{a[substr($0,40,15)]=$0;next}
FILENAME=="${cpm_work_out}/to{$ch_name}/${file_id3}" && /^2/ && a[$3]{print a[$3] > "${ready_dir}/rnd.out1"}
FILENAME=="${cpm_work_out}/to{$ch_name}/${file_id6}" && /^2/ && a[$3]{print a[$3] > "${ready_dir}/rnd.out4"}
FILENAME=="${cpm_work_out}/to{$ch_name}/${file_id9}" && /^2/ && a[$3]{print a[$3] > "${ready_dir}/rnd.out7"}
' ${cpm_work_in}/fr${ch_name}/$file_id ${cpm_work_out}/to{$ch_name}/$file_id3 ${cpm_work_out}/to{$ch_name}/$file_id6 ${cpm_work_out}/to{$ch_name}/$file_id9

total_amnt_01=`awk '{a += (substr($1,10,12))}END{printf a}' ${ready_dir}/rnd.out1`
total_amnt_06=`awk '{a += (substr($1,10,12))}END{printf a}' ${ready_dir}/rnd.out4`
total_amnt_09=`awk '{a += (substr($1,10,12))}END{printf a}' ${ready_dir}/rnd.out7`

rec_cnt_01=`(awk 'END{print NR}' ${ready_dir}/rnd.out1)`
rec_cnt_04=`(awk 'END{print NR}' ${ready_dir}/rnd.out4)`
rec_cnt_07=`(awk 'END{print NR}' ${ready_dir}/rnd.out7)`

sed -n '2p' ${ctrl_dir}/ctrl_${ch_name} >>  ${ready_dir}/tmp.1
sed -n '4p' ${ctrl_dir}/ctrl_${ch_name} >>  ${ready_dir}/tmp.4
sed -n '6p' ${ctrl_dir}/ctrl_${ch_name} >>  ${ready_dir}/tmp.7

cat ${ready_dir}/tmp.1 ${ready_dir}/rnd.out1 >> ${ready_dir}/din_cel_rej_20090804_CU01
cat ${ready_dir}/tmp.4 ${ready_dir}/rnd.out4 >> ${ready_dir}/din_cel_rej_20090804_CU04
cat ${ready_dir}/tmp.7 ${ready_dir}/rnd.out7 >> ${ready_dir}/din_cel_rej_20090804_CU07

rm ${ready_dir}/tmp.1
rm ${ready_dir}/tmp.4
rm ${ready_dir}/tmp.7
rm ${ready_dir}/rnd.out1
rm ${ready_dir}/rnd.out4
rm ${ready_dir}/rnd.out7

count_3=`leading_zeroes $rec_cnt_01 5`
count_6=`leading_zeroes $rec_cnt_04 5`
count_9=`leading_zeroes $rec_cnt_07 5`

amount_3=`leading_zeroes $total_amnt_01 12`
amount_6=`leading_zeroes $total_amnt_01 12`
amount_9=`leading_zeroes $total_amnt_01 12`
tr=3
filler=`trailing_blanks 61`

echo "${tr}${count_3}${amount_3}${filler}" >>  ${ready_dir}/din_cel_rej_20090804_CU01
echo "${tr}${count_6}${amount_6}${filler}" >>  ${ready_dir}/din_cel_rej_20090804_CU04
echo "${tr}${count_9}${amount_9}${filler}" >>  ${ready_dir}/din_cel_rej_20090804_CU07
}

#Main Program start
{
case $read_bank in

         DINER|diner)
                 echo "Splitting files for DINER only..............."
                 split_files DINER
esac

if [ "$read_bank" = "" ] ; then
# ftp scripts to be added  here for all 4 banks
split_files DINER
fi
} >> $log

echo "logs created in $log "
exit 0

And unable to undersatnd this error:
Code:
awk: Input line ¨ cannot be longer than 3,000 bytes.

# 12  
Old 08-12-2009
Quote:
Originally Posted by ss_ss
And unable to undersatnd this error:
Code:
awk: Input line ¨ cannot be longer than 3,000 bytes.

That's the limit of most awk implementations, use (install) gawk or mawk.

Regards
# 13  
Old 08-12-2009
But the generalized version has lesser no of awk input lines than the specific ones as all the paths have been taken into variables Smilie

---------- Post updated at 04:40 AM ---------- Previous update was at 03:50 AM ----------

I again replaced the file name & path variables with another variable as shown below:
Code:
var=`${cpm_work_in}/fr${ch_name}/$file_id`
var1=`${cpm_work_out}/to{$ch_name}/$file_id3`
var4=`${cpm_work_out}/to{$ch_name}/$file_id6`
var7=`${cpm_work_out}/to{$ch_name}/$file_id9`

awk -F" |_" 'NR==FNR && /^2/{a[substr($0,40,15)]=$0;next}
FILENAME=="${cpm_work_out}/to${ch_name}/$file_id3" && /^2/ && a[$3]{a[$3] > "${ready_dir}/din.out1"}
FILENAME=="${cpm_work_out}/to${ch_name}/$file_id6" && /^2/ && a[$3]{a[$3] > "${ready_dir}/din.out4"}
FILENAME=="${cpm_work_out}/to${ch_name}/$file_id9" && /^2/ && a[$3]{a[$3] > "{ready_dir}/din.out7"}
' $var $var1 $var4 $var7

and it resolved that problem Smilie

Thanks & Regards
Login or Register to Ask a Question

Previous Thread | Next Thread

10 More Discussions You Might Find Interesting

1. Shell Programming and Scripting

How to Compare local & remote Files over ssh?

I want to make a script to compare list of files in terms of its size on local & remote server whose names are same & this is required over ssh. How can I accomplish this. Any help would be appreciated. (1 Reply)
Discussion started by: m_raheelahmed
1 Replies

2. Shell Programming and Scripting

Compare files & extract column awk

I have two tab delimited files as given below: File_1: PV16 E1 865 2814 1950 PV16 E2 2756 3853 1098 PV16 E4 3333 3620 288 PV16 E5 3850 4101 252 PV16 E6 83 559 477 PV16 E7 562 858 297 PV16 L2 4237 5658 ... (10 Replies)
Discussion started by: vaibhavvsk
10 Replies

3. Shell Programming and Scripting

Compare & subtract lines in files by column using awk.

I have two files with similar column pattern as given below : 2 sample lines from file1 are given below. 18 12630 . G T 49.97 . AC=2;AF=1.00;AN=2;DP=3;Dels=0.00;FS=0.000;HRun=0;HaplotypeScore=0.0000;MQ=60.00;MQ0=0;NDA=1;QD=16.66;SB=-0.01 GT:AD:DP:GQ:PL ... (2 Replies)
Discussion started by: vaibhavvsk
2 Replies

4. Shell Programming and Scripting

Format & Compare two huge CSV files

I have two csv files having 90K records each & each row has around 50 columns.Lets say the file names are FILE1 and FILE2. I have to compare both the files and generate a new file that has rows from FILE2 if it differs. FILE1 ----- 2001,"John",25,19901130,21211.41,Unix Forum... (3 Replies)
Discussion started by: Sheel
3 Replies

5. Shell Programming and Scripting

How to Read & Compare Two Files

Hi forumers, How is it going. Ok i need some advice on the following problem. I have 2 files to read and compare data.FileA and FileB. FileA will return either status 1 or 0. FileB on the other hand is trickier and has the following details:- Count DeviceID CurrentStatus ... (7 Replies)
Discussion started by: prakash1111
7 Replies

6. Shell Programming and Scripting

Compare two files A & B and accordingly modify file A

Friends, i have two huge complex files (for eg :A & B)as output , the sample contents of the files are as follows : A == ID,DATE,SUM1,SUM2,TOTAL(SUM1+2) A5066,20/04/2010,25000,50000,75000 A5049,20/04/2010,25000,60000,85000 B == ID,DATE,SUM1,SUM2,TOTAL(SUM1+2)... (2 Replies)
Discussion started by: appu2176
2 Replies

7. UNIX for Dummies Questions & Answers

How to compare 2 files & get specific value & replace it in other file.

Hiiii Friends I have 2 files with huge data. I want to compare this 2 files & if they hav same set of vales in specific rows & columns i need to get that value from one file & replace it in other. For example: I have few set data of both files here: a.dat: PDE-W 2009 12 16 5 29 11.11 ... (10 Replies)
Discussion started by: reva
10 Replies

8. Shell Programming and Scripting

PHP read large string & split in multidimensional arrays & assign fieldnames & write into MYSQL

Hi, I hope the title does not scare people to look into this thread but it describes roughly what I'm trying to do. I need a solution in PHP. I'm a programming beginner, so it might be that the approach to solve this, might be easier to solve with an other approach of someone else, so if you... (0 Replies)
Discussion started by: lowmaster
0 Replies

9. Shell Programming and Scripting

How to compare 2 files & get only few columns based on a condition related to both files?

Hiiiii friends I have 2 files which contains huge data & few lines of it are as shown below File1: b.dat(which has 21 columns) SSR 1976 8 12 13 10 44.00 39.0700 70.7800 7.0 0 0.00 0 2.78 0.00 0.00 0 0.00 2.78 0 NULL ISC 1976 8 12 22 32 37.39 36.2942 70.7338... (6 Replies)
Discussion started by: reva
6 Replies

10. Shell Programming and Scripting

How to search & compare paragraphs between two files

Hello Guys, Greetings to All. I am stuck in my work here today while trying to comapre paragraphs between two files, I need your help on urgent basis, without your inputs I can not proceed. Kindly find some time to answer my question, I'll be grateful to you for ever. My detailed issue is as... (10 Replies)
Discussion started by: NARESH1302
10 Replies
Login or Register to Ask a Question