Emergency UNIX and Linux Support

awk: Internal software error in the tostring function on TS1101?05044400?.0085498227?0?.0011041461?.0034752266?.00397045?0?0?0?0?0?0?11/02/10?09/23/10???10?no??0??no?sct_det3_10_20110516_143936.txt

awk: Internal software error in the tostring function on TS1101?05044400?.0085498227?0?.0011041461?.0034752266?.00397045?0?0?0?0?0?0?11/02/10?09/23/10???10?no??0??no?sct_det3_10_20110516_143936.txt

./rem_dups.sh '1,2,3' . in.txt . out.txt dups_archive 009 4

Unix parameter $1: Value: '1,2,3'  Description: This is a comma-separated list of column positions in the input file that together make up the key used
to define a unique record
Unix parameter $2: Value: . (for same directory, it could be any directory) Description: This is the path to the input file.
Unix parameter $3: Value: in.txt (you can give it whatever name)  Description: This is the filename of the input file.
Unix parameter $4: Value: . (for same directory, it could be any directory) Description: This is the path to the output file with duplicates removed.
Unix parameter $5: Value: out.txt (you can give it whatever name)  Description: This is the filename of the output file with duplicates removed.
Unix parameter $6: Value: dups_archive (you can give it whatever name)  Description: This is the filename of the archive file containing the duplicate records.
Unix parameter $7: Value: 009 (this is the ASCII code for the tab character) Description: this is the ASCII code for the delimiter in the input file. 
Unix parameter $8: Value: 4 Description: this is the column position in the input file of the site-datetime column used to determine which record to keep
and which record to move to the duplicates archive file. It is also used to add the site to the key value to make the key complete in order to define a unique record.

31erescca    010240    10    sct_det3_10_20110516_143947.txt
11erescca    010240    10    sct_det3_10_20110516_143936.txt
31erescca    010240    10    sct_det3_10_20110516_143947.txt
21erescca    010240    10    sct_det3_10_20110516_143937.txt
31erescca    010240    10    sct_det3_10_20110516_233947.txt
11erescca    010240    10    sct_det3_10_20110516_143936.txt

11erescca    010240    10    sct_det3_10_20110516_143936.txt
11erescca    010240    10    sct_det3_10_20110516_143936.txt
21erescca    010240    10    sct_det3_10_20110516_143937.txt
31erescca    010240    10    sct_det3_10_20110516_143947.txt
31erescca    010240    10    sct_det3_10_20110516_143947.txt
31erescca    010240    10    sct_det3_10_20110516_233947.txt

11erescca    010240    10    sct_det3_10_20110516_143936.txt
31erescca    010240    10    sct_det3_10_20110516_143947.txt
31erescca    010240    10    sct_det3_10_20110516_143947.txt

11erescca    010240    10    sct_det3_10_20110516_143936.txt
21erescca    010240    10    sct_det3_10_20110516_143937.txt
31erescca    010240    10    sct_det3_10_20110516_233947.txt

#!/usr/bin/sh

LANG=""
LC_CTYPE="C"
LC_COLLATE="C"
LC_MONETARY="C"
LC_NUMERIC="C"
LC_TIME="C"
LC_MESSAGES="C"
LC_ALL=""

export LANG
export LC_CTYPE
export LC_COLLATE
export LC_MONETARY
export LC_NUMERIC
export LC_TIME
export LC_MESSAGES
export LC_ALL

locale

pos="$1"
infile="$2/$3"
outfile="$4/$5"
temp_dups_file="$2/$6"
temp_sort_file="$2/temp_sort_inputfile_$5"
flagfile="$2/duplicates_flagfile_$5"
delimiter="$7"
site_datetime="$8"

awk -v key_cols="$pos" -v delim="$delimiter" -v site_dt="$site_datetime" '
 
BEGIN {
    FS=sprintf("%c", delim);
    numkeys=split(key_cols,k,",");
} 

{ 

n=split($(site_dt), z, "_")
datetime=z[n-1] substr(z[n], 1 , 6) 
site=z[n-2];

for (x = 1; x <= numkeys; x++)
{    
    key=key $(k[x]) "-";
}

keysite=key site;
key="";

if(datetime > m[keysite]) 
{
    m[keysite] = datetime; out[keysite] = $0;
}

next }

END { for (keysite in out) print out[keysite] }' <$infile |sort > $outfile

sort $infile > $temp_sort_file

diff $outfile $temp_sort_file |awk '/^>/ {print $0}'|sed 's/^> //' > $temp_dups_file

LINES=$(wc -l < $temp_dups_file)

if [ "$LINES" -gt 0 ]
then
        echo "There were $LINES duplicate records."
     touch $flagfile
else
        echo "There were $LINES duplicate records. 0 duplicate records."
fi

awk -F "," ' NR == FNR {   cnt[$1] ++ } NR != FNR {   if (cnt[$1] == 1)     print $0 }' your-file your-file