UNIX for Dummies Questions & Answers

72426459560          2010-06-2 ABC                           LC11100619758

95327GNFA4S          2010-06-2 XYZ                           97BCX3AMD10G

95327GNFA4S          2010-06-2 XYZ                           97BCX3AMKLMO

900278VGA4T          2010-06-2 KLM                            QVA697C8LAYMACBF

900278VG567          2010-06-2 LUF                            QVA697C8LAYMACBF

95327GNFA4S 2010-06-2 XYZ 97BCX3AMD10G

95327GNFA4S 2010-06-2 XYZ 97BCX3AMKLMO

900278VGA4T 2010-06-2 KLM QVA697C8LAYMACBF

900278VG567 2010-06-2 LUF QVA697C8LAYMACBF

#!/bin/sh

COL=3

# Break file into sortable chunks xaa - xzz
# Try between 500M and 2G for your gigantic file.
split -C 100K < megadata.txt

# Sort each individual file on given column
for FILE in x??
do
        sort -k $COL < "${FILE}" > "${FILE}.tmp"
        rm -f "${FILE}"
        mv "${FILE}.tmp" "${FILE}"
        shift
done

# Merge all the sorted files into one great big list.
# run it through awk, which will print only the first row which has any
# particular value for column COL.
sort -k $COL -m x?? | awk -v COL=${COL} '{
        if(LAST != $COL)        print;
        LAST=$COL;      }' > megasorted.txt

# Delete splitted files
# rm -f x??

#!/bin/sh

COL=4

# Break file into livable chunks
split -C 100K < megadata.txt

for FILE in ???
do
        sort -k $COL < "${FILE}" > "${FILE}.tmp"
        rm -f "${FILE}"
        mv "${FILE}.tmp" "${FILE}"
        shift
done

sort -k $COL -m ??? | awk -v COL=${COL} '{
        if($COL == LAST)
        {
                if(orig)
                {       print orig;     orig="";        }

                print;
        }
        else
        {
                LAST=$COL;
                orig=$0;
        }

                }' > output.txt
rm -f x??

900278VG567 2010-06-2 ABC QVA697C8LAYMACBF
900278VGA4T 2010-06-2 ABC QVA697C8LAYMACBF