Shell Programming and Scripting

sort -R <inputfile | head -n 10000 | awk -v FS='>' '{
        # Get stuff after second >, and convert it all to uppercase
        $0=toupper($3);
        # Substitute every non-alpha char with space
        gsub(/[^A-Z]/, " ");
        # Split on spaces, into array A
        split($0, A, " ");
        # count words
        for(key in A)   count[A[key]]++;
}
END {
        # print words
        for(key in count)
                printf("%s\t%d\n", key, count[key]);
}' < filename

LINES=`wc -l < random.txt`

awk -v FS='>' -v LINES=$LINES -v COUNT=10000 '
BEGIN {
        SKIP=(LINES/COUNT)-1;
        srand();
        R=rand()*SKIP;
}

{
        if(M++ > R)
        {
                # Get stuff after second >, and convert it all to uppercase
                $0=toupper($3);
                # Substitute every non-alpha char with space
                gsub(/[^A-Z]/, " ");
                # Split on spaces, into array A
                split($0, A, " ");
                # count words
                for(key in A)   count[A[key]]++;

                # Cheat out of the loop
                R=SKIP+1;
        }

        if(M > SKIP)
        {
                M=0
                R=rand()*SKIP;
        }
}

END {
        # print words
        for(key in count)
                printf("%s\t%d\n", key, count[key]);
        }' <random.txt

FILE=$1
COUNT=$2
rl -c $COUNT $FILE | awk -v FS='>' '{
        $0=tolower($3);
        gsub(/[^a-z]/, " ");
        split($0, A, " ");
        for(key in A)   count[A[key]]++;
}
END {
        for(key in count) {
        words += count[key];
        uniq++;
    }
        printf("Total words: %d\tUnique words:%d\n", words, uniq);
}'