Shell Programming and Scripting

for FILE in *.txt
do
        sed 's/[^a-zA-Z \r\t]//g' "$FILE" | 
                tr -s ' \r\t' '\n' | 
                sort |
                uniq -c |
                sort -n |
                head -n 50
done

for FILE in *.txt
do
        sed 's/[^a-zA-Z \r\t]//g' "$FILE" | 
                tr -s ' \r\t' '\n' | 
                grep -v -F -f exclude.txt |
                sort |
                uniq -c |
                sort -n |
                head -n 50
done

cat ${a[$i]} | sed 's/\([[:space:]]\)/\1 \n/g' | sed 's/^[ \t]*//' | sed \
's/[ \t]*$//' | sed '/^$/d' | grep -v -F -f exclude.lst | tr -cd '[:alnum:] [:space:]'| sort \
| uniq -c | sort -nr | head -n 50 >> lsf2dout.txt

#!/bin/bash
# program for data mining
#this program needs a file named exclude.lst

echo "This programs read text files and brings out the"
echo "word count of first 50 most used words in a file."

a=(*.txt)

for (( i = 0; i < ${#a[*]}; i++ ))
do
echo "a[$i] = ${a[$i]}"
done
echo ""

for (( i = 0; i < ${#a[*]}; i++ ))
do
printf "=======================" >> lsf2dout.txt
printf "\n" >> lsf2dout.txt
printf "Name of the file: ${a[$i]}" >> lsf2dout.txt
printf "\n" >> lsf2dout.txt

cat ${a[$i]} | sed 's/\([[:space:]]\)/\1 \n/g' | sed 's/^[ \t]*//' | sed \
's/[ \t]*$//' | sed '/^$/d' | grep -w -v -F -f exclude.lst | tr -cd '[:alnum:] [:space:]'| sort \
| uniq -c | sort -nr | head -n 100 >> lsf2dout.txt

printf "\n\n" >> lsf2dout.txt
done