Shell Programming and Scripting

#head -10 *.lst > headers.txt 
# zip headers.zip headers.txt

#data2imp.sh
#this script will process all files as *.txt in the current directory.
#The result file contains one line for each file, with the first field as the original file name.
#
#we need first to format the input file
function format_item {

awk '
# Filter non blank lines. NF = Number of Fields ( NF = 0 for empty line)
#  replace all consecutive blanks with 1 single blank space
NF {gsub(/  */," ")
# print line without Line Feed
 printf "%s" ,$0 
 istext++
 next
}
# Print a Line Feed when text is followed an empty line (NF==0)
istext {print "";istext=0}' $1
}
# Creation of a sample csv record 
# Take care there is no semicolumn inside the original text. In such case the field separator might be changed
function insert_item {
awk -v OFS=";" -v ITEM=$itemname '
# Record 1 is the item date
NR==1 {DATE=$1 " " $2}
# Record 2 is item title
# if length <= 100 : TITLE is the full record
# else seek for a dot in position between 51 and 100 of the record and cut record to this position
#    if no dot  in position between 51 and 100 : cut to the 100 first characters.           
NR==2 { if (length($0) <= 100) TITLE = $0
        else {
           dotposition=index(substr($0,51),".")
           if (dotposition == 0) {
             TITLE = substr($0,1,100) "..."}
           else {
             TITLE = substr($0,1,50 + dotposition)
           }
        }
       }
# same method as for  TITLE
NR==4{ if (length($0) <= 500) SNIPPET = $0
        else {
           dotposition=index(substr($0,401),".")
           if (dotposition == 0) {
             SNIPPET = substr($0,1,500) "..."}
           else {
             SNIPPET = substr($0,1,400 + dotposition)
           }
        }
print ITEM,DATE,TITLE,SNIPPET
}' $1
}
#--- main -----------------------------------
# Output initialisation
#
>items.csv
#
for i in *.txt
do
itemname=$(basename $i .txt)
echo Processing item $itemname
format_item $i > $itemname.lst
insert_item $itemname.lst >> items.csv
done

$ ./data2imp.sh 
( output remove) 
# check for useless keywords :
$ sort *.tmp |uniq -c |sort -rn | head -20 |cut -c-60
     56
     40 (FOR FY
     18 2017)
     18 2016)
     17 AUGUST 2012 1
     16 2015)
     13 DEPARTMENT OF
     13 AUGUST 2011 1
     10 (FY
      9 AUGUST 2015 TABLE OF CONTENTS
      9 1. MANDATE
      8 1
      7 DEPARTMENT OF ENERGY
      6 AUGUST 2016 1
      6 AUGUST 2013 2
      6 ((FFYY 22001133))
      5 SITUATIONER
      5 SEPTEMBER 2013 1
      5 DEPARTMENT OF TRADE AND INDUSTRY
      5 DEPARTMENT OF FINANCE

# remove comment if required to format files (txt to lst)
# for i in *.txt
# do
# itemname=$(basename $i .txt)
# echo Formatting item $i
# format_item $i > $itemname.lst
# done

rm: cannot remove `*.tmp': No such file or directory
Processing item *.lst
awk: cmd. line:40: fatal: cannot open file `*.lst' for reading (No such file or directory)

NR==2 { if (length($0) <= 100) TITLE = $0
        else {
           dotposition=index(substr($0,51),".")
           if (dotposition == 0) {
             TITLE = substr($0,1,100) "..."}
           else {
             TITLE = substr($0,1,50 + dotposition)
           }
        }
       }