UNIX for Beginners Questions & Answers

#!/bin/sh
MAXTHREAD=30
awk '{print $1}' metadata.csv > nvpipe &
while [ $c -le $MAXTHREAD ]
do
   ${BIN}/parallel_wot.sh &
   PID=$!
   sleep 3
   c=`expr $c + 1`
STRING=$STRING","$PID
done

cd $PIPEDIR/
while IFS=',' read DIR1 DIR2
do
 echo starting at `date`
  ${COMPUSETBIN}/prg1.sh prg2.sh $DIR1 $DIR2
  echo ending at `date`
  echo ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
done < nvpipe

#!/bin/bash
threads=30                                            # How many threads you want to work with

all_lines=$(wc -l < metadata.csv)                     # Count all the lines in your full input file
req_lines=$(printf $all_lines / $threads +1 | bc      # Get the lines required in each split file
                                                      # Rules of BIDMAS apply so the 1 is added after the divide

split -ld $req_lines metadata.csv metadata.           # Note the trailing dot. This will generate $threads files
                                                      # of up to $req_lines each in the format metadata.nn
                                                      # so up to 100 threads if you choose
for split_file in metadata.??
do
   ( while IFS=',' read DIR1 DIR2
   do
     printf "Start at $(date)\n"
     ${COMPUSETBIN}/prg1.sh prg2.sh $DIR1 $DIR2
     printf "Ending at $(date)\n"
     printf "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
   done < $split_file
done ) &

wait               #  All threads must complete before this script will exit