|
|
Help needed on restart-from-point-of-failure in Parallel Processing
04-06-2017
$ cat properties.file Account Customer Address Phone
while IFS= read -r fileLines; do
/path/to/product/script/data_process "${fileLines}" || exit -1 &
dpPIDs+=" $!"
done < properties.file
## logic to check background job status (success or failed)
for chkPIDs in $dpPIDs; do
if ! wait $chkPIDs; then
failCnt=`expr $failCnt + 1`
fi
done
if [ $failCnt -gt 0 ]; then
echo "[`date`][FATAL] There are errors in Data Processing, total number of DP jobs failure is $failCnt. Check log."
exit -1
else
echo "[`date`][SUCCESS] Data Processing for received files has been completed..."
fi
04-07-2017
), don't know how).
$ [] cat bgPIDTest.ksh
# Some function that takes a long time to process
longprocess() {
# Sleep up to 14 seconds
#sleep $((RANDOM % 15))
sleepTime=$((RANDOM % 5))
# Randomly exit with 0 or 1
exitCode=$((RANDOM % 2))
echo "sleeping for: $sleepTime with exit code: $exitCode "
sleep $sleepTime
exit $exitCode
}
pids=""
failCnt=0
# Run five concurrent processes
( longprocess ) &
# store PID of process
pids+=" $!"
echo PID $pids
( longprocess ) &
# store PID of process
pids+=" $!"
echo PID $pids
( longprocess ) &
# store PID of process
pids+=" $!"
echo PID $pids
( longprocess ) &
# store PID of process
pids+=" $!"
echo PID $pids
( longprocess ) &
# store PID of process
pids+=" $!"
echo PID $pids
( longprocess ) &
# store PID of process
pids+=" $!"
echo PID $pids
# Wait for all processes to finish, will take max 14s
echo "initial failCnt is $failCnt"
for p in $pids; do
#if wait $p; then
if ! wait $p; then
cmdJobNM=`ps -p $p -o command=`
failCnt=`expr $failCnt + 1`
echo "failed command is --> $cmdJobNM, PID: $p"
fi
done
echo "total failCnt is $failCnt"
if [ $failCnt -gt 0 ]; then
exit -1
fi$ [] ksh bgPIDTest.ksh PID 15858 sleeping for: 2 with exit code: 1 PID 15858 15859 sleeping for: 4 with exit code: 0 PID 15858 15859 15860 sleeping for: 0 with exit code: 0 PID 15858 15859 15860 15861 sleeping for: 3 with exit code: 0 PID 15858 15859 15860 15861 15862 sleeping for: 3 with exit code: 1 PID 15858 15859 15860 15861 15862 15863 initial failCnt is 0 sleeping for: 1 with exit code: 0 failed command is --> , PID: 15858 failed command is --> , PID: 15862 total failCnt is 2
|
|