Parallel Processing - An al...

#!/bin/sh



# run script to do any preparation before the children are spawned. this could be

# used to create an empty file to hold the collective output of the child threads



commonInitialization

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



# remove the done file for each child process, one for each file in the horsemen folder



for pgm in `ls -1 horsemen`

do



touch ${pgm}done

rm ${pgm}done

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



done



# spawn off each child process, one for each file in the horsemen folder



for pgm in `ls -1 horsemen`

do



./horseman ${pgm} &

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



done

#!/bin/sh



# this thread is going to process each item listed in the file named by the parameter



for item in `cat horsemen/${1}`

do



# it needs to build a command, a piece of SQL in my instance, which will allow it to

# run in isolation from its siblings, both in terms of data processed and files touched



## first it composes a sed command to modify a template, a master command or sql file,  

## to create a specific command or sql file to run for this iteration



## the template contains :::: in place of the thread name and ::: in place of the item name



echo sed -e s/::::/${1}/g -e s/:::/${item}/g \

 ${MYSQLDIR}SampleMaster.sql | sed "s/_/'/g" > ~/${1}scratch

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



## then it runs the sed command to create the specific command for this iteration



sh ~/${1}scratch > ${MYSQLDIR}Sample${1}.sql

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



## then it runs the specific command to process this item in isolation from the other threads



./exec_sqlplus Sample${1} > ~/Sample${item}.log

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



done



# now that all items in the file named for the thread have been processed, this thread

# needs to write a "done" file so that the main script will know that it has finisned



touch ~/${1}done

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi

# first make sure that the file that allows the program to wait exists.

# to break an infinite loop, remove this file to exit the inner loop below



touch rm_this_to_stop

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



# wait for each child process to finish in the same order as it started them



for pgm in `ls -1 horsemen`

do



# inner loop waits for each a specific child process to write its done file



while [ ! -f ${pgm}done ]

do



sleep 300



# and it one of the child processes does not finish it can be made to exit



if [ ! -f rm_this_to_stop ] ; then

 exit 13

fi



done



# run script to perform single threaded processing after a child finishes

# this could be used to append an output file from the newly completed

# thread onto a single merged output file initialized in the beginning



threadCleanup

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi



done



# run script to do any collective processing after all the children are done

# this could be used to concatenate the output file from each completed

# thread into a single output file which would not require initialization



commonCleanup

rc=$?

if [ $rc -ne 0 ] ; then

 exit 13

fi

• horsemen/Numbers would contain:
  

  oneFish
  
  twoFish

• horsemen/Colors would contain:
  

  redFish
  
  blueFish

• The first thread would be called Numbers and would process oneFish and twoFish in turn before writing Numbersdone.
  
  
• The second thread would be called Colors and would process redFish and blueFish in turn before writting Colorsdone
  
  
• The template file could be as simple as:

  #!/bin/sh
  
  
  
  echo :::

  to try to tell the story. The generated sed command will replace the ::: with the item from the file, in this case the camelized line from the book. Note that the actual order of execution will jumble the story.