#!/bin/ksh

#------------------------------------------------------------------#
# This is a script for running PVM jobs under the "LoadLeveler"    #
# batch queuing system at NIST.                                    #
#------------------------------------------------------------------#
# Please report any difficulties you have in running this script   #
# to karin@cam.nist.gov                                            #
#------------------------------------------------------------------#

#------------------------------------------------------------------#
# First, set some variables that will be used to manage jobs/files:
#------------------------------------------------------------------#

export USER=`whoami`
export TMPDIR="/tmp/$USER"
export MYUID=`grep karin /etc/passwd | awk -F: '{print $3}' `
export MYSHELL=`grep karin /etc/passwd | awk -F: '{print $7}' `
HOST=`hostname`
export HOST=${HOST%%.*}
export CMD="${0##*/}"
export INITIALDIR=`pwd`

# This umask needed to give access to PVMLIST - if user's umask is set
# to 077, this file can't be read, and the script fails.
umask 022

# Create the temporary directory on the current host:
if [ ! -d "/tmp/${USER}" ]
then
    mkdir /tmp/${USER}
fi

#-----------------------------------------------------------------#
# Report to standard out and err which host is running the job:
#-----------------------------------------------------------------#

echo `date`
echo `date` >& 2
echo "$CMD: Running $CMD on node $HOST (Job ID: $CLUSTER)"
echo "$CMD: Running $CMD on node $HOST (Job ID: $CLUSTER)" >& 2

#---------------------------#
# PVM environment set-up:
#---------------------------#

# Destroy old pvmd file if left hanging around:

if [ -r /tmp/pvmd.$MYUID ]
then
   echo "Remving old copy of /tmp/pvmd.$MYUID"
   rm /tmp/pvmd.$MYUID
fi

# Set necessary environment variables:

export PVM_ROOT=/usr/local/apps/pvm3
export PVMDPATH=$PVM_ROOT/lib/RS6K
export PVMD=${PVMDPATH}/pvmd3
export PVM=${PVMDPATH}/pvm
export LLHOST=$HOST
export LLCLASS
export TMPDIR
export PVMLIST="${TMPDIR}/pvmnodelist"
export NODELIST=$PVMLIST
export PVM_EXPORT=PVMDPATH:TMPDIR:USER

#  Create a hostlist for this set of batch nodes:

set_pvm_pool

#  Record to standard out the nodes used:

export hostlist=` cat $PVMLIST | awk -F. '{print $1}'`
echo "Node pool:\n$hostlist"
echo "Current working directory: ${INITIALDIR}"

#---------------------------------------------------------#
# Copy any necessary files to the local node directory:
#---------------------------------------------------------#

echo "$CMD: Copying files to node temporary directories (${TMPDIR})" >& 2
echo "$CMD: Files to copy: " >& 2
echo "${PROGRAM} ${NODEPROGRAM} ${INPUT_FILES} ${DEBUG_FILES}" >& 2
for host in $hostlist
do
  rsh ${host} mkdir ${TMPDIR} > /dev/null 2>&1
  rcp ${PROGRAM} ${NODEPROGRAM} ${INPUT_FILES} ${DEBUG_FILES} ${host}:${TMPDIR}
done
#  Use trap command before starting the PVM daemon to handle
#  trapping interrupts and take actions on cleaning up
#  hopelessly lost PVM daemon and user processes by calling
#  pvmtidy utility.

trap 'pvmtidy ${PVMLIST} -u ${PROGRAM} ${NODEPROGRAM}; exit 1 ' 1 2 3 14 15

#---------------------------------------------------#
#  Start the PVM daemon across the node partition:
#---------------------------------------------------#

echo "$CMD: starting $PVMD daemon on $NPROCS nodes" >& 2
(cd ${TMPDIR}; $PVMD $PVMLIST )  &  
sleep 15

#  Successfully started PVM; 

cd ${INITIALDIR}
#---------------------------------------------#
# Invoke a user setup script if available:
#---------------------------------------------#
if [ -n "${USERSETUP}" ] && [ -x "${USERSETUP}" ]
then
    echo "$CMD Invoking user set-up procedure ${USERSETUP}"
    ${USERSETUP}
fi 


#---------------------------------------------#
# Execute the program on the current node:
#---------------------------------------------#
if [ -x ${PROGRAM} ]
then
   if [ "${INPUT}" = "/dev/null" ]
   then
      echo "$CMD: No standard input, will not copy /dev/null." >& 2
   else
      cp ${INPUT} ${TMPDIR}
      INPUT=${TMPDIR}/${INPUT}
   fi

#  Change into the TMPDIR working directory 
   cd ${TMPDIR} 

   echo "$CMD: Starting $PROGRAM $ARGUMENTS < ${INPUT} on $HOST" >& 2
   echo "\n\n<<<<<<<<<<<<<<  Start of program output >>>>>>>>>>>>>>\n\n"
   (${DEBUG} ${TMPDIR}/${PROGRAM} $ARGUMENTS < ${INPUT})
   echo "\n\n<<<<<<<<<<<<<<   End of program output  >>>>>>>>>>>>>>\n\n"
else
   print -u2 "$CMD: error $PROGRAM not found"
fi

if [ "$PVM_LLDEBUG" != "" ]
then
#  Capture PVM logfile for this session:
   echo "Logfile is /tmp/pvml.$MYUID\n"
   if [ -r /tmp/pvml.$MYUID ]
   then
      cat /tmp/pvml.$MYUID
   else
      echo "Logfile not found."
   fi
fi

#-----------------------------------------------#
# For PVM, before exiting, HALT the daemon:
#-----------------------------------------------#

echo "$CMD: Halting the PVM daemon..." >& 2
which ${PVM} >& 2
echo halt | ${PVM} 
echo 


#------------------------------#
# Clean up node directories: 
#------------------------------#

if [ -x ${USERCLEAN} ]
then
    echo "$CMD Invoking user clean-up procedure ${USERCLEAN}"
    ${USERCLEAN}
fi

#------------------------------------------------------------#
# Run pvmtidy utility at the end of the PVM runstream to
# cleanup hopelessly lost PVM daemon and user processes
# if there are any.
#------------------------------------------------------------#

echo "\npvmtidy $PVMLIST -u $PROGRAM $NODEPROGRAM" >& 2
pvmtidy $PVMLIST -u $PROGRAM $NODEPROGRAM >& 2

#--------------------------------------------------------#
# CD out of the directory where files will be removed:
#--------------------------------------------------------#

cd $HOME

echo "$CMD: Cleaning tmp directories on the nodes..." >& 2
for host in $hostlist
do
  rsh ${host} rm -fr ${TMPDIR}
done

echo "$CMD: Done." >& 2

#--------------------------------------#
# End clean-up; Exit shell program:
#--------------------------------------#

exit 0