UNIX for Dummies Questions & Answers

#!/bin/bash 
#SBATCH -o phGB72.scf.out
#SBATCH -J scf-GB72-ph
#SBATCH --partition=superlong
#SBATCH --nodes=4
#SBATCH --cpus-per-task=1
#SBATCH --error=phGB72.scf.err
#SBATCH --nodelist=veredas81,veredas85,veredas86,veredas89
##-----------------espresso-4.3.2-----------------
EXEC_DIR=/home_cluster/fis718/eliemouj/espresso-4.3.2/GB72
##EXEC_BIN=parsec_arpack_seq.x
##EXEC_BIN=parsec_arpack_mpi.x
EXEC_BIN=/etc/profile.d/00-modules.sh
INPUTFILE=GB72ph.scf.in
##OUTPUTFILE=GB72ph.scf.out
##PP_DIR=${HOME}/parsec_pp
##PP_FILES="C_POTRE.DAT"


##-----------------parallel env-----------------
MPI=INTEL
##MPI=BULL
##MPI=SERIAL


if [ "${MPI}" == "INTEL" ]; then 
    source /opt/intel/Compiler/11.1/069/bin/iccvars.sh intel64
    source /opt/intel/Compiler/11.1/069/bin/ifortvars.sh intel64
    source /opt/intel/impi/4.0.0.028/intel64/bin/mpivars.sh
    export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi.so
    export I_MPI_FABRICS=shm:dapl
fi
if [ "${MPI}" == "BULL" ]; then 
    mpibull2_devices -d=ibmr_gen2
fi
ulimit -s unlimited
ulimit -a

if [ -z "$SLURM_NPROCS" ] ; then
  if [ -z "$SLURM_NTASKS_PER_NODE" ] ; then
    SLURM_NTASKS_PER_NODE=1
  fi
  SLURM_NPROCS=$(( $SLURM_JOB_NUM_NODES * $SLURM_NTASKS_PER_NODE ))
fi
echo "SLURM_TASKS_PER_NODE    = " $SLURM_TASKS_PER_NODE   
echo "SLURM_NTASKS_PER_CORE   = " $SLURM_NTASKS_PER_CORE
echo "SLURM_NTASKS_PER_NODE   = " $SLURM_NTASKS_PER_NODE
echo "SLURM_NTASKS_PER_SOCKET = " $SLURM_NTASKS_PER_SOCKET
echo "SLURM_JOB_NUM_NODES     = " $SLURM_JOB_NUM_NODES 
echo "SLURM_NNODES            = " $SLURM_NNODES 
echo "SLURM_CPUS_PER_TASK     = " $SLURM_CPUS_PER_TASK
echo "SLURM_NPROCS            = " $SLURM_NPROCS

#-----------------create directories-----------------
cd ${SLURM_SUBMIT_DIR}                                   

#
# check if we need to restart the calculation

ID=${SLURM_JOB_NAME}.${SLURM_JOB_ID}
STORAGE_DIR=/storage/fis718/ananias/${ID}

stagein()
{
    #----------------------------------------
    # DO NOT TRUST sbcast. USE pdcp instead
    #----------------------------------------

    echo 'starting stagein'
    srun --ntasks=${SLURM_JOB_NUM_NODES} mkdir -p ${STORAGE_DIR}
    for pp in ${PP_FILES}; do
        /bin/cp -f ${PP_DIR}/${pp} ${STORAGE_DIR}/${pp}
    done
    
    /bin/cp -f ${EXEC_DIR}/${EXEC_BIN}          ${STORAGE_DIR}/${EXEC_BIN}
    /bin/cp -f ${SLURM_SUBMIT_DIR}/${INPUTFILE} ${STORAGE_DIR}/${INPUTFILE}
    ln -s ${STORAGE_DIR}/ ${SLURM_SUBMIT_DIR}
    echo 'finalizing stagein'
}

early()
{
    echo ' '
    echo ' ############ WARNING:  EARLY TERMINATION #############'
    echo ' '
    
    touch stop_scf
    sleep 120
    # and so parsec does a clean kill ...
}

trap 'early' SIGHUP SIGINT SIGUSR1 SIGUSR2
#--------------------------EXECUTION--------------------------
cd ${STORAGE_DIR}

if [ "${MPI}" == "BULL" ]; then
    srun  ./${EXEC_BIN}
fi

if [ "${MPI}" == "INTEL" ]; then
    HOSTFILE=/tmp/hosts.$SLURM_JOB_ID
    srun hostname -s | sort -u > ${HOSTFILE}
    mpdboot -n ${SLURM_NNODES} -f ${HOSTFILE} -r ssh
    mpdtrace -l
    echo "start executable"
    mpiexec -np ${SLURM_NPROCS} ${EXEC_DIR}/./${EXEC_BIN} <${INPUTFILE}
    mpdallexit
fi

#rm -f ${EXEC_BIN}
#
# send time checker daemon 

#--------------------------EXECUTION--------------------------

exit