#!/bin/tcsh
#  Sample Batch Script for a OpenMPI-Intel job
#  on forge
#
#  Access openmpi as follows:
#  module load openmpi-1.4.3-intel-12.0.4
#  Avoid aggressive optmizations -O3, -O4, -fast, ipo
#
#  Submit this script using the command: qsub <script_name>
#
#  Use the "qstat" command to check the status of a job.
#
# The following are embedded QSUB options. The syntax is #PBS (the # does
# _not_  denote that the lines are commented out so do not remove).
#
# walltime : maximum wall clock time (hh:mm:ss)
#PBS -l walltime=00:25:00
#
# nodes: number of 16-core nodes
# ppn: how many cores per node to use (1 through 16)
#       (you are always charged for the entire node)
#PBS -l nodes=1:ppn=4
#

# export all my environment variables to the job
#PBS -V
#PBS -q normal
# job name (default = name of script file)
#PBS -N test_mpi_ompi
#
#
# filename for standard output (default = <job_name>.o<job_id>)
# at end of job, it is in directory from which qsub was executed
# remove extra ## from the line below if you want to name your own file
###PBS -o testjob.out
#
# filename for standard error (default = <job_name>.e<job_id>)
# at end of job, it is in directory from which qsub was executed
# remove extra ## from the line below if you want to name your own file
###PBS -e testjob.err
#
# End of embedded QSUB options
#
# set echo               # echo commands before execution; use for debugging
#

# Go to the job scratch directory. Use cdjob <jobid> to go to this
# directory once the job has started.

cd ${HOME}/code/yank/src/
 
setenv NP `wc -l ${PBS_NODEFILE} | cut -d'/' -f1`
#setenv NP 4 # override

module load openmpi-1.4.3-intel-12.0.4

### setenv any other vars you might need
### must setenv these to avoid hanging on ib  
setenv OMPI_MCA_btl_openib_flags 1
setenv OMPI_MCA_btl_mpi_leave_pinned 0
setenv OMPI_MCA_btl_openib_warn_default_gid_prefix 0

### run mpi
#mpirun -np ${NP} -hostfile ${PBS_NODEFILE} python alchemical-repex-volume.py

setenv SYSTEM 1-methylpyrrole
setenv JOBDIR "${HOME}/code/yank/test-systems/T4-lysozyme-L99A/amber-gbsa/amber-gbsa/$SYSTEM/"

# DEBUG
#setenv JOBDIR "${HOME}/code/yank/src/alanine-binding-test"

rm -f ${JOBDIR}/*.nc
mpirun -np ${NP} -hostfile ${PBS_NODEFILE} python yank.py --complex_prmtop $JOBDIR/complex.prmtop  --receptor_prmtop $JOBDIR/receptor.prmtop --ligand_prmtop $JOBDIR/ligand.prmtop --complex_crd $JOBDIR/complex.crd --output $JOBDIR --verbose --mpi