Wednesday, June 26, 2013

Gamess (US) frequently asked questions Part 4: The rungms script

Third guest post by Kirill Berezovsky (Petrozadovsk State University).

Gamess (US) is run though a script provided in the installation package. This script MUST be adapted by the user. The rungms-script below has been adapted to be used with the Gamess (US) installation described in earlier posts. Happy computing :-)



RUNGMS script

#!/bin/csh

set SCR=/scr/$USER
set USERSCR=~$USER/scr
set GMSPATH=/usr/local/gamess

set JOB=$1
set VERNO=$2

if ($VERNO == cpu) set TARGET=mpi
if ($VERNO == gpu) set TARGET=ga
if (null$VERNO == null) set VERNO=cpu

set master=`hostname`
printf "\n * Started at:           `date`"

set DSK=`df -m $SCR | awk 'NR==2{print$4}'`
printf "\n * Available disk space: $DSK MB"

printf "\n * Temporary files in:   $SCR and in $USERSCR\n\n"

limit stacksize 8192

if ($JOB:r.inp == $JOB) set JOB=$JOB:r
if (-e $JOB.inp) then
   cp  $JOB.inp  $SCR/$JOB.F05
else
   echo "Input file $JOB.inp not found"
   exit 4
endif

source $GMSPATH/gms-files.csh
if (-e $HOME/.gmsrc) source $HOME/.gmsrc

set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`
if ($ngddi > 0) then
   set GDDIjob=true
   echo "This is a GDDI run, keeping various output files on local disks"
   set echo
   setenv  OUTPUT $SCR/$JOB.F06
   setenv   PUNCH $SCR/$JOB.F07
   unset echo
else
   set GDDIjob=false
endif

if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then
   echo "Please save, rename, or erase these files from a previous run:"
   echo "     $PUNCH,"
   echo "     $TRAJECT,"
   echo "     $RESTART, and/or"
   echo "     $MAKEFP,"
   echo "and then resubmit this computation."
   exit 4
endif

#----------------------------------------------------------------------

if ($TARGET == mpi) then

 set NCPUS=`grep cores /proc/cpuinfo | wc -l`

 echo " * CPU cores:   $NCPUS "
 echo " * GPU devices: not used"
 echo " "

 setenv TRAJECT $USERSCR/$JOB.trj
 setenv RESTART $USERSCR/$JOB.rst
 setenv INPUT $SCR/$JOB.F05
 setenv PUNCH $USERSCR/$JOB.dat

 if ( -e $TRAJECT ) rm $TRAJECT
 if ( -e  $PUNCH ) rm $PUNCH
 if ( -e  $RESTART ) rm $RESTART

 setenv LD_LIBRARY_PATH /opt/intel/impi/4.0.2.003/intel64/lib:$LD_LIBRARY_PATH
 set path= ( /opt/intel/impi/4.0.2.003/intel64/bin $path )
mpdboot
 mpiexec -n $NCPUS $GMSPATH/gamess.$VERNO.x
mpdallexit
 cp $PUNCH .
endif

#----------------------------------------------------------------------

if ($TARGET == ga) then
 set PPN=1
 set NCPUS=1

 @ NPROCS = $NCPUS

 setenv HOSTFILE $SCR/$JOB.nodes.mpd
 if (-e $HOSTFILE) rm $HOSTFILE
 touch $HOSTFILE
  
 echo `hostname` >> $HOSTFILE
 set NNODES=1

 setenv PROCFILE $SCR/$JOB.processes.mpd
 if (-e $PROCFILE) rm $PROCFILE
 touch $PROCFILE

 echo "-n $NPROCS -host `hostname` $GMSPATH/gamess.$VERNO.x" >> $PROCFILE

 set path=(/opt/intel/impi/4.0.2.003/intel64/bin $path)

   setenv I_MPI_WAIT_MODE enable
   setenv I_MPI_PIN disable
   setenv I_MPI_DEBUG 0
   setenv I_MPI_STATS 0
   setenv I_MPI_DEVICE sock
   setenv I_MPI_NETMASK ib0

 setenv LD_LIBRARY_PATH /opt/intel/impi/4.0.2.003/intel64/lib:$LD_LIBRARY_PATH
 setenv LD_LIBRARY_PATH /opt/intel/composerxe-2011.4.191/compiler/lib/intel64:$LD_LIBRARY_PATH
 setenv LD_LIBRARY_PATH /opt/intel/composer_xe_2013.0.079/mkl/lib/intel64:$LD_LIBRARY_PATH

 setenv MKL_SERIAL YES
 setenv MKL_NUM_THREADS 1

 setenv LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH

 setenv GMS_CCHEM '1'

 @ NUMGPU=1
 setenv CCHEM 'devices=0;memory=4g'

 echo " * CPU cores:   `grep cores /proc/cpuinfo | wc -l` "
 echo " * GPU devices: $NUMGPU (with settings: $CCHEM)"
 echo " "

 chdir $SCR

 set echo
 mpdboot --rsh=ssh -n $NNODES -f $HOSTFILE
  mpiexec -configfile $PROCFILE < /dev/null
 mpdallexit
 unset echo

 rm -f $PROCFILE
endif

#----------------------------------------------------------------------

echo ----- accounting info -----

if ($GDDIjob == true) cp $SCR/$JOB.F07 ~/scr/$JOB.dat

echo Files used on the master node $master were:
ls -lF $SCR/$JOB.*
rm -f  $SCR/$JOB.F*

if (-e $SCR/$JOB.V84)        mv $SCR/$JOB.V84     $USERSCR
if (-e $SCR/$JOB.V80)        rm -f $SCR/$JOB.V*
if (-e $SCR/$JOB.TEMP02)     rm -f $SCR/$JOB.TEMP*
if (-e $SCR/$JOB.orb)        mv $SCR/$JOB.orb     $USERSCR
if (-e $SCR/$JOB.vec)        mv $SCR/$JOB.vec     $USERSCR
if (-e $SCR/$JOB.mol)        mv $SCR/$JOB.mol     $USERSCR
if (-e $SCR/$JOB.molf)       mv $SCR/$JOB.molf    $USERSCR
if (-e $SCR/$JOB.mkl)        mv $SCR/$JOB.mkl     $USERSCR
if (-e $SCR/$JOB.xyz)        mv $SCR/$JOB.xyz     $USERSCR
ls $SCR/${JOB}-*.cube > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.cube $USERSCR
rm -f $SCR/${JOB}.lis
ls $SCR/${JOB}-*.grd > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.grd $USERSCR
rm -f $SCR/${JOB}.lis
ls $SCR/${JOB}-*.csv > $SCR/${JOB}.lis
if (! -z $SCR/${JOB}.lis) mv $SCR/${JOB}*.csv $USERSCR
rm -f $SCR/${JOB}.lis

if ($TARGET == mpi) then
   set nnodes=`wc -l $HOSTFILE`
   set nnodes=$nnodes[1]
   @ n=1
   set master=`hostname`
   set master=$master:r
   while ($n <= $nnodes)
      set host=`sed -n -e "$n p" $HOSTFILE`
      set host=$host[1]
      if ($host != $master) then
         echo Files used on node $host were:
         ssh $host -l $USER "ls -l $SCR/$JOB.*"
         ssh $host -l $USER "rm -f $SCR/$JOB.*"
      endif
      @ n++
   end
   rm -f $HOSTFILE
   if ($?I_MPI_STATS) then
      if ($I_MPI_STATS > 0) mv $SCR/stats.txt ~/$JOB.$NCPUS.stats
   endif
endif

date
time
exit

1 comment:

  1. This comment has been removed by a blog administrator.

    ReplyDelete