PmWikiFisica | Hpc / Compiling and Running QE @ FERMI CINECA browse

Install instruction are reported in the install directory.

Detailed instructions:

Extract the ELPA archive in the QE root directory
Run the configure script:

module purge
module load bgq-xl/1.0
module load essl/5.1
module load lapack/3.4.1--bgq-xl--1.0
module load scalapack/2.0.2--bgq-xl--1.0
module load mass/7.3--bgq-xl--1.0
export BLAS_LIBS="-L$ESSL_LIB -lesslsmpbg"
export LAPACK_LIBS=$LAPACK_LIB/liblapack.a
export SCALAPACK_LIBS=$SCALAPACK_LIB/libscalapack.a
./configure --enable-openmp  --with-scalapack --with-elpa

Check that both LINUX_ESSL and ELPA libraries are present:

DFLAGS         = -D__XLF -D__LINUX_ESSL -D__MASS -D__MPI -D__PARA -D__SCALAPACK -D__OPENMP -D__ELPA
FDFLAGS        = -D__XLF,-D__LINUX_ESSL,-D__MASS,-D__MPI,-D__PARA,-D__SCALAPACK,-D__OPENMP,-D__ELPA

Check LD links to the mpilibrary without debug code:

LD             = /bgsys/drivers/ppcfloor/comm/xl.ndebug/bin/mpixlf90_r

Run the following serial job to compile:

#!/bin/bash 
# @ job_type =serial
# @ job_name = serial.$(jobid)
# @ output = $(job_name).out
# @ error = $(job_name).err
# @ wall_clock_limit = 0:25:00
# @ class = serial
# @ queue

cd /gpfs/scratch/userexternal/pbonfa00/QE5/espresso-5.0.2


module load bgq-xl/1.0
module load essl/5.1
module load lapack/3.4.1--bgq-xl--1.0
module load scalapack/2.0.2--bgq-xl--1.0
module load mass/7.3--bgq-xl--1.0

make clean

make pw 1>makejob.log 2>makejob.err

cd flib
mpixlf90_r -qalias=noaryovrlp:nointptr -O2 -qstrict -qdpc=e -qsuffix=cpp=f90 -WF,-D__XLF,-D__FFTW,-D__MASS,-D__MPI,-D__PARA,-D__SCALAPACK,-D__OPENMP -I../include -I../iotk/src -I../Modules -I. -c functionals.f90 1>>makejob.log 2>>makejob.err

cd ..
make pw 1>>makejob.log 2>>makejob.err

Enjoy PWscf!

Fast & Furious version of the above:

Run the following script

#!/bin/bash 
# @ job_type =serial
# @ job_name = serial.$(jobid)
# @ output = $(job_name).out
# @ error = $(job_name).err
# @ wall_clock_limit = 0:25:00
# @ class = serial
# @ queue

cd /gpfs/scratch/userexternal/pbonfa00/QE5/espresso-5.0.2


module load bgq-xl/1.0
module load essl/5.1
module load lapack/3.4.1--bgq-xl--1.0
module load scalapack/2.0.2--bgq-xl--1.0
module load mass/7.3--bgq-xl--1.0

make distclean

./configure --enable-openmp  --with-scalapack --with-elpa 1>configure.log 2>configure.err


make pw 1>makejob.log 2>makejob.err

echo "===================" >> makejob.log
echo "== ENTERING flib ==" >> makejob.log
echo "===================" >> makejob.log


cd flib
mpixlf90_r -qalias=noaryovrlp:nointptr -O2 -qstrict -qdpc=e -qsuffix=cpp=f90 -WF,-D__XLF,-D__FFTW,-D__MASS,-D__MPI,-D__PARA,-D__SCALAPACK,-D__OPENMP -I../include -I../iotk/src -I../Modules -I. -c functionals.f90 1>>../makejob.log 2>>../makejob.err
cd ..

echo "====================" >> makejob.log
echo "== DONE WITH flib ==" >> makejob.log
echo "====================" >> makejob.log

make pw 1>>makejob.log 2>>makejob.err

Done!

Running jobs through Load Leveler

Simple Load Leveler script

# @ job_type = bluegene
# @ job_name = lfao_t1
# @ initialdir = .
# @ output = $(job_name).$(jobid).out
# @ error = $(job_name).$(jobid).err
# @ wall_clock_limit = 1:30:00
# @ bg_size = 128
# @ account_no = ***
# @ queue

module load bgq-xl/1.0
module load essl/5.1
module load lapack/3.4.1--bgq-xl--1.0
module load scalapack/2.0.2--bgq-xl--1.0
module load mass/7.3--bgq-xl--1.0

QE_HOME=/where/you/compiled/QE5/espresso-5.0.2/

export WORKDIR=/gpfs/scratch/userexternal/someuser/QE/compound/
export BINDIR=$QE_HOME/bin

  export TASKS=512

  # Set the number of MPI task x node
  export TASK_PER_NODE=4

  # Set the number of threads x node
  export THREADS=4

  # Set the number of NPOOL. As a role I suggest using NPOOL=Number of K-points
  export NPOOL=2

  # Set the number of group to run FFT3D over G-vectors in parallel among bands. The number of MPI processes involved in each 3DFFT is equal to TASKS/NPOOL/NTG
  export NTG=8

  # Set the MPI processing that will to perform eigen-solvers problem using scalapack
  export NDIAG=256


export INPUT_FILE=$WORKDIR/input.in
export OUTPUT_FILE=$WORKDIR/test-ELPA-bgq_${TASKS}mpi_.${TASK_PER_NODE}mpixnode.${THREADS}ths.${NPOOL}npool.${NTG}ntg.${NDIAG}ndiag.out.full.fermi

runjob --np $TASKS --ranks-per-node $TASK_PER_NODE --envs OMP_NUM_THREADS=$THREADS --exe $BINDIR/pw.x --args -npool --args $NPOOL --args -ntg --args $NTG --args -ndiag --args $NDIAG --args -input --args $INPUT_FILE > $OUTPUT_FILE

subblock job script:

#!/bin/bash
# @ job_type = bluegene
# @ job_name = lfao_t1
# @ initialdir = .
# @ output = $(job_name).$(jobid).out
# @ error = $(job_name).$(jobid).err
# @ wall_clock_limit = 0:29:00
# @ bg_size = 64
# @ account_no = ***
# @ queue


### Useful functions
# Prints $1 times $2
# example: NPOOL_L=(`neqparms 4 128`)
neqparms () { for (( i=0; i<$1; i++)); do echo -n "$2 "; done }

### Loads application
module load qe/5.0bgq

# Sets DIRECTORIES

export WORKDIR=/gpfs/scratch/userexternal/auser/adir
export BINDIR=$QE_HOME/bin


#################################################################
### FERMI SECTION
### Please modify only the following variables
#################################################################

export N_SUBBLOCK=4                       ### No. of sub-block you want.
                                          ### Choose between 2, 4, 8, 16, 32, 64.

module load subblock
source ${SUBBLOCK_HOME}/npart_${N_SUBBLOCK}.txt


#################################################################
### PARAMS
#################################################################

# Fermi params
TASK_L=(256 256 256 256)
TPN_L=(`neqparms 4 16 `)
TH_L=(2 2 4 4)

#QE params
NPOOL_L=(128 128 128 128)
NTG_L=(1 2 1 2)
NDIAG_L=(1 1 1 1)


### Check params
if [ ${#TASK_L[

} != $N_SUBBLOCK ] ; then echo "OMG! Wrong number of TASKS in TASK_L array"; exit; fi if [ ${#TPN_L

]} != $N_SUBBLOCK ] ; then echo "OMG! Wrong number of TASKS_Per_NODE in TPN_L array"; exit; fi
if [ ${#TH_L[

} != $N_SUBBLOCK ] ; then echo "OMG! Wrong number of THREADS in TH_L array"; exit; fi if [ ${#NPOOL_L

]} != $N_SUBBLOCK ] ; then echo "OMG! Wrong number of NPOOL in NPOOL_L array"; exit; fi
if [ ${#NTG_L[

} != $N_SUBBLOCK ] ; then echo "OMG! Wrong number of NTG in NTG_L array"; exit; fi if [ ${#NDIAG_L

]} != $N_SUBBLOCK ] ; then echo "OMG! Wrong number of NDIAG in NDIAG_L array"; exit; fi

for (( i=0; i<${N_PART}; i++ ));
do
  # Set the global number of MPI task
  export TASKS=${TASK_L[$i]}                         ### No. of MPI tasks in each sub-block.

  # Set the number of MPI task x node
  export TASK_PER_NODE=${TPN_L[$i]}                  ### No. of MPI tasks in each node.

  # Set the number of threads x node
  export THREADS=${TH_L[$i]}

  # Set the number of NPOOL. As a role I suggest using NPOOL=Number of K-points
  export NPOOL=${NPOOL_L[$i]}

  # Set the number of group to run FFT3D over G-vectors in parallel among bands. The number of MPI processes involved in each 3DFFT is equal to TASKS/NTG 
  export NTG=${NTG_L[$i]}

  # Set the MPI processing that will to perform eigen-solvers problem using scalapack 
  export NDIAG=${NDIAG_L[$i]}

  export INPUT_FILE=$WORKDIR/1.in

  export OUTPUT_FILE=$WORKDIR/test_bgq_${TASKS}mpi_.${TASK_PER_NODE}mpixnode.${THREADS}ths.${NPOOL}npool.${NTG}ntg.${NDIAG}ndiag.out.full.fermi

  CC=$(($i+1))

  runjob --block $BLOCK_ALLOCATED --corner $(n_cor $CC) --shape $SHAPE_SB  --np $TASKS \
              --ranks-per-node $TASK_PER_NODE  \
              --envs OMP_NUM_THREADS=$THREADS --exe $BINDIR/pw.x --args -npool --args $NPOOL --args -ntg --args $NTG \
              --args -ndiag --args $NDIAG --args -input --args $INPUT_FILE > $OUTPUT_FILE &

done
wait