CSMP Basics

Here we list some very basic instructions to get CSMP running on your PC or the Heriot-Watt cluster

A basic .bashrc file for Sonora

# .bashrc

# Source global definitions
if [ -f /etc/bashrc ]; then
        . /etc/bashrc
fi

# Aliases
# #######

# Some example alias instructions
# If these are enabled they will be used instead of any instructions
# they may mask.  For example, alias rm='rm -i' will mask the rm
# application.  To override the alias instruction use a \ before, ie
# \rm will call the real rm not the alias.

# Interactive operation...
alias rm='rm -i'
# alias cp='cp -i'
# alias mv='mv -i'

# Default to human readable figures
alias df='df -h'
alias du='du -h'

# Misc :)
alias less='less -r'                          # raw control characters
alias whence='type -a'                        # where, of a sort
alias grep='grep --color'                     # show differences in colour

# Some shortcuts for different directory listings
alias ls='ls -hF --color=tty'                 # classify files in colour
alias dir='ls --color=auto --format=vertical'
alias vdir='ls --color=auto --format=long'
alias ll='ls -l'                              # long list
alias la='ls -A'                              # all but . and ..
alias l='ls -CF'                              #

# set path to CSMP and MPICH directory, replace sebastian with your username
export CSMPHOME="/home/sebastian/csmp";
export MPICHHOME="/usr/local/openmpi/Intel"
export SVD_LICENSE_FILE="[e-mail]"

# Own aliases, fast access to CSP directory, HW cluster and remove gedit backup files
alias csp="cd $CSMPHOME"
alias hwcluster="ssh -l sg100 hwcluster.hw.ac.uk -X"
alias rmtilde="rm *~"

A basic .bashrc file for the HW cluster

# .bashrc

# User specific aliases and functions, replace sg100 with your username

# Source global definitions
if [ -f /etc/bashrc ]; then
        . /etc/bashrc
fi

# set path to SAMG and INTEL libraries
if [ -z "${LD_LIBRARY_PATH}" ]
then
    LD_LIBRARY_PATH="/home/sg100/csp/libraries:/opt/mpich/intel/lib"; export LD_LIBRARY_PATH
else
    LD_LIBRARY_PATH="/home/sg100/csp/libraries:/opt/mpich/intel/lib:$LD_LIBRARY_PATH"; export LD_LIBRARY_PATH
fi

# set path to SAMG license file
if [ -z "${LM_LICENSE_FILE}" ]
then
    LM_LICENSE_FILE="/home/sg100/csp/libraries/FHGSCAI.lic"; export LM_LICENSE_FILE
else
    LM_LICENSE_FILE="/home/sg100/csp/libraries/FHGSCAI.lic:$LM_LICENSE_FILE"; export LM_LICENSE_FILE
fi

# set path to MPICH directory
if [ -z "${PATH}" ]
then
    PATH="/opt/mpich/intel/bin"; export PATH
else
    PATH="/opt/mpich/intel/bin:$PATH"; export PATH
fi

# set path to MPICH directory
if [ -z "${MPICHHOME}" ]
then
    MPICHHOME="/opt/mpich/intel"; export MPICHHOME
else
    MPICHHOME="/opt/mpich/intel:$MPICHHOME"; export MPICHHOME
fi

# set path to Fortran compiler directory
if [ -z "${FCEHOME}" ]
then
    FCEHOME="/opt/intel/fce/9.0"; export FCEHOME
else
    FCEHOME="/opt/intel/fce/9.0:$FCEHOME"; export FCEHOME
fi

# set path to CSMP toplevel directory
if [ -z "${CSMPHOME}" ]
then
    CSMPHOME="/home/sg100/csp"; export CSMPHOME
else
    CSMPHOME="/home/sg100/csp:$CSMPHOME"; export CSMPHOME
fi

# user defined aliases
alias rm='rm -i'
alias rmtilde='rm -i *~*'
alias ll='ls -l --color=always'
alias la='ls -a --color=always'
alias ls='ls --color=always'
alias qload='qstat -f | grep "BIP" | sort +3'
alias lmutil='$CSMPHOME/support_libraries/samg/lmutil'
alias csp='cd $CSMPHOME'

# using ssh for communication across processors
export P4_RSHCOMMAND=ssh

The make.inc file for compiling CSMP with the Intel compiler

#################
# Include paths #
#################
# CSMPHOME (location of toplevel CSMP directory), MPICHHOME (location of MPI), and 
# FCEHOME (location of Fortran compiler) variables must be set in .bashrc
CSP_ALG   = $(CSMPHOME)/source_code/algorithms
CSP_MAIN  = $(CSMPHOME)/source_code/main_library
CSP_FEM   = $(CSMPHOME)/source_code/finite_elements
CSP_MATH  = $(CSMPHOME)/source_code/applied_math
CSP_IFS   = $(CSMPHOME)/source_code/interfaces
CSP_CGEO  = $(CSMPHOME)/source_code/computational_geometry
CSP_EOS   = $(CSMPHOME)/source_code/eos
CSP_H2O   = $(CSMPHOME)/source_code/eos/eos_h2o
CSP_NACL  = $(CSMPHOME)/source_code/eos/eos_nacl_h2o
CSP_TWO   = $(CSMPHOME)/source_code/two_phase_flow
CSP_PDE   = $(CSMPHOME)/source_code/pde_operators
CSP_INT   = $(CSMPHOME)/source_code/interrelations
CSP_SOLV  = $(CSMPHOME)/source_code/solver
CSP_VIS   = $(CSMPHOME)/source_code/visitors
CSP_TFV   = $(CSMPHOME)/source_code/triangular_finite_volumes
CSP_GFVN  = $(CSMPHOME)/source_code/generic_node_centered_finite_volumes
CSP_GFVE  = $(CSMPHOME)/source_code/generic_element_centered_finite_volumes
CSP_PAR   = $(CSMPHOME)/source_code/parallel
CSP_LIB   = $(CSMPHOME)/libraries
JPEG      = $(CSMPHOME)/support_libraries/jpeg
F2C       = $(CSMPHOME)/support_libraries/f2c
MESCHACH  = $(CSMPHOME)/support_libraries/meschach
UTILS     = $(CSMPHOME)/support_libraries/utilities
MPICH     = $(MPICHHOME)/include
MPICHLIB  = $(MPICHHOME)/lib
FCELIB    = $(FCEHOME)/lib
CSP       = ./

#######################
# Include directories #
#######################
INCFLAG = -I$(JPEG) -I$(F2C) -I$(UTILS) -I$(MESCHACH) -I$(CSP_MAIN) -I$(CSP_FEM) -I$(CSP) \
          -I$(CSP_MATH) -I$(CSP_IFS) -I$(CSP_CGEO) -I$(CSP_TWO) -I$(CSP_PDE) -I$(CSP_PAR) \
          -I$(CSP_INT) -I$(CSP_ALG) -I$(CSP_SOLV)  -I$(CSP_GFVN)  -I$(CSP_GFVE)  -I$(CSP_TFV) \
          -I$(CSP_VIS) -I$(CSP_EOS) -I$(CSP_H2O) -I$(CSP_NACL) -I$(MPICH) -I$(MPICHLIB)

##################
# Serial linking #
##################
LIBS = -L$(CSP_LIB) -lcsp3D_basic_intel -lcsp3D_fe_intel -lcsp3D_interfaces_intel \
                    -lcsp3D_interrelations_intel -lcsp3D_cgeom_intel -lcsp3D_applied_math_intel \
                    -lcsp3D_two_phase_intel -lcsp3D_visitors_intel -lcsp3D_eos_intel \
                    -lcsp3D_ecfv_intel -lcsp3D_fv_intel -lcsp3D_ncvfe_intel -lcsp3D_pde_intel \
                    -lcsp3D_solver_intel -lcsp3D_applied_math_intel -lcsp3D_basic_intel -lcsp3D_fe_intel \
                    -lcsp3D_interfaces_intel -lcsp3D_algorithms_intel -lcsp3D_cgeom_intel \
                    -lcsp3D_applied_math_intel -lcsp3D_parallel_intel -lcsp3D_interfaces_intel \
                    -lcsp3D_basic_intel -ljpeg_intel -lf2c_intel -lmeschach_intel \
                    -lamg_serial -lmpidummy

###################
# Paralel linking #
###################
LIBP = -L$(CSP_LIB) -lcsp3D_basic_intel -lcsp3D_fe_intel -lcsp3D_interfaces_intel \
                    -lcsp3D_interrelations_intel -lcsp3D_cgeom_intel -lcsp3D_applied_math_intel \
                    -lcsp3D_two_phase_intel -lcsp3D_visitors_intel -lcsp3D_eos_intel \
                    -lcsp3D_ecfv_intel -lcsp3D_fv_intel -lcsp3D_ncvfe_intel -lcsp3D_pde_intel \
                    -lcsp3D_solver_intel -lcsp3D_applied_math_intel -lcsp3D_basic_intel -lcsp3D_fe_intel \
                    -lcsp3D_interfaces_intel -lcsp3D_algorithms_intel -lcsp3D_cgeom_intel \
                    -lcsp3D_applied_math_intel -lcsp3D_parallel_intel -lcsp3D_interfaces_intel \
                    -lcsp3D_basic_intel -ljpeg_intel -lf2c_intel -lmeschach_intel \
                    -lamg -lmpistubs -lfmpich -L$(FCELIB) -lifcore

#####################################
# C++ compiler and MPI C++ wrappter #
#####################################
#CC = icpc
CC = $(MPICHHOME)/bin/mpicxx

########################################
# Compile Options for the C++ compiler #
########################################
OPTIONS = -O3 -DSAMG_UNIX_LINUX -DSAMG_LCASE_USCORE -DNDEBUG -DCSP_WITH_SAMG_SOLVER

##################
# MAKEFILE RULES #
##################
.SUFFIXES: .f .o .cpp

.cpp.o :
        $(CC) -c $(OPTIONS) $(INCFLAG) $<

.c.o :
        $(CC) -c $(OPTIONS) $(INCFLAG) $<

.o.:
        $(CC) $(OPTIONS) $(INCFLAG) -o $@ $<

An example makefile to compile a serial CSMP executable with the Intel compiler

include ../../source_code/make.inc

##########################
# Source Files
##########################
OBJS =  main_seria.o

#########################
# Linking
#########################
serial_exec : $(OBJS)
        $(CC) -o $@ $(OBJS) $(LIBS)
        mv serial_exec ../../runs/serial_folder

clean :
        -rm $(OBJS)

veryclean :
        -rm $(OBJS)
#########################

An example makefile to compile a parallel CSMP executable with the Intel compiler

include ../../source_code/make.inc

##########################
# Source Files
##########################
OBJS = main_parllel.o

#########################
# Linking
#########################
parallel_exec : $(OBJS)
        $(CC) -o $@ $(OBJS) $(LIBP)
        mv parallel_exec ../../runs/parallel_folder

clean :
        -rm $(OBJS)

veryclean :
        -rm $(OBJS)
#########################

Submitting a job to the HWU cluster

Serial run

qsub serial.sh

Parallel run: Flag are parallel environment (-pe) to instruct the cluster which MPI to use (here mpich) and the number of processors (here 4)

qsub -pe mpich 4 parllel.sh

Note that all screen-output will be written to a file named serial.sh.o* or parallel.sh.o*, respectively, where the * denotes the job number assigned by the cluster

Check if your job is running with (replace userid with your user name on the cluster

qstat -u userid

A shell script for serial runs on HWU cluster

#serial.sh
#
#$ -j y
#$ -S /bin/bash
#$ -cwd
#
# set path to SAMG and INTEL libraries
# !!! REPLACE sg100 WITH YOUR USERNAME !!!
if [ -z "${LD_LIBRARY_PATH}" ]
then
    LD_LIBRARY_PATH="/home/sg100/csp/libraries:/opt/intel/cce/9.0/lib:/opt/intel/fce/9.0/lib:/opt/mpich/intel/lib"; export LD_LIBRARY_PATH
else
    LD_LIBRARY_PATH="/home/sg100/csp/libraries:/opt/intel/cce/9.0/lib:/opt/intel/fce/9.0/lib:/opt/mpich/intel/lib:$LD_LIBRARY_PATH"; export LD_LIBRARY_PATH
fi
#
# set path to SAMG license file
if [ -z "${LM_LICENSE_FILE}" ]
then
    LM_LICENSE_FILE="@hwcluster.hw.ac.uk"; export LM_LICENSE_FILE
else
    LM_LICENSE_FILE="@hwcluster.hw.ac.uk:$LM_LICENSE_FILE"; export LM_LICENSE_FILE
fi
#
# set path to MPICH directory
if [ -z "${PATH}" ]
then
    PATH="/opt/mpich/intel/bin"; export PATH
else
    PATH="/opt/mpich/intel/bin:$PATH"; export PATH
fi
#
# echo the different variables set in this script
echo $PATH
echo $LM_LICENSE_FILE
echo $LD_LIBRARY_PATH
#
# set variable for current directory
CURRENTDIR=`pwd`
#
# execute program
date
$CURRENTDIR/my_serial_executable 
date

A shell script for parallel runs on HWU cluster

#parallell.sh
#
#$ -j y
#$ -S /bin/bash
#$ -cwd
#
# set path to SAMG and INTEL libraries
# !!! REPLACE sg100 WITH YOUR USERNAME !!!
if [ -z "${LD_LIBRARY_PATH}" ]
then
    LD_LIBRARY_PATH="/home/sg100/csp/libraries:/opt/intel/cce/9.0/lib:/opt/intel/fce/9.0/lib:/opt/mpich/intel/lib"; export LD_LIBRARY_PATH
else
    LD_LIBRARY_PATH="/home/sg100/csp/libraries:/opt/intel/cce/9.0/lib:/opt/intel/fce/9.0/lib:/opt/mpich/intel/lib:$LD_LIBRARY_PATH"; export LD_LIBRARY_PATH
fi
#
# set path to SAMG license file
if [ -z "${LM_LICENSE_FILE}" ]
then
    LM_LICENSE_FILE="@hwcluster.hw.ac.uk"; export LM_LICENSE_FILE
else
    LM_LICENSE_FILE="@hwcluster.hw.ac.uk:$LM_LICENSE_FILE"; export LM_LICENSE_FILE
fi
#
# set path to MPICH directory
if [ -z "${PATH}" ]
then
    PATH="/opt/mpich/intel/bin"; export PATH
else
    PATH="/opt/mpich/intel/bin:$PATH"; export PATH
fi
#
# echo the different variables set in this script
echo $PATH
echo $LM_LICENSE_FILE
echo $LD_LIBRARY_PATH
#
# set variable for current directory
CURRENTDIR=`pwd`
#
# execute program
date
/opt/mpich/intel/bin/mpirun -np $NSLOTS -machinefile $TMPDIR/machines $CURRENTDIR/my_parallel_executable 
date

Keeping executables running on Sonora

To avoid that your simulation terminates on Sonora once Sonora logs you out after an extended period of inactivity, start all your simulations with

nohup ./my_executable &

Releasing a blocked SAMG license on HWU cluster

If a parallel simulation crashes, it can happen that FLEXlm, the server that handles the SAMG license, does not check in the SAMG license currently used. Hence you may find yourself running out of SAMG licenses quickly. Here is how to solve this problem

1. Use lmutil lmstat -a to check who is using which SAMG license. The output will look like:

joe nirvana /dev/ttyp5 (v1.000) (cloud9/7654 102), start Fri 10/29 18:40

In this example, the user is “joe,” the user host is “nirvana,” the display is “/dev/typp5,” the server host is “cloud9,” the TCP/IP port is “7654,” and the license handle is “102.”

2. Use lmutil lmremove to remove a blocked license that was not checked back in. There are two options

lmremove [-c license_file_list] feature user user_host display or lmremove [-c license_file_list] -h feature server_host port handle

Where license_file_list is the samg-license-file

Installing new SAMGp on the cluster

A new SAMGp version does not contain any MPI calls inside it anymore. Instead it uses the additional wrapper-functions that allow SAMGp to be independent of the existing MPI implementation. The trade-off is that you have to compile those wrapper-functions yourself, using the MPI implementation installed on your computer. Here is how you do this. Note: the following instructions assume that you have a new SAMGp ".tar.gz" file.

1. Make sure that MPICHHOME and FCEHOME variables are defined in your .bashrc file:

if [ -z "${MPICHHOME}" ]
then
    MPICHHOME="/opt/mpich/intel"; export MPICHHOME
else
    MPICHHOME="/opt/mpich/intel:$MPICHHOME"; export MPICHHOME
fi

if [ -z "${FCEHOME}" ]
then
    FCEHOME="/opt/intel/fce/9.0"; export FCEHOME
else
    FCEHOME="/opt/intel/fce/9.0:$FCEHOME"; export FCEHOME
fi

This is merely a path to the MPI and Fortran libraries used for the subsequent compilation procedure.

1. Unzip the archive file and open the "make.inc" file (do not mix it with the "make.inc" for CSMP). Make sure that the following things are set properly (values in brackets correspond to HWU cluster settings):
   • FC - Fortran compiler (ifort)
   • MPIF90 - MPI wrapper for Fortran compiler ($(MPICHHOME)/bin/mpif90)
   • RTS_PATH - path to the Fortran Runtime libraries (commented out, as it is part of the $LD_LIBRARY_PATH environment variable)
   • CC - C++ compiler (icpc)
   • MPICC - MPI wrapper for the C++ compiler ($(MPICHHOME)/bin/mpicxx)
   • EXTRA - contains the additional libraries for linking (-ldl -lstdc++ -L$(FCEHOME)/lib -lifcore, the path to libifcore is explicitly specified because in case both icpc and ifort compilers are installed in the system, the compiler cannot automatically locate this library for some reason)
2. Go inside the "mpi" folder and run "make". Two libraries should appear: "libmpidummy.a" and "libmpistubd.a". Dummy is needed for the serial calculations and stubs for the parallel. Copy those 2 files along with the "libamg.so" and "libamg_serial.so" from the "samg" folder into the csp/libraries folder.
3. Recompile the "Solver" and "Parallel" CSMP libraries using the -DSAMG_LCASE_USCORE compile option, ideally by including the above make.inc in the makefiles
4. Run the above makefiles, along with the make.inc file, for your (serial or parallel) project and pray to Cthulhu!
5. If you want to keep using your old project makefiles (i.e., the one for creating an executable) you need to do the following
   • Use the flag -DSAMG_LCASE_USCORE
   • Include the paths to /opt/intel/mpich/lib and /opt/intel/mpich/include
   • Link the following libraries after the CSMP libraries: -lamg_serial -lmpidummy for serial projects and -lamg -lmpistubs -lfmpich -L$(FCEHOME)/lib -lifcore for parallel projects

Resolving license issues with CodeWarrior IDE

On the IPE network, licenses are hosted on a server. CodeWarrior, however, uses a license that is installed on our local PC. Per default, it looks for its license on the IPE licenserver (instead of the local license on your PC), which can cause it to respond slowly or hang for a couple of minutes from time to time. Here is how this issue can be resolved:

1. Right-click on "My Computer", select Properties, and go to tab "Advanced"
2. Click on "Environment Variables"
   
3. Click on the entry "LM_LICENSE_FILE" in the top Window (your personal user variables) and click "Edit"
   
4. Change the entry to C:\Program Files\Metrowerks\CodeWarrior\license.dat;33778@localhost;7321@localhost;27000@localhost;@petlicense This must be the exact entry (without any quotes etc.). Make sure that you actually installed CodeWarrior in the directory C:\Program Files\Metrowerks\CodeWarrior, otherwise you need to amend this such that the path before "license.dat" points to the directory where your CodeWarrior license is located.

Using Qt Creator IDE for compiling and debugging CSMP

We have been able to compile CSMP under Windows 32 bit with the free Qt Creator IDE and its bundled mingw (gcc 4.4) compiler. The relevant projects are now in the SVN (for the CSMP libraries, the support libraries, and the three tutorials to demonstrate how executables can be built). The debugger is also on par with CodeWarrior. As Qt Creator is a cross platform IDE, the project files should be working on Linux too, with amending the config.prj file in csmp/source_code. Qt Creator has not been built for Windows 64 bit yet but it appears that there is no problem creating 32-bit executables under Windows 64 bit and we keep investigating. Below are some basic instructions how to get Qt Creator and CSMP running on you

1. Download Qt Creator from Nokia's Qt website and install it on your PC.
2. Update your CSMP version from the SVN to obtain the project files and latest changes to the source code (very minor).
3. In each subdirectory of source_code, support_libraries, and tutorials you will find a new directory which bears the same name as the subdirectory. It contains the Qt Creator *.pro project file for the library/executable to be built.
4. Open the config.prj file in source_code and set the CSMP_HOME variable to your csmp directory. If you have built MPI on your system too, then you may want to set the MPICHHOME variable as well. Otherwise it is fine to have it point to the MPI directory of CSMP in support_libraries. NB: This config.prj file can be amended as needed to change settings for your PC (e.g. to create trial versions of CSMP, add further test directories etc.)
5. Open a *.pro file in, say, one of the subdirectories of source_code. A window will pop up and look like this (Qt Creator 2.0)
   

   or this (Qt Creator 2.2, click on image to enlarge)

   

   For Qt Creator 2.0 uncheck the "Qt in PATH" option if it appears (it may not be present, on one PC I have it, on the other one I don't…). For Qt Creator 2.2 you may wish to uncheck the shadow build option.

6. Compile the release and debug version of the library by changing the build setting in Qt Creator (button on the lower left panel of Qt Creator). The compiled library will be moved to the subdirectories debug and release of csmp/libraries. Note that there are no *.pro files for computer_science, generic_element_centered_finite_volumes, and distributed_examples. One should create a Qt project file for the latter. The two former libraries are not really needed to run CSMP.
7. Compile all CSMP and support libraries (f2c, jpeg, and meschach — if run in parallel, MPI and Metis come with their own makefiles), as well as the three tutorials. Note that the order at which you compile the (support-)libraries does not matter as long as all the libraries are compiled (both in debug and release mode) before your compile any executable.
8. Note that the output directory (defined by the variable DESTDIR) is overwritten in the three tutorials to move the executables into the tutorial directories and not into the libraries directory, as specified originally in the config.prj file.
9. You can run your executables directly within Qt Creator. If you want to run them from the DOS command prompt, cygwin, via double clicking, etc. you need to set a PATH to the mingwm10.dll, which is needed at run time. The file is located in the subdirectory qt\bin of your Qt Creator installation directory for QtCreator version 2.0 or mingw\bin for QtCreator version 2.2 (in my case C:\Qt\qt\bin or C:\Qt\mingw\bin). To add the PATH, right-click on My Computer, go to Properties. Open the tab "Advanced".
   

   Click on "Environment Variables", edit the PATH setting in the user variables for your account and add the path to the location of the mingwm10.dll file.

   

   Note that this may differ for Windows 7 or Vista. Note that samg_dyn_noomp.dll must be located in the Windows system directory (e.g. C:\WINDOWS\system32 for XP 32-bit) where Windows is looking for dll librarys per default at runtime. Alternatively, you can add the directory where your samg_dyn_noomp.dll is located to PATH too (obviousyl, this only applies for Windows, not for Linux).

10. Enjoy!
11. PS: Gillian pointed out a very few minor issues, see the relevant forum section for further tips and add discussions, e.g. how to compile native 64-bit code with Qt Creator under Windows and the like.
12. PPS: Qt Creator 2.2 allows you straightforwardly to include compilers other than the MinGW that is shipped with Qt Creator. This opens the door for easy compilation of 64 bit executables under Windows. Make sure that you have downloaded Qt Creator 2.2 for 64 bit Windows. Also download the 64 bit build for MinGW from Equation solution. Install the 64 bit MinGW in C:\MinGW, it will automatically add itself to the PATH and you can check if it is available on your Windows PC by opening the command prompt and entering gcc -v, which should show that you have gcc and g++ version 4.6.x installed. When you launch Qt Creator 2.2, go to Tools -> Options -> Tool Chains and you should see the MinGW you just installed as an alternative to the MinGW compiler that comes with Qt Creator. Note that restarting the computer may be needed, it's Windows after all… (click on image to enlarge).
    

    You can then select your favorite compiler under the settings of the particluar project (click on image to enlarge).

    

    There may be some way to choose the default compiler, but I have not worked this out. Also, see this explanation for further information. In principle, you can add any C++ compiler that you want. Importantly, don't forget that you will need to link a 64 bit CSMP version against a 64 bit SAMG version.