WRF (GNU+MPICH) v1
Background Material
- https://www.mmm.ucar.edu/weather-research-and-forecasting-model
- http://www2.mmm.ucar.edu/wrf/users/supports/tutorial.html
- http://www2.mmm.ucar.edu/wrf/users/tutorial/tutorial_presentation_winter...
Staring with http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compilation_tutorial.php
customized to the GreenPlanet environment $HOME (your home directory).
Copy-paste friendly instructions (copy-paste "paragraphs" and inspect the output of each series of commands)
http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compilation_tutorial.php#STEP1
### SYSTEM ENVIRONMENT TESTS ###
# should have named WRF Build_WRF to match original docs!!! - ugh
# use one of the these (SCRATCH => proof of concept, DATA => once it works):
#
# cd /beegfs/SCRATCH/$(id -gn)/$(id -un)
# cd /beegfs/DATA/$(id -gn)/$(id -un)
### for gplogin2 ###
ml purge
ml gnu/7.3.0
### for gplogin2 ###
### for gplogin1/3 ###
module purge
module load compiler/gnu/7.1.0
### for gplogin1/3 ###
### for hpc-login-X-Y ###
module purge
module load gcc/4.8.1
module load mpich-3.0.4/gcc-4.8.1
### for hpc-login-X-Y ###
(mkdir WRF 2>&1 )> /dev/null || echo "WRF already exists"
cd WRF
(mkdir Fortran_tests 2>&1 )> /dev/null || echo "Fortran_tests already exists"
cd Fortran_tests
wget -c http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/Fortran_C_tests.tar
tar xvf Fortran_C_tests.tar
gfortran TEST_1_fortran_only_fixed.f
./a.out
gfortran TEST_2_fortran_only_free.f90
./a.out
gfortran TEST_3_c_only.c
./a.out
gcc -c -m64 TEST_4_fortran+c_c.c
gfortran -c -m64 TEST_4_fortran+c_f.f90
gfortran -m64 TEST_4_fortran+c_f.o TEST_4_fortran+c_c.o
./a.outhttp://www2.mmm.ucar.edu/wrf/OnLineTutorial/compilation_tutorial.php#STEP2
#### BUILDING LIBRARIES ####
cd ..
wget -c http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/mpich-3.0.4.tar.gz \
http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/netcdf-4.1.3.tar.gz \
http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/jasper-1.900.1.tar.gz \
http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/libpng-1.2.50.tar.gz \
http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/zlib-1.2.7.tar.gz
(mkdir LIBRARIES 2>&1 )> /dev/null || echo "LIBRARIES already exists"
mv jasper-1.900.1.tar.gz libpng-1.2.50.tar.gz mpich-3.0.4.tar.gz netcdf-4.1.3.tar.gz zlib-1.2.7.tar.gz LIBRARIES/
cd LIBRARIES
#### NetCDF ####
## first section should be realized as follows since our default shell is bash
## GP-HPC: one of these
#export DIR=~/WRF/LIBRARIES
#export DIR=/beegfs/SCRATCH/$(id -gn)/$(id -un)/WRF/LIBRARIES
#export DIR=/beegfs/DATA/$(id -gn)/$(id -un)/WRF/LIBRARIES
## OIT-HPC: one of these
#export DIR=/fast-scratch/$(id -un)/WRF/LIBRARIES
#export DIR=/data/users/$(id -un)/WRF/LIBRARIES
# check
echo $DIR
echo $CC
echo $CXX
echo $FC
echo $FCFLAGS
echo $F77
echo $FFLAGS
# set
export CC=$(which gcc)
export CXX=$(which g++)
export FC=$(which gfortran)
export FCFLAGS=-m64
export F77=$(which gfortran)
export FFLAGS=-m64
# trust but verify
echo $DIR
echo $CC
echo $CXX
echo $FC
echo $FCFLAGS
echo $F77
echo $FFLAGS
# NetCDF
tar xzvf netcdf-4.1.3.tar.gz #or just .tar if no .gz present
cd netcdf-4.1.3
./configure --prefix=$DIR/netcdf --disable-dap --disable-netcdf-4 --disable-shared 2>&1 | tee configure-netcdf.txt
make 2>&1 | tee make-netcdf.txt
make install 2>&1 | tee make-install-netcdf.txt
export PATH=$DIR/netcdf/bin:$PATH
export NETCDF=$DIR/netcdf
echo $PATH
echo $NETCDF
cd ..
# MPICH
tar xzvf mpich-3.0.4.tar.gz #or just .tar if no .gz present
cd mpich-3.0.4
### ADDITIONAL STEPS REQUIRED - this is where we deviate from the provided instructions
# this failed:
#
# ./configure --prefix=$DIR/mpich
# failed with this being the last line
# configure: error: F90 and F90FLAGS are replaced by FC and FCFLAGS respectively in this configure, please unset F90/F90FLAGS and set FC/FCFLAGS instead and rerun configure again.
echo $F90
echo $F90FLAGS
export F90=
FC=$(which gfortran) FCFLAGS=-m64 ./configure --prefix=$DIR/mpich 2>&1 | tee ./configure-mpich.txt
make 2>&1 | tee make-mphich.txt
make install 2>&1 | tee make-install-mphich.txt
export PATH=$DIR/mpich/bin:$PATH
echo $PATH
cd ..
# zlib
export LDFLAGS=-L$DIR/grib2/lib
export CPPFLAGS=-I$DIR/grib2/include
tar xzvf zlib-1.2.7.tar.gz #or just .tar if no .gz present
cd zlib-1.2.7
./configure --prefix=$DIR/grib2
make
make install
cd ..
# libpng
tar xzvf libpng-1.2.50.tar.gz #or just .tar if no .gz present
cd libpng-1.2.50
./configure --prefix=$DIR/grib2
make
make install
cd ..
# JasPer
tar xzvf jasper-1.900.1.tar.gz #or just .tar if no .gz present
cd jasper-1.900.1
./configure --prefix=$DIR/grib2
make
make install
cd .. http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compilation_tutorial.php#STEP3
#### LIBRARY COMPATIBILITY TESTS ####
(mkdir TESTS 2>&1 )> /dev/null || echo "TESTS already exists"
cd TESTS
wget -c http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compile_tutorial/tar_files/Fortran_C_NETCDF_MPI_tests.tar
tar -xvf Fortran_C_NETCDF_MPI_tests.tar
# Test #1: Fortran + C + NetCDF
cp ${NETCDF}/include/netcdf.inc .
gfortran -c 01_fortran+c+netcdf_f.f
gcc -c 01_fortran+c+netcdf_c.c
gfortran 01_fortran+c+netcdf_f.o 01_fortran+c+netcdf_c.o \
-L${NETCDF}/lib -lnetcdff -lnetcdf
./a.out
#Should display
#C function called by Fortran
#Values are xx = 2.00 and ii = 1
#SUCCESS test 1 fortran + c + netcdf
# Test #2: Fortran + C + NetCDF + MPI
cp ${NETCDF}/include/netcdf.inc .
mpif90 -c 02_fortran+c+netcdf+mpi_f.f
mpicc -c 02_fortran+c+netcdf+mpi_c.c
mpif90 02_fortran+c+netcdf+mpi_f.o \
02_fortran+c+netcdf+mpi_c.o \
-L${NETCDF}/lib -lnetcdff -lnetcdf
mpirun ./a.out
#Should display
#C function called by Fortran
#Values are xx = 2.00 and ii = 1
#status = 2
#SUCCESS test 2 fortran + c + netcdf + mpi
http://www2.mmm.ucar.edu/wrf/OnLineTutorial/compilation_tutorial.php#STEP4
#### Building WRFV3 #### -
cd ..
wget -c http://www2.mmm.ucar.edu/wrf/src/WRFV3.9.1.1.TAR.gz
tar zxvf WRFV3.9.1.1.TAR.gz
cd WRFV3
Exceptions to the initially referenced
# if file output > 2GB
export WRFIO_NCD_LARGE_FILE_SUPPORT=1
# else
#export WRFIO_NCD_LARGE_FILE_SUPPORT=0
export WRF_CHEM=0
export EM_CORE=1
export NMM_CORE=0
echo $WRF_CHEM
echo $EM_CORE
echo $NMM_CORE
# for WRF>=3.9
#./configure -hyb
# else
#./configure
./configure -hyb
checking for perl5... no
checking for perl... found /bin/perl (perl)
Will use NETCDF in dir: /export/home/mgorris/WRF/LIBRARIES/netcdf
HDF5 not set in environment. Will configure WRF for use without.
PHDF5 not set in environment. Will configure WRF for use without.
Will use 'time' to report timing information
If you REALLY want Grib2 output from WRF, modify the arch/Config_new.pl script.
Right now you are not getting the Jasper lib, from the environment, compiled into WRF.
------------------------------------------------------------------------
Please select from among the following Linux x86_64 options:
1. (serial) 2. (smpar) 3. (dmpar) 4. (dm+sm) PGI (pgf90/gcc)
5. (serial) 6. (smpar) 7. (dmpar) 8. (dm+sm) PGI (pgf90/pgcc): SGI MPT
9. (serial) 10. (smpar) 11. (dmpar) 12. (dm+sm) PGI (pgf90/gcc): PGI accelerator
13. (serial) 14. (smpar) 15. (dmpar) 16. (dm+sm) INTEL (ifort/icc)
17. (dm+sm) INTEL (ifort/icc): Xeon Phi (MIC architecture)
18. (serial) 19. (smpar) 20. (dmpar) 21. (dm+sm) INTEL (ifort/icc): Xeon (SNB with AVX mods)
22. (serial) 23. (smpar) 24. (dmpar) 25. (dm+sm) INTEL (ifort/icc): SGI MPT
26. (serial) 27. (smpar) 28. (dmpar) 29. (dm+sm) INTEL (ifort/icc): IBM POE
30. (serial) 31. (dmpar) PATHSCALE (pathf90/pathcc)
32. (serial) 33. (smpar) 34. (dmpar) 35. (dm+sm) GNU (gfortran/gcc)
36. (serial) 37. (smpar) 38. (dmpar) 39. (dm+sm) IBM (xlf90_r/cc_r)
40. (serial) 41. (smpar) 42. (dmpar) 43. (dm+sm) PGI (ftn/gcc): Cray XC CLE
44. (serial) 45. (smpar) 46. (dmpar) 47. (dm+sm) CRAY CCE (ftn $(NOOMP)/cc): Cray XE and XC
48. (serial) 49. (smpar) 50. (dmpar) 51. (dm+sm) INTEL (ftn/icc): Cray XC
52. (serial) 53. (smpar) 54. (dmpar) 55. (dm+sm) PGI (pgf90/pgcc)
56. (serial) 57. (smpar) 58. (dmpar) 59. (dm+sm) PGI (pgf90/gcc): -f90=pgf90
60. (serial) 61. (smpar) 62. (dmpar) 63. (dm+sm) PGI (pgf90/pgcc): -f90=pgf90
64. (serial) 65. (smpar) 66. (dmpar) 67. (dm+sm) INTEL (ifort/icc): HSW/BDW
68. (serial) 69. (smpar) 70. (dmpar) 71. (dm+sm) INTEL (ifort/icc): KNL MIC
72. (serial) 73. (smpar) 74. (dmpar) 75. (dm+sm) FUJITSU (frtpx/fccpx): FX10/FX100 SPARC64 IXfx/Xlfx
Enter selection [1-75] : 34
------------------------------------------------------------------------
Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]: 1
Configuration successful!
------------------------------------------------------------------------
testing for MPI_Comm_f2c and MPI_Comm_c2f
MPI_Comm_f2c and MPI_Comm_c2f are supported
testing for fseeko and fseeko64
fseeko64 is supported
------------------------------------------------------------------------
# Settings for Linux x86_64 ppc64le, gfortran compiler with gcc (dmpar)
#
DESCRIPTION = GNU ($SFC/$SCC)
DMPARALLEL = 1
OMPCPP = # -D_OPENMP
OMP = # -fopenmp
OMPCC = # -fopenmp
SFC = gfortran
SCC = gcc
CCOMP = gcc
DM_FC = mpif90 -f90=$(SFC)
DM_CC = mpicc -cc=$(SCC) -DMPI2_SUPPORT
FC = time $(DM_FC)
CC = $(DM_CC) -DFSEEKO64_OK
LD = $(FC)
RWORDSIZE = $(NATIVE_RWORDSIZE)
PROMOTION = #-fdefault-real-8
ARCH_LOCAL = -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM
CFLAGS_LOCAL = -w -O3 -c
LDFLAGS_LOCAL =
CPLUSPLUSLIB =
ESMF_LDFLAG = $(CPLUSPLUSLIB)
FCOPTIM = -O2 -ftree-vectorize -funroll-loops
FCREDUCEDOPT = $(FCOPTIM)
FCNOOPT = -O0
FCDEBUG = # -g $(FCNOOPT) # -ggdb -fbacktrace -fcheck=bounds,do,mem,pointer -ffpe-trap=invalid,zero,overflow
FORMAT_FIXED = -ffixed-form
FORMAT_FREE = -ffree-form -ffree-line-length-none
FCSUFFIX =
BYTESWAPIO = -fconvert=big-endian -frecord-marker=4
FCBASEOPTS_NO_G = -w $(FORMAT_FREE) $(BYTESWAPIO)
FCBASEOPTS = $(FCBASEOPTS_NO_G) $(FCDEBUG)
MODULE_SRCH_FLAG =
TRADFLAG =
CPP = /lib/cpp -P -C -nostdinc
AR = ar
ARFLAGS = ru
M4 = m4 -G
RANLIB = ranlib
RLFLAGS =
CC_TOOLS = $(SCC)
###########################################################
######################
# POSTAMBLE
FGREP = fgrep -iq
ARCHFLAGS = $(COREDEFS) -DIWORDSIZE=$(IWORDSIZE) -DDWORDSIZE=$(DWORDSIZE) -DRWORDSIZE=$(RWORDSIZE) -DLWORDSIZE=$(LWORDSIZE) \
$(ARCH_LOCAL) \
$(DA_ARCHFLAGS) \
-DDM_PARALLEL \
\
-DNETCDF \
\
\
\
\
\
\
\
\
\
\
-DUSE_ALLOCATABLES \
-DGRIB1 \
-DINTIO \
-DKEEP_INT_AROUND \
-DLIMIT_ARGS \
-DCONFIG_BUF_LEN=$(CONFIG_BUF_LEN) \
-DMAX_DOMAINS_F=$(MAX_DOMAINS) \
-DMAX_HISTORY=$(MAX_HISTORY) \
-DNMM_NEST=$(WRF_NMM_NEST)
CFLAGS = $(CFLAGS_LOCAL) -DDM_PARALLEL \
-DMAX_HISTORY=$(MAX_HISTORY) -DNMM_CORE=$(WRF_NMM_CORE)
FCFLAGS = $(FCOPTIM) $(FCBASEOPTS)
ESMF_LIB_FLAGS =
# ESMF 5 -- these are defined in esmf.mk, included above
ESMF_IO_LIB = -L$(WRF_SRC_ROOT_DIR)/external/esmf_time_f90 -lesmf_time
ESMF_IO_LIB_EXT = -L$(WRF_SRC_ROOT_DIR)/external/esmf_time_f90 -lesmf_time
INCLUDE_MODULES = $(MODULE_SRCH_FLAG) \
$(ESMF_MOD_INC) $(ESMF_LIB_FLAGS) \
-I$(WRF_SRC_ROOT_DIR)/main \
-I$(WRF_SRC_ROOT_DIR)/external/io_netcdf \
-I$(WRF_SRC_ROOT_DIR)/external/io_int \
-I$(WRF_SRC_ROOT_DIR)/frame \
-I$(WRF_SRC_ROOT_DIR)/share \
-I$(WRF_SRC_ROOT_DIR)/phys \
-I$(WRF_SRC_ROOT_DIR)/chem -I$(WRF_SRC_ROOT_DIR)/inc \
-I$(NETCDFPATH)/include \
REGISTRY = Registry
CC_TOOLS_CFLAGS = -DNMM_CORE=$(WRF_NMM_CORE)
LIB_BUNDLED = \
$(WRF_SRC_ROOT_DIR)/external/fftpack/fftpack5/libfftpack.a \
$(WRF_SRC_ROOT_DIR)/external/io_grib1/libio_grib1.a \
$(WRF_SRC_ROOT_DIR)/external/io_grib_share/libio_grib_share.a \
$(WRF_SRC_ROOT_DIR)/external/io_int/libwrfio_int.a \
$(ESMF_IO_LIB) \
$(WRF_SRC_ROOT_DIR)/external/RSL_LITE/librsl_lite.a \
$(WRF_SRC_ROOT_DIR)/frame/module_internal_header_util.o \
$(WRF_SRC_ROOT_DIR)/frame/pack_utils.o
LIB_EXTERNAL = \
-L$(WRF_SRC_ROOT_DIR)/external/io_netcdf -lwrfio_nf -L/export/home/mgorris/WRF/LIBRARIES/netcdf/lib -lnetcdff -lnetcdf
LIB = $(LIB_BUNDLED) $(LIB_EXTERNAL) $(LIB_LOCAL) $(LIB_WRF_HYDRO)
LDFLAGS = $(OMP) $(FCFLAGS) $(LDFLAGS_LOCAL)
ENVCOMPDEFS = -DHYBRID_COORD=1
WRF_CHEM = 0
CPPFLAGS = $(ARCHFLAGS) $(ENVCOMPDEFS) -I$(LIBINCLUDE) $(TRADFLAG)
NETCDFPATH = /export/home/mgorris/WRF/LIBRARIES/netcdf
HDF5PATH =
WRFPLUSPATH =
RTTOVPATH =
PNETCDFPATH =
bundled: io_only
external: io_only $(WRF_SRC_ROOT_DIR)/external/RSL_LITE/librsl_lite.a gen_comms_rsllite module_dm_rsllite $(ESMF_TARGET)
io_only: esmf_time wrfio_nf \
wrf_ioapi_includes wrfio_grib_share wrfio_grib1 wrfio_int fftpack
######################
------------------------------------------------------------------------
Settings listed above are written to configure.wrf.
If you wish to change settings, please edit that file.
If you wish to change the default options, edit the file:
arch/configure_new.defaults
NetCDF users note:
This installation of NetCDF supports large file support. To DISABLE large file
support in NetCDF, set the environment variable WRFIO_NCD_NO_LARGE_FILE_SUPPORT
to 1 and run configure again. Set to any other value to avoid this message.
Testing for NetCDF, C and Fortran compiler
This installation of NetCDF is 64-bit
C compiler is 64-bit
Fortran compiler is 64-bit
It will build in 64-bit
./compile em_real 2>&1 | tee log.compile
cd ../..Building WPS
wget http://www2.mmm.ucar.edu/wrf/src/WPSV3.9.1.TAR.gz
tar zxvf WPSV3.9.1.TAR.gz
cd WPS
./clean
export JASPERLIB=$DIR/grib2/lib
export JASPERINC=$DIR/grib2/include
./configure
# enter 1
./compile 2>&1 | tee log.compile
cd ..Acquire Static Geography Data
wget http://www2.mmm.ucar.edu/wrf/src/wps_files/geog_complete.tar.gz
wget http://www2.mmm.ucar.edu/wrf/src/wps_files/geog_new3.9.tar.bz2
tar zxvf geog_complete.tar.gz
mv geog WPS_GEOG
cd WPS_GEOG
tar jxvf ../geog_new3.9.tar.bz2
Skipping "Real-Time Data" section since it already exists on disk locally
cd ../WPS
Change namelists.wps so
geog_data_path = '/beegfs/DATA/GROUP/USER/WRF/WPS_GEOG/'where GROUP is replaced with output from $(id -gn)
where USER is replaced with output from $(id -un)
Testing
./geogrid.exe 2>&1 | tee log.geogrid
./link_grib.csh /beegfs/DATA/zender/mgorris/WRF.old/wx_data/gfs.0p25.20170524to20170528/
ln -sf ungrib/Variable_Tables/Vtable.GFS Vtable
export LD_LIBRARY_PATH=$DIR/grib2/lib:$LD_LIBRARY_PATH
./ungrib.exe
./metgrid.exe 2>&1 | tee log.metgridRunning WRFV3
cd ../WRFV3/test/em_real
cp -p /beegfs/DATA/zender/mgorris/WRF.old/WRFV3/test/em_real/namelist.input .Edit copied namelist.input by removing &chem section at the end
Need to copy or link your met_em* files into the working directory
ln -sf ../../../WPS/met_em* .
ls -l met_em*
./real.exeThis should produce some output files
wrfinput_d01
wrfbdy_d01
rsl.out.0000
rsl.error.0000da
mpirun -np 1 ./real.exeThis should produce some output files
mpirun -np 8 ./wrf.exe This should produce some output files
Environment variables added which will be needed in a Slurm job
# One of the following depending on where the libraries resides
export DIR=/beegfs/DATA/$(id -gn)/$(id -un)/WRF/LIBRARIES
export DIR=/beegfs/SCRATCH/$(id -gn)/$(id -un)/WRF/LIBRARIES
# all of these verbatim
export PATH=$DIR/mpich/bin:$PATH
export JASPERLIB=$DIR/grib2/lib
export JASPERINC=$DIR/grib2/include
export LDFLAGS=-L$DIR/WRF/LIBRARIES/grib2/lib
export LD_LIBRARY_PATH=$DIR/grib2/lib:$LD_LIBRARY_PATH