如何在Linux上为netCDF4安装I/O并行？

Question

我正在尝试为netCDF4安装并行I/O。我正在运行的脚本如下：

#!/bin/bash

module purge
module load gcc/5.3.1
module load mpich/3.2

# zlib
cd /storage/home/htn5098/
mkdir -p local_lib/R40
cd local_lib/R40  
wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4/zlib-1.2.8.tar.gz
tar -xf zlib-1.2.8.tar.gz && cd zlib-1.2.8
./configure --prefix=../zlib
make install
cd ../zlib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib
cd ..
rm zlib-1.2.8.tar.gz
rm zlib-1.2.8 -rf

# HDF5
wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4/hdf5-1.8.13.tar.gz
tar -xf hdf5-1.8.13.tar.gz && cd hdf5-1.8.13
prefix="/storage/work/htn5098/local_lib/R40/hdf5"
export HDF5_DIR=$prefix
./configure --enable-shared --enable-hl --enable-parallel --prefix=$HDF5_DIR CFLAGS=-fPIC # important for parallel processing 
#./configure --enable-shared --enable-hl --prefix=$HDF5_DIR
make -j4
make install
cd $HDF5_DIR
export CPPFLAGS=-I$HDF5_DIR/include 
export LDFLAGS=-L$HDF5_DIR/lib 
cd ..
rm hdf5-1.8.13.tar.gz
rm hdf5-1.8.13 -rf

# NETCDF4
wget http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-c-4.7.3.tar.gz
tar -xf netcdf-c-4.7.3.tar.gz && cd netcdf-c-4.7.3
prefix="/storage/work/htn5098/local_lib/R40/netcdf"
export NETCDF4_DIR=$prefix 
./configure --enable-netcdf-4 -with-gnu-ld --enable-logging --enable-shared --enable-dap --enable-parallel-tests --prefix=$NETCDF4_DIR 
make -j4
make install
cd $prefix
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib
export PATH=$PATH:$PWD/bin
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/storage/work/htn5098/local_lib/R40/netcdf/lib/pkgconfig
cd ..
rm netcdf-c-4.7.3.tar.gz
rm netcdf-c-4.7.3 -rf

# R ncdf4 package
module load r
cd ~/work/local_lib/R40
wget http://cirrus.ucsd.edu/~pierce/ncdf/ncdf4_1.13.tar.gz
R CMD INSTALL ncdf4_1.13.tar.gz --library=.
rm ncdf4_1.13.tar.gz

然而，我一直收到这个错误：

checking hdf5.h presence... no
checking for hdf5.h... no
configure: error: Compiling a test with HDF5 failed.  Either hdf5.h cannot be found, or config.log should be checked for other reason.

我认为这可能是HDF5库路径不与netCDF4共享的情况。你能告诉我如何为netCDF4编译HDF5来实现并行吗？

非常感谢。

Answer 1

这是我通常在我的Ubuntu服务器上使用的。它最近也在一台OpenSuse机器上工作，只需最少的修改。

请注意，有多次手动插入安装目录、/opt/software作为LDFLAGS中的参数，等等。您还需要根据您的特定情况进行调整。

# export installation directorie
export INSDIR=/opt/software

1) SZIP v2.1.1 - https://support.hdfgroup.org/doc_resource/SZIP

./configure --prefix=${INSDIR} \
    CC=gcc \
    CFLAGS="-march=native -g -O2 -ansi -pedantic -Wall -W -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wcast-align -Wwrite-strings -Wconversion -Wsign-compare -Waggregate-return -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wnested-externs -Winline -O -fomit-frame-pointer -finline-functions"

make && sudo make install && make all check

-----------------------------------------------------------------------------------
2) HDF5 v1.12.0 - https://www.hdfgroup.org/downloads/hdf5/source-code

# parallel version
./configure --prefix=${INSDIR} --disable-dependency-tracking \
    --disable-maintainer-mode --disable-silent-rules \
    --build=x86_64-linux-gnu --host=x86_64-linux-gnu \
    --with-pthread --enable-hl --enable-shared \
    --enable-build-mode=production --with-zlib \
    --enable-parallel --with-szlib=${INSDIR} \
    --disable-threadsafe --disable-fortran --disable-cxx \
    CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx FC=/usr/bin/mpifort \
    "CFLAGS=-march=native -g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
    "FCFLAGS=-march=native -g -O2 -fstack-protector-strong" \
    "CXXFLAGS=-march=native -g -O2 -fstack-protector-strong -Wformat -Werror=format-security"

make && sudo make install && sudo make check-install

-----------------------------------------------------------------------------------
3) NetCDF C v4.7.4 - https://www.unidata.ucar.edu/downloads/netcdf/index.jsp

# parallel version
./configure -prefix=${INSDIR} --enable-shared --enable-parallel-tests \
    CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx \
    "CFLAGS=-march=native -g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64" \
    "CPPFLAGS=-I/opt/software/include" \
    "LDFLAGS=-L/opt/software/lib" \
    "LIBS=-lz -ldl -lm -lcurl -lpthread -L/opt/software/lib -lsz -lhdf5_hl -lhdf5"

make && sudo make install && sudo make check

-----------------------------------------------------------------------------------
4) NetCDF Fortran v4.5.3 - https://www.unidata.ucar.edu/downloads/netcdf/index.jsp

# parallel version
./configure --prefix=${INSDIR} --enable-parallel-tests \
    CC=/usr/bin/mpicc FC=/usr/bin/mpifort F77=/usr/bin/mpifort \
    "CFLAGS=-march=native -g -O2 -Wdate-time -D_FORTIFY_SOURCE=2" \
    "FCFLAGS=-march=native -g -O2 " \
    "CPPFLAGS=-I/opt/software/include" \
    "LDFLAGS=-L/opt/software/lib" \
    "LIBS=-L/opt/software/lib -lnetcdf -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now"

make && sudo make install && sudo make check

此外，请注意，在这种情况下启用并行特性的关键步骤不是使用make -j4，而是手动通知您希望使用MPI编译器来构建库。这意味着必须显式地将CC、CXX、FC等标志定义为mpicc版本(mpicc、mpicxx、mpifort等)。

这应该是可行的，但是请注意make check的输出，看看构建的库到底发生了什么。