HDF5: make much more robust across platforms
This addresses various real-world problems with HDF5 pkg-config, including * hdf*.pc with package versions as part of the filename * malformed hdf*.pc missing the commonly-used HDF5 HL module --- Additionally, this refactors more complicated dependencies such as HDF5 and OpenMPI. This may help us deduplicate internal dependency code in the future. HDF5 selftest: improve platform-agnostic test ci: init demo github action for HDF5 framework ci Actions: hold off on MSYS2 for now [skip ci] hdf5: ensure C libraries always included ci: mac hdf5--use clang+gfortran
This commit is contained in:
Michael Hirsch, Ph.D
Jussi Pakkanen
parent
dd15c47ea8
commit
c02593fddc
|
|
@ -0,0 +1,98 @@
|
|||
# at first, we demo HDF5 framework. More can be added.
|
||||
name: ci_frameworks
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "mesonbuild/dependencies/**"
|
||||
- ".github/workflows/frameworks.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "mesonbuild/dependencies/**"
|
||||
- ".github/workflows/frameworks.yml"
|
||||
|
||||
jobs:
|
||||
|
||||
linux:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- run: python -m pip install -e .
|
||||
- run: sudo apt install -yq --no-install-recommends ninja-build g++ gfortran libhdf5-dev
|
||||
- run: meson setup "test cases/frameworks/25 hdf5" build
|
||||
env:
|
||||
FC: gfortran
|
||||
CXX: g++
|
||||
CC: gcc
|
||||
- run: ninja -C build
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: Linux_Log
|
||||
path: build/meson-logs/meson-log.txt
|
||||
- run: meson test -C build -v
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: Linux_Test
|
||||
path: build/meson-logs/testlog.txt
|
||||
|
||||
mac:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- run: python -m pip install -e .
|
||||
- run: brew install pkg-config ninja gcc hdf5
|
||||
- run: meson setup "test cases/frameworks/25 hdf5" build
|
||||
- run: ninja -C build
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: Mac_Log
|
||||
path: build/meson-logs/meson-log.txt
|
||||
- run: meson test -C build -v
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: Mac_Test
|
||||
path: build/meson-logs/testlog.txt
|
||||
|
||||
windows:
|
||||
# as usual, start in MSYS to install packages, then switch to MINGW64 to build.
|
||||
if: false
|
||||
# MSYS2 GitHub Action is still being developed--appears to have PATH bugs stopping this from working.
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: '3.x'
|
||||
- uses: numworks/setup-msys2@v1
|
||||
with:
|
||||
msystem: MSYS
|
||||
- run: msys2do pacman -S mingw64/mingw-w64-x86_64-ninja mingw64/mingw-w64-x86_64-pkg-config mingw64/mingw-w64-x86_64-gcc mingw64/mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-hdf5 --noprogressbar --noconfirm
|
||||
- run: set MSYSTEM=MINGW64
|
||||
- run: python -m pip install -e .
|
||||
- run: meson setup "test cases/frameworks/25 hdf5" build
|
||||
env:
|
||||
FC: gfortran
|
||||
CXX: g++
|
||||
CC: gcc
|
||||
- run: ninja -C build
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: MSYS_Log
|
||||
path: build/meson-logs/meson-log.txt
|
||||
- run: meson test -C build -v
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: failure()
|
||||
with:
|
||||
name: MSYS2_Test
|
||||
path: build/meson-logs/testlog.txt
|
||||
|
|
@ -30,4 +30,4 @@ jobs:
|
|||
with:
|
||||
python-version: '3.x'
|
||||
- run: python -m pip install mypy
|
||||
- run: mypy --follow-imports=skip mesonbuild/mtest.py mesonbuild/minit.py mesonbuild/msetup.py mesonbuild/wrap tools/ mesonbuild/modules/fs.py mesonbuild/dependencies/mpi.py
|
||||
- run: mypy --follow-imports=skip mesonbuild/mtest.py mesonbuild/minit.py mesonbuild/msetup.py mesonbuild/wrap tools/ mesonbuild/modules/fs.py mesonbuild/dependencies/mpi.py mesonbuild/dependencies/hdf5.py
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
from .boost import BoostDependency
|
||||
from .cuda import CudaDependency
|
||||
from .hdf5 import HDF5Dependency
|
||||
from .base import ( # noqa: F401
|
||||
Dependency, DependencyException, DependencyMethods, ExternalProgram, EmptyExternalProgram, NonExistingExternalProgram,
|
||||
ExternalDependency, NotFoundDependency, ExternalLibrary, ExtraFrameworkDependency, InternalDependency,
|
||||
|
|
@ -21,7 +22,7 @@ from .base import ( # noqa: F401
|
|||
from .dev import GMockDependency, GTestDependency, LLVMDependency, ValgrindDependency
|
||||
from .coarrays import CoarrayDependency
|
||||
from .mpi import MPIDependency
|
||||
from .misc import (BlocksDependency, HDF5Dependency, NetCDFDependency, OpenMPDependency, Python3Dependency, ThreadDependency, PcapDependency, CupsDependency, LibWmfDependency, LibGCryptDependency, GpgmeDependency, ShadercDependency)
|
||||
from .misc import (BlocksDependency, NetCDFDependency, OpenMPDependency, Python3Dependency, ThreadDependency, PcapDependency, CupsDependency, LibWmfDependency, LibGCryptDependency, GpgmeDependency, ShadercDependency)
|
||||
from .platform import AppleFrameworks
|
||||
from .ui import GLDependency, GnuStepDependency, Qt4Dependency, Qt5Dependency, SDL2Dependency, WxDependency, VulkanDependency
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
# Copyright 2013-2019 The Meson development team
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This file contains the detection logic for miscellaneous external dependencies.
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from .. import mlog
|
||||
from ..mesonlib import split_args
|
||||
from .base import DependencyException, ExternalDependency, ExternalProgram, PkgConfigDependency
|
||||
|
||||
class HDF5Dependency(ExternalDependency):
|
||||
|
||||
def __init__(self, environment, kwargs):
|
||||
language = kwargs.get('language', 'c')
|
||||
super().__init__('hdf5', environment, language, kwargs)
|
||||
kwargs['required'] = False
|
||||
kwargs['silent'] = True
|
||||
self.is_found = False
|
||||
|
||||
# 1. pkg-config
|
||||
pkgconfig_files = ['hdf5', 'hdf5-serial']
|
||||
# some distros put hdf5-1.2.3.pc with version number in .pc filename.
|
||||
ret = subprocess.run(['pkg-config', '--list-all'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
||||
universal_newlines=True)
|
||||
if ret.returncode == 0:
|
||||
for pkg in ret.stdout.split('\n'):
|
||||
if pkg.startswith(('hdf5')):
|
||||
pkgconfig_files.append(pkg.split(' ', 1)[0])
|
||||
pkgconfig_files = list(set(pkgconfig_files)) # dedupe
|
||||
|
||||
if language not in ('c', 'cpp', 'fortran'):
|
||||
raise DependencyException('Language {} is not supported with HDF5.'.format(language))
|
||||
|
||||
for pkg in pkgconfig_files:
|
||||
pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
|
||||
if not pkgdep.found():
|
||||
continue
|
||||
|
||||
self.compile_args = pkgdep.get_compile_args()
|
||||
# some broken pkgconfig don't actually list the full path to the needed includes
|
||||
newinc = []
|
||||
for arg in self.compile_args:
|
||||
if arg.startswith('-I'):
|
||||
stem = 'static' if kwargs.get('static', False) else 'shared'
|
||||
if (Path(arg[2:]) / stem).is_dir():
|
||||
newinc.append('-I' + str(Path(arg[2:]) / stem))
|
||||
self.compile_args += newinc
|
||||
|
||||
# derive needed libraries by language
|
||||
pd_link_args = pkgdep.get_link_args()
|
||||
link_args = []
|
||||
for larg in pd_link_args:
|
||||
lpath = Path(larg)
|
||||
# some pkg-config hdf5.pc (e.g. Ubuntu) don't include the commonly-used HL HDF5 libraries,
|
||||
# so let's add them if they exist
|
||||
# additionally, some pkgconfig HDF5 HL files are malformed so let's be sure to find HL anyway
|
||||
if lpath.is_file():
|
||||
hl = []
|
||||
if language == 'cpp':
|
||||
hl += ['_hl_cpp', '_cpp']
|
||||
elif language == 'fortran':
|
||||
hl += ['_hl_fortran', 'hl_fortran', '_fortran']
|
||||
hl += ['_hl'] # C HL library, always needed
|
||||
|
||||
suffix = '.' + lpath.name.split('.', 1)[1] # in case of .dll.a
|
||||
for h in hl:
|
||||
hlfn = lpath.parent / (lpath.name.split('.', 1)[0] + h + suffix)
|
||||
if hlfn.is_file():
|
||||
link_args.append(str(hlfn))
|
||||
# HDF5 C libs are required by other HDF5 languages
|
||||
link_args.append(larg)
|
||||
else:
|
||||
link_args.append(larg)
|
||||
|
||||
self.link_args = link_args
|
||||
self.version = pkgdep.get_version()
|
||||
self.is_found = True
|
||||
self.pcdep = pkgdep
|
||||
return
|
||||
|
||||
# 2. compiler wrapper fallback
|
||||
wrappers = {'c': 'h5cc', 'cpp': 'h5c++', 'fortran': 'h5fc'}
|
||||
comp_args = []
|
||||
link_args = []
|
||||
# have to always do C as well as desired language
|
||||
for lang in set([language, 'c']):
|
||||
prog = ExternalProgram(wrappers[lang], silent=True)
|
||||
if not prog.found():
|
||||
return
|
||||
cmd = prog.get_command() + ['-show']
|
||||
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
|
||||
if p.returncode != 0:
|
||||
mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
|
||||
mlog.debug(mlog.bold('Standard output\n'), p.stdout)
|
||||
mlog.debug(mlog.bold('Standard error\n'), p.stderr)
|
||||
return
|
||||
args = split_args(p.stdout)
|
||||
for arg in args[1:]:
|
||||
if arg.startswith(('-I', '-f', '-D')) or arg == '-pthread':
|
||||
comp_args.append(arg)
|
||||
elif arg.startswith(('-L', '-l', '-Wl')):
|
||||
link_args.append(arg)
|
||||
elif Path(arg).is_file():
|
||||
link_args.append(arg)
|
||||
self.compile_args = comp_args
|
||||
self.link_args = link_args
|
||||
self.is_found = True
|
||||
|
|
@ -30,53 +30,6 @@ from .base import (
|
|||
)
|
||||
|
||||
|
||||
class HDF5Dependency(ExternalDependency):
|
||||
|
||||
def __init__(self, environment, kwargs):
|
||||
language = kwargs.get('language', 'c')
|
||||
super().__init__('hdf5', environment, language, kwargs)
|
||||
kwargs['required'] = False
|
||||
kwargs['silent'] = True
|
||||
self.is_found = False
|
||||
|
||||
pkgconfig_files = ['hdf5']
|
||||
|
||||
if language not in ('c', 'cpp', 'fortran'):
|
||||
raise DependencyException('Language {} is not supported with HDF5.'.format(language))
|
||||
|
||||
for pkg in pkgconfig_files:
|
||||
try:
|
||||
pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
|
||||
if pkgdep.found():
|
||||
self.compile_args = pkgdep.get_compile_args()
|
||||
# derive needed libraries by language
|
||||
pd_link_args = pkgdep.get_link_args()
|
||||
link_args = []
|
||||
for larg in pd_link_args:
|
||||
lpath = Path(larg)
|
||||
if lpath.is_file():
|
||||
if language == 'cpp':
|
||||
link_args.append(str(lpath.parent / (lpath.stem + '_hl_cpp' + lpath.suffix)))
|
||||
link_args.append(str(lpath.parent / (lpath.stem + '_cpp' + lpath.suffix)))
|
||||
elif language == 'fortran':
|
||||
link_args.append(str(lpath.parent / (lpath.stem + 'hl_fortran' + lpath.suffix)))
|
||||
link_args.append(str(lpath.parent / (lpath.stem + '_fortran' + lpath.suffix)))
|
||||
|
||||
# HDF5 C libs are required by other HDF5 languages
|
||||
link_args.append(str(lpath.parent / (lpath.stem + '_hl' + lpath.suffix)))
|
||||
link_args.append(larg)
|
||||
else:
|
||||
link_args.append(larg)
|
||||
|
||||
self.link_args = link_args
|
||||
self.version = pkgdep.get_version()
|
||||
self.is_found = True
|
||||
self.pcdep = pkgdep
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class NetCDFDependency(ExternalDependency):
|
||||
|
||||
def __init__(self, environment, kwargs):
|
||||
|
|
|
|||
|
|
@ -1,40 +1,27 @@
|
|||
project('hdf5_test', 'c', 'cpp')
|
||||
|
||||
if build_machine.system() == 'darwin'
|
||||
error('MESON_SKIP_TEST: HDF5 CI image not setup for OSX.')
|
||||
endif
|
||||
|
||||
if build_machine.system() == 'cygwin'
|
||||
error('MESON_SKIP_TEST: HDF5 CI image not setup for Cygwin.')
|
||||
endif
|
||||
project('hdf5_framework', 'c')
|
||||
|
||||
# NOTE: all HDF5 languages must have HDF5 C library working.
|
||||
|
||||
# --- C tests
|
||||
h5c = dependency('hdf5', language : 'c', required : false)
|
||||
if not h5c.found()
|
||||
error('MESON_SKIP_TEST: HDF5 C library not found, skipping HDF5 framework tests.')
|
||||
error('MESON_SKIP_TEST: HDF5 C library not found.')
|
||||
endif
|
||||
exec = executable('exec', 'main.c', dependencies : h5c)
|
||||
|
||||
test('HDF5 C', exec)
|
||||
test('HDF5 C', exec, timeout: 30)
|
||||
|
||||
# --- C++ tests
|
||||
h5cpp = dependency('hdf5', language : 'cpp', required : false)
|
||||
if h5cpp.found()
|
||||
if add_languages('cpp')
|
||||
h5cpp = dependency('hdf5', language : 'cpp', required : false, disabler: true)
|
||||
execpp = executable('execpp', 'main.cpp', dependencies : h5cpp)
|
||||
test('HDF5 C++', execpp)
|
||||
test('HDF5 C++', execpp, timeout: 30)
|
||||
endif
|
||||
|
||||
# --- Fortran tests
|
||||
if build_machine.system() != 'windows'
|
||||
add_languages('fortran')
|
||||
|
||||
h5f = dependency('hdf5', language : 'fortran', required : false)
|
||||
if h5f.found()
|
||||
exef = executable('exef', 'main.f90', dependencies : h5f)
|
||||
|
||||
test('HDF5 Fortran', exef)
|
||||
endif
|
||||
if add_languages('fortran')
|
||||
h5f = dependency('hdf5', language : 'fortran', required : false, disabler: true)
|
||||
exef = executable('exef', 'main.f90', dependencies : h5f)
|
||||
test('HDF5 Fortran', exef, timeout: 30)
|
||||
endif
|
||||
|
||||
# Check we can apply a version constraint
|
||||
|
|
|
|||
Loading…
Reference in New Issue