From a3db9f6ae3592f863b95c1c32b56ff322812694f Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Wed, 3 Oct 2018 09:38:11 -0400 Subject: [PATCH 1/8] wrap: Some cleanups and fixes in resolver - Use CongifParser instead of parsing ourself, this will give more flexibility in the future. - Always read the .wrap file first, because if it contains the 'directory' key we should use that value instead of packagename for the path where do download/lookup for the subproject. - If we download the subproject (git/submodule/tarball/etc) we should still check it contains a meson.build file. --- mesonbuild/wrap/wrap.py | 132 ++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 5cc22256b..be8b04fd6 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -17,6 +17,7 @@ import contextlib import urllib.request, os, hashlib, shutil, tempfile, stat import subprocess import sys +import configparser from pathlib import Path from . import WrapMode from ..mesonlib import Popen_safe @@ -70,28 +71,13 @@ def open_wrapdburl(urlstring): class PackageDefinition: def __init__(self, fname): - self.values = {} - with open(fname) as ifile: - first = ifile.readline().strip() - - if first == '[wrap-file]': - self.type = 'file' - elif first == '[wrap-git]': - self.type = 'git' - elif first == '[wrap-hg]': - self.type = 'hg' - elif first == '[wrap-svn]': - self.type = 'svn' - else: - raise RuntimeError('Invalid format of package file') - for line in ifile: - line = line.strip() - if line == '': - continue - (k, v) = line.split('=', 1) - k = k.strip() - v = v.strip() - self.values[k] = v + self.config = configparser.ConfigParser() + self.config.read(fname) + self.wrap_section = self.config.sections()[0] + if not self.wrap_section.startswith('wrap-'): + raise RuntimeError('Invalid format of package file') + self.type = self.wrap_section[5:] + self.values = dict(self.config[self.wrap_section]) def get(self, key): return self.values[key] @@ -100,60 +86,71 @@ class PackageDefinition: return 'patch_url' in self.values class Resolver: - def __init__(self, subdir_root, wrap_mode=WrapMode(1)): + def __init__(self, subdir_root, wrap_mode=WrapMode.default): self.wrap_mode = wrap_mode self.subdir_root = subdir_root self.cachedir = os.path.join(self.subdir_root, 'packagecache') def resolve(self, packagename): - # Check if the directory is already resolved - dirname = Path(os.path.join(self.subdir_root, packagename)) - subprojdir = os.path.join(*dirname.parts[-2:]) - if dirname.is_dir(): - if (dirname / 'meson.build').is_file(): - # The directory is there and has meson.build? Great, use it. - return packagename - # Is the dir not empty and also not a git submodule dir that is - # not checkout properly? Can't do anything, exception! - elif next(dirname.iterdir(), None) and not (dirname / '.git').is_file(): - m = '{!r} is not empty and has no meson.build files' + # We always have to load the wrap file, if it exists, because it could + # override the default directory name. + p = self.load_wrap(packagename) + directory = packagename + if p and 'directory' in p.values: + directory = p.get('directory') + dirname = os.path.join(self.subdir_root, directory) + subprojdir = os.path.join(*Path(dirname).parts[-2:]) + meson_file = os.path.join(dirname, 'meson.build') + + # The directory is there and has meson.build? Great, use it. + if os.path.exists(meson_file): + return directory + + # Check if the subproject is a git submodule + self.resolve_git_submodule(dirname) + + if os.path.exists(dirname): + if not os.path.isdir(dirname): + m = '{!r} already exists and is not a dir; cannot use as subproject' raise RuntimeError(m.format(subprojdir)) - elif dirname.exists(): - m = '{!r} already exists and is not a dir; cannot use as subproject' + else: + # Don't download subproject data based on wrap file if requested. + # Git submodules are ok (see above)! + if self.wrap_mode is WrapMode.nodownload: + m = 'Automatic wrap-based subproject downloading is disabled' + raise RuntimeError(m) + + # A wrap file is required to download + if not p: + m = 'No {}.wrap found for {!r}' + raise RuntimeError(m.format(packagename, subprojdir)) + + if p.type == 'file': + if not os.path.isdir(self.cachedir): + os.mkdir(self.cachedir) + self.download(p, packagename) + self.extract_package(p) + elif p.type == 'git': + self.get_git(p) + elif p.type == "hg": + self.get_hg(p) + elif p.type == "svn": + self.get_svn(p) + else: + raise AssertionError('Unreachable code.') + + # A meson.build file is required in the directory + if not os.path.exists(meson_file): + m = '{!r} is not empty and has no meson.build files' raise RuntimeError(m.format(subprojdir)) - dirname = str(dirname) - # Check if the subproject is a git submodule - if self.resolve_git_submodule(dirname): - return packagename + return directory - # Don't download subproject data based on wrap file if requested. - # Git submodules are ok (see above)! - if self.wrap_mode is WrapMode.nodownload: - m = 'Automatic wrap-based subproject downloading is disabled' - raise RuntimeError(m) - - # Check if there's a .wrap file for this subproject + def load_wrap(self, packagename): fname = os.path.join(self.subdir_root, packagename + '.wrap') - if not os.path.isfile(fname): - # No wrap file with this name? Give up. - m = 'No {}.wrap found for {!r}' - raise RuntimeError(m.format(packagename, subprojdir)) - p = PackageDefinition(fname) - if p.type == 'file': - if not os.path.isdir(self.cachedir): - os.mkdir(self.cachedir) - self.download(p, packagename) - self.extract_package(p) - elif p.type == 'git': - self.get_git(p) - elif p.type == "hg": - self.get_hg(p) - elif p.type == "svn": - self.get_svn(p) - else: - raise AssertionError('Unreachable code.') - return p.get('directory') + if os.path.isfile(fname): + return PackageDefinition(fname) + return None def resolve_git_submodule(self, dirname): # Are we in a git repository? @@ -181,6 +178,9 @@ class Resolver: # Even if checkout failed, try building it anyway and let the user # handle any problems manually. return True + elif out == b'': + # It is not a submodule, just a folder that exists in the main repository. + return False m = 'Unknown git submodule output: {!r}' raise RuntimeError(m.format(out)) From d6fba7f01ce3effebd18d75ebe3102774353cee0 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 5 Oct 2018 12:07:46 -0400 Subject: [PATCH 2/8] wrap: Allow source and patch to be local files It is sometimes important to be able to build projects offline, in that case subproject tarballs and patches could be shipped directly within the project's repository. --- .../markdown/Wrap-dependency-system-manual.md | 10 ++ mesonbuild/wrap/wrap.py | 163 +++++++++--------- .../meson.build | 4 +- 3 files changed, 90 insertions(+), 87 deletions(-) diff --git a/docs/markdown/Wrap-dependency-system-manual.md b/docs/markdown/Wrap-dependency-system-manual.md index 2e977b2ea..7da4be541 100644 --- a/docs/markdown/Wrap-dependency-system-manual.md +++ b/docs/markdown/Wrap-dependency-system-manual.md @@ -45,6 +45,11 @@ If you then use this subproject in your build, Meson will automatically download and extract it during build. This makes subproject embedding extremely easy. +Since *0.49.0* if `source_filename` is found in project's +`subprojects/packagecache` directory, it will be used instead of downloading the +source, even if `wrap-mode` option is set to `nodownload`. The file's hash will +be checked. + Unfortunately most software projects in the world do not build with Meson. Because of this Meson allows you to specify a patch URL. This works in much the same way as Debian's distro patches. That is, they @@ -76,6 +81,11 @@ thousands of lines of code. Once you have a working build definition, just zip up the Meson build files (and others you have changed) and put them somewhere where you can download them. +Since *0.49.0* if `patch_filename` is found in project's +`subprojects/packagecache` directory, it will be used instead of downloading the +patch, even if `wrap-mode` option is set to `nodownload`. The file's hash will +be checked. + ## Branching subprojects directly from git The above mentioned scheme assumes that your subproject is working off diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index be8b04fd6..a0eeed1a9 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -92,9 +92,10 @@ class Resolver: self.cachedir = os.path.join(self.subdir_root, 'packagecache') def resolve(self, packagename): + self.packagename = packagename # We always have to load the wrap file, if it exists, because it could # override the default directory name. - p = self.load_wrap(packagename) + p = self.load_wrap() directory = packagename if p and 'directory' in p.values: directory = p.get('directory') @@ -114,30 +115,23 @@ class Resolver: m = '{!r} already exists and is not a dir; cannot use as subproject' raise RuntimeError(m.format(subprojdir)) else: - # Don't download subproject data based on wrap file if requested. - # Git submodules are ok (see above)! - if self.wrap_mode is WrapMode.nodownload: - m = 'Automatic wrap-based subproject downloading is disabled' - raise RuntimeError(m) - # A wrap file is required to download if not p: m = 'No {}.wrap found for {!r}' raise RuntimeError(m.format(packagename, subprojdir)) if p.type == 'file': - if not os.path.isdir(self.cachedir): - os.mkdir(self.cachedir) - self.download(p, packagename) - self.extract_package(p) - elif p.type == 'git': - self.get_git(p) - elif p.type == "hg": - self.get_hg(p) - elif p.type == "svn": - self.get_svn(p) + self.get_file(p) else: - raise AssertionError('Unreachable code.') + self.check_can_download() + if p.type == 'git': + self.get_git(p) + elif p.type == "hg": + self.get_hg(p) + elif p.type == "svn": + self.get_svn(p) + else: + raise AssertionError('Unreachable code.') # A meson.build file is required in the directory if not os.path.exists(meson_file): @@ -146,12 +140,19 @@ class Resolver: return directory - def load_wrap(self, packagename): - fname = os.path.join(self.subdir_root, packagename + '.wrap') + def load_wrap(self): + fname = os.path.join(self.subdir_root, self.packagename + '.wrap') if os.path.isfile(fname): return PackageDefinition(fname) return None + def check_can_download(self): + # Don't download subproject data based on wrap file if requested. + # Git submodules are ok (see above)! + if self.wrap_mode is WrapMode.nodownload: + m = 'Automatic wrap-based subproject downloading is disabled' + raise RuntimeError(m) + def resolve_git_submodule(self, dirname): # Are we in a git repository? ret, out = quiet_git(['rev-parse'], self.subdir_root) @@ -184,6 +185,22 @@ class Resolver: m = 'Unknown git submodule output: {!r}' raise RuntimeError(m.format(out)) + def get_file(self, p): + path = self.get_file_internal(p, 'source') + target_dir = os.path.join(self.subdir_root, p.get('directory')) + extract_dir = self.subdir_root + # Some upstreams ship packages that do not have a leading directory. + # Create one for them. + try: + p.get('lead_directory_missing') + os.mkdir(target_dir) + extract_dir = target_dir + except KeyError: + pass + shutil.unpack_archive(path, extract_dir) + if p.has_patch(): + self.apply_patch(p) + def get_git(self, p): checkoutdir = os.path.join(self.subdir_root, p.get('directory')) revno = p.get('revision') @@ -312,41 +329,48 @@ class Resolver: hashvalue = h.hexdigest() return hashvalue, tmpfile.name - def get_hash(self, data): + def check_hash(self, p, what, path): + expected = p.get(what + '_hash') h = hashlib.sha256() - h.update(data) - hashvalue = h.hexdigest() - return hashvalue + with open(path, 'rb') as f: + h.update(f.read()) + dhash = h.hexdigest() + if dhash != expected: + raise RuntimeError('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) - def download(self, p, packagename): - ofname = os.path.join(self.cachedir, p.get('source_filename')) - if os.path.exists(ofname): - mlog.log('Using', mlog.bold(packagename), 'from cache.') - else: - srcurl = p.get('source_url') - mlog.log('Downloading', mlog.bold(packagename), 'from', mlog.bold(srcurl)) - dhash, tmpfile = self.get_data(srcurl) - expected = p.get('source_hash') - if dhash != expected: - os.remove(tmpfile) - raise RuntimeError('Incorrect hash for source %s:\n %s expected\n %s actual.' % (packagename, expected, dhash)) - os.rename(tmpfile, ofname) - if p.has_patch(): - patch_filename = p.get('patch_filename') - filename = os.path.join(self.cachedir, patch_filename) - if os.path.exists(filename): - mlog.log('Using', mlog.bold(patch_filename), 'from cache.') - else: - purl = p.get('patch_url') - mlog.log('Downloading patch from', mlog.bold(purl)) - phash, tmpfile = self.get_data(purl) - expected = p.get('patch_hash') - if phash != expected: - os.remove(tmpfile) - raise RuntimeError('Incorrect hash for patch %s:\n %s expected\n %s actual' % (packagename, expected, phash)) - os.rename(tmpfile, filename) - else: - mlog.log('Package does not require patch.') + def download(self, p, what, ofname): + self.check_can_download() + srcurl = p.get(what + '_url') + mlog.log('Downloading', mlog.bold(self.packagename), what, 'from', mlog.bold(srcurl)) + dhash, tmpfile = self.get_data(srcurl) + expected = p.get(what + '_hash') + if dhash != expected: + os.remove(tmpfile) + raise RuntimeError('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) + os.rename(tmpfile, ofname) + + def get_file_internal(self, p, what): + filename = p.get(what + '_filename') + cache_path = os.path.join(self.cachedir, filename) + + if os.path.exists(cache_path): + self.check_hash(p, what, cache_path) + mlog.log('Using', mlog.bold(self.packagename), what, 'from cache.') + return cache_path + + if not os.path.isdir(self.cachedir): + os.mkdir(self.cachedir) + self.download(p, what, cache_path) + return cache_path + + def apply_patch(self, p): + path = self.get_file_internal(p, 'patch') + try: + shutil.unpack_archive(path, self.subdir_root) + except Exception: + with tempfile.TemporaryDirectory() as workdir: + shutil.unpack_archive(path, workdir) + self.copy_tree(workdir, self.subdir_root) def copy_tree(self, root_src_dir, root_dst_dir): """ @@ -366,36 +390,3 @@ class Resolver: os.chmod(dst_file, stat.S_IWUSR) os.remove(dst_file) shutil.copy2(src_file, dst_dir) - - def extract_package(self, package): - if sys.version_info < (3, 5): - try: - import lzma # noqa: F401 - del lzma - except ImportError: - pass - else: - try: - shutil.register_unpack_format('xztar', ['.tar.xz', '.txz'], shutil._unpack_tarfile, [], "xz'ed tar-file") - except shutil.RegistryError: - pass - target_dir = os.path.join(self.subdir_root, package.get('directory')) - if os.path.isdir(target_dir): - return - extract_dir = self.subdir_root - # Some upstreams ship packages that do not have a leading directory. - # Create one for them. - try: - package.get('lead_directory_missing') - os.mkdir(target_dir) - extract_dir = target_dir - except KeyError: - pass - shutil.unpack_archive(os.path.join(self.cachedir, package.get('source_filename')), extract_dir) - if package.has_patch(): - try: - shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), self.subdir_root) - except Exception: - with tempfile.TemporaryDirectory() as workdir: - shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), workdir) - self.copy_tree(workdir, self.subdir_root) diff --git a/test cases/common/158 wrap file should not failed/meson.build b/test cases/common/158 wrap file should not failed/meson.build index 9cf4e9a6b..9d707eec5 100644 --- a/test cases/common/158 wrap file should not failed/meson.build +++ b/test cases/common/158 wrap file should not failed/meson.build @@ -1,4 +1,6 @@ -project('mainproj', 'c') +project('mainproj', 'c', + default_options : ['wrap_mode=nodownload'], +) subproject('zlib') From f0b1f0340cbcdc503b281a547ab0159347283cff Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Wed, 17 Oct 2018 21:48:18 -0400 Subject: [PATCH 3/8] msetup: Remove dead code --- mesonbuild/msetup.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/mesonbuild/msetup.py b/mesonbuild/msetup.py index 95885554d..425627260 100644 --- a/mesonbuild/msetup.py +++ b/mesonbuild/msetup.py @@ -24,7 +24,6 @@ from . import environment, interpreter, mesonlib from . import build from . import mlog, coredata from .mesonlib import MesonException -from .wrap import WrapMode def add_arguments(parser): coredata.register_builtin_arguments(parser) @@ -43,14 +42,6 @@ def add_arguments(parser): parser.add_argument('builddir', nargs='?', default=None) parser.add_argument('sourcedir', nargs='?', default=None) -def wrapmodetype(string): - try: - return getattr(WrapMode, string) - except AttributeError: - msg = ', '.join([t.name.lower() for t in WrapMode]) - msg = 'invalid argument {!r}, use one of {}'.format(string, msg) - raise argparse.ArgumentTypeError(msg) - class MesonApp: def __init__(self, options): (self.source_dir, self.build_dir) = self.validate_dirs(options.builddir, From 1889e3338918b3bef22838d6de92d6b748ee01aa Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 18 Oct 2018 10:49:50 -0400 Subject: [PATCH 4/8] wrap: Remove dead code trying to update git/hg/svn repositories If the directory exists we early return or raise exception in resolve() method. It was already like that even before the recent refactoring of the code. It is safer like that anyway, we don't want a project reconfigure to silently pull new code. Updating subprojects should be an explicit action of the user. For example gst-build has a 'git-update' script that does that. In the future we could add a 'meson subprojects update' command. --- mesonbuild/wrap/wrap.py | 93 ++++++++++------------------------------- 1 file changed, 21 insertions(+), 72 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index a0eeed1a9..017142944 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -20,7 +20,6 @@ import sys import configparser from pathlib import Path from . import WrapMode -from ..mesonlib import Popen_safe try: import ssl @@ -204,86 +203,36 @@ class Resolver: def get_git(self, p): checkoutdir = os.path.join(self.subdir_root, p.get('directory')) revno = p.get('revision') - is_there = os.path.isdir(checkoutdir) - if is_there: - try: - subprocess.check_call(['git', 'rev-parse'], cwd=checkoutdir) - except subprocess.CalledProcessError: - raise RuntimeError('%s is not empty but is not a valid ' - 'git repository, we can not work with it' - ' as a subproject directory.' % ( - checkoutdir)) - - if revno.lower() == 'head': - # Failure to do pull is not a fatal error, - # because otherwise you can't develop without - # a working net connection. - subprocess.call(['git', 'pull'], cwd=checkoutdir) - else: - if subprocess.call(['git', 'checkout', revno], cwd=checkoutdir) != 0: - subprocess.check_call(['git', 'fetch', p.get('url'), revno], cwd=checkoutdir) - subprocess.check_call(['git', 'checkout', revno], - cwd=checkoutdir) + if p.values.get('clone-recursive', '').lower() == 'true': + subprocess.check_call(['git', 'clone', '--recursive', p.get('url'), + p.get('directory')], cwd=self.subdir_root) else: - if p.values.get('clone-recursive', '').lower() == 'true': - subprocess.check_call(['git', 'clone', '--recursive', p.get('url'), - p.get('directory')], cwd=self.subdir_root) - else: - subprocess.check_call(['git', 'clone', p.get('url'), - p.get('directory')], cwd=self.subdir_root) - if revno.lower() != 'head': - if subprocess.call(['git', 'checkout', revno], cwd=checkoutdir) != 0: - subprocess.check_call(['git', 'fetch', p.get('url'), revno], cwd=checkoutdir) - subprocess.check_call(['git', 'checkout', revno], - cwd=checkoutdir) - push_url = p.values.get('push-url') - if push_url: - subprocess.check_call(['git', 'remote', 'set-url', - '--push', 'origin', push_url], + subprocess.check_call(['git', 'clone', p.get('url'), + p.get('directory')], cwd=self.subdir_root) + if revno.lower() != 'head': + if subprocess.call(['git', 'checkout', revno], cwd=checkoutdir) != 0: + subprocess.check_call(['git', 'fetch', p.get('url'), revno], cwd=checkoutdir) + subprocess.check_call(['git', 'checkout', revno], cwd=checkoutdir) + push_url = p.values.get('push-url') + if push_url: + subprocess.check_call(['git', 'remote', 'set-url', + '--push', 'origin', push_url], + cwd=checkoutdir) def get_hg(self, p): checkoutdir = os.path.join(self.subdir_root, p.get('directory')) revno = p.get('revision') - is_there = os.path.isdir(checkoutdir) - if is_there: - if revno.lower() == 'tip': - # Failure to do pull is not a fatal error, - # because otherwise you can't develop without - # a working net connection. - subprocess.call(['hg', 'pull'], cwd=checkoutdir) - else: - if subprocess.call(['hg', 'checkout', revno], cwd=checkoutdir) != 0: - subprocess.check_call(['hg', 'pull'], cwd=checkoutdir) - subprocess.check_call(['hg', 'checkout', revno], - cwd=checkoutdir) - else: - subprocess.check_call(['hg', 'clone', p.get('url'), - p.get('directory')], cwd=self.subdir_root) - if revno.lower() != 'tip': - subprocess.check_call(['hg', 'checkout', revno], - cwd=checkoutdir) + subprocess.check_call(['hg', 'clone', p.get('url'), + p.get('directory')], cwd=self.subdir_root) + if revno.lower() != 'tip': + subprocess.check_call(['hg', 'checkout', revno], + cwd=checkoutdir) def get_svn(self, p): - checkoutdir = os.path.join(self.subdir_root, p.get('directory')) revno = p.get('revision') - is_there = os.path.isdir(checkoutdir) - if is_there: - p, out = Popen_safe(['svn', 'info', '--show-item', 'revision', checkoutdir]) - current_revno = out - if current_revno == revno: - return - - if revno.lower() == 'head': - # Failure to do pull is not a fatal error, - # because otherwise you can't develop without - # a working net connection. - subprocess.call(['svn', 'update'], cwd=checkoutdir) - else: - subprocess.check_call(['svn', 'update', '-r', revno], cwd=checkoutdir) - else: - subprocess.check_call(['svn', 'checkout', '-r', revno, p.get('url'), - p.get('directory')], cwd=self.subdir_root) + subprocess.check_call(['svn', 'checkout', '-r', revno, p.get('url'), + p.get('directory')], cwd=self.subdir_root) def get_data(self, url): blocksize = 10 * 1024 From 3f98ad8eed598d5502157d06334e5c3857ab0f8b Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 18 Oct 2018 11:18:35 -0400 Subject: [PATCH 5/8] wrap: Improve error handling and logging --- mesonbuild/interpreter.py | 45 ++++++++++++-------------- mesonbuild/wrap/wrap.py | 68 ++++++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 9b3fdc7df..98424ec60 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2261,19 +2261,21 @@ external dependencies (including libraries) must go to "dependencies".''') r = wrap.Resolver(subproject_dir_abs, self.coredata.get_builtin_option('wrap_mode')) try: resolved = r.resolve(dirname) - except RuntimeError as e: - # if the reason subproject execution failed was because - # the directory doesn't exist, try to give some helpful - # advice if it's a nested subproject that needs - # promotion... - self.print_nested_info(dirname) + except wrap.WrapException as e: + subprojdir = os.path.join(self.subproject_dir, r.directory) + if not required: + mlog.log('\nSubproject ', mlog.bold(subprojdir), 'is buildable:', mlog.red('NO'), '(disabling)\n') + return self.disabled_subproject(dirname) - if required: - msg = 'Subproject directory {!r} does not exist and cannot be downloaded:\n{}' - raise InterpreterException(msg.format(os.path.join(self.subproject_dir, dirname), e)) + if isinstance(e, wrap.WrapNotFoundException): + # if the reason subproject execution failed was because + # the directory doesn't exist, try to give some helpful + # advice if it's a nested subproject that needs + # promotion... + self.print_nested_info(dirname) - mlog.log('\nSubproject ', mlog.bold(dirname), 'is buildable:', mlog.red('NO'), '(disabling)\n') - return self.disabled_subproject(dirname) + msg = 'Failed to initialize {!r}:\n{}' + raise InterpreterException(msg.format(subprojdir, e)) subdir = os.path.join(self.subproject_dir, resolved) os.makedirs(os.path.join(self.build.environment.get_build_dir(), subdir), exist_ok=True) @@ -2979,26 +2981,21 @@ external dependencies (including libraries) must go to "dependencies".''') return Disabler() def print_nested_info(self, dependency_name): - message_templ = '''\nDependency %s not found but it is available in a sub-subproject. -To use it in the current project, promote it by going in the project source -root and issuing %s. - -''' + message = ['Dependency', mlog.bold(dependency_name), 'not found but it is available in a sub-subproject.\n' + + 'To use it in the current project, promote it by going in the project source\n' + 'root and issuing'] sprojs = mesonlib.detect_subprojects('subprojects', self.source_root) if dependency_name not in sprojs: return found = sprojs[dependency_name] if len(found) > 1: - suffix = 'one of the following commands' + message.append('one of the following commands:') else: - suffix = 'the following command' - message = message_templ % (dependency_name, suffix) - cmds = [] - command_templ = 'meson wrap promote ' + message.append('the following command:') + command_templ = '\nmeson wrap promote {}' for l in found: - cmds.append(command_templ + l[len(self.source_root) + 1:]) - final_message = message + '\n'.join(cmds) - print(final_message) + message.append(mlog.bold(command_templ.format(l[len(self.source_root) + 1:]))) + mlog.warning(*message) def get_subproject_infos(self, kwargs): fbinfo = kwargs['fallback'] diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 017142944..67da23c6d 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -18,8 +18,8 @@ import urllib.request, os, hashlib, shutil, tempfile, stat import subprocess import sys import configparser -from pathlib import Path from . import WrapMode +from ..mesonlib import MesonException try: import ssl @@ -67,19 +67,35 @@ def open_wrapdburl(urlstring): urlstring = 'http' + urlstring[5:] return urllib.request.urlopen(urlstring, timeout=req_timeout) +class WrapException(MesonException): + pass + +class WrapNotFoundException(WrapException): + pass class PackageDefinition: def __init__(self, fname): - self.config = configparser.ConfigParser() - self.config.read(fname) + self.basename = os.path.basename(fname) + try: + self.config = configparser.ConfigParser() + self.config.read(fname) + except: + raise WrapException('Failed to parse {}'.format(self.basename)) + if len(self.config.sections()) < 1: + raise WrapException('Missing sections in {}'.format(self.basename)) self.wrap_section = self.config.sections()[0] if not self.wrap_section.startswith('wrap-'): - raise RuntimeError('Invalid format of package file') + m = '{!r} is not a valid first section in {}' + raise WrapException(m.format(self.wrap_section, self.basename)) self.type = self.wrap_section[5:] self.values = dict(self.config[self.wrap_section]) def get(self, key): - return self.values[key] + try: + return self.values[key] + except KeyError: + m = 'Missing key {!r} in {}' + raise WrapException(m.format(key, self.basename)) def has_patch(self): return 'patch_url' in self.values @@ -92,32 +108,30 @@ class Resolver: def resolve(self, packagename): self.packagename = packagename + self.directory = packagename # We always have to load the wrap file, if it exists, because it could # override the default directory name. p = self.load_wrap() - directory = packagename if p and 'directory' in p.values: - directory = p.get('directory') - dirname = os.path.join(self.subdir_root, directory) - subprojdir = os.path.join(*Path(dirname).parts[-2:]) + self.directory = p.get('directory') + dirname = os.path.join(self.subdir_root, self.directory) meson_file = os.path.join(dirname, 'meson.build') # The directory is there and has meson.build? Great, use it. if os.path.exists(meson_file): - return directory + return self.directory # Check if the subproject is a git submodule self.resolve_git_submodule(dirname) if os.path.exists(dirname): if not os.path.isdir(dirname): - m = '{!r} already exists and is not a dir; cannot use as subproject' - raise RuntimeError(m.format(subprojdir)) + raise WrapException('Path already exists but is not a directory') else: # A wrap file is required to download if not p: - m = 'No {}.wrap found for {!r}' - raise RuntimeError(m.format(packagename, subprojdir)) + m = 'Subproject directory not found and {}.wrap file not found' + raise WrapNotFoundException(m.format(self.packagename)) if p.type == 'file': self.get_file(p) @@ -130,14 +144,13 @@ class Resolver: elif p.type == "svn": self.get_svn(p) else: - raise AssertionError('Unreachable code.') + raise WrapException('Unknown wrap type {!r}'.format(p.type)) # A meson.build file is required in the directory if not os.path.exists(meson_file): - m = '{!r} is not empty and has no meson.build files' - raise RuntimeError(m.format(subprojdir)) + raise WrapException('Subproject exists but has no meson.build file') - return directory + return self.directory def load_wrap(self): fname = os.path.join(self.subdir_root, self.packagename + '.wrap') @@ -150,7 +163,7 @@ class Resolver: # Git submodules are ok (see above)! if self.wrap_mode is WrapMode.nodownload: m = 'Automatic wrap-based subproject downloading is disabled' - raise RuntimeError(m) + raise WrapException(m) def resolve_git_submodule(self, dirname): # Are we in a git repository? @@ -163,15 +176,15 @@ class Resolver: return False # Submodule has not been added, add it if out.startswith(b'+'): - mlog.warning('git submodule {} might be out of date'.format(dirname)) + mlog.warning('git submodule might be out of date') return True elif out.startswith(b'U'): - raise RuntimeError('submodule {} has merge conflicts'.format(dirname)) + raise WrapException('git submodule has merge conflicts') # Submodule exists, but is deinitialized or wasn't initialized elif out.startswith(b'-'): if subprocess.call(['git', '-C', self.subdir_root, 'submodule', 'update', '--init', dirname]) == 0: return True - raise RuntimeError('Failed to git submodule init {!r}'.format(dirname)) + raise WrapException('git submodule failed to init') # Submodule looks fine, but maybe it wasn't populated properly. Do a checkout. elif out.startswith(b' '): subprocess.call(['git', 'checkout', '.'], cwd=dirname) @@ -182,7 +195,7 @@ class Resolver: # It is not a submodule, just a folder that exists in the main repository. return False m = 'Unknown git submodule output: {!r}' - raise RuntimeError(m.format(out)) + raise WrapException(m.format(out)) def get_file(self, p): path = self.get_file_internal(p, 'source') @@ -190,12 +203,9 @@ class Resolver: extract_dir = self.subdir_root # Some upstreams ship packages that do not have a leading directory. # Create one for them. - try: - p.get('lead_directory_missing') + if 'lead_directory_missing' in p.values: os.mkdir(target_dir) extract_dir = target_dir - except KeyError: - pass shutil.unpack_archive(path, extract_dir) if p.has_patch(): self.apply_patch(p) @@ -285,7 +295,7 @@ class Resolver: h.update(f.read()) dhash = h.hexdigest() if dhash != expected: - raise RuntimeError('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) + raise WrapException('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) def download(self, p, what, ofname): self.check_can_download() @@ -295,7 +305,7 @@ class Resolver: expected = p.get(what + '_hash') if dhash != expected: os.remove(tmpfile) - raise RuntimeError('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) + raise WrapException('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) os.rename(tmpfile, ofname) def get_file_internal(self, p, what): From b26eb3d3845e25b66a777fafd3598b9e32c955fc Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 18 Oct 2018 18:25:05 -0400 Subject: [PATCH 6/8] wrap: Disable interpolation --- mesonbuild/wrap/wrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 67da23c6d..cd113c65c 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -77,7 +77,7 @@ class PackageDefinition: def __init__(self, fname): self.basename = os.path.basename(fname) try: - self.config = configparser.ConfigParser() + self.config = configparser.ConfigParser(interpolation=None) self.config.read(fname) except: raise WrapException('Failed to parse {}'.format(self.basename)) From d9ea7b16a8677c30acb7489e284b6b8217e1c1a0 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 18 Oct 2018 18:29:35 -0400 Subject: [PATCH 7/8] wrap: Rename p to self.wrap --- mesonbuild/wrap/wrap.py | 94 ++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index cd113c65c..2fd1ad3e2 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -111,9 +111,9 @@ class Resolver: self.directory = packagename # We always have to load the wrap file, if it exists, because it could # override the default directory name. - p = self.load_wrap() - if p and 'directory' in p.values: - self.directory = p.get('directory') + self.wrap = self.load_wrap() + if self.wrap and 'directory' in self.wrap.values: + self.directory = self.wrap.get('directory') dirname = os.path.join(self.subdir_root, self.directory) meson_file = os.path.join(dirname, 'meson.build') @@ -129,22 +129,22 @@ class Resolver: raise WrapException('Path already exists but is not a directory') else: # A wrap file is required to download - if not p: + if not self.wrap: m = 'Subproject directory not found and {}.wrap file not found' raise WrapNotFoundException(m.format(self.packagename)) - if p.type == 'file': - self.get_file(p) + if self.wrap.type == 'file': + self.get_file() else: self.check_can_download() - if p.type == 'git': - self.get_git(p) - elif p.type == "hg": - self.get_hg(p) - elif p.type == "svn": - self.get_svn(p) + if self.wrap.type == 'git': + self.get_git() + elif self.wrap.type == "hg": + self.get_hg() + elif self.wrap.type == "svn": + self.get_svn() else: - raise WrapException('Unknown wrap type {!r}'.format(p.type)) + raise WrapException('Unknown wrap type {!r}'.format(self.wrap.type)) # A meson.build file is required in the directory if not os.path.exists(meson_file): @@ -197,52 +197,52 @@ class Resolver: m = 'Unknown git submodule output: {!r}' raise WrapException(m.format(out)) - def get_file(self, p): - path = self.get_file_internal(p, 'source') - target_dir = os.path.join(self.subdir_root, p.get('directory')) + def get_file(self): + path = self.get_file_internal(self.wrap, 'source') + target_dir = os.path.join(self.subdir_root, self.wrap.get('directory')) extract_dir = self.subdir_root # Some upstreams ship packages that do not have a leading directory. # Create one for them. - if 'lead_directory_missing' in p.values: + if 'lead_directory_missing' in self.wrap.values: os.mkdir(target_dir) extract_dir = target_dir shutil.unpack_archive(path, extract_dir) - if p.has_patch(): - self.apply_patch(p) + if self.wrap.has_patch(): + self.apply_patch() - def get_git(self, p): - checkoutdir = os.path.join(self.subdir_root, p.get('directory')) - revno = p.get('revision') - if p.values.get('clone-recursive', '').lower() == 'true': - subprocess.check_call(['git', 'clone', '--recursive', p.get('url'), - p.get('directory')], cwd=self.subdir_root) + def get_git(self): + checkoutdir = os.path.join(self.subdir_root, self.wrap.get('directory')) + revno = self.wrap.get('revision') + if self.wrap.values.get('clone-recursive', '').lower() == 'true': + subprocess.check_call(['git', 'clone', '--recursive', self.wrap.get('url'), + self.wrap.get('directory')], cwd=self.subdir_root) else: - subprocess.check_call(['git', 'clone', p.get('url'), - p.get('directory')], cwd=self.subdir_root) + subprocess.check_call(['git', 'clone', self.wrap.get('url'), + self.wrap.get('directory')], cwd=self.subdir_root) if revno.lower() != 'head': if subprocess.call(['git', 'checkout', revno], cwd=checkoutdir) != 0: - subprocess.check_call(['git', 'fetch', p.get('url'), revno], cwd=checkoutdir) + subprocess.check_call(['git', 'fetch', self.wrap.get('url'), revno], cwd=checkoutdir) subprocess.check_call(['git', 'checkout', revno], cwd=checkoutdir) - push_url = p.values.get('push-url') + push_url = self.wrap.values.get('push-url') if push_url: subprocess.check_call(['git', 'remote', 'set-url', '--push', 'origin', push_url], cwd=checkoutdir) - def get_hg(self, p): - checkoutdir = os.path.join(self.subdir_root, p.get('directory')) - revno = p.get('revision') - subprocess.check_call(['hg', 'clone', p.get('url'), - p.get('directory')], cwd=self.subdir_root) + def get_hg(self): + checkoutdir = os.path.join(self.subdir_root, self.wrap.get('directory')) + revno = self.wrap.get('revision') + subprocess.check_call(['hg', 'clone', self.wrap.get('url'), + self.wrap.get('directory')], cwd=self.subdir_root) if revno.lower() != 'tip': subprocess.check_call(['hg', 'checkout', revno], cwd=checkoutdir) - def get_svn(self, p): - revno = p.get('revision') - subprocess.check_call(['svn', 'checkout', '-r', revno, p.get('url'), - p.get('directory')], cwd=self.subdir_root) + def get_svn(self): + revno = self.wrap.get('revision') + subprocess.check_call(['svn', 'checkout', '-r', revno, self.wrap.get('url'), + self.wrap.get('directory')], cwd=self.subdir_root) def get_data(self, url): blocksize = 10 * 1024 @@ -288,8 +288,8 @@ class Resolver: hashvalue = h.hexdigest() return hashvalue, tmpfile.name - def check_hash(self, p, what, path): - expected = p.get(what + '_hash') + def check_hash(self, what, path): + expected = self.wrap.get(what + '_hash') h = hashlib.sha256() with open(path, 'rb') as f: h.update(f.read()) @@ -297,29 +297,29 @@ class Resolver: if dhash != expected: raise WrapException('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) - def download(self, p, what, ofname): + def download(self, what, ofname): self.check_can_download() - srcurl = p.get(what + '_url') + srcurl = self.wrap.get(what + '_url') mlog.log('Downloading', mlog.bold(self.packagename), what, 'from', mlog.bold(srcurl)) dhash, tmpfile = self.get_data(srcurl) - expected = p.get(what + '_hash') + expected = self.wrap.get(what + '_hash') if dhash != expected: os.remove(tmpfile) raise WrapException('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) os.rename(tmpfile, ofname) - def get_file_internal(self, p, what): - filename = p.get(what + '_filename') + def get_file_internal(self, what): + filename = self.wrap.get(what + '_filename') cache_path = os.path.join(self.cachedir, filename) if os.path.exists(cache_path): - self.check_hash(p, what, cache_path) + self.check_hash(what, cache_path) mlog.log('Using', mlog.bold(self.packagename), what, 'from cache.') return cache_path if not os.path.isdir(self.cachedir): os.mkdir(self.cachedir) - self.download(p, what, cache_path) + self.download(what, cache_path) return cache_path def apply_patch(self, p): From 76ac4f568988d405e65c500fca61259b75ffe396 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Thu, 18 Oct 2018 18:36:32 -0400 Subject: [PATCH 8/8] wrap: Make 'directory' key optional wrap files can now omit the 'directory' key, in that case it will default to the packagename. --- mesonbuild/wrap/wrap.py | 43 +++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 2fd1ad3e2..a4dd7496f 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -114,18 +114,18 @@ class Resolver: self.wrap = self.load_wrap() if self.wrap and 'directory' in self.wrap.values: self.directory = self.wrap.get('directory') - dirname = os.path.join(self.subdir_root, self.directory) - meson_file = os.path.join(dirname, 'meson.build') + self.dirname = os.path.join(self.subdir_root, self.directory) + meson_file = os.path.join(self.dirname, 'meson.build') # The directory is there and has meson.build? Great, use it. if os.path.exists(meson_file): return self.directory # Check if the subproject is a git submodule - self.resolve_git_submodule(dirname) + self.resolve_git_submodule() - if os.path.exists(dirname): - if not os.path.isdir(dirname): + if os.path.exists(self.dirname): + if not os.path.isdir(self.dirname): raise WrapException('Path already exists but is not a directory') else: # A wrap file is required to download @@ -165,13 +165,13 @@ class Resolver: m = 'Automatic wrap-based subproject downloading is disabled' raise WrapException(m) - def resolve_git_submodule(self, dirname): + def resolve_git_submodule(self): # Are we in a git repository? ret, out = quiet_git(['rev-parse'], self.subdir_root) if not ret: return False # Is `dirname` a submodule? - ret, out = quiet_git(['submodule', 'status', dirname], self.subdir_root) + ret, out = quiet_git(['submodule', 'status', self.dirname], self.subdir_root) if not ret: return False # Submodule has not been added, add it @@ -182,12 +182,12 @@ class Resolver: raise WrapException('git submodule has merge conflicts') # Submodule exists, but is deinitialized or wasn't initialized elif out.startswith(b'-'): - if subprocess.call(['git', '-C', self.subdir_root, 'submodule', 'update', '--init', dirname]) == 0: + if subprocess.call(['git', '-C', self.subdir_root, 'submodule', 'update', '--init', self.dirname]) == 0: return True raise WrapException('git submodule failed to init') # Submodule looks fine, but maybe it wasn't populated properly. Do a checkout. elif out.startswith(b' '): - subprocess.call(['git', 'checkout', '.'], cwd=dirname) + subprocess.call(['git', 'checkout', '.'], cwd=self.dirname) # Even if checkout failed, try building it anyway and let the user # handle any problems manually. return True @@ -199,50 +199,47 @@ class Resolver: def get_file(self): path = self.get_file_internal(self.wrap, 'source') - target_dir = os.path.join(self.subdir_root, self.wrap.get('directory')) extract_dir = self.subdir_root # Some upstreams ship packages that do not have a leading directory. # Create one for them. if 'lead_directory_missing' in self.wrap.values: - os.mkdir(target_dir) - extract_dir = target_dir + os.mkdir(self.dirname) + extract_dir = self.dirname shutil.unpack_archive(path, extract_dir) if self.wrap.has_patch(): self.apply_patch() def get_git(self): - checkoutdir = os.path.join(self.subdir_root, self.wrap.get('directory')) revno = self.wrap.get('revision') if self.wrap.values.get('clone-recursive', '').lower() == 'true': subprocess.check_call(['git', 'clone', '--recursive', self.wrap.get('url'), - self.wrap.get('directory')], cwd=self.subdir_root) + self.directory], cwd=self.subdir_root) else: subprocess.check_call(['git', 'clone', self.wrap.get('url'), - self.wrap.get('directory')], cwd=self.subdir_root) + self.directory], cwd=self.subdir_root) if revno.lower() != 'head': - if subprocess.call(['git', 'checkout', revno], cwd=checkoutdir) != 0: - subprocess.check_call(['git', 'fetch', self.wrap.get('url'), revno], cwd=checkoutdir) + if subprocess.call(['git', 'checkout', revno], cwd=self.dirname) != 0: + subprocess.check_call(['git', 'fetch', self.wrap.get('url'), revno], cwd=self.dirname) subprocess.check_call(['git', 'checkout', revno], - cwd=checkoutdir) + cwd=self.dirname) push_url = self.wrap.values.get('push-url') if push_url: subprocess.check_call(['git', 'remote', 'set-url', '--push', 'origin', push_url], - cwd=checkoutdir) + cwd=self.dirname) def get_hg(self): - checkoutdir = os.path.join(self.subdir_root, self.wrap.get('directory')) revno = self.wrap.get('revision') subprocess.check_call(['hg', 'clone', self.wrap.get('url'), - self.wrap.get('directory')], cwd=self.subdir_root) + self.directory], cwd=self.subdir_root) if revno.lower() != 'tip': subprocess.check_call(['hg', 'checkout', revno], - cwd=checkoutdir) + cwd=self.dirname) def get_svn(self): revno = self.wrap.get('revision') subprocess.check_call(['svn', 'checkout', '-r', revno, self.wrap.get('url'), - self.wrap.get('directory')], cwd=self.subdir_root) + self.directory], cwd=self.subdir_root) def get_data(self, url): blocksize = 10 * 1024