From 6776bfad3ce0601b9ee0c604b5d51031726824b3 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 28 Jun 2021 11:47:16 -0400 Subject: [PATCH 1/4] Add "meson wrap update-db" command It downloads releases.json from wrapdb and store it in subprojects/wrapdb.json. That file will be used by Meson to find dependency fallbacks offline. --- mesonbuild/wrap/wraptool.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py index ec2ac3edc..80a58ab1e 100644 --- a/mesonbuild/wrap/wraptool.py +++ b/mesonbuild/wrap/wraptool.py @@ -21,6 +21,7 @@ import typing as T from glob import glob from urllib.parse import urlparse from .wrap import open_wrapdburl, WrapException +from pathlib import Path from .. import mesonlib @@ -69,9 +70,18 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None: p.add_argument('project_path') p.set_defaults(wrap_func=promote) -def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: + p = subparsers.add_parser('update-db', help='Update list of projects available in WrapDB (Since 0.61.0)') + p.add_argument('--allow-insecure', default=False, action='store_true', + help='Allow insecure server connections.') + p.set_defaults(wrap_func=update_db) + +def get_releases_data(allow_insecure: bool) -> bytes: url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True) - return T.cast('T.Dict[str, T.Any]', json.loads(url.read().decode())) + return url.read() + +def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]: + data = get_releases_data(allow_insecure) + return T.cast('T.Dict[str, T.Any]', json.loads(data.decode())) def list_projects(options: 'argparse.Namespace') -> None: releases = get_releases(options.allow_insecure) @@ -244,6 +254,12 @@ def status(options: 'argparse.Namespace') -> None: else: print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.') +def update_db(options: 'argparse.Namespace') -> None: + data = get_releases_data(options.allow_insecure) + Path('subprojects').mkdir(exist_ok=True) + with Path('subprojects/wrapdb.json').open('wb') as f: + f.write(data) + def run(options: 'argparse.Namespace') -> int: options.wrap_func(options) return 0 From ced9efb5793ff932a0132e79378ded4e393c200d Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 28 Jun 2021 11:53:34 -0400 Subject: [PATCH 2/4] Get wrap from wrapdb when not found locally Download wrap file from wrapdb automatically when it is not found locally but we have it in wrapdb.json. This makes for example `dependency('glib-2.0')` work out of the box simply by running `meson wrap update-db`, even if the project does not provide any wraps. --- mesonbuild/wrap/wrap.py | 68 +++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 10aa1b80e..1cc55ee7f 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -29,10 +29,12 @@ import configparser import time import typing as T import textwrap +import json from base64 import b64encode from netrc import netrc from pathlib import Path + from . import WrapMode from .. import coredata from ..mesonlib import quiet_git, GIT, ProgressBar, MesonException, windows_proof_rmtree, Popen_safe @@ -262,8 +264,12 @@ class Resolver: self.netrc: T.Optional[netrc] = None self.provided_deps = {} # type: T.Dict[str, PackageDefinition] self.provided_programs = {} # type: T.Dict[str, PackageDefinition] + self.wrapdb: T.Dict[str, T.Any] = {} + self.wrapdb_provided_deps: T.Dict[str, str] = {} + self.wrapdb_provided_programs: T.Dict[str, str] = {} self.load_wraps() self.load_netrc() + self.load_wrapdb() def load_netrc(self) -> None: try: @@ -294,18 +300,48 @@ class Resolver: self.wraps[wrap.name] = wrap for wrap in self.wraps.values(): - for k in wrap.provided_deps.keys(): - if k in self.provided_deps: - prev_wrap = self.provided_deps[k] - m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}' - raise WrapException(m) - self.provided_deps[k] = wrap - for k in wrap.provided_programs: - if k in self.provided_programs: - prev_wrap = self.provided_programs[k] - m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}' - raise WrapException(m) - self.provided_programs[k] = wrap + self.add_wrap(wrap) + + def add_wrap(self, wrap: PackageDefinition) -> None: + for k in wrap.provided_deps.keys(): + if k in self.provided_deps: + prev_wrap = self.provided_deps[k] + m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}' + raise WrapException(m) + self.provided_deps[k] = wrap + for k in wrap.provided_programs: + if k in self.provided_programs: + prev_wrap = self.provided_programs[k] + m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}' + raise WrapException(m) + self.provided_programs[k] = wrap + + def load_wrapdb(self) -> None: + try: + with Path(self.subdir_root, 'wrapdb.json').open('r', encoding='utf-8') as f: + self.wrapdb = json.load(f) + except FileNotFoundError: + return + for name, info in self.wrapdb.items(): + self.wrapdb_provided_deps.update({i: name for i in info.get('dependency_names', [])}) + self.wrapdb_provided_programs.update({i: name for i in info.get('program_names', [])}) + + def get_from_wrapdb(self, subp_name: str) -> PackageDefinition: + info = self.wrapdb.get(subp_name) + if not info: + return None + self.check_can_download() + latest_version = info['versions'][0] + version, revision = latest_version.rsplit('-', 1) + url = urllib.request.urlopen(f'https://wrapdb.mesonbuild.com/v2/{subp_name}_{version}-{revision}/{subp_name}.wrap') + fname = Path(self.subdir_root, f'{subp_name}.wrap') + with fname.open('wb') as f: + f.write(url.read()) + mlog.log(f'Installed {subp_name} version {version} revision {revision}') + wrap = PackageDefinition(str(fname)) + self.wraps[wrap.name] = wrap + self.add_wrap(wrap) + return wrap def merge_wraps(self, other_resolver: 'Resolver') -> None: for k, v in other_resolver.wraps.items(): @@ -323,7 +359,8 @@ class Resolver: if wrap: dep_var = wrap.provided_deps.get(packagename) return wrap.name, dep_var - return None, None + wrap_name = self.wrapdb_provided_deps.get(packagename) + return wrap_name, None def get_varname(self, subp_name: str, depname: str) -> T.Optional[str]: wrap = self.wraps.get(subp_name) @@ -334,12 +371,17 @@ class Resolver: wrap = self.provided_programs.get(name) if wrap: return wrap.name + wrap_name = self.wrapdb_provided_programs.get(name) + if wrap_name: + return wrap_name return None def resolve(self, packagename: str, method: str) -> str: self.packagename = packagename self.directory = packagename self.wrap = self.wraps.get(packagename) + if not self.wrap: + self.wrap = self.get_from_wrapdb(packagename) if not self.wrap: m = f'Neither a subproject directory nor a {self.packagename}.wrap file was found.' raise WrapNotFoundException(m) From 39d825fabfde54268b990aaec23539800c95ca89 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 28 Jun 2021 12:00:40 -0400 Subject: [PATCH 3/4] Get fallback varname from wrap file after it has been configured When _subproject_impl() is called the wrap file could not have been downloaded form wrapdb yet, it is done when fetching the subproject. Delay getting the variable name to when we actually need it, at that point we are sure the wrap file has been downloaded. --- mesonbuild/interpreter/dependencyfallbacks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mesonbuild/interpreter/dependencyfallbacks.py b/mesonbuild/interpreter/dependencyfallbacks.py index 3fbce42f0..54be99056 100644 --- a/mesonbuild/interpreter/dependencyfallbacks.py +++ b/mesonbuild/interpreter/dependencyfallbacks.py @@ -66,14 +66,6 @@ class DependencyFallbacksHolder(MesonInterpreterObject): self._subproject_impl(subp_name, varname) def _subproject_impl(self, subp_name: str, varname: str) -> None: - if not varname: - # If no variable name is specified, check if the wrap file has one. - # If the wrap file has a variable name, better use it because the - # subproject most probably is not using meson.override_dependency(). - for name in self.names: - varname = self.wrap_resolver.get_varname(subp_name, name) - if varname: - break assert self.subproject_name is None self.subproject_name = subp_name self.subproject_varname = varname @@ -174,6 +166,14 @@ class DependencyFallbacksHolder(MesonInterpreterObject): # Legacy: Use the variable name if provided instead of relying on the # subproject to override one of our dependency names + if not varname: + # If no variable name is specified, check if the wrap file has one. + # If the wrap file has a variable name, better use it because the + # subproject most probably is not using meson.override_dependency(). + for name in self.names: + varname = self.wrap_resolver.get_varname(subp_name, name) + if varname: + break if not varname: mlog.warning(f'Subproject {subp_name!r} did not override {self._display_name!r} dependency and no variable name specified') mlog.log('Dependency', mlog.bold(self._display_name), 'from subproject', From 8c3a87847e8469c4b49a1ed130247e4ff023072a Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 28 Jun 2021 12:20:32 -0400 Subject: [PATCH 4/4] Document and test new WrapDB auto fallback --- docs/markdown/Using-wraptool.md | 14 ++++++++++++++ docs/markdown/snippets/wrapdb.md | 6 ++++++ docs/yaml/functions/dependency.yaml | 5 +++++ unittests/platformagnostictests.py | 19 +++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 docs/markdown/snippets/wrapdb.md diff --git a/docs/markdown/Using-wraptool.md b/docs/markdown/Using-wraptool.md index cabdc0efd..edbceaada 100644 --- a/docs/markdown/Using-wraptool.md +++ b/docs/markdown/Using-wraptool.md @@ -82,3 +82,17 @@ straightforward: Wraptool can do other things besides these. Documentation for these can be found in the command line help, which can be accessed by `meson wrap --help`. + +## Automatic dependency fallback + +Since *0.64.0* Meson can use WrapDB to automatically find missing dependencies. + +The user simply needs to download latest database, the following command stores +it in `subprojects/wrapdb.json`: + $ meson wrap update-db + +Once the database is available locally, any dependency not found on the system +but available in WrapDB will automatically be downloaded. + +Automatic fetch of WrapDB subprojects can be disabled by removing the file +`subprojects/wrapdb.json`, or by using `--wrap-mode=nodownload`. diff --git a/docs/markdown/snippets/wrapdb.md b/docs/markdown/snippets/wrapdb.md new file mode 100644 index 000000000..d5caf4f66 --- /dev/null +++ b/docs/markdown/snippets/wrapdb.md @@ -0,0 +1,6 @@ +## Automatic fallback using WrapDB + +A new command has been added: `meson wrap update-db`. It downloads the list of +wraps available in [WrapDB](wrapdb.mesonbuild.com) and stores it locally in +`subprojects/wrapdb.json`. When that file exists and a dependency is not found +on the system but is available in WrapDB, Meson will automatically download it. diff --git a/docs/yaml/functions/dependency.yaml b/docs/yaml/functions/dependency.yaml index dcb696d36..3a4d2e8fb 100644 --- a/docs/yaml/functions/dependency.yaml +++ b/docs/yaml/functions/dependency.yaml @@ -15,6 +15,11 @@ description: | of those name will return the same value. This is useful in case a dependency could have different names, such as `png` and `libpng`. + * Since *0.64.0* a dependency fallback can be provided by WrapDB. Simply download + the database locally using `meson wrap update-db` command and Meson will + automatically fallback to subprojects provided by WrapDB if the dependency is + not found on the system and the project does not ship their own `.wrap` file. + Dependencies can also be resolved in two other ways: * if the same name was used in a `meson.override_dependency` prior to diff --git a/unittests/platformagnostictests.py b/unittests/platformagnostictests.py index a2d664019..ed5d96b8a 100644 --- a/unittests/platformagnostictests.py +++ b/unittests/platformagnostictests.py @@ -14,7 +14,10 @@ import os import tempfile +import subprocess +import textwrap from unittest import skipIf +from pathlib import Path from .baseplatformtests import BasePlatformTests from .helpers import is_ci @@ -94,3 +97,19 @@ class PlatformAgnosticTests(BasePlatformTests): # https://github.com/mesonbuild/meson/issues/10225. self.setconf('-Dfoo=enabled') self.build('reconfigure') + + def test_update_wrapdb(self): + # Write the project into a temporary directory because it will add files + # into subprojects/ and we don't want to pollute meson source tree. + with tempfile.TemporaryDirectory() as testdir: + with Path(testdir, 'meson.build').open('w', encoding='utf-8') as f: + f.write(textwrap.dedent( + ''' + project('wrap update-db', + default_options: ['wrap_mode=forcefallback']) + + zlib_dep = dependency('zlib') + assert(zlib_dep.type_name() == 'internal') + ''')) + subprocess.check_call(self.wrap_command + ['update-db'], cwd=testdir) + self.init(testdir, workdir=testdir)