cargo/interpreter: Implement an interpreter for Cargo TOML
This converts a Cargo TOML file into Meson AST Co-Authored-By: Thibault Saunier <tsaunier@igalia.com>
This commit is contained in:
Dylan Baker
parent
4017dab484
commit
f02e26877d
|
|
@ -0,0 +1,451 @@
|
|||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright © 2022-2023 Intel Corporation
|
||||
|
||||
"""Interpreter for converting Cargo Toml definitions to Meson AST
|
||||
|
||||
There are some notable limits here. We don't even try to convert something with
|
||||
a build.rs: there's so few limits on what Cargo allows a build.rs (basically
|
||||
none), and no good way for us to convert them. In that case, an actual meson
|
||||
port will be required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import dataclasses
|
||||
import glob
|
||||
import importlib
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import typing as T
|
||||
|
||||
from . import builder
|
||||
from . import version
|
||||
from .. import mparser
|
||||
from .._pathlib import Path
|
||||
from ..mesonlib import MesonException, Popen_safe
|
||||
|
||||
if T.TYPE_CHECKING:
|
||||
from types import ModuleType
|
||||
|
||||
from . import manifest
|
||||
from ..environment import Environment
|
||||
|
||||
# tomllib is present in python 3.11, before that it is a pypi module called tomli,
|
||||
# we try to import tomllib, then tomli,
|
||||
# TODO: add a fallback to toml2json?
|
||||
tomllib: T.Optional[ModuleType] = None
|
||||
toml2json: T.Optional[str] = None
|
||||
for t in ['tomllib', 'tomli']:
|
||||
try:
|
||||
tomllib = importlib.import_module(t)
|
||||
break
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
# TODO: it would be better to use an Executable here, which could be looked
|
||||
# up in the cross file or provided by a wrap. However, that will have to be
|
||||
# passed in externally, since we don't have (and I don't think we should),
|
||||
# have access to the `Environment` for that in this module.
|
||||
toml2json = shutil.which('toml2json')
|
||||
|
||||
|
||||
def load_toml(filename: str) -> T.Dict[object, object]:
|
||||
if tomllib:
|
||||
with open(filename, 'rb') as f:
|
||||
raw = tomllib.load(f)
|
||||
else:
|
||||
if toml2json is None:
|
||||
raise MesonException('Could not find an implementation of tomllib, nor toml2json')
|
||||
|
||||
p, out, err = Popen_safe([toml2json, filename])
|
||||
if p.returncode != 0:
|
||||
raise MesonException('toml2json failed to decode output\n', err)
|
||||
|
||||
raw = json.loads(out)
|
||||
|
||||
if not isinstance(raw, dict):
|
||||
raise MesonException("Cargo.toml isn't a dictionary? How did that happen?")
|
||||
|
||||
return raw
|
||||
|
||||
|
||||
def fixup_meson_varname(name: str) -> str:
|
||||
"""Fixup a meson variable name
|
||||
|
||||
:param name: The name to fix
|
||||
:return: the fixed name
|
||||
"""
|
||||
return name.replace('-', '_')
|
||||
|
||||
# Pylance can figure out that these do not, in fact, overlap, but mypy can't
|
||||
@T.overload
|
||||
def _fixup_raw_mappings(d: manifest.BuildTarget) -> manifest.FixedBuildTarget: ... # type: ignore
|
||||
|
||||
@T.overload
|
||||
def _fixup_raw_mappings(d: manifest.LibTarget) -> manifest.FixedLibTarget: ... # type: ignore
|
||||
|
||||
@T.overload
|
||||
def _fixup_raw_mappings(d: manifest.Dependency) -> manifest.FixedDependency: ...
|
||||
|
||||
def _fixup_raw_mappings(d: T.Union[manifest.BuildTarget, manifest.LibTarget, manifest.Dependency]
|
||||
) -> T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget,
|
||||
manifest.FixedDependency]:
|
||||
"""Fixup raw cargo mappings to ones more suitable for python to consume.
|
||||
|
||||
This does the following:
|
||||
* replaces any `-` with `_`, cargo likes the former, but python dicts make
|
||||
keys with `-` in them awkward to work with
|
||||
* Convert Dependndency versions from the cargo format to something meson
|
||||
understands
|
||||
|
||||
:param d: The mapping to fix
|
||||
:return: the fixed string
|
||||
"""
|
||||
raw = {fixup_meson_varname(k): v for k, v in d.items()}
|
||||
if 'version' in raw:
|
||||
assert isinstance(raw['version'], str), 'for mypy'
|
||||
raw['version'] = version.convert(raw['version'])
|
||||
return T.cast('T.Union[manifest.FixedBuildTarget, manifest.FixedLibTarget, manifest.FixedDependency]', raw)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Package:
|
||||
|
||||
"""Representation of a Cargo Package entry, with defaults filled in."""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
description: str
|
||||
resolver: T.Optional[str] = None
|
||||
authors: T.List[str] = dataclasses.field(default_factory=list)
|
||||
edition: manifest.EDITION = '2015'
|
||||
rust_version: T.Optional[str] = None
|
||||
documentation: T.Optional[str] = None
|
||||
readme: T.Optional[str] = None
|
||||
homepage: T.Optional[str] = None
|
||||
repository: T.Optional[str] = None
|
||||
license: T.Optional[str] = None
|
||||
license_file: T.Optional[str] = None
|
||||
keywords: T.List[str] = dataclasses.field(default_factory=list)
|
||||
categories: T.List[str] = dataclasses.field(default_factory=list)
|
||||
workspace: T.Optional[str] = None
|
||||
build: T.Optional[str] = None
|
||||
links: T.Optional[str] = None
|
||||
exclude: T.List[str] = dataclasses.field(default_factory=list)
|
||||
include: T.List[str] = dataclasses.field(default_factory=list)
|
||||
publish: bool = True
|
||||
metadata: T.Dict[str, T.Dict[str, str]] = dataclasses.field(default_factory=dict)
|
||||
default_run: T.Optional[str] = None
|
||||
autobins: bool = True
|
||||
autoexamples: bool = True
|
||||
autotests: bool = True
|
||||
autobenches: bool = True
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Dependency:
|
||||
|
||||
"""Representation of a Cargo Dependency Entry."""
|
||||
|
||||
version: T.List[str]
|
||||
registry: T.Optional[str] = None
|
||||
git: T.Optional[str] = None
|
||||
branch: T.Optional[str] = None
|
||||
rev: T.Optional[str] = None
|
||||
path: T.Optional[str] = None
|
||||
optional: bool = False
|
||||
package: T.Optional[str] = None
|
||||
default_features: bool = False
|
||||
features: T.List[str] = dataclasses.field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def from_raw(cls, raw: manifest.DependencyV) -> Dependency:
|
||||
"""Create a dependency from a raw cargo dictionary"""
|
||||
if isinstance(raw, str):
|
||||
return cls(version.convert(raw))
|
||||
return cls(**_fixup_raw_mappings(raw))
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class BuildTarget:
|
||||
|
||||
name: str
|
||||
crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib'])
|
||||
path: dataclasses.InitVar[T.Optional[str]] = None
|
||||
|
||||
# https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-test-field
|
||||
# True for lib, bin, test
|
||||
test: bool = True
|
||||
|
||||
# https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doctest-field
|
||||
# True for lib
|
||||
doctest: bool = False
|
||||
|
||||
# https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-bench-field
|
||||
# True for lib, bin, benchmark
|
||||
bench: bool = True
|
||||
|
||||
# https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-doc-field
|
||||
# True for libraries and binaries
|
||||
doc: bool = False
|
||||
|
||||
harness: bool = True
|
||||
edition: manifest.EDITION = '2015'
|
||||
required_features: T.List[str] = dataclasses.field(default_factory=list)
|
||||
plugin: bool = False
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Library(BuildTarget):
|
||||
|
||||
"""Representation of a Cargo Library Entry."""
|
||||
|
||||
doctest: bool = True
|
||||
doc: bool = True
|
||||
proc_macro: bool = False
|
||||
crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['lib'])
|
||||
doc_scrape_examples: bool = True
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Binary(BuildTarget):
|
||||
|
||||
"""Representation of a Cargo Bin Entry."""
|
||||
|
||||
doc: bool = True
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Test(BuildTarget):
|
||||
|
||||
"""Representation of a Cargo Test Entry."""
|
||||
|
||||
bench: bool = True
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Benchmark(BuildTarget):
|
||||
|
||||
"""Representation of a Cargo Benchmark Entry."""
|
||||
|
||||
test: bool = True
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Example(BuildTarget):
|
||||
|
||||
"""Representation of a Cargo Example Entry."""
|
||||
|
||||
crate_type: T.List[manifest.CRATE_TYPE] = dataclasses.field(default_factory=lambda: ['bin'])
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Manifest:
|
||||
|
||||
"""Cargo Manifest definition.
|
||||
|
||||
Most of these values map up to the Cargo Manifest, but with default values
|
||||
if not provided.
|
||||
|
||||
Cargo subprojects can contain what Meson wants to treat as multiple,
|
||||
interdependent, subprojects.
|
||||
|
||||
:param subdir: the subdirectory that this cargo project is in
|
||||
:param path: the path within the cargo subproject.
|
||||
"""
|
||||
|
||||
package: Package
|
||||
dependencies: T.Dict[str, Dependency]
|
||||
dev_dependencies: T.Dict[str, Dependency]
|
||||
build_dependencies: T.Dict[str, Dependency]
|
||||
lib: Library
|
||||
bin: T.List[Binary]
|
||||
test: T.List[Test]
|
||||
bench: T.List[Benchmark]
|
||||
example: T.List[Example]
|
||||
features: T.Dict[str, T.List[str]]
|
||||
target: T.Dict[str, T.Dict[str, Dependency]]
|
||||
subdir: str
|
||||
path: str = ''
|
||||
|
||||
|
||||
def _create_project(package: Package, build: builder.Builder, env: Environment) -> mparser.FunctionNode:
|
||||
"""Create a function call
|
||||
|
||||
:param package: The Cargo package to generate from
|
||||
:param filename: The full path to the file
|
||||
:param meson_version: The generating meson version
|
||||
:return: a FunctionNode
|
||||
"""
|
||||
args: T.List[mparser.BaseNode] = []
|
||||
args.extend([
|
||||
build.string(package.name),
|
||||
build.string('rust'),
|
||||
])
|
||||
kwargs: T.Dict[str, mparser.BaseNode] = {
|
||||
'version': build.string(package.version),
|
||||
# Always assume that the generated meson is using the latest features
|
||||
# This will warn when when we generate deprecated code, which is helpful
|
||||
# for the upkeep of the module
|
||||
'meson_version': build.string(f'>= {env.coredata.version}'),
|
||||
'default_options': build.array([build.string(f'rust_std={package.edition}')]),
|
||||
}
|
||||
if package.license:
|
||||
kwargs['license'] = build.string(package.license)
|
||||
elif package.license_file:
|
||||
kwargs['license_files'] = build.string(package.license_file)
|
||||
|
||||
return build.function('project', args, kwargs)
|
||||
|
||||
|
||||
def _convert_manifest(raw_manifest: manifest.Manifest, subdir: str, path: str = '') -> Manifest:
|
||||
# This cast is a bit of a hack to deal with proc-macro
|
||||
lib = _fixup_raw_mappings(raw_manifest.get('lib', {}))
|
||||
|
||||
# We need to set the name field if it's not set manually,
|
||||
# including if other fields are set in the lib section
|
||||
lib.setdefault('name', raw_manifest['package']['name'])
|
||||
|
||||
pkg = T.cast('manifest.FixedPackage',
|
||||
{fixup_meson_varname(k): v for k, v in raw_manifest['package'].items()})
|
||||
|
||||
return Manifest(
|
||||
Package(**pkg),
|
||||
{k: Dependency.from_raw(v) for k, v in raw_manifest.get('dependencies', {}).items()},
|
||||
{k: Dependency.from_raw(v) for k, v in raw_manifest.get('dev-dependencies', {}).items()},
|
||||
{k: Dependency.from_raw(v) for k, v in raw_manifest.get('build-dependencies', {}).items()},
|
||||
Library(**lib),
|
||||
[Binary(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bin', {})],
|
||||
[Test(**_fixup_raw_mappings(b)) for b in raw_manifest.get('test', {})],
|
||||
[Benchmark(**_fixup_raw_mappings(b)) for b in raw_manifest.get('bench', {})],
|
||||
[Example(**_fixup_raw_mappings(b)) for b in raw_manifest.get('example', {})],
|
||||
raw_manifest.get('features', {}),
|
||||
{k: {k2: Dependency.from_raw(v2) for k2, v2 in v['dependencies'].items()}
|
||||
for k, v in raw_manifest.get('target', {}).items()},
|
||||
subdir,
|
||||
path,
|
||||
)
|
||||
|
||||
|
||||
def _load_manifests(subdir: str) -> T.Dict[str, Manifest]:
|
||||
filename = os.path.join(subdir, 'Cargo.toml')
|
||||
raw = load_toml(filename)
|
||||
|
||||
manifests: T.Dict[str, Manifest] = {}
|
||||
|
||||
raw_manifest: T.Union[manifest.Manifest, manifest.VirtualManifest]
|
||||
if 'package' in raw:
|
||||
raw_manifest = T.cast('manifest.Manifest', raw)
|
||||
manifest_ = _convert_manifest(raw_manifest, subdir)
|
||||
manifests[manifest_.package.name] = manifest_
|
||||
else:
|
||||
raw_manifest = T.cast('manifest.VirtualManifest', raw)
|
||||
|
||||
if 'workspace' in raw_manifest:
|
||||
# XXX: need to verify that python glob and cargo globbing are the
|
||||
# same and probably write a glob implementation. Blarg
|
||||
|
||||
# We need to chdir here to make the glob work correctly
|
||||
pwd = os.getcwd()
|
||||
os.chdir(subdir)
|
||||
members: T.Iterable[str]
|
||||
try:
|
||||
members = itertools.chain.from_iterable(
|
||||
glob.glob(m) for m in raw_manifest['workspace']['members'])
|
||||
finally:
|
||||
os.chdir(pwd)
|
||||
if 'exclude' in raw_manifest['workspace']:
|
||||
members = (x for x in members if x not in raw_manifest['workspace']['exclude'])
|
||||
|
||||
for m in members:
|
||||
filename = os.path.join(subdir, m, 'Cargo.toml')
|
||||
raw = load_toml(filename)
|
||||
|
||||
raw_manifest = T.cast('manifest.Manifest', raw)
|
||||
man = _convert_manifest(raw_manifest, subdir, m)
|
||||
manifests[man.package.name] = man
|
||||
|
||||
return manifests
|
||||
|
||||
|
||||
def load_all_manifests(subproject_dir: str) -> T.Dict[str, Manifest]:
|
||||
"""Find all cargo subprojects, and load them
|
||||
|
||||
:param subproject_dir: Directory to look for subprojects in
|
||||
:return: A dictionary of rust project names to Manifests
|
||||
"""
|
||||
manifests: T.Dict[str, Manifest] = {}
|
||||
for p in Path(subproject_dir).iterdir():
|
||||
if p.is_dir() and (p / 'Cargo.toml').exists():
|
||||
manifests.update(_load_manifests(str(p)))
|
||||
return manifests
|
||||
|
||||
|
||||
def _create_lib(cargo: Manifest, build: builder.Builder) -> T.List[mparser.BaseNode]:
|
||||
kw: T.Dict[str, mparser.BaseNode] = {}
|
||||
if cargo.dependencies:
|
||||
ids = [build.identifier(f'dep_{n}') for n in cargo.dependencies]
|
||||
kw['dependencies'] = build.array(
|
||||
[build.method('get_variable', i, [build.string('dep')]) for i in ids])
|
||||
|
||||
# FIXME: currently assuming that an rlib is being generated, which is
|
||||
# the most common.
|
||||
return [
|
||||
build.assign(
|
||||
build.function(
|
||||
'static_library',
|
||||
[
|
||||
build.string(fixup_meson_varname(cargo.package.name)),
|
||||
build.string(os.path.join('src', 'lib.rs')),
|
||||
],
|
||||
kw,
|
||||
),
|
||||
'lib'
|
||||
),
|
||||
|
||||
build.assign(
|
||||
build.function(
|
||||
'declare_dependency',
|
||||
kw={'link_with': build.identifier('lib'), **kw},
|
||||
),
|
||||
'dep'
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def interpret(cargo: Manifest, env: Environment) -> mparser.CodeBlockNode:
|
||||
filename = os.path.join(cargo.subdir, cargo.path, 'Cargo.toml')
|
||||
build = builder.Builder(filename)
|
||||
|
||||
ast: T.List[mparser.BaseNode] = [
|
||||
_create_project(cargo.package, build, env),
|
||||
build.assign(build.function('import', [build.string('rust')]), 'rust'),
|
||||
]
|
||||
|
||||
if cargo.dependencies:
|
||||
for name, dep in cargo.dependencies.items():
|
||||
kw = {
|
||||
'version': build.array([build.string(s) for s in dep.version]),
|
||||
}
|
||||
ast.extend([
|
||||
build.assign(
|
||||
build.method(
|
||||
'cargo',
|
||||
build.identifier('rust'),
|
||||
[build.string(name)],
|
||||
kw,
|
||||
),
|
||||
f'dep_{fixup_meson_varname(name)}',
|
||||
),
|
||||
])
|
||||
|
||||
# Libs are always auto-discovered and there's no other way to handle them,
|
||||
# which is unfortunate for reproducability
|
||||
if os.path.exists(os.path.join(env.source_dir, cargo.subdir, cargo.path, 'src', 'lib.rs')):
|
||||
ast.extend(_create_lib(cargo, build))
|
||||
|
||||
# XXX: make this not awful
|
||||
block = builder.block(filename)
|
||||
block.lines = ast
|
||||
return block
|
||||
Loading…
Reference in New Issue