| # SPDX-License-Identifier: Apache-2.0 | |
| # Copyright © 2022-2024 Intel Corporation | |
| """Interpreter for converting Cargo Toml definitions to Meson AST | |
| There are some notable limits here. We don't even try to convert something with | |
| a build.rs: there's so few limits on what Cargo allows a build.rs (basically | |
| none), and no good way for us to convert them. In that case, an actual meson | |
| port will be required. | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| import os | |
| import collections | |
| import urllib.parse | |
| import itertools | |
| import typing as T | |
| from . import builder, version, cfg | |
| from .toml import load_toml, TomlImplementationMissing | |
| from .manifest import Manifest, CargoLock, fixup_meson_varname | |
| from ..mesonlib import MesonException, MachineChoice | |
| from .. import coredata, mlog | |
| from ..wrap.wrap import PackageDefinition | |
| if T.TYPE_CHECKING: | |
| from . import raw | |
| from .. import mparser | |
| from .manifest import Dependency, SystemDependency | |
| from ..environment import Environment | |
| from ..interpreterbase import SubProject | |
| from ..compilers.rust import RustCompiler | |
| def _dependency_name(package_name: str, api: str, suffix: str = '-rs') -> str: | |
| basename = package_name[:-len(suffix)] if package_name.endswith(suffix) else package_name | |
| return f'{basename}-{api}{suffix}' | |
| def _dependency_varname(package_name: str) -> str: | |
| return f'{fixup_meson_varname(package_name)}_dep' | |
| def _extra_args_varname() -> str: | |
| return 'extra_args' | |
| def _extra_deps_varname() -> str: | |
| return 'extra_deps' | |
| @dataclasses.dataclass | |
| class PackageState: | |
| manifest: Manifest | |
| downloaded: bool = False | |
| features: T.Set[str] = dataclasses.field(default_factory=set) | |
| required_deps: T.Set[str] = dataclasses.field(default_factory=set) | |
| optional_deps_features: T.Dict[str, T.Set[str]] = dataclasses.field(default_factory=lambda: collections.defaultdict(set)) | |
| @dataclasses.dataclass(frozen=True) | |
| class PackageKey: | |
| package_name: str | |
| api: str | |
| class Interpreter: | |
| def __init__(self, env: Environment) -> None: | |
| self.environment = env | |
| self.host_rustc = T.cast('RustCompiler', self.environment.coredata.compilers[MachineChoice.HOST]['rust']) | |
| # Map Cargo.toml's subdir to loaded manifest. | |
| self.manifests: T.Dict[str, Manifest] = {} | |
| # Map of cargo package (name + api) to its state | |
| self.packages: T.Dict[PackageKey, PackageState] = {} | |
| # Rustc's config | |
| self.cfgs = self._get_cfgs() | |
| def get_build_def_files(self) -> T.List[str]: | |
| return [os.path.join(subdir, 'Cargo.toml') for subdir in self.manifests] | |
| def interpret(self, subdir: str) -> mparser.CodeBlockNode: | |
| manifest = self._load_manifest(subdir) | |
| pkg, cached = self._fetch_package(manifest.package.name, manifest.package.api) | |
| if not cached: | |
| # This is an entry point, always enable the 'default' feature. | |
| # FIXME: We should have a Meson option similar to `cargo build --no-default-features` | |
| self._enable_feature(pkg, 'default') | |
| # Build an AST for this package | |
| filename = os.path.join(self.environment.source_dir, subdir, 'Cargo.toml') | |
| build = builder.Builder(filename) | |
| ast = self._create_project(pkg, build) | |
| ast += [ | |
| build.assign(build.function('import', [build.string('rust')]), 'rust'), | |
| build.function('message', [ | |
| build.string('Enabled features:'), | |
| build.array([build.string(f) for f in pkg.features]), | |
| ]), | |
| ] | |
| ast += self._create_dependencies(pkg, build) | |
| ast += self._create_meson_subdir(build) | |
| if pkg.manifest.lib: | |
| for crate_type in pkg.manifest.lib.crate_type: | |
| ast.extend(self._create_lib(pkg, build, crate_type)) | |
| return build.block(ast) | |
| def _fetch_package(self, package_name: str, api: str) -> T.Tuple[PackageState, bool]: | |
| key = PackageKey(package_name, api) | |
| pkg = self.packages.get(key) | |
| if pkg: | |
| return pkg, True | |
| meson_depname = _dependency_name(package_name, api) | |
| subdir, _ = self.environment.wrap_resolver.resolve(meson_depname) | |
| subprojects_dir = os.path.join(subdir, 'subprojects') | |
| self.environment.wrap_resolver.load_and_merge(subprojects_dir, T.cast('SubProject', meson_depname)) | |
| manifest = self._load_manifest(subdir) | |
| downloaded = \ | |
| meson_depname in self.environment.wrap_resolver.wraps and \ | |
| self.environment.wrap_resolver.wraps[meson_depname].type is not None | |
| pkg = PackageState(manifest, downloaded) | |
| self.packages[key] = pkg | |
| # Merge target specific dependencies that are enabled | |
| for condition, dependencies in manifest.target.items(): | |
| if cfg.eval_cfg(condition, self.cfgs): | |
| manifest.dependencies.update(dependencies) | |
| # Fetch required dependencies recursively. | |
| for depname, dep in manifest.dependencies.items(): | |
| if not dep.optional: | |
| self._add_dependency(pkg, depname) | |
| return pkg, False | |
| def _dep_package(self, dep: Dependency) -> PackageState: | |
| return self.packages[PackageKey(dep.package, dep.api)] | |
| def _load_manifest(self, subdir: str) -> Manifest: | |
| manifest_ = self.manifests.get(subdir) | |
| if not manifest_: | |
| path = os.path.join(self.environment.source_dir, subdir) | |
| filename = os.path.join(path, 'Cargo.toml') | |
| toml = load_toml(filename) | |
| if 'package' in toml: | |
| raw_manifest = T.cast('raw.Manifest', toml) | |
| manifest_ = Manifest.from_raw(raw_manifest, path) | |
| self.manifests[subdir] = manifest_ | |
| else: | |
| raise MesonException(f'{subdir}/Cargo.toml does not have [package] section') | |
| return manifest_ | |
| def _add_dependency(self, pkg: PackageState, depname: str) -> None: | |
| if depname in pkg.required_deps: | |
| return | |
| dep = pkg.manifest.dependencies.get(depname) | |
| if not dep: | |
| if depname in itertools.chain(pkg.manifest.dev_dependencies, pkg.manifest.build_dependencies): | |
| # FIXME: Not supported yet | |
| return | |
| raise MesonException(f'Dependency {depname} not defined in {pkg.manifest.package.name} manifest') | |
| pkg.required_deps.add(depname) | |
| dep_pkg, _ = self._fetch_package(dep.package, dep.api) | |
| if dep.default_features: | |
| self._enable_feature(dep_pkg, 'default') | |
| for f in dep.features: | |
| self._enable_feature(dep_pkg, f) | |
| for f in pkg.optional_deps_features[depname]: | |
| self._enable_feature(dep_pkg, f) | |
| def _enable_feature(self, pkg: PackageState, feature: str) -> None: | |
| if feature in pkg.features: | |
| return | |
| pkg.features.add(feature) | |
| # A feature can also be a dependency. | |
| if feature in pkg.manifest.dependencies: | |
| self._add_dependency(pkg, feature) | |
| # Recurse on extra features and dependencies this feature pulls. | |
| # https://doc.rust-lang.org/cargo/reference/features.html#the-features-section | |
| for f in pkg.manifest.features.get(feature, []): | |
| if '/' in f: | |
| depname, dep_f = f.split('/', 1) | |
| if depname[-1] == '?': | |
| depname = depname[:-1] | |
| if depname in pkg.required_deps: | |
| dep = pkg.manifest.dependencies[depname] | |
| dep_pkg = self._dep_package(dep) | |
| self._enable_feature(dep_pkg, dep_f) | |
| else: | |
| # This feature will be enabled only if that dependency | |
| # is later added. | |
| pkg.optional_deps_features[depname].add(dep_f) | |
| else: | |
| self._add_dependency(pkg, depname) | |
| dep = pkg.manifest.dependencies.get(depname) | |
| if dep: | |
| dep_pkg = self._dep_package(dep) | |
| self._enable_feature(dep_pkg, dep_f) | |
| elif f.startswith('dep:'): | |
| self._add_dependency(pkg, f[4:]) | |
| else: | |
| self._enable_feature(pkg, f) | |
| def _get_cfgs(self) -> T.Dict[str, str]: | |
| cfgs = self.host_rustc.get_cfgs().copy() | |
| rustflags = self.environment.coredata.get_external_args(MachineChoice.HOST, 'rust') | |
| rustflags_i = iter(rustflags) | |
| for i in rustflags_i: | |
| if i == '--cfg': | |
| cfgs.append(next(rustflags_i)) | |
| return dict(self._split_cfg(i) for i in cfgs) | |
| @staticmethod | |
| def _split_cfg(cfg: str) -> T.Tuple[str, str]: | |
| pair = cfg.split('=', maxsplit=1) | |
| value = pair[1] if len(pair) > 1 else '' | |
| if value and value[0] == '"': | |
| value = value[1:-1] | |
| return pair[0], value | |
| def _create_project(self, pkg: PackageState, build: builder.Builder) -> T.List[mparser.BaseNode]: | |
| """Create the project() function call | |
| :param pkg: The package to generate from | |
| :param build: The AST builder | |
| :return: a list nodes | |
| """ | |
| default_options: T.List[mparser.BaseNode] = [] | |
| default_options.append(build.string(f'rust_std={pkg.manifest.package.edition}')) | |
| default_options.append(build.string(f'build.rust_std={pkg.manifest.package.edition}')) | |
| if pkg.downloaded: | |
| default_options.append(build.string('warning_level=0')) | |
| args: T.List[mparser.BaseNode] = [] | |
| args.extend([ | |
| build.string(pkg.manifest.package.name), | |
| build.string('rust'), | |
| ]) | |
| kwargs: T.Dict[str, mparser.BaseNode] = { | |
| 'version': build.string(pkg.manifest.package.version), | |
| # Always assume that the generated meson is using the latest features | |
| # This will warn when when we generate deprecated code, which is helpful | |
| # for the upkeep of the module | |
| 'meson_version': build.string(f'>= {coredata.stable_version}'), | |
| 'default_options': build.array(default_options), | |
| } | |
| if pkg.manifest.package.license: | |
| kwargs['license'] = build.string(pkg.manifest.package.license) | |
| elif pkg.manifest.package.license_file: | |
| kwargs['license_files'] = build.string(pkg.manifest.package.license_file) | |
| return [build.function('project', args, kwargs)] | |
| def _create_dependencies(self, pkg: PackageState, build: builder.Builder) -> T.List[mparser.BaseNode]: | |
| ast: T.List[mparser.BaseNode] = [] | |
| for depname in pkg.required_deps: | |
| dep = pkg.manifest.dependencies[depname] | |
| ast += self._create_dependency(dep, build) | |
| ast.append(build.assign(build.array([]), 'system_deps_args')) | |
| for name, sys_dep in pkg.manifest.system_dependencies.items(): | |
| if sys_dep.enabled(pkg.features): | |
| ast += self._create_system_dependency(name, sys_dep, build) | |
| return ast | |
| def _create_system_dependency(self, name: str, dep: SystemDependency, build: builder.Builder) -> T.List[mparser.BaseNode]: | |
| # TODO: handle feature_overrides | |
| kw = { | |
| 'version': build.array([build.string(s) for s in dep.meson_version]), | |
| 'required': build.bool(not dep.optional), | |
| } | |
| varname = f'{fixup_meson_varname(name)}_system_dep' | |
| cfg = f'system_deps_have_{fixup_meson_varname(name)}' | |
| return [ | |
| build.assign( | |
| build.function( | |
| 'dependency', | |
| [build.string(dep.name)], | |
| kw, | |
| ), | |
| varname, | |
| ), | |
| build.if_( | |
| build.method('found', build.identifier(varname)), build.block([ | |
| build.plusassign( | |
| build.array([build.string('--cfg'), build.string(cfg)]), | |
| 'system_deps_args' | |
| ), | |
| ]) | |
| ), | |
| ] | |
| def _create_dependency(self, dep: Dependency, build: builder.Builder) -> T.List[mparser.BaseNode]: | |
| pkg = self._dep_package(dep) | |
| kw = { | |
| 'version': build.array([build.string(s) for s in dep.meson_version]), | |
| } | |
| # Lookup for this dependency with the features we want in default_options kwarg. | |
| # | |
| # However, this subproject could have been previously configured with a | |
| # different set of features. Cargo collects the set of features globally | |
| # but Meson can only use features enabled by the first call that triggered | |
| # the configuration of that subproject. | |
| # | |
| # Verify all features that we need are actually enabled for that dependency, | |
| # otherwise abort with an error message. The user has to set the corresponding | |
| # option manually with -Dxxx-rs:feature-yyy=true, or the main project can do | |
| # that in its project(..., default_options: ['xxx-rs:feature-yyy=true']). | |
| return [ | |
| # xxx_dep = dependency('xxx', version : ...) | |
| build.assign( | |
| build.function( | |
| 'dependency', | |
| [build.string(_dependency_name(dep.package, dep.api))], | |
| kw, | |
| ), | |
| _dependency_varname(dep.package), | |
| ), | |
| # actual_features = xxx_dep.get_variable('features', default_value : '').split(',') | |
| build.assign( | |
| build.method( | |
| 'split', | |
| build.method( | |
| 'get_variable', | |
| build.identifier(_dependency_varname(dep.package)), | |
| [build.string('features')], | |
| {'default_value': build.string('')} | |
| ), | |
| [build.string(',')], | |
| ), | |
| 'actual_features' | |
| ), | |
| # needed_features = [f1, f2, ...] | |
| # foreach f : needed_features | |
| # if f not in actual_features | |
| # error() | |
| # endif | |
| # endforeach | |
| build.assign(build.array([build.string(f) for f in pkg.features]), 'needed_features'), | |
| build.foreach(['f'], build.identifier('needed_features'), build.block([ | |
| build.if_(build.not_in(build.identifier('f'), build.identifier('actual_features')), build.block([ | |
| build.function('error', [ | |
| build.string('Dependency'), | |
| build.string(_dependency_name(dep.package, dep.api)), | |
| build.string('previously configured with features'), | |
| build.identifier('actual_features'), | |
| build.string('but need'), | |
| build.identifier('needed_features'), | |
| ]) | |
| ])) | |
| ])), | |
| ] | |
| def _create_meson_subdir(self, build: builder.Builder) -> T.List[mparser.BaseNode]: | |
| # Allow Cargo subprojects to add extra Rust args in meson/meson.build file. | |
| # This is used to replace build.rs logic. | |
| # extra_args = [] | |
| # extra_deps = [] | |
| # fs = import('fs') | |
| # if fs.is_dir('meson') | |
| # subdir('meson') | |
| # endif | |
| return [ | |
| build.assign(build.array([]), _extra_args_varname()), | |
| build.assign(build.array([]), _extra_deps_varname()), | |
| build.assign(build.function('import', [build.string('fs')]), 'fs'), | |
| build.if_(build.method('is_dir', build.identifier('fs'), [build.string('meson')]), | |
| build.block([build.function('subdir', [build.string('meson')])])) | |
| ] | |
| def _create_lib(self, pkg: PackageState, build: builder.Builder, crate_type: raw.CRATE_TYPE) -> T.List[mparser.BaseNode]: | |
| dependencies: T.List[mparser.BaseNode] = [] | |
| dependency_map: T.Dict[mparser.BaseNode, mparser.BaseNode] = {} | |
| for name in pkg.required_deps: | |
| dep = pkg.manifest.dependencies[name] | |
| dependencies.append(build.identifier(_dependency_varname(dep.package))) | |
| if name != dep.package: | |
| dep_pkg = self._dep_package(dep) | |
| dep_lib_name = dep_pkg.manifest.lib.name | |
| dependency_map[build.string(fixup_meson_varname(dep_lib_name))] = build.string(name) | |
| for name, sys_dep in pkg.manifest.system_dependencies.items(): | |
| if sys_dep.enabled(pkg.features): | |
| dependencies.append(build.identifier(f'{fixup_meson_varname(name)}_system_dep')) | |
| rust_args: T.List[mparser.BaseNode] = [ | |
| build.identifier('features_args'), | |
| build.identifier(_extra_args_varname()), | |
| build.identifier('system_deps_args'), | |
| ] | |
| dependencies.append(build.identifier(_extra_deps_varname())) | |
| posargs: T.List[mparser.BaseNode] = [ | |
| build.string(fixup_meson_varname(pkg.manifest.lib.name)), | |
| build.string(pkg.manifest.lib.path), | |
| ] | |
| kwargs: T.Dict[str, mparser.BaseNode] = { | |
| 'dependencies': build.array(dependencies), | |
| 'rust_dependency_map': build.dict(dependency_map), | |
| 'rust_args': build.array(rust_args), | |
| } | |
| depname_suffix = '-rs' if crate_type in {'lib', 'rlib', 'proc-macro'} else f'-{crate_type}' | |
| depname = _dependency_name(pkg.manifest.package.name, pkg.manifest.package.api, depname_suffix) | |
| lib: mparser.BaseNode | |
| if pkg.manifest.lib.proc_macro or crate_type == 'proc-macro': | |
| lib = build.method('proc_macro', build.identifier('rust'), posargs, kwargs) | |
| else: | |
| if crate_type in {'lib', 'rlib', 'staticlib'}: | |
| target_type = 'static_library' | |
| elif crate_type in {'dylib', 'cdylib'}: | |
| target_type = 'shared_library' | |
| else: | |
| raise MesonException(f'Unsupported crate type {crate_type}') | |
| if crate_type in {'staticlib', 'cdylib'}: | |
| kwargs['rust_abi'] = build.string('c') | |
| lib = build.function(target_type, posargs, kwargs) | |
| features_args: T.List[mparser.BaseNode] = [] | |
| for f in pkg.features: | |
| features_args += [build.string('--cfg'), build.string(f'feature="{f}"')] | |
| # features_args = ['--cfg', 'feature="f1"', ...] | |
| # lib = xxx_library() | |
| # dep = declare_dependency() | |
| # meson.override_dependency() | |
| return [ | |
| build.assign(build.array(features_args), 'features_args'), | |
| build.assign(lib, 'lib'), | |
| build.assign( | |
| build.function( | |
| 'declare_dependency', | |
| kw={ | |
| 'link_with': build.identifier('lib'), | |
| 'variables': build.dict({ | |
| build.string('features'): build.string(','.join(pkg.features)), | |
| }), | |
| 'version': build.string(pkg.manifest.package.version), | |
| }, | |
| ), | |
| 'dep' | |
| ), | |
| build.method( | |
| 'override_dependency', | |
| build.identifier('meson'), | |
| [ | |
| build.string(depname), | |
| build.identifier('dep'), | |
| ], | |
| ), | |
| ] | |
| def load_wraps(source_dir: str, subproject_dir: str) -> T.List[PackageDefinition]: | |
| """ Convert Cargo.lock into a list of wraps """ | |
| wraps: T.List[PackageDefinition] = [] | |
| filename = os.path.join(source_dir, 'Cargo.lock') | |
| if os.path.exists(filename): | |
| try: | |
| toml = load_toml(filename) | |
| except TomlImplementationMissing as e: | |
| mlog.warning('Failed to load Cargo.lock:', str(e), fatal=False) | |
| return wraps | |
| raw_cargolock = T.cast('raw.CargoLock', toml) | |
| cargolock = CargoLock.from_raw(raw_cargolock) | |
| for package in cargolock.package: | |
| subp_name = _dependency_name(package.name, version.api(package.version)) | |
| if package.source is None: | |
| # This is project's package, or one of its workspace members. | |
| pass | |
| elif package.source == 'registry+https://github.com/rust-lang/crates.io-index': | |
| checksum = package.checksum | |
| if checksum is None: | |
| checksum = cargolock.metadata[f'checksum {package.name} {package.version} ({package.source})'] | |
| url = f'https://crates.io/api/v1/crates/{package.name}/{package.version}/download' | |
| directory = f'{package.name}-{package.version}' | |
| wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'file', { | |
| 'directory': directory, | |
| 'source_url': url, | |
| 'source_filename': f'{directory}.tar.gz', | |
| 'source_hash': checksum, | |
| 'method': 'cargo', | |
| })) | |
| elif package.source.startswith('git+'): | |
| parts = urllib.parse.urlparse(package.source[4:]) | |
| query = urllib.parse.parse_qs(parts.query) | |
| branch = query['branch'][0] if 'branch' in query else '' | |
| revision = parts.fragment or branch | |
| url = urllib.parse.urlunparse(parts._replace(params='', query='', fragment='')) | |
| wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'git', { | |
| 'directory': package.name, | |
| 'url': url, | |
| 'revision': revision, | |
| 'method': 'cargo', | |
| })) | |
| else: | |
| mlog.warning(f'Unsupported source URL in {filename}: {package.source}') | |
| return wraps |