| # SPDX-License-Identifier: Apache-2. |
| # Copyright 2022 The Meson development team |
| |
| from __future__ import annotations |
| |
| import functools, json, operator, os, textwrap |
| from pathlib import Path |
| import typing as T |
| |
| from .. import mesonlib, mlog |
| from .base import process_method_kw, DependencyCandidate, DependencyException, DependencyMethods, ExternalDependency, SystemDependency |
| from .configtool import ConfigToolDependency |
| from .detect import packages |
| from .factory import DependencyFactory |
| from .framework import ExtraFrameworkDependency |
| from .pkgconfig import PkgConfigDependency |
| from ..envconfig import detect_cpu_family |
| from ..mesonlib import MachineChoice |
| from ..programs import ExternalProgram |
| from ..options import OptionKey |
| |
| if T.TYPE_CHECKING: |
| from typing_extensions import Final, TypedDict |
| |
| from .factory import DependencyGenerator |
| from ..environment import Environment |
| from .base import DependencyObjectKWs |
| |
| class PythonIntrospectionDict(TypedDict): |
| |
| install_paths: T.Dict[str, str] |
| is_pypy: bool |
| is_venv: bool |
| is_freethreaded: bool |
| link_libpython: bool |
| sysconfig_paths: T.Dict[str, str] |
| paths: T.Dict[str, str] |
| platform: str |
| suffix: str |
| limited_api_suffix: str |
| variables: T.Dict[str, str] |
| version: str |
| |
| _Base = ExternalDependency |
| else: |
| _Base = object |
| |
| |
| class Pybind11ConfigToolDependency(ConfigToolDependency): |
| |
| tools = ['pybind11-config'] |
| |
| # any version of the tool is valid, since this is header-only |
| allow_default_for_cross = True |
| |
| # pybind11 in 2.10.4 added --version, sanity-check another flag unique to it |
| # in the meantime |
| skip_version = '--pkgconfigdir' |
| |
| def __init__(self, name: str, environment: Environment, kwargs: DependencyObjectKWs): |
| super().__init__(name, environment, kwargs) |
| if not self.is_found: |
| return |
| self.compile_args = self.get_config_value(['--includes'], 'compile_args') |
| |
| |
| class NumPyConfigToolDependency(ConfigToolDependency): |
| |
| tools = ['numpy-config'] |
| |
| def __init__(self, name: str, environment: Environment, kwargs: DependencyObjectKWs): |
| super().__init__(name, environment, kwargs) |
| if not self.is_found: |
| return |
| self.compile_args = self.get_config_value(['--cflags'], 'compile_args') |
| |
| |
| class PythonBuildConfig: |
| """PEP 739 build-details.json config file.""" |
| |
| IMPLEMENTED_VERSION: Final[str] = '1.0' |
| """Schema version currently implemented.""" |
| _PATH_KEYS = ( |
| 'base_interpreter', |
| 'libpython.dynamic', |
| 'libpython.dynamic_stableabi', |
| 'libpython.static', |
| 'c_api.headers', |
| 'c_api.pkgconfig_path', |
| ) |
| """Path keys — may be relative, need to be expanded.""" |
| |
| def __init__(self, path: str) -> None: |
| self._path = Path(path) |
| |
| try: |
| self._data = json.loads(self._path.read_text(encoding='utf8')) |
| except OSError as e: |
| raise DependencyException(f'Failed to read python.build_config: {e}') from e |
| |
| self._validate_data() |
| self._expand_paths() |
| |
| def __getitem__(self, key: str) -> T.Any: |
| return functools.reduce(operator.getitem, key.split('.'), self._data) |
| |
| def __contains__(self, key: str) -> bool: |
| try: |
| self[key] |
| except KeyError: |
| return False |
| else: |
| return True |
| |
| def get(self, key: str, default: T.Any = None) -> T.Any: |
| try: |
| return self[key] |
| except KeyError: |
| return default |
| |
| def _validate_data(self) -> None: |
| schema_version = self._data['schema_version'] |
| if mesonlib.version_compare(schema_version, '< 1.0'): |
| raise DependencyException(f'Invalid schema_version in python.build_config: {schema_version}') |
| if mesonlib.version_compare(schema_version, '>= 2.0'): |
| raise DependencyException( |
| f'Unsupported schema_version {schema_version!r} in python.build_config, ' |
| f'but we only implement support for {self.IMPLEMENTED_VERSION!r}' |
| ) |
| # Schema version that we currently understand |
| if mesonlib.version_compare(schema_version, f'> {self.IMPLEMENTED_VERSION}'): |
| mlog.log( |
| f'python.build_config has schema_version {schema_version!r}, ' |
| f'but we only implement support for {self.IMPLEMENTED_VERSION!r}, ' |
| 'new functionality might be missing' |
| ) |
| |
| def _expand_paths(self) -> None: |
| """Expand relative path (they're relative to base_prefix).""" |
| for key in self._PATH_KEYS: |
| if key not in self: |
| continue |
| parent, _, child = key.rpartition('.') |
| container = self[parent] if parent else self._data |
| path = Path(container[child]) |
| if not path.is_absolute(): |
| container[child] = os.fspath(self.base_prefix / path) |
| |
| @property |
| def config_path(self) -> Path: |
| return self._path |
| |
| @mesonlib.lazy_property |
| def base_prefix(self) -> Path: |
| path = Path(self._data['base_prefix']) |
| if path.is_absolute(): |
| return path |
| # Non-absolute paths are relative to the build config directory |
| return self.config_path.parent / path |
| |
| |
| class BasicPythonExternalProgram(ExternalProgram): |
| def __init__(self, name: str, command: T.Optional[T.List[str]] = None, |
| ext_prog: T.Optional[ExternalProgram] = None, |
| build_config_path: T.Optional[str] = None): |
| if ext_prog is None: |
| super().__init__(name, command=command, silent=True) |
| else: |
| self.name = name |
| self.command = ext_prog.command |
| self.path = ext_prog.path |
| self.cached_version = None |
| self.version_arg = '--version' |
| |
| self.build_config = PythonBuildConfig(build_config_path) if build_config_path else None |
| |
| # We want strong key values, so we always populate this with bogus data. |
| # Otherwise to make the type checkers happy we'd have to do .get() for |
| # everycall, even though we know that the introspection data will be |
| # complete |
| self.info: 'PythonIntrospectionDict' = { |
| 'install_paths': {}, |
| 'is_pypy': False, |
| 'is_venv': False, |
| 'is_freethreaded': False, |
| 'link_libpython': False, |
| 'sysconfig_paths': {}, |
| 'paths': {}, |
| 'platform': 'sentinel', |
| 'suffix': 'sentinel', |
| 'limited_api_suffix': 'sentinel', |
| 'variables': {}, |
| 'version': '0.0', |
| } |
| self.pure: bool = True |
| |
| @property |
| def version(self) -> str: |
| if self.build_config: |
| value = self.build_config['language']['version'] |
| else: |
| value = self.info['variables'].get('LDVERSION') or self.info['version'] |
| assert isinstance(value, str) |
| return value |
| |
| def _check_version(self, version: str) -> bool: |
| if self.name == 'python2': |
| return mesonlib.version_compare(version, '< 3.0') |
| elif self.name == 'python3': |
| return mesonlib.version_compare(version, '>= 3.0') |
| return True |
| |
| def sanity(self) -> bool: |
| # Sanity check, we expect to have something that at least quacks in tune |
| |
| if self.build_config: |
| if not self.build_config['libpython']: |
| mlog.debug('This Python installation does not provide a libpython') |
| return False |
| if not self.build_config['c_api']: |
| mlog.debug('This Python installation does support the C API') |
| return False |
| |
| import importlib.resources |
| |
| with importlib.resources.path('mesonbuild.scripts', 'python_info.py') as f: |
| cmd = self.get_command() + [str(f)] |
| env = os.environ.copy() |
| env['SETUPTOOLS_USE_DISTUTILS'] = 'stdlib' |
| p, stdout, stderr = mesonlib.Popen_safe(cmd, env=env) |
| |
| try: |
| info = json.loads(stdout) |
| except json.JSONDecodeError: |
| info = None |
| mlog.debug('Could not introspect Python (%s): exit code %d' % (str(p.args), p.returncode)) |
| mlog.debug('Program stdout:\n') |
| mlog.debug(stdout) |
| mlog.debug('Program stderr:\n') |
| mlog.debug(stderr) |
| |
| if info is not None and self._check_version(info['version']): |
| self.info = T.cast('PythonIntrospectionDict', info) |
| return True |
| else: |
| return False |
| |
| |
| class _PythonDependencyBase(_Base): |
| |
| for_machine: MachineChoice |
| |
| def __init__(self, python_holder: 'BasicPythonExternalProgram', embed: bool): |
| self.embed = embed |
| self.build_config = python_holder.build_config |
| |
| if self.build_config: |
| self.version = self.build_config['language']['version'] |
| self.platform = self.build_config['platform'] |
| self.is_freethreaded = 't' in self.build_config['abi']['flags'] |
| self.link_libpython = self.build_config['libpython']['link_extensions'] |
| # TODO: figure out how to deal with frameworks |
| # see the logic at the bottom of PythonPkgConfigDependency.__init__() |
| if self.env.machines.host.is_darwin(): |
| raise DependencyException('--python.build-config is not supported on Darwin') |
| else: |
| self.version = python_holder.info['version'] |
| self.platform = python_holder.info['platform'] |
| self.is_freethreaded = python_holder.info['is_freethreaded'] |
| self.link_libpython = python_holder.info['link_libpython'] |
| # This data shouldn't be needed when build_config is set |
| self.is_pypy = python_holder.info['is_pypy'] |
| self.variables = python_holder.info['variables'] |
| |
| self.paths = python_holder.info['paths'] |
| |
| # The "-embed" version of python.pc / python-config was introduced in 3.8, |
| # and distutils extension linking was changed to be considered a non embed |
| # usage. Before then, this dependency always uses the embed=True handling |
| # because that is the only one that exists. |
| # |
| # On macOS and some Linux distros (Debian) distutils doesn't link extensions |
| # against libpython, even on 3.7 and below. We call into distutils and |
| # mirror its behavior. See https://github.com/mesonbuild/meson/issues/4117 |
| if not self.link_libpython: |
| self.link_libpython = embed |
| |
| self.info: T.Optional[T.Dict[str, str]] = None |
| if mesonlib.version_compare(self.version, '>= 3.0'): |
| self.major_version = 3 |
| else: |
| self.major_version = 2 |
| |
| # pyconfig.h is shared between regular and free-threaded builds in the |
| # Windows installer from python.org, and hence does not define |
| # Py_GIL_DISABLED correctly. So do it here: |
| if mesonlib.is_windows() and self.is_freethreaded: |
| self.compile_args += ['-DPy_GIL_DISABLED'] |
| |
| def find_libpy(self, environment: 'Environment') -> None: |
| if self.build_config: |
| path = self.build_config['libpython'].get('dynamic') |
| if not path: |
| raise DependencyException('Python does not provide a dynamic libpython library') |
| sysroot = environment.properties[self.for_machine].get_sys_root() or '' |
| path = sysroot + path |
| if not os.path.isfile(path): |
| raise DependencyException('Python dynamic library does not exist or is not a file') |
| self.link_args = [path] |
| self.is_found = True |
| return |
| |
| if self.is_pypy: |
| if self.major_version == 3: |
| libname = 'pypy3-c' |
| else: |
| libname = 'pypy-c' |
| libdir = os.path.join(self.variables.get('base'), 'bin') |
| libdirs = [libdir] |
| else: |
| libname = f'python{self.version}' |
| if 'DEBUG_EXT' in self.variables: |
| libname += self.variables['DEBUG_EXT'] |
| if 'ABIFLAGS' in self.variables: |
| libname += self.variables['ABIFLAGS'] |
| libdirs = [] |
| |
| largs = self.clib_compiler.find_library(libname, libdirs) |
| if largs is not None: |
| self.link_args = largs |
| self.is_found = True |
| |
| def get_windows_python_arch(self) -> str: |
| if self.platform.startswith('mingw'): |
| if 'x86_64' in self.platform: |
| return 'x86_64' |
| elif 'i686' in self.platform: |
| return 'x86' |
| elif 'aarch64' in self.platform: |
| return 'aarch64' |
| else: |
| raise DependencyException(f'MinGW Python built with unknown platform {self.platform!r}, please file a bug') |
| elif self.platform == 'win32': |
| return 'x86' |
| elif self.platform in {'win64', 'win-amd64'}: |
| return 'x86_64' |
| elif self.platform in {'win-arm64'}: |
| return 'aarch64' |
| raise DependencyException('Unknown Windows Python platform {self.platform!r}') |
| |
| def get_windows_link_args(self, limited_api: bool, environment: 'Environment') -> T.Optional[T.List[str]]: |
| if self.build_config: |
| if self.static: |
| key = 'static' |
| elif limited_api: |
| key = 'dynamic-stableabi' |
| else: |
| key = 'dynamic' |
| sysroot = environment.properties[self.for_machine].get_sys_root() or '' |
| return [sysroot + self.build_config['libpython'][key]] |
| |
| if self.platform.startswith('win'): |
| vernum = self.variables.get('py_version_nodot') |
| verdot = self.variables.get('py_version_short') |
| imp_lower = self.variables.get('implementation_lower', 'python') |
| if self.static: |
| libpath = Path('libs') / f'libpython{vernum}.a' |
| else: |
| if limited_api: |
| vernum = vernum[0] |
| comp = self.get_compiler() |
| if comp.id == "gcc": |
| if imp_lower == 'pypy' and verdot == '3.8': |
| # The naming changed between 3.8 and 3.9 |
| libpath = Path('libpypy3-c.dll') |
| elif imp_lower == 'pypy': |
| libpath = Path(f'libpypy{verdot}-c.dll') |
| else: |
| if self.is_freethreaded: |
| libpath = Path(f'python{vernum}t.dll') |
| else: |
| libpath = Path(f'python{vernum}.dll') |
| else: |
| if self.is_freethreaded: |
| libpath = Path('libs') / f'python{vernum}t.lib' |
| else: |
| libpath = Path('libs') / f'python{vernum}.lib' |
| # For a debug build, pyconfig.h may force linking with |
| # pythonX_d.lib (see meson#10776). This cannot be avoided |
| # and won't work unless we also have a debug build of |
| # Python itself (except with pybind11, which has an ugly |
| # hack to work around this) - so emit a warning to explain |
| # the cause of the expected link error. |
| buildtype = self.env.coredata.optstore.get_value_for(OptionKey('buildtype')) |
| assert isinstance(buildtype, str) |
| debug = self.env.coredata.optstore.get_value_for(OptionKey('debug')) |
| # `debugoptimized` buildtype may not set debug=True currently, see gh-11645 |
| is_debug_build = debug or buildtype == 'debug' |
| vscrt_debug = False |
| if OptionKey('b_vscrt') in self.env.coredata.optstore: |
| vscrt = self.env.coredata.optstore.get_value_for('b_vscrt') |
| if vscrt in {'mdd', 'mtd', 'from_buildtype', 'static_from_buildtype'}: |
| vscrt_debug = True |
| if is_debug_build and vscrt_debug and not self.variables.get('Py_DEBUG'): |
| mlog.warning(textwrap.dedent('''\ |
| Using a debug build type with MSVC or an MSVC-compatible compiler |
| when the Python interpreter is not also a debug build will almost |
| certainly result in a failed build. Prefer using a release build |
| type or a debug Python interpreter. |
| ''')) |
| # base_prefix to allow for virtualenvs. |
| lib = Path(self.variables.get('base_prefix')) / libpath |
| elif self.platform.startswith('mingw'): |
| if self.static: |
| if limited_api: |
| libname = self.variables.get('ABI3DLLLIBRARY') |
| else: |
| libname = self.variables.get('LIBRARY') |
| else: |
| if limited_api: |
| libname = self.variables.get('ABI3LDLIBRARY') |
| else: |
| libname = self.variables.get('LDLIBRARY') |
| lib = Path(self.variables.get('LIBDIR')) / libname |
| else: |
| raise mesonlib.MesonBugException( |
| 'On a Windows path, but the OS doesn\'t appear to be Windows or MinGW.') |
| if not lib.exists(): |
| mlog.log('Could not find Python3 library {!r}'.format(str(lib))) |
| return None |
| return [str(lib)] |
| |
| def find_libpy_windows(self, env: 'Environment', limited_api: bool = False) -> None: |
| ''' |
| Find python3 libraries on Windows and also verify that the arch matches |
| what we are building for. |
| ''' |
| try: |
| pyarch = self.get_windows_python_arch() |
| except DependencyException as e: |
| mlog.log(str(e)) |
| self.is_found = False |
| return |
| arch = detect_cpu_family(env.coredata.compilers.host) |
| if arch != pyarch: |
| mlog.log('Need', mlog.bold(self.name), f'for {arch}, but found {pyarch}') |
| self.is_found = False |
| return |
| # This can fail if the library is not found |
| largs = self.get_windows_link_args(limited_api, env) |
| if largs is None: |
| self.is_found = False |
| return |
| self.link_args = largs |
| self.is_found = True |
| |
| |
| class PythonPkgConfigDependency(PkgConfigDependency, _PythonDependencyBase): |
| |
| # name is needed for polymorphism |
| def __init__(self, name: str, environment: Environment, kwargs: DependencyObjectKWs, |
| installation: 'BasicPythonExternalProgram'): |
| embed = kwargs.get('embed', False) |
| pkg_embed = '-embed' if embed and mesonlib.version_compare(installation.info['version'], '>=3.8') else '' |
| pkg_name = f'python-{installation.version}{pkg_embed}' |
| |
| if installation.build_config: |
| pkg_libdir = installation.build_config.get('c_api.pkgconfig_path') |
| pkg_libdir_origin = 'c_api.pkgconfig_path from the Python build config' |
| else: |
| pkg_libdir = installation.info['variables'].get('LIBPC') |
| pkg_libdir_origin = 'LIBPC' |
| if pkg_libdir is None: |
| # we do not fall back to system directories, since this could lead |
| # to using pkg-config of another Python installation, for example |
| # we could end up using CPython .pc file for PyPy |
| mlog.debug(f'Skipping pkgconfig lookup, {pkg_libdir_origin} is unset') |
| self.is_found = False |
| return |
| |
| for_machine = kwargs['native'] |
| sysroot = environment.properties[for_machine].get_sys_root() or '' |
| pkg_libdir = sysroot + pkg_libdir |
| |
| mlog.debug(f'Searching for {pkg_libdir!r} via pkgconfig lookup in {pkg_libdir_origin}') |
| pkgconfig_paths = [pkg_libdir] if pkg_libdir else [] |
| |
| PkgConfigDependency.__init__(self, pkg_name, environment, kwargs, extra_paths=pkgconfig_paths) |
| _PythonDependencyBase.__init__(self, installation, embed) |
| |
| if pkg_libdir and not self.is_found: |
| mlog.debug(f'{pkg_name!r} could not be found in {pkg_libdir_origin}, ' |
| 'this is likely due to a relocated python installation') |
| return |
| |
| # pkg-config files are usually accurate starting with python 3.8 |
| if not self.link_libpython and mesonlib.version_compare(self.version, '< 3.8'): |
| self.link_args = [] |
| |
| # But not Apple, because it's a framework |
| if self.env.machines.host.is_darwin() and 'PYTHONFRAMEWORKPREFIX' in self.variables: |
| framework_prefix = self.variables['PYTHONFRAMEWORKPREFIX'] |
| # Add rpath, will be de-duplicated if necessary |
| if framework_prefix.startswith('/Applications/Xcode.app/'): |
| self.link_args += ['-Wl,-rpath,' + framework_prefix] |
| if self.raw_link_args is not None: |
| # When None, self.link_args is used |
| self.raw_link_args += ['-Wl,-rpath,' + framework_prefix] |
| |
| |
| class PythonFrameworkDependency(ExtraFrameworkDependency, _PythonDependencyBase): |
| |
| def __init__(self, name: str, environment: 'Environment', |
| kwargs: DependencyObjectKWs, installation: 'BasicPythonExternalProgram'): |
| ExtraFrameworkDependency.__init__(self, name, environment, kwargs) |
| _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) |
| |
| |
| class PythonSystemDependency(SystemDependency, _PythonDependencyBase): |
| |
| def __init__(self, name: str, environment: 'Environment', |
| kwargs: DependencyObjectKWs, installation: BasicPythonExternalProgram): |
| SystemDependency.__init__(self, name, environment, kwargs) |
| _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) |
| |
| # For most platforms, match pkg-config behavior. iOS is a special case; |
| # check for that first, so that check takes priority over |
| # `link_libpython` (which *shouldn't* be set, but just in case) |
| if self.platform.startswith('ios-'): |
| # iOS doesn't use link_libpython - it links with the *framework*. |
| self.link_args = ['-framework', 'Python', '-F', self.variables.get('base_prefix')] |
| self.is_found = True |
| elif self.link_libpython: |
| # link args |
| if mesonlib.is_windows(): |
| self.find_libpy_windows(environment, limited_api=False) |
| else: |
| self.find_libpy(environment) |
| else: |
| self.is_found = True |
| |
| # compile args |
| if self.build_config: |
| sysroot = environment.properties[self.for_machine].get_sys_root() or '' |
| inc_paths = mesonlib.OrderedSet([sysroot + self.build_config['c_api']['headers']]) |
| else: |
| inc_paths = mesonlib.OrderedSet([ |
| self.variables.get('INCLUDEPY'), |
| self.paths.get('include'), |
| self.paths.get('platinclude')]) |
| |
| self.compile_args += ['-I' + path for path in inc_paths if path] |
| |
| # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/ |
| # https://github.com/python/cpython/pull/100137 |
| if mesonlib.is_windows() and self.get_windows_python_arch().endswith('64') and mesonlib.version_compare(self.version, '<3.12'): |
| self.compile_args += ['-DMS_WIN64='] |
| |
| if not self.clib_compiler.has_header('Python.h', '', extra_args=self.compile_args)[0]: |
| self.is_found = False |
| |
| def python_factory(env: Environment, kwargs: DependencyObjectKWs, |
| installation: T.Optional['BasicPythonExternalProgram'] = None) -> T.List['DependencyGenerator']: |
| # We can't use the factory_methods decorator here, as we need to pass the |
| # extra installation argument |
| methods = process_method_kw({DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM}, kwargs) |
| candidates: T.List['DependencyGenerator'] = [] |
| from_installation = installation is not None |
| # When not invoked through the python module, default installation. |
| if installation is None: |
| installation = BasicPythonExternalProgram('python3', mesonlib.python_command) |
| installation.sanity() |
| |
| if DependencyMethods.PKGCONFIG in methods: |
| if from_installation: |
| candidates.append(DependencyCandidate( |
| functools.partial(PythonPkgConfigDependency, installation=installation), |
| 'python3', PythonPkgConfigDependency.type_name, arguments=(env, kwargs))) |
| else: |
| candidates.append(DependencyCandidate.from_dependency( |
| 'python3', PkgConfigDependency, (env, kwargs))) |
| |
| if DependencyMethods.SYSTEM in methods: |
| # This is a unique log-tried. |
| candidates.append(DependencyCandidate( |
| functools.partial(PythonSystemDependency, installation=installation), |
| 'python', 'sysconfig', arguments=(env, kwargs))) |
| |
| if DependencyMethods.EXTRAFRAMEWORK in methods: |
| nkwargs = kwargs.copy() |
| if mesonlib.version_compare(installation.version, '>= 3'): |
| # There is a python in /System/Library/Frameworks, but that's python 2.x, |
| # Python 3 will always be in /Library |
| nkwargs['paths'] = ['/Library/Frameworks'] |
| candidates.append(DependencyCandidate( |
| functools.partial(PythonFrameworkDependency, installation=installation), |
| 'python', PythonPkgConfigDependency.type_name, arguments=(env, nkwargs))) |
| |
| return candidates |
| |
| packages['python3'] = python_factory |
| |
| packages['pybind11'] = pybind11_factory = DependencyFactory( |
| 'pybind11', |
| [DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.CMAKE], |
| configtool=Pybind11ConfigToolDependency, |
| ) |
| |
| packages['numpy'] = numpy_factory = DependencyFactory( |
| 'numpy', |
| [DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL], |
| configtool=NumPyConfigToolDependency, |
| ) |
