| """ | |
| mkvenv - QEMU pyvenv bootstrapping utility | |
| usage: mkvenv [-h] command ... | |
| QEMU pyvenv bootstrapping utility | |
| options: | |
| -h, --help show this help message and exit | |
| Commands: | |
| command Description | |
| create create a venv | |
| post_init | |
| post-venv initialization | |
| ensuregroup | |
| Ensure that the specified package group is installed. | |
| -------------------------------------------------- | |
| usage: mkvenv create [-h] target | |
| positional arguments: | |
| target Target directory to install virtual environment into. | |
| options: | |
| -h, --help show this help message and exit | |
| -------------------------------------------------- | |
| usage: mkvenv post_init [-h] | |
| options: | |
| -h, --help show this help message and exit | |
| -------------------------------------------------- | |
| usage: mkvenv ensuregroup [-h] [--online] [--dir DIR] file group... | |
| positional arguments: | |
| file pointer to a TOML file | |
| group section name in the TOML file | |
| options: | |
| -h, --help show this help message and exit | |
| --online Install packages from PyPI, if necessary. | |
| --dir DIR Path to vendored packages where we may install from. | |
| """ | |
| # Copyright (C) 2022-2023 Red Hat, Inc. | |
| # | |
| # Authors: | |
| # John Snow <jsnow@redhat.com> | |
| # Paolo Bonzini <pbonzini@redhat.com> | |
| # | |
| # This work is licensed under the terms of the GNU GPL, version 2 or | |
| # later. See the COPYING file in the top-level directory. | |
| import argparse | |
| from importlib.metadata import ( | |
| Distribution, | |
| EntryPoint, | |
| PackageNotFoundError, | |
| distribution, | |
| version, | |
| ) | |
| from importlib.util import find_spec | |
| import logging | |
| import os | |
| from pathlib import Path | |
| import re | |
| import shutil | |
| import site | |
| import subprocess | |
| import sys | |
| import sysconfig | |
| from types import SimpleNamespace | |
| from typing import ( | |
| Any, | |
| Dict, | |
| Iterator, | |
| Optional, | |
| Sequence, | |
| Tuple, | |
| Union, | |
| cast, | |
| ) | |
| import venv | |
| # Try to load distlib, with a fallback to pip's vendored version. | |
| # HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail | |
| # outside the venv or before a potential call to ensurepip in checkpip(). | |
| _import_ok = True | |
| try: | |
| import distlib.scripts | |
| except ImportError: | |
| try: | |
| # Reach into pip's cookie jar. pylint and flake8 don't understand | |
| # that these imports will be used via distlib.xxx. | |
| from pip._vendor import distlib | |
| import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import | |
| except ImportError: | |
| _import_ok = False | |
| HAVE_DISTLIB = _import_ok | |
| # pip 25.2 does not vendor distlib.version, but it uses vendored | |
| # packaging.version | |
| _import_ok = True | |
| try: | |
| import distlib.version # pylint: disable=ungrouped-imports | |
| except ImportError: | |
| try: | |
| # pylint: disable=unused-import,ungrouped-imports | |
| import pip._vendor.distlib.version # noqa | |
| except ImportError: | |
| _import_ok = False | |
| HAVE_DISTLIB_VERSION = _import_ok | |
| _import_ok = True | |
| try: | |
| # Do not bother importing non-vendored packaging, because it is not | |
| # in stdlib. | |
| from pip._vendor import packaging | |
| # pylint: disable=unused-import | |
| import pip._vendor.packaging.requirements # noqa | |
| import pip._vendor.packaging.version # noqa | |
| except ImportError: | |
| _import_ok = False | |
| HAVE_PACKAGING_VERSION = _import_ok | |
| # Try to load tomllib, with a fallback to tomli. | |
| # HAVE_TOMLLIB is checked below, just-in-time, so that mkvenv does not fail | |
| # outside the venv or before a potential call to ensurepip in checkpip(). | |
| _import_ok = True | |
| try: | |
| import tomllib | |
| except ImportError: | |
| try: | |
| import tomli as tomllib | |
| except ImportError: | |
| _import_ok = False | |
| HAVE_TOMLLIB = _import_ok | |
| # Do not add any mandatory dependencies from outside the stdlib: | |
| # This script *must* be usable standalone! | |
| DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] | |
| logger = logging.getLogger("mkvenv") | |
| def inside_a_venv() -> bool: | |
| """Returns True if it is executed inside of a virtual environment.""" | |
| return sys.prefix != sys.base_prefix | |
| class Ouch(RuntimeError): | |
| """An Exception class we can't confuse with a builtin.""" | |
| class Matcher: | |
| """Compatibility appliance for version/requirement string parsing.""" | |
| def __init__(self, name_and_constraint: str): | |
| """Create a matcher from a requirement-like string.""" | |
| if HAVE_DISTLIB_VERSION: | |
| self._m = distlib.version.LegacyMatcher(name_and_constraint) | |
| elif HAVE_PACKAGING_VERSION: | |
| self._m = packaging.requirements.Requirement(name_and_constraint) | |
| else: | |
| raise Ouch("found neither distlib.version nor packaging.version") | |
| self.name = self._m.name | |
| def match(self, version_str: str) -> bool: | |
| """Return True if `version` satisfies the stored constraint.""" | |
| if HAVE_DISTLIB_VERSION: | |
| return cast( | |
| bool, | |
| self._m.match(distlib.version.LegacyVersion(version_str)) | |
| ) | |
| assert HAVE_PACKAGING_VERSION | |
| return cast( | |
| bool, | |
| self._m.specifier.contains( | |
| packaging.version.Version(version_str), prereleases=True | |
| ) | |
| ) | |
| def __str__(self) -> str: | |
| """String representation delegated to the backend.""" | |
| return str(self._m) | |
| def __repr__(self) -> str: | |
| """Stable debug representation delegated to the backend.""" | |
| return repr(self._m) | |
| class QemuEnvBuilder(venv.EnvBuilder): | |
| """ | |
| An extension of venv.EnvBuilder for building QEMU's configure-time venv. | |
| The primary difference is that it emulates a "nested" virtual | |
| environment when invoked from inside of an existing virtual | |
| environment by including packages from the parent. Also, | |
| "ensurepip" is replaced if possible with just recreating pip's | |
| console_scripts inside the virtual environment. | |
| Parameters for base class init: | |
| - system_site_packages: bool = False | |
| - clear: bool = False | |
| - symlinks: bool = False | |
| - upgrade: bool = False | |
| - with_pip: bool = False | |
| - prompt: Optional[str] = None | |
| - upgrade_deps: bool = False (Since 3.9) | |
| """ | |
| def __init__(self, *args: Any, **kwargs: Any) -> None: | |
| logger.debug("QemuEnvBuilder.__init__(...)") | |
| # For nested venv emulation: | |
| self.use_parent_packages = False | |
| if inside_a_venv(): | |
| # Include parent packages only if we're in a venv and | |
| # system_site_packages was True. | |
| self.use_parent_packages = kwargs.pop( | |
| "system_site_packages", False | |
| ) | |
| # Include system_site_packages only when the parent, | |
| # The venv we are currently in, also does so. | |
| kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES | |
| # ensurepip is slow: venv creation can be very fast for cases where | |
| # we allow the use of system_site_packages. Therefore, ensurepip is | |
| # replaced with our own script generation once the virtual environment | |
| # is setup. | |
| self.want_pip = kwargs.get("with_pip", False) | |
| if self.want_pip: | |
| if ( | |
| kwargs.get("system_site_packages", False) | |
| and not need_ensurepip() | |
| ): | |
| kwargs["with_pip"] = False | |
| else: | |
| check_ensurepip() | |
| super().__init__(*args, **kwargs) | |
| # Make the context available post-creation: | |
| self._context: Optional[SimpleNamespace] = None | |
| def get_parent_libpath(self) -> Optional[str]: | |
| """Return the libpath of the parent venv, if applicable.""" | |
| if self.use_parent_packages: | |
| return sysconfig.get_path("purelib") | |
| return None | |
| @staticmethod | |
| def compute_venv_libpath(context: SimpleNamespace) -> str: | |
| """ | |
| Compatibility wrapper for context.lib_path for Python < 3.12 | |
| """ | |
| # Python 3.12+, not strictly necessary because it's documented | |
| # to be the same as 3.10 code below: | |
| if sys.version_info >= (3, 12): | |
| return context.lib_path | |
| # Python 3.10+ | |
| if "venv" in sysconfig.get_scheme_names(): | |
| lib_path = sysconfig.get_path( | |
| "purelib", scheme="venv", vars={"base": context.env_dir} | |
| ) | |
| assert lib_path is not None | |
| return lib_path | |
| # For Python <= 3.9 we need to hardcode this. Fortunately the | |
| # code below was the same in Python 3.6-3.10, so there is only | |
| # one case. | |
| if sys.platform == "win32": | |
| return os.path.join(context.env_dir, "Lib", "site-packages") | |
| return os.path.join( | |
| context.env_dir, | |
| "lib", | |
| "python%d.%d" % sys.version_info[:2], | |
| "site-packages", | |
| ) | |
| def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: | |
| logger.debug("ensure_directories(env_dir=%s)", env_dir) | |
| self._context = super().ensure_directories(env_dir) | |
| return self._context | |
| def create(self, env_dir: DirType) -> None: | |
| logger.debug("create(env_dir=%s)", env_dir) | |
| super().create(env_dir) | |
| assert self._context is not None | |
| self.post_post_setup(self._context) | |
| def post_post_setup(self, context: SimpleNamespace) -> None: | |
| """ | |
| The final, final hook. Enter the venv and run commands inside of it. | |
| """ | |
| if self.use_parent_packages: | |
| # We're inside of a venv and we want to include the parent | |
| # venv's packages. | |
| parent_libpath = self.get_parent_libpath() | |
| assert parent_libpath is not None | |
| logger.debug("parent_libpath: %s", parent_libpath) | |
| our_libpath = self.compute_venv_libpath(context) | |
| logger.debug("our_libpath: %s", our_libpath) | |
| pth_file = os.path.join(our_libpath, "nested.pth") | |
| with open(pth_file, "w", encoding="UTF-8") as file: | |
| file.write(parent_libpath + os.linesep) | |
| if self.want_pip: | |
| args = [ | |
| context.env_exe, | |
| __file__, | |
| "post_init", | |
| ] | |
| subprocess.run(args, check=True) | |
| def get_value(self, field: str) -> str: | |
| """ | |
| Get a string value from the context namespace after a call to build. | |
| For valid field names, see: | |
| https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories | |
| """ | |
| ret = getattr(self._context, field) | |
| assert isinstance(ret, str) | |
| return ret | |
| def need_ensurepip() -> bool: | |
| """ | |
| Tests for the presence of setuptools and pip. | |
| :return: `True` if we do not detect both packages. | |
| """ | |
| # Don't try to actually import them, it's fraught with danger: | |
| # https://github.com/pypa/setuptools/issues/2993 | |
| if find_spec("setuptools") and find_spec("pip"): | |
| return False | |
| return True | |
| def check_ensurepip() -> None: | |
| """ | |
| Check that we have ensurepip. | |
| Raise a fatal exception with a helpful hint if it isn't available. | |
| """ | |
| if not find_spec("ensurepip"): | |
| msg = ( | |
| "Python's ensurepip module is not found.\n" | |
| "It's normally part of the Python standard library, " | |
| "maybe your distribution packages it separately?\n" | |
| "Either install ensurepip, or alleviate the need for it in the " | |
| "first place by installing pip and setuptools for " | |
| f"'{sys.executable}'.\n" | |
| "(Hint: Debian puts ensurepip in its python3-venv package.)" | |
| ) | |
| raise Ouch(msg) | |
| # ensurepip uses pyexpat, which can also go missing on us: | |
| if not find_spec("pyexpat"): | |
| msg = ( | |
| "Python's pyexpat module is not found.\n" | |
| "It's normally part of the Python standard library, " | |
| "maybe your distribution packages it separately?\n" | |
| "Either install pyexpat, or alleviate the need for it in the " | |
| "first place by installing pip and setuptools for " | |
| f"'{sys.executable}'.\n\n" | |
| "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" | |
| ) | |
| raise Ouch(msg) | |
| def make_venv( # pylint: disable=too-many-arguments | |
| env_dir: Union[str, Path], | |
| system_site_packages: bool = False, | |
| clear: bool = True, | |
| symlinks: Optional[bool] = None, | |
| with_pip: bool = True, | |
| ) -> None: | |
| """ | |
| Create a venv using `QemuEnvBuilder`. | |
| This is analogous to the `venv.create` module-level convenience | |
| function that is part of the Python stdblib, except it uses | |
| `QemuEnvBuilder` instead. | |
| :param env_dir: The directory to create/install to. | |
| :param system_site_packages: | |
| Allow inheriting packages from the system installation. | |
| :param clear: When True, fully remove any prior venv and files. | |
| :param symlinks: | |
| Whether to use symlinks to the target interpreter or not. If | |
| left unspecified, it will use symlinks except on Windows to | |
| match behavior with the "venv" CLI tool. | |
| :param with_pip: | |
| Whether to install "pip" binaries or not. | |
| """ | |
| logger.debug( | |
| "%s: make_venv(env_dir=%s, system_site_packages=%s, " | |
| "clear=%s, symlinks=%s, with_pip=%s)", | |
| __file__, | |
| str(env_dir), | |
| system_site_packages, | |
| clear, | |
| symlinks, | |
| with_pip, | |
| ) | |
| if symlinks is None: | |
| # Default behavior of standard venv CLI | |
| symlinks = os.name != "nt" | |
| builder = QemuEnvBuilder( | |
| system_site_packages=system_site_packages, | |
| clear=clear, | |
| symlinks=symlinks, | |
| with_pip=with_pip, | |
| ) | |
| style = "non-isolated" if builder.system_site_packages else "isolated" | |
| nested = "" | |
| if builder.use_parent_packages: | |
| nested = f"(with packages from '{builder.get_parent_libpath()}') " | |
| print( | |
| f"mkvenv: Creating {style} virtual environment" | |
| f" {nested}at '{str(env_dir)}'", | |
| file=sys.stderr, | |
| ) | |
| try: | |
| logger.debug("Invoking builder.create()") | |
| try: | |
| builder.create(str(env_dir)) | |
| except SystemExit as exc: | |
| # pylint 3.3 bug: | |
| # pylint: disable=raising-non-exception, raise-missing-from | |
| # Some versions of the venv module raise SystemExit; *nasty*! | |
| # We want the exception that prompted it. It might be a subprocess | |
| # error that has output we *really* want to see. | |
| logger.debug("Intercepted SystemExit from EnvBuilder.create()") | |
| raise exc.__cause__ or exc.__context__ or exc | |
| logger.debug("builder.create() finished") | |
| except subprocess.CalledProcessError as exc: | |
| logger.error("mkvenv subprocess failed:") | |
| logger.error("cmd: %s", exc.cmd) | |
| logger.error("returncode: %d", exc.returncode) | |
| def _stringify(data: Union[str, bytes]) -> str: | |
| if isinstance(data, bytes): | |
| return data.decode() | |
| return data | |
| lines = [] | |
| if exc.stdout: | |
| lines.append("========== stdout ==========") | |
| lines.append(_stringify(exc.stdout)) | |
| lines.append("============================") | |
| if exc.stderr: | |
| lines.append("========== stderr ==========") | |
| lines.append(_stringify(exc.stderr)) | |
| lines.append("============================") | |
| if lines: | |
| logger.error(os.linesep.join(lines)) | |
| raise Ouch("VENV creation subprocess failed.") from exc | |
| # print the python executable to stdout for configure. | |
| print(builder.get_value("env_exe")) | |
| def _get_entry_points(packages: Sequence[str]) -> Iterator[str]: | |
| def _generator() -> Iterator[str]: | |
| for package in packages: | |
| try: | |
| entry_points: Iterator[EntryPoint] = \ | |
| iter(distribution(package).entry_points) | |
| except PackageNotFoundError: | |
| continue | |
| # The EntryPoints type is only available in 3.10+, | |
| # treat this as a vanilla list and filter it ourselves. | |
| entry_points = filter( | |
| lambda ep: ep.group == "console_scripts", entry_points | |
| ) | |
| for entry_point in entry_points: | |
| yield f"{entry_point.name} = {entry_point.value}" | |
| return _generator() | |
| def generate_console_scripts( | |
| packages: Sequence[str], | |
| python_path: Optional[str] = None, | |
| bin_path: Optional[str] = None, | |
| ) -> None: | |
| """ | |
| Generate script shims for console_script entry points in @packages. | |
| """ | |
| if python_path is None: | |
| python_path = sys.executable | |
| if bin_path is None: | |
| bin_path = sysconfig.get_path("scripts") | |
| assert bin_path is not None | |
| logger.debug( | |
| "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", | |
| packages, | |
| python_path, | |
| bin_path, | |
| ) | |
| if not packages: | |
| return | |
| maker = distlib.scripts.ScriptMaker(None, bin_path) | |
| maker.variants = {""} | |
| maker.clobber = False | |
| for entry_point in _get_entry_points(packages): | |
| for filename in maker.make(entry_point): | |
| logger.debug("wrote console_script '%s'", filename) | |
| def pkgname_from_depspec(dep_spec: str) -> str: | |
| """ | |
| Parse package name out of a PEP-508 depspec. | |
| See https://peps.python.org/pep-0508/#names | |
| """ | |
| match = re.match( | |
| r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE | |
| ) | |
| if not match: | |
| raise ValueError( | |
| f"dep_spec '{dep_spec}'" | |
| " does not appear to contain a valid package name" | |
| ) | |
| return match.group(0) | |
| def _path_is_prefix(prefix: Optional[str], path: str) -> bool: | |
| try: | |
| return ( | |
| prefix is not None and os.path.commonpath([prefix, path]) == prefix | |
| ) | |
| except ValueError: | |
| return False | |
| def _is_system_package(dist: Distribution) -> bool: | |
| path = str(dist.locate_file(".")) | |
| return not ( | |
| _path_is_prefix(sysconfig.get_path("purelib"), path) | |
| or _path_is_prefix(sysconfig.get_path("platlib"), path) | |
| ) | |
| def diagnose( | |
| dep_spec: str, | |
| online: bool, | |
| wheels_dir: Optional[Union[str, Path]], | |
| prog: Optional[str], | |
| ) -> Tuple[str, bool]: | |
| """ | |
| Offer a summary to the user as to why a package failed to be installed. | |
| :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' | |
| :param online: Did we allow PyPI access? | |
| :param prog: | |
| Optionally, a shell program name that can be used as a | |
| bellwether to detect if this program is installed elsewhere on | |
| the system. This is used to offer advice when a program is | |
| detected for a different python version. | |
| :param wheels_dir: | |
| Optionally, a directory that was searched for vendored packages. | |
| """ | |
| # pylint: disable=too-many-branches | |
| # Some errors are not particularly serious | |
| bad = False | |
| pkg_name = pkgname_from_depspec(dep_spec) | |
| pkg_version: Optional[str] = None | |
| try: | |
| pkg_version = version(pkg_name) | |
| except PackageNotFoundError: | |
| pass | |
| lines = [] | |
| if pkg_version: | |
| lines.append( | |
| f"Python package '{pkg_name}' version '{pkg_version}' was found," | |
| " but isn't suitable." | |
| ) | |
| else: | |
| lines.append( | |
| f"Python package '{pkg_name}' was not found nor installed." | |
| ) | |
| if wheels_dir: | |
| lines.append( | |
| "No suitable version found in, or failed to install from" | |
| f" '{wheels_dir}'." | |
| ) | |
| bad = True | |
| if online: | |
| lines.append("A suitable version could not be obtained from PyPI.") | |
| bad = True | |
| else: | |
| lines.append( | |
| "mkvenv was configured to operate offline and did not check PyPI." | |
| ) | |
| if prog and not pkg_version: | |
| which = shutil.which(prog) | |
| if which: | |
| if sys.base_prefix in site.PREFIXES: | |
| pypath = Path(sys.executable).resolve() | |
| lines.append( | |
| f"'{prog}' was detected on your system at '{which}', " | |
| f"but the Python package '{pkg_name}' was not found by " | |
| f"this Python interpreter ('{pypath}'). " | |
| f"Typically this means that '{prog}' has been installed " | |
| "against a different Python interpreter on your system." | |
| ) | |
| else: | |
| lines.append( | |
| f"'{prog}' was detected on your system at '{which}', " | |
| "but the build is using an isolated virtual environment." | |
| ) | |
| bad = True | |
| lines = [f" • {line}" for line in lines] | |
| if bad: | |
| lines.insert(0, f"Could not provide build dependency '{dep_spec}':") | |
| else: | |
| lines.insert(0, f"'{dep_spec}' not found:") | |
| return os.linesep.join(lines), bad | |
| def pip_install( | |
| args: Sequence[str], | |
| online: bool = False, | |
| wheels_dir: Optional[Union[str, Path]] = None, | |
| env: Optional[Dict[str, str]] = None, | |
| ) -> None: | |
| """ | |
| Use pip to install a package or package(s) as specified in @args. | |
| """ | |
| loud = bool( | |
| os.environ.get("DEBUG") | |
| or os.environ.get("GITLAB_CI") | |
| or os.environ.get("V") | |
| ) | |
| full_args = [ | |
| sys.executable, | |
| "-m", | |
| "pip", | |
| "install", | |
| "--disable-pip-version-check", | |
| "-v" if loud else "-q", | |
| ] | |
| if not online: | |
| full_args += ["--no-index"] | |
| if wheels_dir: | |
| full_args += ["--find-links", str(wheels_dir)] | |
| full_args += list(args) | |
| subprocess.run( | |
| full_args, | |
| env=env, | |
| check=True, | |
| ) | |
| def _make_version_constraint(info: Dict[str, str], install: bool) -> str: | |
| """ | |
| Construct the version constraint part of a PEP 508 dependency | |
| specification (for example '>=0.61.5') from the accepted and | |
| installed keys of the provided dictionary. | |
| :param info: A dictionary corresponding to a TOML key-value list. | |
| :param install: True generates install constraints, False generates | |
| presence constraints | |
| """ | |
| if install and "installed" in info: | |
| return "==" + info["installed"] | |
| dep_spec = info.get("accepted", "") | |
| dep_spec = dep_spec.strip() | |
| # Double check that they didn't just use a version number | |
| if dep_spec and dep_spec[0] not in "!~><=(": | |
| raise Ouch( | |
| "invalid dependency specifier " + dep_spec + " in dependency file" | |
| ) | |
| return dep_spec | |
| def _do_ensure( | |
| group: Dict[str, Dict[str, str]], | |
| online: bool = False, | |
| wheels_dir: Optional[Union[str, Path]] = None, | |
| ) -> Optional[Tuple[str, bool]]: | |
| """ | |
| Use pip to ensure we have the packages specified in @group. | |
| If the packages are already installed, do nothing. If online and | |
| wheels_dir are both provided, prefer packages found in wheels_dir | |
| first before connecting to PyPI. | |
| :param group: A dictionary of dictionaries, corresponding to a | |
| section in a pythondeps.toml file. | |
| :param online: If True, fall back to PyPI. | |
| :param wheels_dir: If specified, search this path for packages. | |
| """ | |
| absent = [] | |
| local_packages = [] | |
| present = [] | |
| canary = None | |
| for name, info in group.items(): | |
| if "path" in info: | |
| pkgpath = Path(__file__).parents[2].joinpath(info["path"]) | |
| local_packages.append(str(pkgpath)) | |
| continue | |
| constraint = _make_version_constraint(info, False) | |
| matcher = Matcher(name + constraint) | |
| print(f"mkvenv: checking for {matcher}", file=sys.stderr) | |
| dist: Optional[Distribution] = None | |
| try: | |
| dist = distribution(matcher.name) | |
| except PackageNotFoundError: | |
| pass | |
| if ( | |
| dist is None | |
| # Always pass installed package to pip, so that they can be | |
| # updated if the requested version changes | |
| or not _is_system_package(dist) | |
| or not matcher.match(dist.version) | |
| ): | |
| absent.append(name + _make_version_constraint(info, True)) | |
| if len(absent) == 1: | |
| canary = info.get("canary", None) | |
| else: | |
| logger.info("found %s %s", name, dist.version) | |
| present.append(name) | |
| if present: | |
| generate_console_scripts(present) | |
| if absent: | |
| if online or wheels_dir: | |
| # Some packages are missing or aren't a suitable version, | |
| # install a suitable (possibly vendored) package. | |
| print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) | |
| try: | |
| pip_install(args=absent, online=online, wheels_dir=wheels_dir) | |
| absent = [] | |
| except subprocess.CalledProcessError: | |
| pass | |
| if absent: | |
| return diagnose( | |
| absent[0], | |
| online, | |
| wheels_dir, | |
| canary, | |
| ) | |
| # Handle local packages separately and last so we can use different | |
| # installation arguments (-e), and so that any dependencies that may | |
| # be covered above will be handled according to the depfile | |
| # specifications. | |
| if local_packages: | |
| print(f"mkvenv: installing {', '.join(local_packages)}", | |
| file=sys.stderr) | |
| env = dict(os.environ) | |
| env['PIP_CONFIG_SETTINGS'] = "editable_mode=compat" | |
| pip_install( | |
| args=["--no-build-isolation", | |
| "-e"] + local_packages, | |
| online=online, | |
| wheels_dir=wheels_dir, | |
| env=env, | |
| ) | |
| return None | |
| def _parse_groups(file: str) -> Dict[str, Dict[str, Any]]: | |
| if not HAVE_TOMLLIB: | |
| if sys.version_info < (3, 11): | |
| raise Ouch("found no usable tomli, please install it") | |
| raise Ouch( | |
| "Python >=3.11 does not have tomllib... what have you done!?" | |
| ) | |
| # Use loads() to support both tomli v1.2.x (Ubuntu 22.04, | |
| # Debian bullseye-backports) and v2.0.x | |
| with open(file, "r", encoding="ascii") as depfile: | |
| contents = depfile.read() | |
| return tomllib.loads(contents) # type: ignore | |
| def ensure_group( | |
| file: str, | |
| groups: Sequence[str], | |
| online: bool = False, | |
| wheels_dir: Optional[Union[str, Path]] = None, | |
| ) -> None: | |
| """ | |
| Use pip to ensure we have the package specified by @dep_specs. | |
| If the package is already installed, do nothing. If online and | |
| wheels_dir are both provided, prefer packages found in wheels_dir | |
| first before connecting to PyPI. | |
| :param dep_specs: | |
| PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. | |
| :param online: If True, fall back to PyPI. | |
| :param wheels_dir: If specified, search this path for packages. | |
| """ | |
| if not HAVE_DISTLIB: | |
| raise Ouch("found no usable distlib, please install it") | |
| parsed_deps = _parse_groups(file) | |
| to_install: Dict[str, Dict[str, str]] = {} | |
| for group in groups: | |
| try: | |
| to_install.update(parsed_deps[group]) | |
| except KeyError as exc: | |
| raise Ouch(f"group {group} not defined") from exc | |
| result = _do_ensure(to_install, online, wheels_dir) | |
| if result: | |
| # Well, that's not good. | |
| if result[1]: | |
| raise Ouch(result[0]) | |
| raise SystemExit(f"\n{result[0]}\n\n") | |
| if inside_a_venv(): | |
| for group in groups: | |
| path = Path(sys.prefix).joinpath(f"{group}.group") | |
| with open(path, "w", encoding="UTF8"): | |
| pass | |
| def post_venv_setup() -> None: | |
| """ | |
| This is intended to be run *inside the venv* after it is created. | |
| """ | |
| logger.debug("post_venv_setup()") | |
| # Generate a 'pip' script so the venv is usable in a normal | |
| # way from the CLI. This only happens when we inherited pip from a | |
| # parent/system-site and haven't run ensurepip in some way. | |
| generate_console_scripts(["pip"]) | |
| def _add_create_subcommand(subparsers: Any) -> None: | |
| subparser = subparsers.add_parser("create", help="create a venv") | |
| subparser.add_argument( | |
| "target", | |
| type=str, | |
| action="store", | |
| help="Target directory to install virtual environment into.", | |
| ) | |
| def _add_post_init_subcommand(subparsers: Any) -> None: | |
| subparsers.add_parser("post_init", help="post-venv initialization") | |
| def _add_ensuregroup_subcommand(subparsers: Any) -> None: | |
| subparser = subparsers.add_parser( | |
| "ensuregroup", | |
| help="Ensure that the specified package group is installed.", | |
| ) | |
| subparser.add_argument( | |
| "--online", | |
| action="store_true", | |
| help="Install packages from PyPI, if necessary.", | |
| ) | |
| subparser.add_argument( | |
| "--dir", | |
| type=str, | |
| action="store", | |
| help="Path to vendored packages where we may install from.", | |
| ) | |
| subparser.add_argument( | |
| "file", | |
| type=str, | |
| action="store", | |
| help=("Path to a TOML file describing package groups"), | |
| ) | |
| subparser.add_argument( | |
| "group", | |
| type=str, | |
| action="store", | |
| help="One or more package group names", | |
| nargs="+", | |
| ) | |
| def main() -> int: | |
| """CLI interface to make_qemu_venv. See module docstring.""" | |
| if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): | |
| # You're welcome. | |
| logging.basicConfig(level=logging.DEBUG) | |
| else: | |
| if os.environ.get("V"): | |
| logging.basicConfig(level=logging.INFO) | |
| parser = argparse.ArgumentParser( | |
| prog="mkvenv", | |
| description="QEMU pyvenv bootstrapping utility", | |
| ) | |
| subparsers = parser.add_subparsers( | |
| title="Commands", | |
| dest="command", | |
| required=True, | |
| metavar="command", | |
| help="Description", | |
| ) | |
| _add_create_subcommand(subparsers) | |
| _add_post_init_subcommand(subparsers) | |
| _add_ensuregroup_subcommand(subparsers) | |
| args = parser.parse_args() | |
| try: | |
| if args.command == "create": | |
| make_venv( | |
| args.target, | |
| system_site_packages=True, | |
| clear=True, | |
| ) | |
| if args.command == "post_init": | |
| post_venv_setup() | |
| if args.command == "ensuregroup": | |
| ensure_group( | |
| file=args.file, | |
| groups=args.group, | |
| online=args.online, | |
| wheels_dir=args.dir, | |
| ) | |
| logger.debug("mkvenv.py %s: exiting", args.command) | |
| except Ouch as exc: | |
| print("\n*** Ouch! ***\n", file=sys.stderr) | |
| print(str(exc), "\n\n", file=sys.stderr) | |
| return 1 | |
| except SystemExit: | |
| raise | |
| except: # pylint: disable=bare-except | |
| logger.exception("mkvenv did not complete successfully:") | |
| return 2 | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) |