| # SPDX-License-Identifier: Apache-2.0 |
| # Copyright 2017 The Meson development team |
| # Copyright © 2023 Intel Corporation |
| |
| from __future__ import annotations |
| |
| |
| import abc |
| import argparse |
| import gzip |
| import os |
| import sys |
| import shlex |
| import shutil |
| import subprocess |
| import tarfile |
| import tempfile |
| import hashlib |
| import typing as T |
| |
| from dataclasses import dataclass |
| from glob import glob |
| from pathlib import Path |
| from mesonbuild.environment import Environment, detect_ninja |
| from mesonbuild.mesonlib import (MesonException, RealPathAction, get_meson_command, quiet_git, |
| windows_proof_rmtree, setup_vsenv, OptionKey) |
| from mesonbuild.msetup import add_arguments as msetup_argparse |
| from mesonbuild.wrap import wrap |
| from mesonbuild import mlog, build, coredata |
| from .scripts.meson_exe import run_exe |
| |
| if T.TYPE_CHECKING: |
| from ._typing import ImmutableListProtocol |
| from .mesonlib import ExecutableSerialisation |
| |
| archive_choices = ['bztar', 'gztar', 'xztar', 'zip'] |
| |
| archive_extension = {'bztar': '.tar.bz2', |
| 'gztar': '.tar.gz', |
| 'xztar': '.tar.xz', |
| 'zip': '.zip'} |
| |
| # Note: when adding arguments, please also add them to the completion |
| # scripts in $MESONSRC/data/shell-completions/ |
| def add_arguments(parser: argparse.ArgumentParser) -> None: |
| parser.add_argument('-C', dest='wd', action=RealPathAction, |
| help='directory to cd into before running') |
| parser.add_argument('--allow-dirty', action='store_true', |
| help='Allow even when repository contains uncommitted changes.') |
| parser.add_argument('--formats', default='xztar', |
| help='Comma separated list of archive types to create. Supports xztar (default), bztar, gztar, and zip.') |
| parser.add_argument('--include-subprojects', action='store_true', |
| help='Include source code of subprojects that have been used for the build.') |
| parser.add_argument('--no-tests', action='store_true', |
| help='Do not build and test generated packages.') |
| |
| |
| def create_hash(fname: str) -> None: |
| hashname = fname + '.sha256sum' |
| m = hashlib.sha256() |
| m.update(open(fname, 'rb').read()) |
| with open(hashname, 'w', encoding='utf-8') as f: |
| # A space and an asterisk because that is the format defined by GNU coreutils |
| # and accepted by busybox and the Perl shasum tool. |
| f.write('{} *{}\n'.format(m.hexdigest(), os.path.basename(fname))) |
| |
| |
| msg_uncommitted_changes = 'Repository has uncommitted changes that will not be included in the dist tarball' |
| |
| def handle_dirty_opt(msg: str, allow_dirty: bool) -> None: |
| if allow_dirty: |
| mlog.warning(msg) |
| else: |
| mlog.error(msg + '\n' + 'Use --allow-dirty to ignore the warning and proceed anyway') |
| sys.exit(1) |
| |
| def is_git(src_root: str) -> bool: |
| ''' |
| Checks if meson.build file at the root source directory is tracked by git. |
| It could be a subproject part of the parent project git repository. |
| ''' |
| return quiet_git(['ls-files', '--error-unmatch', 'meson.build'], src_root)[0] |
| |
| def is_hg(src_root: str) -> bool: |
| return os.path.isdir(os.path.join(src_root, '.hg')) |
| |
| |
| @dataclass |
| class Dist(metaclass=abc.ABCMeta): |
| dist_name: str |
| src_root: str |
| bld_root: str |
| dist_scripts: T.List[ExecutableSerialisation] |
| subprojects: T.Dict[str, str] |
| options: argparse.Namespace |
| |
| def __post_init__(self) -> None: |
| self.dist_sub = os.path.join(self.bld_root, 'meson-dist') |
| self.distdir = os.path.join(self.dist_sub, self.dist_name) |
| |
| @abc.abstractmethod |
| def create_dist(self, archives: T.List[str]) -> T.List[str]: |
| pass |
| |
| def run_dist_scripts(self) -> None: |
| assert os.path.isabs(self.distdir) |
| mesonrewrite = Environment.get_build_command() + ['rewrite'] |
| env = {'MESON_DIST_ROOT': self.distdir, |
| 'MESON_SOURCE_ROOT': self.src_root, |
| 'MESON_BUILD_ROOT': self.bld_root, |
| 'MESONREWRITE': ' '.join(shlex.quote(x) for x in mesonrewrite), |
| } |
| for d in self.dist_scripts: |
| if d.subproject and d.subproject not in self.subprojects: |
| continue |
| subdir = self.subprojects.get(d.subproject, '') |
| env['MESON_PROJECT_DIST_ROOT'] = os.path.join(self.distdir, subdir) |
| env['MESON_PROJECT_SOURCE_ROOT'] = os.path.join(self.src_root, subdir) |
| env['MESON_PROJECT_BUILD_ROOT'] = os.path.join(self.bld_root, subdir) |
| name = ' '.join(d.cmd_args) |
| print(f'Running custom dist script {name!r}') |
| try: |
| rc = run_exe(d, env) |
| if rc != 0: |
| sys.exit('Dist script errored out') |
| except OSError: |
| print(f'Failed to run dist script {name!r}') |
| sys.exit(1) |
| |
| |
| class GitDist(Dist): |
| def git_root(self, dir_: str) -> Path: |
| # Cannot use --show-toplevel here because git in our CI prints cygwin paths |
| # that python cannot resolve. Workaround this by taking parent of src_root. |
| prefix = quiet_git(['rev-parse', '--show-prefix'], dir_, check=True)[1].strip() |
| if not prefix: |
| return Path(dir_) |
| prefix_level = len(Path(prefix).parents) |
| return Path(dir_).parents[prefix_level - 1] |
| |
| def have_dirty_index(self) -> bool: |
| '''Check whether there are uncommitted changes in git''' |
| ret = subprocess.call(['git', '-C', self.src_root, 'diff-index', '--quiet', 'HEAD']) |
| return ret == 1 |
| |
| def copy_git(self, src: T.Union[str, os.PathLike], distdir: str, revision: str = 'HEAD', |
| prefix: T.Optional[str] = None, subdir: T.Optional[str] = None) -> None: |
| cmd = ['git', 'archive', '--format', 'tar', revision] |
| if prefix is not None: |
| cmd.insert(2, f'--prefix={prefix}/') |
| if subdir is not None: |
| cmd.extend(['--', subdir]) |
| with tempfile.TemporaryFile() as f: |
| subprocess.check_call(cmd, cwd=src, stdout=f) |
| f.seek(0) |
| t = tarfile.open(fileobj=f) # [ignore encoding] |
| t.extractall(path=distdir) |
| |
| def process_git_project(self, src_root: str, distdir: str) -> None: |
| if self.have_dirty_index(): |
| handle_dirty_opt(msg_uncommitted_changes, self.options.allow_dirty) |
| if os.path.exists(distdir): |
| windows_proof_rmtree(distdir) |
| repo_root = self.git_root(src_root) |
| if repo_root.samefile(src_root): |
| os.makedirs(distdir) |
| self.copy_git(src_root, distdir) |
| else: |
| subdir = Path(src_root).relative_to(repo_root) |
| tmp_distdir = distdir + '-tmp' |
| if os.path.exists(tmp_distdir): |
| windows_proof_rmtree(tmp_distdir) |
| os.makedirs(tmp_distdir) |
| self.copy_git(repo_root, tmp_distdir, subdir=str(subdir)) |
| Path(tmp_distdir, subdir).rename(distdir) |
| windows_proof_rmtree(tmp_distdir) |
| self.process_submodules(src_root, distdir) |
| |
| def process_submodules(self, src: str, distdir: str) -> None: |
| module_file = os.path.join(src, '.gitmodules') |
| if not os.path.exists(module_file): |
| return |
| cmd = ['git', 'submodule', 'status', '--cached', '--recursive'] |
| modlist = subprocess.check_output(cmd, cwd=src, universal_newlines=True).splitlines() |
| for submodule in modlist: |
| status = submodule[:1] |
| sha1, rest = submodule[1:].split(' ', 1) |
| subpath = rest.rsplit(' ', 1)[0] |
| |
| if status == '-': |
| mlog.warning(f'Submodule {subpath!r} is not checked out and cannot be added to the dist') |
| continue |
| elif status in {'+', 'U'}: |
| handle_dirty_opt(f'Submodule {subpath!r} has uncommitted changes that will not be included in the dist tarball', self.options.allow_dirty) |
| |
| self.copy_git(os.path.join(src, subpath), distdir, revision=sha1, prefix=subpath) |
| |
| def create_dist(self, archives: T.List[str]) -> T.List[str]: |
| self.process_git_project(self.src_root, self.distdir) |
| for path in self.subprojects.values(): |
| sub_src_root = os.path.join(self.src_root, path) |
| sub_distdir = os.path.join(self.distdir, path) |
| if os.path.exists(sub_distdir): |
| continue |
| if is_git(sub_src_root): |
| self.process_git_project(sub_src_root, sub_distdir) |
| else: |
| shutil.copytree(sub_src_root, sub_distdir) |
| self.run_dist_scripts() |
| output_names = [] |
| for a in archives: |
| compressed_name = self.distdir + archive_extension[a] |
| shutil.make_archive(self.distdir, a, root_dir=self.dist_sub, base_dir=self.dist_name) |
| output_names.append(compressed_name) |
| windows_proof_rmtree(self.distdir) |
| return output_names |
| |
| |
| class HgDist(Dist): |
| def have_dirty_index(self) -> bool: |
| '''Check whether there are uncommitted changes in hg''' |
| out = subprocess.check_output(['hg', '-R', self.src_root, 'summary']) |
| return b'commit: (clean)' not in out |
| |
| def create_dist(self, archives: T.List[str]) -> T.List[str]: |
| if self.have_dirty_index(): |
| handle_dirty_opt(msg_uncommitted_changes, self.options.allow_dirty) |
| if self.dist_scripts: |
| mlog.warning('dist scripts are not supported in Mercurial projects') |
| |
| os.makedirs(self.dist_sub, exist_ok=True) |
| tarname = os.path.join(self.dist_sub, self.dist_name + '.tar') |
| xzname = tarname + '.xz' |
| bz2name = tarname + '.bz2' |
| gzname = tarname + '.gz' |
| zipname = os.path.join(self.dist_sub, self.dist_name + '.zip') |
| # Note that -X interprets relative paths using the current working |
| # directory, not the repository root, so this must be an absolute path: |
| # https://bz.mercurial-scm.org/show_bug.cgi?id=6267 |
| # |
| # .hg[a-z]* is used instead of .hg* to keep .hg_archival.txt, which may |
| # be useful to link the tarball to the Mercurial revision for either |
| # manual inspection or in case any code interprets it for a --version or |
| # similar. |
| subprocess.check_call(['hg', 'archive', '-R', self.src_root, '-S', '-t', 'tar', |
| '-X', self.src_root + '/.hg[a-z]*', tarname]) |
| output_names = [] |
| if 'xztar' in archives: |
| import lzma |
| with lzma.open(xzname, 'wb') as xf, open(tarname, 'rb') as tf: |
| shutil.copyfileobj(tf, xf) |
| output_names.append(xzname) |
| if 'bztar' in archives: |
| import bz2 |
| with bz2.open(bz2name, 'wb') as bf, open(tarname, 'rb') as tf: |
| shutil.copyfileobj(tf, bf) |
| output_names.append(bz2name) |
| if 'gztar' in archives: |
| with gzip.open(gzname, 'wb') as zf, open(tarname, 'rb') as tf: |
| shutil.copyfileobj(tf, zf) |
| output_names.append(gzname) |
| os.unlink(tarname) |
| if 'zip' in archives: |
| subprocess.check_call(['hg', 'archive', '-R', self.src_root, '-S', '-t', 'zip', zipname]) |
| output_names.append(zipname) |
| return output_names |
| |
| |
| def run_dist_steps(meson_command: T.List[str], unpacked_src_dir: str, builddir: str, installdir: str, ninja_args: T.List[str]) -> int: |
| if subprocess.call(meson_command + ['--backend=ninja', unpacked_src_dir, builddir]) != 0: |
| print('Running Meson on distribution package failed') |
| return 1 |
| if subprocess.call(ninja_args, cwd=builddir) != 0: |
| print('Compiling the distribution package failed') |
| return 1 |
| if subprocess.call(ninja_args + ['test'], cwd=builddir) != 0: |
| print('Running unit tests on the distribution package failed') |
| return 1 |
| myenv = os.environ.copy() |
| myenv['DESTDIR'] = installdir |
| if subprocess.call(ninja_args + ['install'], cwd=builddir, env=myenv) != 0: |
| print('Installing the distribution package failed') |
| return 1 |
| return 0 |
| |
| def check_dist(packagename: str, meson_command: ImmutableListProtocol[str], extra_meson_args: T.List[str], bld_root: str, privdir: str) -> int: |
| print(f'Testing distribution package {packagename}') |
| unpackdir = os.path.join(privdir, 'dist-unpack') |
| builddir = os.path.join(privdir, 'dist-build') |
| installdir = os.path.join(privdir, 'dist-install') |
| for p in (unpackdir, builddir, installdir): |
| if os.path.exists(p): |
| windows_proof_rmtree(p) |
| os.mkdir(p) |
| ninja_args = detect_ninja() |
| shutil.unpack_archive(packagename, unpackdir) |
| unpacked_files = glob(os.path.join(unpackdir, '*')) |
| assert len(unpacked_files) == 1 |
| unpacked_src_dir = unpacked_files[0] |
| meson_command += ['setup'] |
| meson_command += create_cmdline_args(bld_root) |
| meson_command += extra_meson_args |
| |
| ret = run_dist_steps(meson_command, unpacked_src_dir, builddir, installdir, ninja_args) |
| if ret > 0: |
| print(f'Dist check build directory was {builddir}') |
| else: |
| windows_proof_rmtree(unpackdir) |
| windows_proof_rmtree(builddir) |
| windows_proof_rmtree(installdir) |
| print(f'Distribution package {packagename} tested') |
| return ret |
| |
| def create_cmdline_args(bld_root: str) -> T.List[str]: |
| parser = argparse.ArgumentParser() |
| msetup_argparse(parser) |
| args = T.cast('coredata.SharedCMDOptions', parser.parse_args([])) |
| coredata.parse_cmd_line_options(args) |
| coredata.read_cmd_line_file(bld_root, args) |
| args.cmd_line_options.pop(OptionKey('backend'), '') |
| return shlex.split(coredata.format_cmd_line_options(args)) |
| |
| def determine_archives_to_generate(options: argparse.Namespace) -> T.List[str]: |
| result = [] |
| for i in options.formats.split(','): |
| if i not in archive_choices: |
| sys.exit(f'Value "{i}" not one of permitted values {archive_choices}.') |
| result.append(i) |
| if len(i) == 0: |
| sys.exit('No archive types specified.') |
| return result |
| |
| def run(options: argparse.Namespace) -> int: |
| buildfile = Path(options.wd) / 'meson-private' / 'build.dat' |
| if not buildfile.is_file(): |
| raise MesonException(f'Directory {options.wd!r} does not seem to be a Meson build directory.') |
| b = build.load(options.wd) |
| need_vsenv = T.cast('bool', b.environment.coredata.get_option(OptionKey('vsenv'))) |
| setup_vsenv(need_vsenv) |
| src_root = b.environment.source_dir |
| bld_root = b.environment.build_dir |
| priv_dir = os.path.join(bld_root, 'meson-private') |
| |
| dist_name = b.project_name + '-' + b.project_version |
| |
| archives = determine_archives_to_generate(options) |
| |
| subprojects = {} |
| extra_meson_args = [] |
| if options.include_subprojects: |
| subproject_dir = os.path.join(src_root, b.subproject_dir) |
| for sub in b.subprojects: |
| directory = wrap.get_directory(subproject_dir, sub) |
| subprojects[sub] = os.path.join(b.subproject_dir, directory) |
| extra_meson_args.append('-Dwrap_mode=nodownload') |
| |
| cls: T.Type[Dist] |
| if is_git(src_root): |
| cls = GitDist |
| elif is_hg(src_root): |
| if subprojects: |
| print('--include-subprojects option currently not supported with Mercurial') |
| return 1 |
| cls = HgDist |
| else: |
| print('Dist currently only works with Git or Mercurial repos') |
| return 1 |
| |
| project = cls(dist_name, src_root, bld_root, b.dist_scripts, subprojects, options) |
| names = project.create_dist(archives) |
| |
| if names is None: |
| return 1 |
| rc = 0 |
| if not options.no_tests: |
| # Check only one. |
| rc = check_dist(names[0], get_meson_command(), extra_meson_args, bld_root, priv_dir) |
| if rc == 0: |
| for name in names: |
| create_hash(name) |
| print('Created', name) |
| return rc |
