Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 1 | # TestEnv class to manage test environment variables. |
| 2 | # |
| 3 | # Copyright (c) 2020-2021 Virtuozzo International GmbH |
| 4 | # |
| 5 | # This program is free software; you can redistribute it and/or modify |
| 6 | # it under the terms of the GNU General Public License as published by |
| 7 | # the Free Software Foundation; either version 2 of the License, or |
| 8 | # (at your option) any later version. |
| 9 | # |
| 10 | # This program is distributed in the hope that it will be useful, |
| 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | # GNU General Public License for more details. |
| 14 | # |
| 15 | # You should have received a copy of the GNU General Public License |
| 16 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 | # |
| 18 | |
| 19 | import os |
| 20 | import sys |
| 21 | import tempfile |
| 22 | from pathlib import Path |
| 23 | import shutil |
| 24 | import collections |
| 25 | import random |
| 26 | import subprocess |
| 27 | import glob |
John Snow | e389004 | 2024-06-26 19:22:29 -0400 | [diff] [blame] | 28 | from typing import List, Dict, Any, Optional |
| 29 | |
| 30 | if sys.version_info >= (3, 9): |
| 31 | from contextlib import AbstractContextManager as ContextManager |
| 32 | else: |
| 33 | from typing import ContextManager |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 34 | |
Emanuele Giuseppe Esposito | cfb9b0b | 2021-08-09 11:01:03 +0200 | [diff] [blame] | 35 | DEF_GDB_OPTIONS = 'localhost:12345' |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 36 | |
| 37 | def isxfile(path: str) -> bool: |
| 38 | return os.path.isfile(path) and os.access(path, os.X_OK) |
| 39 | |
| 40 | |
| 41 | def get_default_machine(qemu_prog: str) -> str: |
| 42 | outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True, |
| 43 | universal_newlines=True, |
| 44 | stdout=subprocess.PIPE).stdout |
| 45 | |
| 46 | machines = outp.split('\n') |
| 47 | try: |
Andrey Drobyshev | 3b7094f | 2023-11-22 14:15:38 +0200 | [diff] [blame] | 48 | default_machine = next(m for m in machines if ' (default)' in m) |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 49 | except StopIteration: |
| 50 | return '' |
| 51 | default_machine = default_machine.split(' ', 1)[0] |
| 52 | |
| 53 | alias_suf = ' (alias of {})'.format(default_machine) |
| 54 | alias = next((m for m in machines if m.endswith(alias_suf)), None) |
| 55 | if alias is not None: |
| 56 | default_machine = alias.split(' ', 1)[0] |
| 57 | |
| 58 | return default_machine |
| 59 | |
| 60 | |
| 61 | class TestEnv(ContextManager['TestEnv']): |
| 62 | """ |
| 63 | Manage system environment for running tests |
| 64 | |
| 65 | The following variables are supported/provided. They are represented by |
| 66 | lower-cased TestEnv attributes. |
| 67 | """ |
| 68 | |
| 69 | # We store environment variables as instance attributes, and there are a |
| 70 | # lot of them. Silence pylint: |
| 71 | # pylint: disable=too-many-instance-attributes |
| 72 | |
| 73 | env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR', |
Hanna Reitz | 1a8fcca | 2022-02-21 18:29:09 +0100 | [diff] [blame] | 74 | 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG', |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 75 | 'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG', |
John Snow | c163c72 | 2021-09-22 20:49:31 -0400 | [diff] [blame] | 76 | 'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS', |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 77 | 'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT', |
| 78 | 'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO', |
| 79 | 'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU', |
| 80 | 'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX', |
Emanuele Giuseppe Esposito | cfb9b0b | 2021-08-09 11:01:03 +0200 | [diff] [blame] | 81 | 'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_', |
Emanuele Giuseppe Esposito | eb7a91d | 2021-08-09 11:01:13 +0200 | [diff] [blame] | 82 | 'GDB_OPTIONS', 'PRINT_QEMU'] |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 83 | |
Paolo Bonzini | c64430d | 2021-05-03 13:01:08 +0200 | [diff] [blame] | 84 | def prepare_subprocess(self, args: List[str]) -> Dict[str, str]: |
| 85 | if self.debug: |
| 86 | args.append('-d') |
| 87 | |
| 88 | with open(args[0], encoding="utf-8") as f: |
| 89 | try: |
| 90 | if f.readline().rstrip() == '#!/usr/bin/env python3': |
| 91 | args.insert(0, self.python) |
| 92 | except UnicodeDecodeError: # binary test? for future. |
| 93 | pass |
| 94 | |
| 95 | os_env = os.environ.copy() |
| 96 | os_env.update(self.get_env()) |
| 97 | return os_env |
| 98 | |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 99 | def get_env(self) -> Dict[str, str]: |
| 100 | env = {} |
| 101 | for v in self.env_variables: |
| 102 | val = getattr(self, v.lower(), None) |
| 103 | if val is not None: |
| 104 | env[v] = val |
| 105 | |
| 106 | return env |
| 107 | |
| 108 | def init_directories(self) -> None: |
| 109 | """Init directory variables: |
| 110 | PYTHONPATH |
| 111 | TEST_DIR |
| 112 | SOCK_DIR |
| 113 | SAMPLE_IMG_DIR |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 114 | """ |
John Snow | af6d4c5 | 2021-09-23 14:07:10 -0400 | [diff] [blame] | 115 | |
| 116 | # Path where qemu goodies live in this source tree. |
| 117 | qemu_srctree_path = Path(__file__, '../../../python').resolve() |
| 118 | |
| 119 | self.pythonpath = os.pathsep.join(filter(None, ( |
| 120 | self.source_iotests, |
| 121 | str(qemu_srctree_path), |
| 122 | os.getenv('PYTHONPATH'), |
| 123 | ))) |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 124 | |
| 125 | self.test_dir = os.getenv('TEST_DIR', |
| 126 | os.path.join(os.getcwd(), 'scratch')) |
| 127 | Path(self.test_dir).mkdir(parents=True, exist_ok=True) |
| 128 | |
| 129 | try: |
| 130 | self.sock_dir = os.environ['SOCK_DIR'] |
| 131 | self.tmp_sock_dir = False |
Paolo Bonzini | c3d479a | 2021-05-03 13:01:10 +0200 | [diff] [blame] | 132 | Path(self.sock_dir).mkdir(parents=True, exist_ok=True) |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 133 | except KeyError: |
Daniel P. Berrangé | 7d2faf0 | 2024-02-05 15:51:58 +0000 | [diff] [blame] | 134 | self.sock_dir = tempfile.mkdtemp(prefix="qemu-iotests-") |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 135 | self.tmp_sock_dir = True |
| 136 | |
| 137 | self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR', |
| 138 | os.path.join(self.source_iotests, |
| 139 | 'sample_images')) |
| 140 | |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 141 | def init_binaries(self) -> None: |
| 142 | """Init binary path variables: |
| 143 | PYTHON (for bash tests) |
| 144 | QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 145 | """ |
| 146 | self.python = sys.executable |
| 147 | |
| 148 | def root(*names: str) -> str: |
| 149 | return os.path.join(self.build_root, *names) |
| 150 | |
| 151 | arch = os.uname().machine |
| 152 | if 'ppc64' in arch: |
| 153 | arch = 'ppc64' |
| 154 | |
| 155 | self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}')) |
| 156 | if not os.path.exists(self.qemu_prog): |
| 157 | pattern = root('qemu-system-*') |
| 158 | try: |
Kevin Wolf | ca502ca | 2021-02-02 15:28:02 +0100 | [diff] [blame] | 159 | progs = sorted(glob.iglob(pattern)) |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 160 | self.qemu_prog = next(p for p in progs if isxfile(p)) |
| 161 | except StopIteration: |
| 162 | sys.exit("Not found any Qemu executable binary by pattern " |
| 163 | f"'{pattern}'") |
| 164 | |
| 165 | self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img')) |
| 166 | self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io')) |
| 167 | self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd')) |
| 168 | self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon', |
| 169 | 'qemu-storage-daemon')) |
| 170 | |
| 171 | for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog, |
| 172 | self.qemu_prog, self.qsd_prog]: |
| 173 | if not os.path.exists(b): |
| 174 | sys.exit('No such file: ' + b) |
| 175 | if not isxfile(b): |
| 176 | sys.exit('Not executable: ' + b) |
| 177 | |
Daniel P. Berrangé | 0c8076b | 2023-03-15 17:43:17 +0000 | [diff] [blame] | 178 | def __init__(self, source_dir: str, build_dir: str, |
| 179 | imgfmt: str, imgproto: str, aiomode: str, |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 180 | cachemode: Optional[str] = None, |
| 181 | imgopts: Optional[str] = None, |
| 182 | misalign: bool = False, |
| 183 | debug: bool = False, |
Emanuele Giuseppe Esposito | cfb9b0b | 2021-08-09 11:01:03 +0200 | [diff] [blame] | 184 | valgrind: bool = False, |
Emanuele Giuseppe Esposito | eb7a91d | 2021-08-09 11:01:13 +0200 | [diff] [blame] | 185 | gdb: bool = False, |
Daniel P. Berrangé | a9e2178 | 2023-03-15 17:43:18 +0000 | [diff] [blame] | 186 | qprint: bool = False, |
| 187 | dry_run: bool = False) -> None: |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 188 | self.imgfmt = imgfmt |
| 189 | self.imgproto = imgproto |
| 190 | self.aiomode = aiomode |
| 191 | self.imgopts = imgopts |
| 192 | self.misalign = misalign |
| 193 | self.debug = debug |
| 194 | |
Emanuele Giuseppe Esposito | eb7a91d | 2021-08-09 11:01:13 +0200 | [diff] [blame] | 195 | if qprint: |
| 196 | self.print_qemu = 'y' |
| 197 | |
Emanuele Giuseppe Esposito | cfb9b0b | 2021-08-09 11:01:03 +0200 | [diff] [blame] | 198 | if gdb: |
| 199 | self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS) |
| 200 | if not self.gdb_options: |
| 201 | # cover the case 'export GDB_OPTIONS=' |
| 202 | self.gdb_options = DEF_GDB_OPTIONS |
| 203 | elif 'GDB_OPTIONS' in os.environ: |
| 204 | # to not propagate it in prepare_subprocess() |
| 205 | del os.environ['GDB_OPTIONS'] |
| 206 | |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 207 | if valgrind: |
| 208 | self.valgrind_qemu = 'y' |
| 209 | |
| 210 | if cachemode is None: |
| 211 | self.cachemode_is_default = 'true' |
| 212 | self.cachemode = 'writeback' |
| 213 | else: |
| 214 | self.cachemode_is_default = 'false' |
| 215 | self.cachemode = cachemode |
| 216 | |
| 217 | # Initialize generic paths: build_root, build_iotests, source_iotests, |
| 218 | # which are needed to initialize some environment variables. They are |
| 219 | # used by init_*() functions as well. |
| 220 | |
Daniel P. Berrangé | 0c8076b | 2023-03-15 17:43:17 +0000 | [diff] [blame] | 221 | self.source_iotests = source_dir |
| 222 | self.build_iotests = build_dir |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 223 | |
John Snow | 9444e5f | 2023-06-20 20:21:18 -0400 | [diff] [blame] | 224 | self.build_root = Path(self.build_iotests).parent.parent |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 225 | |
| 226 | self.init_directories() |
Daniel P. Berrangé | a9e2178 | 2023-03-15 17:43:18 +0000 | [diff] [blame] | 227 | |
| 228 | if dry_run: |
| 229 | return |
| 230 | |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 231 | self.init_binaries() |
| 232 | |
| 233 | self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_', |
| 234 | str(random.randrange(1, 255))) |
| 235 | |
| 236 | # QEMU_OPTIONS |
| 237 | self.qemu_options = '-nodefaults -display none -accel qtest' |
| 238 | machine_map = ( |
| 239 | ('arm', 'virt'), |
| 240 | ('aarch64', 'virt'), |
| 241 | ('avr', 'mega2560'), |
Laurent Vivier | 7033f1f | 2021-03-23 16:53:06 +0000 | [diff] [blame] | 242 | ('m68k', 'virt'), |
Thomas Huth | 798f35f | 2024-09-17 21:30:28 +0200 | [diff] [blame] | 243 | ('or1k', 'virt'), |
laokz | 3213bba | 2022-03-08 12:33:39 +0800 | [diff] [blame] | 244 | ('riscv32', 'virt'), |
| 245 | ('riscv64', 'virt'), |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 246 | ('rx', 'gdbsim-r5f562n8'), |
Thomas Huth | fd72943 | 2024-09-17 21:43:50 +0200 | [diff] [blame] | 247 | ('sh4', 'r2d'), |
Thomas Huth | 51cdb68 | 2024-10-24 10:27:34 +0200 | [diff] [blame] | 248 | ('sh4eb', 'r2d'), |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 249 | ('tricore', 'tricore_testboard') |
| 250 | ) |
| 251 | for suffix, machine in machine_map: |
| 252 | if self.qemu_prog.endswith(f'qemu-system-{suffix}'): |
| 253 | self.qemu_options += f' -machine {machine}' |
| 254 | |
| 255 | # QEMU_DEFAULT_MACHINE |
| 256 | self.qemu_default_machine = get_default_machine(self.qemu_prog) |
| 257 | |
| 258 | self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS') |
| 259 | self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS') |
| 260 | |
Amjad Alsharafi | c8f60bf | 2024-07-20 18:13:34 +0800 | [diff] [blame] | 261 | is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg', 'vvfat'] |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 262 | self.imgfmt_generic = 'true' if is_generic else 'false' |
| 263 | |
| 264 | self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}' |
| 265 | if self.misalign: |
| 266 | self.qemu_io_options += ' --misalign' |
| 267 | |
| 268 | self.qemu_io_options_no_fmt = self.qemu_io_options |
| 269 | |
| 270 | if self.imgfmt == 'luks': |
| 271 | self.imgoptssyntax = 'true' |
| 272 | self.imgkeysecret = '123456' |
| 273 | if not self.imgopts: |
| 274 | self.imgopts = 'iter-time=10' |
| 275 | elif 'iter-time=' not in self.imgopts: |
| 276 | self.imgopts += ',iter-time=10' |
| 277 | else: |
| 278 | self.imgoptssyntax = 'false' |
| 279 | self.qemu_io_options += ' -f ' + self.imgfmt |
| 280 | |
| 281 | if self.imgfmt == 'vmdk': |
| 282 | if not self.imgopts: |
| 283 | self.imgopts = 'zeroed_grain=on' |
| 284 | elif 'zeroed_grain=' not in self.imgopts: |
| 285 | self.imgopts += ',zeroed_grain=on' |
| 286 | |
| 287 | def close(self) -> None: |
| 288 | if self.tmp_sock_dir: |
| 289 | shutil.rmtree(self.sock_dir) |
| 290 | |
| 291 | def __enter__(self) -> 'TestEnv': |
| 292 | return self |
| 293 | |
| 294 | def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: |
| 295 | self.close() |
| 296 | |
Paolo Bonzini | d316859 | 2022-01-07 13:18:11 +0100 | [diff] [blame] | 297 | def print_env(self, prefix: str = '') -> None: |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 298 | template = """\ |
Paolo Bonzini | d316859 | 2022-01-07 13:18:11 +0100 | [diff] [blame] | 299 | {prefix}QEMU -- "{QEMU_PROG}" {QEMU_OPTIONS} |
| 300 | {prefix}QEMU_IMG -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS} |
| 301 | {prefix}QEMU_IO -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS} |
| 302 | {prefix}QEMU_NBD -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS} |
| 303 | {prefix}IMGFMT -- {IMGFMT}{imgopts} |
| 304 | {prefix}IMGPROTO -- {IMGPROTO} |
| 305 | {prefix}PLATFORM -- {platform} |
| 306 | {prefix}TEST_DIR -- {TEST_DIR} |
| 307 | {prefix}SOCK_DIR -- {SOCK_DIR} |
| 308 | {prefix}GDB_OPTIONS -- {GDB_OPTIONS} |
| 309 | {prefix}VALGRIND_QEMU -- {VALGRIND_QEMU} |
| 310 | {prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU} |
| 311 | {prefix}""" |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 312 | |
| 313 | args = collections.defaultdict(str, self.get_env()) |
| 314 | |
| 315 | if 'IMGOPTS' in args: |
| 316 | args['imgopts'] = f" ({args['IMGOPTS']})" |
| 317 | |
| 318 | u = os.uname() |
| 319 | args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}' |
Paolo Bonzini | d316859 | 2022-01-07 13:18:11 +0100 | [diff] [blame] | 320 | args['prefix'] = prefix |
Vladimir Sementsov-Ogievskiy | 2e5a2f5 | 2021-01-25 21:50:53 +0300 | [diff] [blame] | 321 | print(template.format_map(args)) |