blob: 6326e46b7b100d20a1299d6d62405a959336af14 [file] [log] [blame]
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +03001# TestEnv class to manage test environment variables.
2#
3# Copyright (c) 2020-2021 Virtuozzo International GmbH
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program. If not, see <http://www.gnu.org/licenses/>.
17#
18
19import os
20import sys
21import tempfile
22from pathlib import Path
23import shutil
24import collections
25import random
26import subprocess
27import glob
John Snowe3890042024-06-26 19:22:29 -040028from typing import List, Dict, Any, Optional
29
30if sys.version_info >= (3, 9):
31 from contextlib import AbstractContextManager as ContextManager
32else:
33 from typing import ContextManager
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030034
Emanuele Giuseppe Espositocfb9b0b2021-08-09 11:01:03 +020035DEF_GDB_OPTIONS = 'localhost:12345'
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030036
37def isxfile(path: str) -> bool:
38 return os.path.isfile(path) and os.access(path, os.X_OK)
39
40
41def get_default_machine(qemu_prog: str) -> str:
42 outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
43 universal_newlines=True,
44 stdout=subprocess.PIPE).stdout
45
46 machines = outp.split('\n')
47 try:
Andrey Drobyshev3b7094f2023-11-22 14:15:38 +020048 default_machine = next(m for m in machines if ' (default)' in m)
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030049 except StopIteration:
50 return ''
51 default_machine = default_machine.split(' ', 1)[0]
52
53 alias_suf = ' (alias of {})'.format(default_machine)
54 alias = next((m for m in machines if m.endswith(alias_suf)), None)
55 if alias is not None:
56 default_machine = alias.split(' ', 1)[0]
57
58 return default_machine
59
60
61class TestEnv(ContextManager['TestEnv']):
62 """
63 Manage system environment for running tests
64
65 The following variables are supported/provided. They are represented by
66 lower-cased TestEnv attributes.
67 """
68
69 # We store environment variables as instance attributes, and there are a
70 # lot of them. Silence pylint:
71 # pylint: disable=too-many-instance-attributes
72
73 env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
Hanna Reitz1a8fcca2022-02-21 18:29:09 +010074 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG',
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030075 'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG',
John Snowc163c722021-09-22 20:49:31 -040076 'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS',
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030077 'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT',
78 'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO',
79 'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU',
80 'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',
Emanuele Giuseppe Espositocfb9b0b2021-08-09 11:01:03 +020081 'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_',
Emanuele Giuseppe Espositoeb7a91d2021-08-09 11:01:13 +020082 'GDB_OPTIONS', 'PRINT_QEMU']
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030083
Paolo Bonzinic64430d2021-05-03 13:01:08 +020084 def prepare_subprocess(self, args: List[str]) -> Dict[str, str]:
85 if self.debug:
86 args.append('-d')
87
88 with open(args[0], encoding="utf-8") as f:
89 try:
90 if f.readline().rstrip() == '#!/usr/bin/env python3':
91 args.insert(0, self.python)
92 except UnicodeDecodeError: # binary test? for future.
93 pass
94
95 os_env = os.environ.copy()
96 os_env.update(self.get_env())
97 return os_env
98
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +030099 def get_env(self) -> Dict[str, str]:
100 env = {}
101 for v in self.env_variables:
102 val = getattr(self, v.lower(), None)
103 if val is not None:
104 env[v] = val
105
106 return env
107
108 def init_directories(self) -> None:
109 """Init directory variables:
110 PYTHONPATH
111 TEST_DIR
112 SOCK_DIR
113 SAMPLE_IMG_DIR
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300114 """
John Snowaf6d4c52021-09-23 14:07:10 -0400115
116 # Path where qemu goodies live in this source tree.
117 qemu_srctree_path = Path(__file__, '../../../python').resolve()
118
119 self.pythonpath = os.pathsep.join(filter(None, (
120 self.source_iotests,
121 str(qemu_srctree_path),
122 os.getenv('PYTHONPATH'),
123 )))
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300124
125 self.test_dir = os.getenv('TEST_DIR',
126 os.path.join(os.getcwd(), 'scratch'))
127 Path(self.test_dir).mkdir(parents=True, exist_ok=True)
128
129 try:
130 self.sock_dir = os.environ['SOCK_DIR']
131 self.tmp_sock_dir = False
Paolo Bonzinic3d479a2021-05-03 13:01:10 +0200132 Path(self.sock_dir).mkdir(parents=True, exist_ok=True)
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300133 except KeyError:
Daniel P. Berrangé7d2faf02024-02-05 15:51:58 +0000134 self.sock_dir = tempfile.mkdtemp(prefix="qemu-iotests-")
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300135 self.tmp_sock_dir = True
136
137 self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR',
138 os.path.join(self.source_iotests,
139 'sample_images'))
140
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300141 def init_binaries(self) -> None:
142 """Init binary path variables:
143 PYTHON (for bash tests)
144 QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300145 """
146 self.python = sys.executable
147
148 def root(*names: str) -> str:
149 return os.path.join(self.build_root, *names)
150
151 arch = os.uname().machine
152 if 'ppc64' in arch:
153 arch = 'ppc64'
154
155 self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}'))
156 if not os.path.exists(self.qemu_prog):
157 pattern = root('qemu-system-*')
158 try:
Kevin Wolfca502ca2021-02-02 15:28:02 +0100159 progs = sorted(glob.iglob(pattern))
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300160 self.qemu_prog = next(p for p in progs if isxfile(p))
161 except StopIteration:
162 sys.exit("Not found any Qemu executable binary by pattern "
163 f"'{pattern}'")
164
165 self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img'))
166 self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io'))
167 self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd'))
168 self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon',
169 'qemu-storage-daemon'))
170
171 for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog,
172 self.qemu_prog, self.qsd_prog]:
173 if not os.path.exists(b):
174 sys.exit('No such file: ' + b)
175 if not isxfile(b):
176 sys.exit('Not executable: ' + b)
177
Daniel P. Berrangé0c8076b2023-03-15 17:43:17 +0000178 def __init__(self, source_dir: str, build_dir: str,
179 imgfmt: str, imgproto: str, aiomode: str,
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300180 cachemode: Optional[str] = None,
181 imgopts: Optional[str] = None,
182 misalign: bool = False,
183 debug: bool = False,
Emanuele Giuseppe Espositocfb9b0b2021-08-09 11:01:03 +0200184 valgrind: bool = False,
Emanuele Giuseppe Espositoeb7a91d2021-08-09 11:01:13 +0200185 gdb: bool = False,
Daniel P. Berrangéa9e21782023-03-15 17:43:18 +0000186 qprint: bool = False,
187 dry_run: bool = False) -> None:
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300188 self.imgfmt = imgfmt
189 self.imgproto = imgproto
190 self.aiomode = aiomode
191 self.imgopts = imgopts
192 self.misalign = misalign
193 self.debug = debug
194
Emanuele Giuseppe Espositoeb7a91d2021-08-09 11:01:13 +0200195 if qprint:
196 self.print_qemu = 'y'
197
Emanuele Giuseppe Espositocfb9b0b2021-08-09 11:01:03 +0200198 if gdb:
199 self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS)
200 if not self.gdb_options:
201 # cover the case 'export GDB_OPTIONS='
202 self.gdb_options = DEF_GDB_OPTIONS
203 elif 'GDB_OPTIONS' in os.environ:
204 # to not propagate it in prepare_subprocess()
205 del os.environ['GDB_OPTIONS']
206
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300207 if valgrind:
208 self.valgrind_qemu = 'y'
209
210 if cachemode is None:
211 self.cachemode_is_default = 'true'
212 self.cachemode = 'writeback'
213 else:
214 self.cachemode_is_default = 'false'
215 self.cachemode = cachemode
216
217 # Initialize generic paths: build_root, build_iotests, source_iotests,
218 # which are needed to initialize some environment variables. They are
219 # used by init_*() functions as well.
220
Daniel P. Berrangé0c8076b2023-03-15 17:43:17 +0000221 self.source_iotests = source_dir
222 self.build_iotests = build_dir
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300223
John Snow9444e5f2023-06-20 20:21:18 -0400224 self.build_root = Path(self.build_iotests).parent.parent
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300225
226 self.init_directories()
Daniel P. Berrangéa9e21782023-03-15 17:43:18 +0000227
228 if dry_run:
229 return
230
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300231 self.init_binaries()
232
233 self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_',
234 str(random.randrange(1, 255)))
235
236 # QEMU_OPTIONS
237 self.qemu_options = '-nodefaults -display none -accel qtest'
238 machine_map = (
239 ('arm', 'virt'),
240 ('aarch64', 'virt'),
241 ('avr', 'mega2560'),
Laurent Vivier7033f1f2021-03-23 16:53:06 +0000242 ('m68k', 'virt'),
Thomas Huth798f35f2024-09-17 21:30:28 +0200243 ('or1k', 'virt'),
laokz3213bba2022-03-08 12:33:39 +0800244 ('riscv32', 'virt'),
245 ('riscv64', 'virt'),
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300246 ('rx', 'gdbsim-r5f562n8'),
Thomas Huthfd729432024-09-17 21:43:50 +0200247 ('sh4', 'r2d'),
Thomas Huth51cdb682024-10-24 10:27:34 +0200248 ('sh4eb', 'r2d'),
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300249 ('tricore', 'tricore_testboard')
250 )
251 for suffix, machine in machine_map:
252 if self.qemu_prog.endswith(f'qemu-system-{suffix}'):
253 self.qemu_options += f' -machine {machine}'
254
255 # QEMU_DEFAULT_MACHINE
256 self.qemu_default_machine = get_default_machine(self.qemu_prog)
257
258 self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS')
259 self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS')
260
Amjad Alsharafic8f60bf2024-07-20 18:13:34 +0800261 is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg', 'vvfat']
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300262 self.imgfmt_generic = 'true' if is_generic else 'false'
263
264 self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}'
265 if self.misalign:
266 self.qemu_io_options += ' --misalign'
267
268 self.qemu_io_options_no_fmt = self.qemu_io_options
269
270 if self.imgfmt == 'luks':
271 self.imgoptssyntax = 'true'
272 self.imgkeysecret = '123456'
273 if not self.imgopts:
274 self.imgopts = 'iter-time=10'
275 elif 'iter-time=' not in self.imgopts:
276 self.imgopts += ',iter-time=10'
277 else:
278 self.imgoptssyntax = 'false'
279 self.qemu_io_options += ' -f ' + self.imgfmt
280
281 if self.imgfmt == 'vmdk':
282 if not self.imgopts:
283 self.imgopts = 'zeroed_grain=on'
284 elif 'zeroed_grain=' not in self.imgopts:
285 self.imgopts += ',zeroed_grain=on'
286
287 def close(self) -> None:
288 if self.tmp_sock_dir:
289 shutil.rmtree(self.sock_dir)
290
291 def __enter__(self) -> 'TestEnv':
292 return self
293
294 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
295 self.close()
296
Paolo Bonzinid3168592022-01-07 13:18:11 +0100297 def print_env(self, prefix: str = '') -> None:
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300298 template = """\
Paolo Bonzinid3168592022-01-07 13:18:11 +0100299{prefix}QEMU -- "{QEMU_PROG}" {QEMU_OPTIONS}
300{prefix}QEMU_IMG -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS}
301{prefix}QEMU_IO -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS}
302{prefix}QEMU_NBD -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS}
303{prefix}IMGFMT -- {IMGFMT}{imgopts}
304{prefix}IMGPROTO -- {IMGPROTO}
305{prefix}PLATFORM -- {platform}
306{prefix}TEST_DIR -- {TEST_DIR}
307{prefix}SOCK_DIR -- {SOCK_DIR}
308{prefix}GDB_OPTIONS -- {GDB_OPTIONS}
309{prefix}VALGRIND_QEMU -- {VALGRIND_QEMU}
310{prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU}
311{prefix}"""
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300312
313 args = collections.defaultdict(str, self.get_env())
314
315 if 'IMGOPTS' in args:
316 args['imgopts'] = f" ({args['IMGOPTS']})"
317
318 u = os.uname()
319 args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}'
Paolo Bonzinid3168592022-01-07 13:18:11 +0100320 args['prefix'] = prefix
Vladimir Sementsov-Ogievskiy2e5a2f52021-01-25 21:50:53 +0300321 print(template.format_map(args))