blob: 59e7314ff6f5aa223240f22108449616acc016a6 [file] [log] [blame]
Vladimir Sementsov-Ogievskiy7cc8e0a2020-02-28 10:19:11 +03001#!/usr/bin/env python
2#
3# Simple benchmarking framework
4#
5# Copyright (c) 2019 Virtuozzo International GmbH.
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 2 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program. If not, see <http://www.gnu.org/licenses/>.
19#
20
21
22def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
23 """Benchmark one test-case
24
25 test_func -- benchmarking function with prototype
26 test_func(env, case), which takes test_env and test_case
27 arguments and returns {'seconds': int} (which is benchmark
28 result) on success and {'error': str} on error. Returned
29 dict may contain any other additional fields.
30 test_env -- test environment - opaque first argument for test_func
31 test_case -- test case - opaque second argument for test_func
32 count -- how many times to call test_func, to calculate average
33 initial_run -- do initial run of test_func, which don't get into result
34
35 Returns dict with the following fields:
36 'runs': list of test_func results
37 'average': average seconds per run (exists only if at least one run
38 succeeded)
39 'delta': maximum delta between test_func result and the average
40 (exists only if at least one run succeeded)
41 'n-failed': number of failed runs (exists only if at least one run
42 failed)
43 """
44 if initial_run:
45 print(' #initial run:')
46 print(' ', test_func(test_env, test_case))
47
48 runs = []
49 for i in range(count):
50 print(' #run {}'.format(i+1))
51 res = test_func(test_env, test_case)
52 print(' ', res)
53 runs.append(res)
54
55 result = {'runs': runs}
56
57 successed = [r for r in runs if ('seconds' in r)]
58 if successed:
59 avg = sum(r['seconds'] for r in successed) / len(successed)
60 result['average'] = avg
61 result['delta'] = max(abs(r['seconds'] - avg) for r in successed)
62
63 if len(successed) < count:
64 result['n-failed'] = count - len(successed)
65
66 return result
67
68
69def ascii_one(result):
70 """Return ASCII representation of bench_one() returned dict."""
71 if 'average' in result:
72 s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta'])
73 if 'n-failed' in result:
74 s += '\n({} failed)'.format(result['n-failed'])
75 return s
76 else:
77 return 'FAILED'
78
79
80def bench(test_func, test_envs, test_cases, *args, **vargs):
81 """Fill benchmark table
82
83 test_func -- benchmarking function, see bench_one for description
84 test_envs -- list of test environments, see bench_one
85 test_cases -- list of test cases, see bench_one
86 args, vargs -- additional arguments for bench_one
87
88 Returns dict with the following fields:
89 'envs': test_envs
90 'cases': test_cases
91 'tab': filled 2D array, where cell [i][j] is bench_one result for
92 test_cases[i] for test_envs[j] (i.e., rows are test cases and
93 columns are test environments)
94 """
95 tab = {}
96 results = {
97 'envs': test_envs,
98 'cases': test_cases,
99 'tab': tab
100 }
101 n = 1
102 n_tests = len(test_envs) * len(test_cases)
103 for env in test_envs:
104 for case in test_cases:
105 print('Testing {}/{}: {} :: {}'.format(n, n_tests,
106 env['id'], case['id']))
107 if case['id'] not in tab:
108 tab[case['id']] = {}
109 tab[case['id']][env['id']] = bench_one(test_func, env, case,
110 *args, **vargs)
111 n += 1
112
113 print('Done')
114 return results
115
116
117def ascii(results):
118 """Return ASCII representation of bench() returned dict."""
119 from tabulate import tabulate
120
121 tab = [[""] + [c['id'] for c in results['envs']]]
122 for case in results['cases']:
123 row = [case['id']]
124 for env in results['envs']:
125 row.append(ascii_one(results['tab'][case['id']][env['id']]))
126 tab.append(row)
127
128 return tabulate(tab)