| #!/usr/bin/env python2 |
| # |
| # Copyright 2016 Jeremy Kerr <jk@ozlabs.org> |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
| # implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import os.path |
| import re |
| import sys |
| import string |
| import json |
| import argparse |
| import subprocess |
| from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \ |
| lineno |
| |
| json_params = { |
| 'indent': 1, |
| 'sort_keys': True, |
| 'separators': (',', ': '), |
| } |
| |
| def create_parser(): |
| # Match a C-style comment starting with two *s |
| comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL) |
| |
| # Match an @fwts-<tag> annotation (within the comment), plus the proceeding |
| # text |
| annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)', |
| re.DOTALL) |
| |
| # Match the following prlog() call |
| log_call = (((Literal("prerror") + Literal('(').suppress()) | |
| (Literal("prlog") + Literal('(').suppress() + |
| Word(string.letters + string.digits + '_') + |
| Literal(',').suppress())) + |
| Combine(OneOrMore(QuotedString('"')), adjacent=False) + |
| (Literal(')') | Literal(',')).suppress() |
| ) |
| |
| pattern = comment + log_call |
| pattern.setWhitespaceChars(string.whitespace + '\n') |
| |
| def comment_action(tok): |
| patterns = {} |
| for result in annotation.scanString(tok['content']): |
| patterns.update(result[0][0]) |
| return patterns |
| |
| def annotation_action(tok): |
| return { |
| tok['tag']: cleanup_content(tok['text']) |
| } |
| |
| comment.setParseAction(comment_action) |
| annotation.setParseAction(annotation_action) |
| pattern.parseWithTabs() |
| |
| return pattern |
| |
| def find_sources(dirname): |
| sources = [] |
| |
| def is_source(fname): |
| return fname.endswith('.c') |
| |
| def add_fn(s, dname, fnames): |
| s.extend([ os.path.join(dname, fname) |
| for fname in fnames if is_source(fname) ]) |
| |
| os.path.walk(dirname, add_fn, sources) |
| return sources |
| |
| def cleanup_content(content): |
| comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE) |
| whitespace_re = re.compile(r'\s+') |
| |
| content = comment_prefix_re.sub(' ', content) |
| content = whitespace_re.sub(' ', content) |
| return content.strip() |
| |
| def warn(loc, message): |
| print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message) |
| |
| def log_level_to_fwts(level): |
| level_map = { |
| 'PR_EMERG': 'LOG_LEVEL_CRITICAL', |
| 'PR_ALERT': 'LOG_LEVEL_CRITICAL', |
| 'PR_CRIT': 'LOG_LEVEL_CRITICAL', |
| 'PR_ERR': 'LOG_LEVEL_CRITICAL', |
| 'PR_WARNING': 'LOG_LEVEL_HIGH', |
| 'PR_NOTICE': 'LOG_LEVEL_MEDIUM', |
| 'PR_PRINTF': 'LOG_LEVEL_MEDIUM', |
| } |
| return level_map.get(level, 'LOG_LEVEL_LOW') |
| |
| def message_to_pattern(loc, msg): |
| """ Convert a C printf()-style template to a pattern suitable for fwts """ |
| |
| # Somewhat-simplified match for a %-template |
| template_re = re.compile( |
| '%(?P<flag>[-#0 +]*)' |
| '(?P<width>(?:[0-9]*|\*))?' |
| '(?P<precision>\.*(?:[1-9][0-9]*|\*))?' |
| '(?:hh|h|ll|l|L|j|z|t)?' |
| '(?P<conversion>[a-zA-Z%])') |
| global is_regex |
| is_regex = False |
| |
| def expand_template(match): |
| global is_regex |
| c = match.group('conversion').lower() |
| if c == '%': |
| return '%' |
| is_regex = True |
| if c in ['d', 'i', 'u']: |
| return '[0-9]+' |
| elif c == 'o': |
| return '[0-7]+' |
| elif c == 'x': |
| return '[0-9a-f]+' |
| elif c == 'p': |
| return '(0x[0-9a-f]+|nil)' |
| elif c == 's': |
| return '.*' |
| else: |
| warn(loc, "Unknown template conversion '%s'" % match.group(0)) |
| return '.*' |
| |
| escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL) |
| def expand_escape(match): |
| global is_regex |
| c = match.group('char') |
| if c == 'n': |
| return '\n' |
| elif c in ['\\', '"']: |
| return c |
| else: |
| warn(loc, "Unhandled escape sequence '%s'" % match.group(0)) |
| is_regex = True |
| return '.' |
| |
| pattern = template_re.sub(expand_template, msg) |
| pattern = escape_re.sub(expand_escape, pattern) |
| pattern = pattern.strip() |
| |
| compare_mode = "string" |
| if is_regex: |
| compare_mode = "regex" |
| |
| return (compare_mode, pattern) |
| |
| def parse_patterns(parser, fname, tag): |
| patterns = [] |
| data = open(fname).read() |
| i = 1 |
| for result in parser.scanString(data): |
| (token, loc, _) = result |
| if token[1] == 'prlog': |
| (annotations, logfn, level, msg) = token |
| else: |
| (annotations, logfn, msg) = token |
| level = 'PR_ERR' |
| |
| loc = (fname, lineno(loc, data)) |
| |
| if logfn != 'prlog' and logfn != 'prerror': |
| warn(loc, "unknown log output function '%s'" % logfn) |
| |
| compare_mode, pattern_str = message_to_pattern(loc, msg) |
| |
| pattern = { |
| 'log_level': log_level_to_fwts(level), |
| 'compare_mode': compare_mode, |
| 'pattern': pattern_str, |
| 'last_tag': tag, |
| } |
| |
| pattern.update(annotations) |
| |
| if not 'label' in pattern: |
| warn(loc, "missing label") |
| pattern['label'] = '%s:%d' % (fname, i) |
| i += 1 |
| |
| if not 'advice' in pattern: |
| warn(loc, "missing advice") |
| |
| allowed_data = ['compare_mode', 'log_level', |
| 'pattern', 'advice', 'label', 'last_tag'] |
| extras = set(pattern.keys()) - set(allowed_data) |
| if extras: |
| warn(loc, "unknown pattern annotation: %s" % |
| ','.join([ "'%s'" % e for e in extras])) |
| for e in extras: |
| del pattern[e] |
| |
| patterns.append(pattern) |
| |
| return patterns |
| |
| if __name__ == '__main__': |
| argparser = argparse.ArgumentParser( |
| description='Generate FWTS olog definitions from the skiboot ' |
| 'source tree') |
| argparser.add_argument('directories', metavar='DIR', nargs='*', |
| help='path to source files (default .)', default=['.']) |
| argparser.add_argument('--output', '-o', metavar='FILE', |
| type=argparse.FileType('w'), default=sys.stdout, |
| help='output to FILE (default to stdout)', nargs='?') |
| args = argparser.parse_args() |
| |
| sources = [] |
| for directory in args.directories: |
| try: |
| git_tag = subprocess.check_output(["git","-C", directory, "describe", "--abbrev=0" ]) |
| except: |
| git_tag = "???" |
| git_tag = git_tag.replace("\n", "") |
| sources.extend([ (x, git_tag) for x in find_sources(directory)]) |
| |
| parser = create_parser() |
| patterns = [] |
| for source, tag in sources: |
| patterns.extend(parse_patterns(parser, source, tag)) |
| |
| data = {'olog_error_warning_patterns': patterns} |
| |
| args.output.write(json.dumps(data, **json_params) + '\n') |
| |