blob: 5fb436e08ff6927f58f384007f85aa81613a666b [file] [log] [blame]
# SPDX-License-Identifier: GPL-2.0+
#
# Copyright 2020 Google LLC
#
"""Talks to the patchwork service to figure out what patches have been reviewed
and commented on. Provides a way to display review tags and comments.
Allows creation of a new branch based on the old but with the review tags
collected from patchwork.
"""
import collections
import concurrent.futures
from itertools import repeat
import re
import pygit2
import requests
from patman import patchstream
from patman.patchstream import PatchStream
from u_boot_pylib import terminal
from u_boot_pylib import tout
# Patches which are part of a multi-patch series are shown with a prefix like
# [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last
# part is optional. This decodes the string into groups. For single patches
# the [] part is not present:
# Groups: (ignore, ignore, ignore, prefix, version, sequence, subject)
RE_PATCH = re.compile(r'(\[(((.*),)?(.*),)?(.*)\]\s)?(.*)$')
# This decodes the sequence string into a patch number and patch count
RE_SEQ = re.compile(r'(\d+)/(\d+)')
def to_int(vals):
"""Convert a list of strings into integers, using 0 if not an integer
Args:
vals (list): List of strings
Returns:
list: List of integers, one for each input string
"""
out = [int(val) if val.isdigit() else 0 for val in vals]
return out
class Patch(dict):
"""Models a patch in patchwork
This class records information obtained from patchwork
Some of this information comes from the 'Patch' column:
[RFC,v2,1/3] dm: Driver and uclass changes for tiny-dm
This shows the prefix, version, seq, count and subject.
The other properties come from other columns in the display.
Properties:
pid (str): ID of the patch (typically an integer)
seq (int): Sequence number within series (1=first) parsed from sequence
string
count (int): Number of patches in series, parsed from sequence string
raw_subject (str): Entire subject line, e.g.
"[1/2,v2] efi_loader: Sort header file ordering"
prefix (str): Prefix string or None (e.g. 'RFC')
version (str): Version string or None (e.g. 'v2')
raw_subject (str): Raw patch subject
subject (str): Patch subject with [..] part removed (same as commit
subject)
"""
def __init__(self, pid):
super().__init__()
self.id = pid # Use 'id' to match what the Rest API provides
self.seq = None
self.count = None
self.prefix = None
self.version = None
self.raw_subject = None
self.subject = None
# These make us more like a dictionary
def __setattr__(self, name, value):
self[name] = value
def __getattr__(self, name):
return self[name]
def __hash__(self):
return hash(frozenset(self.items()))
def __str__(self):
return self.raw_subject
def parse_subject(self, raw_subject):
"""Parse the subject of a patch into its component parts
See RE_PATCH for details. The parsed info is placed into seq, count,
prefix, version, subject
Args:
raw_subject (str): Subject string to parse
Raises:
ValueError: the subject cannot be parsed
"""
self.raw_subject = raw_subject.strip()
mat = RE_PATCH.search(raw_subject.strip())
if not mat:
raise ValueError("Cannot parse subject '%s'" % raw_subject)
self.prefix, self.version, seq_info, self.subject = mat.groups()[3:]
mat_seq = RE_SEQ.match(seq_info) if seq_info else False
if mat_seq is None:
self.version = seq_info
seq_info = None
if self.version and not self.version.startswith('v'):
self.prefix = self.version
self.version = None
if seq_info:
if mat_seq:
self.seq = int(mat_seq.group(1))
self.count = int(mat_seq.group(2))
else:
self.seq = 1
self.count = 1
class Review:
"""Represents a single review email collected in Patchwork
Patches can attract multiple reviews. Each consists of an author/date and
a variable number of 'snippets', which are groups of quoted and unquoted
text.
"""
def __init__(self, meta, snippets):
"""Create new Review object
Args:
meta (str): Text containing review author and date
snippets (list): List of snippets in th review, each a list of text
lines
"""
self.meta = ' : '.join([line for line in meta.splitlines() if line])
self.snippets = snippets
def compare_with_series(series, patches):
"""Compare a list of patches with a series it came from
This prints any problems as warnings
Args:
series (Series): Series to compare against
patches (:type: list of Patch): list of Patch objects to compare with
Returns:
tuple
dict:
key: Commit number (0...n-1)
value: Patch object for that commit
dict:
key: Patch number (0...n-1)
value: Commit object for that patch
"""
# Check the names match
warnings = []
patch_for_commit = {}
all_patches = set(patches)
for seq, cmt in enumerate(series.commits):
pmatch = [p for p in all_patches if p.subject == cmt.subject]
if len(pmatch) == 1:
patch_for_commit[seq] = pmatch[0]
all_patches.remove(pmatch[0])
elif len(pmatch) > 1:
warnings.append("Multiple patches match commit %d ('%s'):\n %s" %
(seq + 1, cmt.subject,
'\n '.join([p.subject for p in pmatch])))
else:
warnings.append("Cannot find patch for commit %d ('%s')" %
(seq + 1, cmt.subject))
# Check the names match
commit_for_patch = {}
all_commits = set(series.commits)
for seq, patch in enumerate(patches):
cmatch = [c for c in all_commits if c.subject == patch.subject]
if len(cmatch) == 1:
commit_for_patch[seq] = cmatch[0]
all_commits.remove(cmatch[0])
elif len(cmatch) > 1:
warnings.append("Multiple commits match patch %d ('%s'):\n %s" %
(seq + 1, patch.subject,
'\n '.join([c.subject for c in cmatch])))
else:
warnings.append("Cannot find commit for patch %d ('%s')" %
(seq + 1, patch.subject))
return patch_for_commit, commit_for_patch, warnings
def call_rest_api(url, subpath):
"""Call the patchwork API and return the result as JSON
Args:
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
subpath (str): URL subpath to use
Returns:
dict: Json result
Raises:
ValueError: the URL could not be read
"""
full_url = '%s/api/1.2/%s' % (url, subpath)
response = requests.get(full_url)
if response.status_code != 200:
raise ValueError("Could not read URL '%s'" % full_url)
return response.json()
def collect_patches(series, series_id, url, rest_api=call_rest_api):
"""Collect patch information about a series from patchwork
Uses the Patchwork REST API to collect information provided by patchwork
about the status of each patch.
Args:
series (Series): Series object corresponding to the local branch
containing the series
series_id (str): Patch series ID number
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
rest_api (function): API function to call to access Patchwork, for
testing
Returns:
list: List of patches sorted by sequence number, each a Patch object
Raises:
ValueError: if the URL could not be read or the web page does not follow
the expected structure
"""
data = rest_api(url, 'series/%s/' % series_id)
# Get all the rows, which are patches
patch_dict = data['patches']
count = len(patch_dict)
num_commits = len(series.commits)
if count != num_commits:
tout.warning('Warning: Patchwork reports %d patches, series has %d' %
(count, num_commits))
patches = []
# Work through each row (patch) one at a time, collecting the information
warn_count = 0
for pw_patch in patch_dict:
patch = Patch(pw_patch['id'])
patch.parse_subject(pw_patch['name'])
patches.append(patch)
if warn_count > 1:
tout.warning(' (total of %d warnings)' % warn_count)
# Sort patches by patch number
patches = sorted(patches, key=lambda x: x.seq)
return patches
def find_new_responses(new_rtag_list, review_list, seq, cmt, patch, url,
rest_api=call_rest_api):
"""Find new rtags collected by patchwork that we don't know about
This is designed to be run in parallel, once for each commit/patch
Args:
new_rtag_list (list): New rtags are written to new_rtag_list[seq]
list, each a dict:
key: Response tag (e.g. 'Reviewed-by')
value: Set of people who gave that response, each a name/email
string
review_list (list): New reviews are written to review_list[seq]
list, each a
List of reviews for the patch, each a Review
seq (int): Position in new_rtag_list to update
cmt (Commit): Commit object for this commit
patch (Patch): Corresponding Patch object for this patch
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
rest_api (function): API function to call to access Patchwork, for
testing
"""
if not patch:
return
# Get the content for the patch email itself as well as all comments
data = rest_api(url, 'patches/%s/' % patch.id)
pstrm = PatchStream.process_text(data['content'], True)
rtags = collections.defaultdict(set)
for response, people in pstrm.commit.rtags.items():
rtags[response].update(people)
data = rest_api(url, 'patches/%s/comments/' % patch.id)
reviews = []
for comment in data:
pstrm = PatchStream.process_text(comment['content'], True)
if pstrm.snippets:
submitter = comment['submitter']
person = '%s <%s>' % (submitter['name'], submitter['email'])
reviews.append(Review(person, pstrm.snippets))
for response, people in pstrm.commit.rtags.items():
rtags[response].update(people)
# Find the tags that are not in the commit
new_rtags = collections.defaultdict(set)
base_rtags = cmt.rtags
for tag, people in rtags.items():
for who in people:
is_new = (tag not in base_rtags or
who not in base_rtags[tag])
if is_new:
new_rtags[tag].add(who)
new_rtag_list[seq] = new_rtags
review_list[seq] = reviews
def show_responses(rtags, indent, is_new):
"""Show rtags collected
Args:
rtags (dict): review tags to show
key: Response tag (e.g. 'Reviewed-by')
value: Set of people who gave that response, each a name/email string
indent (str): Indentation string to write before each line
is_new (bool): True if this output should be highlighted
Returns:
int: Number of review tags displayed
"""
col = terminal.Color()
count = 0
for tag in sorted(rtags.keys()):
people = rtags[tag]
for who in sorted(people):
terminal.tprint(indent + '%s %s: ' % ('+' if is_new else ' ', tag),
newline=False, colour=col.GREEN, bright=is_new)
terminal.tprint(who, colour=col.WHITE, bright=is_new)
count += 1
return count
def create_branch(series, new_rtag_list, branch, dest_branch, overwrite,
repo=None):
"""Create a new branch with review tags added
Args:
series (Series): Series object for the existing branch
new_rtag_list (list): List of review tags to add, one for each commit,
each a dict:
key: Response tag (e.g. 'Reviewed-by')
value: Set of people who gave that response, each a name/email
string
branch (str): Existing branch to update
dest_branch (str): Name of new branch to create
overwrite (bool): True to force overwriting dest_branch if it exists
repo (pygit2.Repository): Repo to use (use None unless testing)
Returns:
int: Total number of review tags added across all commits
Raises:
ValueError: if the destination branch name is the same as the original
branch, or it already exists and @overwrite is False
"""
if branch == dest_branch:
raise ValueError(
'Destination branch must not be the same as the original branch')
if not repo:
repo = pygit2.Repository('.')
count = len(series.commits)
new_br = repo.branches.get(dest_branch)
if new_br:
if not overwrite:
raise ValueError("Branch '%s' already exists (-f to overwrite)" %
dest_branch)
new_br.delete()
if not branch:
branch = 'HEAD'
target = repo.revparse_single('%s~%d' % (branch, count))
repo.branches.local.create(dest_branch, target)
num_added = 0
for seq in range(count):
parent = repo.branches.get(dest_branch)
cherry = repo.revparse_single('%s~%d' % (branch, count - seq - 1))
repo.merge_base(cherry.oid, parent.target)
base_tree = cherry.parents[0].tree
index = repo.merge_trees(base_tree, parent, cherry)
tree_id = index.write_tree(repo)
lines = []
if new_rtag_list[seq]:
for tag, people in new_rtag_list[seq].items():
for who in people:
lines.append('%s: %s' % (tag, who))
num_added += 1
message = patchstream.insert_tags(cherry.message.rstrip(),
sorted(lines))
repo.create_commit(
parent.name, cherry.author, cherry.committer, message, tree_id,
[parent.target])
return num_added
def check_patchwork_status(series, series_id, branch, dest_branch, force,
show_comments, url, rest_api=call_rest_api,
test_repo=None):
"""Check the status of a series on Patchwork
This finds review tags and comments for a series in Patchwork, displaying
them to show what is new compared to the local series.
Args:
series (Series): Series object for the existing branch
series_id (str): Patch series ID number
branch (str): Existing branch to update, or None
dest_branch (str): Name of new branch to create, or None
force (bool): True to force overwriting dest_branch if it exists
show_comments (bool): True to show the comments on each patch
url (str): URL of patchwork server, e.g. 'https://patchwork.ozlabs.org'
rest_api (function): API function to call to access Patchwork, for
testing
test_repo (pygit2.Repository): Repo to use (use None unless testing)
"""
patches = collect_patches(series, series_id, url, rest_api)
col = terminal.Color()
count = len(series.commits)
new_rtag_list = [None] * count
review_list = [None] * count
patch_for_commit, _, warnings = compare_with_series(series, patches)
for warn in warnings:
tout.warning(warn)
patch_list = [patch_for_commit.get(c) for c in range(len(series.commits))]
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
futures = executor.map(
find_new_responses, repeat(new_rtag_list), repeat(review_list),
range(count), series.commits, patch_list, repeat(url),
repeat(rest_api))
for fresponse in futures:
if fresponse:
raise fresponse.exception()
num_to_add = 0
for seq, cmt in enumerate(series.commits):
patch = patch_for_commit.get(seq)
if not patch:
continue
terminal.tprint('%3d %s' % (patch.seq, patch.subject[:50]),
colour=col.BLUE)
cmt = series.commits[seq]
base_rtags = cmt.rtags
new_rtags = new_rtag_list[seq]
indent = ' ' * 2
show_responses(base_rtags, indent, False)
num_to_add += show_responses(new_rtags, indent, True)
if show_comments:
for review in review_list[seq]:
terminal.tprint('Review: %s' % review.meta, colour=col.RED)
for snippet in review.snippets:
for line in snippet:
quoted = line.startswith('>')
terminal.tprint(' %s' % line,
colour=col.MAGENTA if quoted else None)
terminal.tprint()
terminal.tprint("%d new response%s available in patchwork%s" %
(num_to_add, 's' if num_to_add != 1 else '',
'' if dest_branch
else ' (use -d to write them to a new branch)'))
if dest_branch:
num_added = create_branch(series, new_rtag_list, branch,
dest_branch, force, test_repo)
terminal.tprint(
"%d response%s added from patchwork into new branch '%s'" %
(num_added, 's' if num_added != 1 else '', dest_branch))