| # @file EccCheck.py | |
| # | |
| # Copyright (c) 2021, Arm Limited. All rights reserved.<BR> | |
| # Copyright (c) 2020, Intel Corporation. All rights reserved.<BR> | |
| # SPDX-License-Identifier: BSD-2-Clause-Patent | |
| ## | |
| import os | |
| import shutil | |
| import re | |
| import csv | |
| import xml.dom.minidom | |
| from typing import List, Dict, Tuple | |
| import logging | |
| from io import StringIO | |
| from edk2toolext.environment import shell_environment | |
| from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin | |
| from edk2toolext.environment.var_dict import VarDict | |
| from edk2toollib.utility_functions import RunCmd | |
| class EccCheck(ICiBuildPlugin): | |
| """ | |
| A CiBuildPlugin that finds the Ecc issues of newly added code in pull request. | |
| Configuration options: | |
| "EccCheck": { | |
| "ExceptionList": [], | |
| "IgnoreFiles": [] | |
| }, | |
| """ | |
| FindModifyFile = re.compile(r'\+\+\+ b\/(.*)') | |
| LineScopePattern = (r'@@ -\d*\,*\d* \+\d*\,*\d* @@.*') | |
| LineNumRange = re.compile(r'@@ -\d*\,*\d* \+(\d*)\,*(\d*) @@.*') | |
| def GetTestName(self, packagename: str, environment: VarDict) -> tuple: | |
| """ Provide the testcase name and classname for use in reporting | |
| testclassname: a descriptive string for the testcase can include whitespace | |
| classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> | |
| Args: | |
| packagename: string containing name of package to build | |
| environment: The VarDict for the test to run in | |
| Returns: | |
| a tuple containing the testcase name and the classname | |
| (testcasename, classname) | |
| """ | |
| return ("Check for efi coding style for " + packagename, packagename + ".EccCheck") | |
| ## | |
| # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin | |
| # | |
| # - package is the edk2 path to package. This means workspace/packagepath relative. | |
| # - edk2path object configured with workspace and packages path | |
| # - PkgConfig Object (dict) for the pkg | |
| # - EnvConfig Object | |
| # - Plugin Manager Instance | |
| # - Plugin Helper Obj Instance | |
| # - Junit Logger | |
| # - output_stream the StringIO output stream from this plugin via logging | |
| def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): | |
| workspace_path = Edk2pathObj.WorkspacePath | |
| basetools_path = environment.GetValue("EDK_TOOLS_PATH") | |
| python_path = os.path.join(basetools_path, "Source", "Python") | |
| env = shell_environment.GetEnvironment() | |
| env.set_shell_var('PYTHONPATH', python_path) | |
| env.set_shell_var('WORKSPACE', workspace_path) | |
| env.set_shell_var('PACKAGES_PATH', os.pathsep.join(Edk2pathObj.PackagePathList)) | |
| self.ECC_PASS = True | |
| abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(packagename) | |
| if abs_pkg_path is None: | |
| tc.SetSkipped() | |
| tc.LogStdError("No Package folder {0}".format(abs_pkg_path)) | |
| return 0 | |
| # Create temp directory | |
| temp_path = os.path.join(workspace_path, 'Build', '.pytool', 'Plugin', 'EccCheck') | |
| try: | |
| # Delete temp directory | |
| if os.path.exists(temp_path): | |
| shutil.rmtree(temp_path) | |
| # Copy package being scanned to temp_path | |
| shutil.copytree ( | |
| abs_pkg_path, | |
| os.path.join(temp_path, packagename), | |
| symlinks=True | |
| ) | |
| # Copy exception.xml to temp_path | |
| shutil.copyfile ( | |
| os.path.join(basetools_path, "Source", "Python", "Ecc", "exception.xml"), | |
| os.path.join(temp_path, "exception.xml") | |
| ) | |
| # Output file to use for git diff operations | |
| temp_diff_output = os.path.join (temp_path, 'diff.txt') | |
| self.ApplyConfig(pkgconfig, temp_path, packagename) | |
| modify_dir_list = self.GetModifyDir(packagename, temp_diff_output) | |
| patch = self.GetDiff(packagename, temp_diff_output) | |
| ecc_diff_range = self.GetDiffRange(patch, packagename, temp_path) | |
| # | |
| # Use temp_path as working directory when running ECC tool | |
| # | |
| self.GenerateEccReport(modify_dir_list, ecc_diff_range, temp_path, basetools_path) | |
| ecc_log = os.path.join(temp_path, "Ecc.log") | |
| if self.ECC_PASS: | |
| # Delete temp directory | |
| if os.path.exists(temp_path): | |
| shutil.rmtree(temp_path) | |
| tc.SetSuccess() | |
| return 0 | |
| else: | |
| with open(ecc_log, encoding='utf8') as output: | |
| ecc_output = output.readlines() | |
| for line in ecc_output: | |
| logging.error(line.strip()) | |
| # Delete temp directory | |
| if os.path.exists(temp_path): | |
| shutil.rmtree(temp_path) | |
| tc.SetFailed("EccCheck failed for {0}".format(packagename), "CHECK FAILED") | |
| return 1 | |
| except KeyboardInterrupt: | |
| # If EccCheck is interrupted by keybard interrupt, then return failure | |
| # Delete temp directory | |
| if os.path.exists(temp_path): | |
| shutil.rmtree(temp_path) | |
| tc.SetFailed("EccCheck interrupted for {0}".format(packagename), "CHECK FAILED") | |
| return 1 | |
| else: | |
| # If EccCheck fails for any other exception type, raise the exception | |
| # Delete temp directory | |
| if os.path.exists(temp_path): | |
| shutil.rmtree(temp_path) | |
| tc.SetFailed("EccCheck exception for {0}".format(packagename), "CHECK FAILED") | |
| raise | |
| return 1 | |
| def GetDiff(self, pkg: str, temp_diff_output: str) -> List[str]: | |
| patch = [] | |
| # | |
| # Generate unified diff between origin/master and HEAD. | |
| # | |
| params = "diff --output={} --unified=0 origin/master HEAD".format(temp_diff_output) | |
| RunCmd("git", params) | |
| with open(temp_diff_output) as file: | |
| patch = file.read().strip().split('\n') | |
| return patch | |
| def GetModifyDir(self, pkg: str, temp_diff_output: str) -> List[str]: | |
| # | |
| # Generate diff between origin/master and HEAD using --diff-filter to | |
| # exclude deleted and renamed files that do not need to be scanned by | |
| # ECC. Also use --name-status to only generate the names of the files | |
| # with differences. The output format of this git diff command is a | |
| # list of files with the change status and the filename. The filename | |
| # is always at the end of the line. Examples: | |
| # | |
| # M MdeModulePkg/Application/CapsuleApp/CapsuleApp.h | |
| # M MdeModulePkg/Application/UiApp/FrontPage.h | |
| # | |
| params = "diff --output={} --diff-filter=dr --name-status origin/master HEAD".format(temp_diff_output) | |
| RunCmd("git", params) | |
| dir_list = [] | |
| with open(temp_diff_output) as file: | |
| dir_list = file.read().strip().split('\n') | |
| modify_dir_list = [] | |
| for modify_dir in dir_list: | |
| # | |
| # Parse file name from the end of the line | |
| # | |
| file_path = modify_dir.strip().split() | |
| # | |
| # Skip lines that do not have at least 2 elements (status and file name) | |
| # | |
| if len(file_path) < 2: | |
| continue | |
| # | |
| # Parse the directory name from the file name | |
| # | |
| file_dir = os.path.dirname(file_path[-1]) | |
| # | |
| # Skip directory names that do not start with the package being scanned. | |
| # | |
| if file_dir.split('/')[0] != pkg: | |
| continue | |
| # | |
| # Skip directory names that are identical to the package being scanned. | |
| # The assumption here is that there are no source files at the package | |
| # root. Instead, the only expected files in the package root are | |
| # EDK II meta data files (DEC, DSC, FDF). | |
| # | |
| if file_dir == pkg: | |
| continue | |
| # | |
| # Skip directory names that are already in the modified dir list | |
| # | |
| if file_dir in modify_dir_list: | |
| continue | |
| # | |
| # Add the candidate directory to scan to the modified dir list | |
| # | |
| modify_dir_list.append(file_dir) | |
| # | |
| # Remove duplicates from modify_dir_list | |
| # Given a folder path, ECC performs a recursive scan of that folder. | |
| # If a parent and child folder are both present in modify_dir_list, | |
| # then ECC will perform redudanct scans of source files. In order | |
| # to prevent redundant scans, if a parent and child folder are both | |
| # present, then remove all the child folders. | |
| # | |
| # For example, if modified_dir_list contains the following elements: | |
| # MdeModulePkg/Core/Dxe | |
| # MdeModulePkg/Core/Dxe/Hand | |
| # MdeModulePkg/Core/Dxe/Mem | |
| # | |
| # Then MdeModulePkg/Core/Dxe/Hand and MdeModulePkg/Core/Dxe/Mem should | |
| # be removed because the files in those folders are covered by a scan | |
| # of MdeModulePkg/Core/Dxe. | |
| # | |
| filtered_list = [] | |
| for dir1 in modify_dir_list: | |
| Append = True | |
| for dir2 in modify_dir_list: | |
| if dir1 == dir2: | |
| continue | |
| common = os.path.commonpath([dir1, dir2]) | |
| if os.path.normpath(common) == os.path.normpath(dir2): | |
| Append = False | |
| break | |
| if Append and dir1 not in filtered_list: | |
| filtered_list.append(dir1) | |
| return filtered_list | |
| def GetDiffRange(self, patch_diff: List[str], pkg: str, temp_path: str) -> Dict[str, List[Tuple[int, int]]]: | |
| IsDelete = True | |
| StartCheck = False | |
| range_directory: Dict[str, List[Tuple[int, int]]] = {} | |
| for line in patch_diff: | |
| modify_file = self.FindModifyFile.findall(line) | |
| if modify_file and pkg in modify_file[0] and not StartCheck and os.path.isfile(modify_file[0]): | |
| modify_file_comment_dic = self.GetCommentRange(modify_file[0], temp_path) | |
| IsDelete = False | |
| StartCheck = True | |
| modify_file_dic = modify_file[0] | |
| modify_file_dic = modify_file_dic.replace("/", os.sep) | |
| range_directory[modify_file_dic] = [] | |
| elif line.startswith('--- '): | |
| StartCheck = False | |
| elif re.match(self.LineScopePattern, line, re.I) and not IsDelete and StartCheck: | |
| start_line = self.LineNumRange.search(line).group(1) | |
| line_range = self.LineNumRange.search(line).group(2) | |
| if not line_range: | |
| line_range = '1' | |
| range_directory[modify_file_dic].append((int(start_line), int(start_line) + int(line_range) - 1)) | |
| for i in modify_file_comment_dic: | |
| if int(i[0]) <= int(start_line) <= int(i[1]): | |
| range_directory[modify_file_dic].append(i) | |
| return range_directory | |
| def GetCommentRange(self, modify_file: str, temp_path: str) -> List[Tuple[int, int]]: | |
| comment_range: List[Tuple[int, int]] = [] | |
| modify_file_path = os.path.join(temp_path, modify_file) | |
| if not os.path.exists (modify_file_path): | |
| return comment_range | |
| with open(modify_file_path) as f: | |
| line_no = 1 | |
| Start = False | |
| for line in f: | |
| if line.startswith('/**'): | |
| start_no = line_no | |
| Start = True | |
| if line.startswith('**/') and Start: | |
| end_no = line_no | |
| Start = False | |
| comment_range.append((int(start_no), int(end_no))) | |
| line_no += 1 | |
| if comment_range and comment_range[0][0] == 1: | |
| del comment_range[0] | |
| return comment_range | |
| def GenerateEccReport(self, modify_dir_list: List[str], ecc_diff_range: Dict[str, List[Tuple[int, int]]], | |
| temp_path: str, basetools_path: str) -> None: | |
| ecc_need = False | |
| ecc_run = True | |
| config = os.path.normpath(os.path.join(basetools_path, "Source", "Python", "Ecc", "config.ini")) | |
| exception = os.path.normpath(os.path.join(temp_path, "exception.xml")) | |
| report = os.path.normpath(os.path.join(temp_path, "Ecc.csv")) | |
| for modify_dir in modify_dir_list: | |
| target = os.path.normpath(os.path.join(temp_path, modify_dir)) | |
| logging.info('Run ECC tool for the commit in %s' % modify_dir) | |
| ecc_need = True | |
| ecc_params = "-c {0} -e {1} -t {2} -r {3}".format(config, exception, target, report) | |
| return_code = RunCmd("Ecc", ecc_params, workingdir=temp_path) | |
| if return_code != 0: | |
| ecc_run = False | |
| break | |
| if not ecc_run: | |
| logging.error('Fail to run ECC tool') | |
| self.ParseEccReport(ecc_diff_range, temp_path) | |
| if not ecc_need: | |
| logging.info("Doesn't need run ECC check") | |
| return | |
| def ParseEccReport(self, ecc_diff_range: Dict[str, List[Tuple[int, int]]], temp_path: str) -> None: | |
| ecc_log = os.path.join(temp_path, "Ecc.log") | |
| ecc_csv = os.path.join(temp_path, "Ecc.csv") | |
| row_lines = [] | |
| ignore_error_code = self.GetIgnoreErrorCode() | |
| if os.path.exists(ecc_csv): | |
| with open(ecc_csv) as csv_file: | |
| reader = csv.reader(csv_file) | |
| for row in reader: | |
| for modify_file in ecc_diff_range: | |
| if modify_file in row[3]: | |
| for i in ecc_diff_range[modify_file]: | |
| line_no = int(row[4]) | |
| if i[0] <= line_no <= i[1] and row[1] not in ignore_error_code: | |
| row[0] = '\nEFI coding style error' | |
| row[1] = 'Error code: ' + row[1] | |
| row[3] = 'file: ' + row[3] | |
| row[4] = 'Line number: ' + row[4] | |
| row_line = '\n *'.join(row) | |
| row_lines.append(row_line) | |
| break | |
| break | |
| if row_lines: | |
| self.ECC_PASS = False | |
| with open(ecc_log, 'a') as log: | |
| all_line = '\n'.join(row_lines) | |
| all_line = all_line + '\n' | |
| log.writelines(all_line) | |
| return | |
| def ApplyConfig(self, pkgconfig: Dict[str, List[str]], temp_path: str, pkg: str) -> None: | |
| if "IgnoreFiles" in pkgconfig: | |
| for a in pkgconfig["IgnoreFiles"]: | |
| a = os.path.join(temp_path, pkg, a) | |
| a = a.replace(os.sep, "/") | |
| logging.info("Ignoring Files {0}".format(a)) | |
| if os.path.exists(a): | |
| if os.path.isfile(a): | |
| os.remove(a) | |
| elif os.path.isdir(a): | |
| shutil.rmtree(a) | |
| else: | |
| logging.error("EccCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore files".format(a)) | |
| if "ExceptionList" in pkgconfig: | |
| exception_list = pkgconfig["ExceptionList"] | |
| exception_xml = os.path.join(temp_path, "exception.xml") | |
| try: | |
| logging.info("Appending exceptions") | |
| self.AppendException(exception_list, exception_xml) | |
| except Exception as e: | |
| logging.error("Fail to apply exceptions") | |
| raise e | |
| return | |
| def AppendException(self, exception_list: List[str], exception_xml: str) -> None: | |
| error_code_list = exception_list[::2] | |
| keyword_list = exception_list[1::2] | |
| dom_tree = xml.dom.minidom.parse(exception_xml) | |
| root_node = dom_tree.documentElement | |
| for error_code, keyword in zip(error_code_list, keyword_list): | |
| customer_node = dom_tree.createElement("Exception") | |
| keyword_node = dom_tree.createElement("KeyWord") | |
| keyword_node_text_value = dom_tree.createTextNode(keyword) | |
| keyword_node.appendChild(keyword_node_text_value) | |
| customer_node.appendChild(keyword_node) | |
| error_code_node = dom_tree.createElement("ErrorID") | |
| error_code_text_value = dom_tree.createTextNode(error_code) | |
| error_code_node.appendChild(error_code_text_value) | |
| customer_node.appendChild(error_code_node) | |
| root_node.appendChild(customer_node) | |
| with open(exception_xml, 'w') as f: | |
| dom_tree.writexml(f, indent='', addindent='', newl='\n', encoding='UTF-8') | |
| return | |
| def GetIgnoreErrorCode(self) -> set: | |
| """ | |
| Below are kinds of error code that are accurate in ecc scanning of edk2 level. | |
| But EccCheck plugin is partial scanning so they are always false positive issues. | |
| The mapping relationship of error code and error message is listed BaseTools/Sourc/Python/Ecc/EccToolError.py | |
| """ | |
| ignore_error_code = { | |
| "10000", | |
| "10001", | |
| "10002", | |
| "10003", | |
| "10004", | |
| "10005", | |
| "10006", | |
| "10007", | |
| "10008", | |
| "10009", | |
| "10010", | |
| "10011", | |
| "10012", | |
| "10013", | |
| "10015", | |
| "10016", | |
| "10017", | |
| "10022", | |
| } | |
| return ignore_error_code |