BaseTools/Edk2ToolsBuild.py

# @file Edk2ToolsBuild.py
# Invocable class that builds the basetool c files.
#
# Supports VS2019, VS2022, and GCC5
##
# Copyright (c) Microsoft Corporation
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import os
import sys
import logging
import argparse
import multiprocessing
import shutil
from edk2toolext import edk2_logging
from edk2toolext.environment import self_describing_environment, shell_environment
from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
from edk2toollib.utility_functions import RunCmd, GetHostInfo
from edk2toollib.windows.locate_tools import QueryVcVariables


class Edk2ToolsBuild(BaseAbstractInvocable):

    def ParseCommandLineOptions(self):
        ''' parse arguments '''
        ParserObj = argparse.ArgumentParser()
        ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",
                               help="Set the toolchain used to compile the build tools")
        ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
                               help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
                               "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
        args = ParserObj.parse_args()
        self.tool_chain_tag = args.tct
        self.target_arch = args.arch

    def GetWorkspaceRoot(self):
        ''' Return the workspace root for initializing the SDE '''

        # this is the bastools dir...not the traditional EDK2 workspace root
        return os.path.dirname(os.path.abspath(__file__))

    def GetActiveScopes(self):
        ''' return tuple containing scopes that should be active for this process '''

        # Adding scope for cross compilers when building for ARM/AARCH64
        scopes = ('global',)
        if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"):
            if self.target_arch is None:
                return scopes
            if "AARCH64" in self.target_arch:
                scopes += ("gcc_aarch64_linux",)
            if "ARM" in self.target_arch:
                scopes += ("gcc_arm_linux",)
        return scopes

    def GetLoggingLevel(self, loggerType):
        ''' Get the logging level for a given type (return Logging.Level)
        base == lowest logging level supported
        con  == Screen logging
        txt  == plain text file logging
        md   == markdown file logging
        '''
        if(loggerType == "con"):
            return logging.ERROR
        else:
            return logging.DEBUG

    def GetLoggingFolderRelativeToRoot(self):
        ''' Return a path to folder for log files '''
        return "BaseToolsBuild"

    def GetVerifyCheckRequired(self):
        ''' Will call self_describing_environment.VerifyEnvironment if this returns True '''
        return True

    def GetLoggingFileName(self, loggerType):
        ''' Get the logging file name for the type.
        Return None if the logger shouldn't be created

        base == lowest logging level supported
        con  == Screen logging
        txt  == plain text file logging
        md   == markdown file logging
        '''
        return "BASETOOLS_BUILD"

    def WritePathEnvFile(self, OutputDir):
        ''' Write a PyTool path env file for future PyTool based edk2 builds'''
        content = '''##
# Set shell variable EDK_TOOLS_BIN to this folder
#
# Autogenerated by Edk2ToolsBuild.py
#
# Copyright (c), Microsoft Corporation
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
  "id": "You-Built-BaseTools",
  "scope": "edk2-build",
  "flags": ["set_shell_var", "set_path"],
  "var_name": "EDK_TOOLS_BIN"
}
'''
        with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f:
            f.write(content)

    def Go(self):
        logging.info("Running Python version: " + str(sys.version_info))

        (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
            self.GetWorkspaceRoot(), self.GetActiveScopes())

        # # Bind our current execution environment into the shell vars.
        ph = os.path.dirname(sys.executable)
        if " " in ph:
            ph = '"' + ph + '"'
        shell_env.set_shell_var("PYTHON_HOME", ph)
        # PYTHON_COMMAND is required to be set for using edk2 python builds.
        pc = sys.executable
        if " " in pc:
            pc = '"' + pc + '"'
        shell_env.set_shell_var("PYTHON_COMMAND", pc)

        if self.tool_chain_tag.lower().startswith("vs"):
            if self.target_arch is None:
                # Put a default as IA32
                self.target_arch = "IA32"

            if self.target_arch == "IA32":
                VcToolChainArch = "x86"
                TargetInfoArch = "x86"
                OutputDir = "Win32"
            elif self.target_arch == "ARM":
                VcToolChainArch = "x86_arm"
                TargetInfoArch = "ARM"
                OutputDir = "Win32"
            elif self.target_arch == "X64":
                VcToolChainArch = "amd64"
                TargetInfoArch = "x86"
                OutputDir = "Win64"
            elif self.target_arch == "AARCH64":
                VcToolChainArch = "amd64_arm64"
                TargetInfoArch = "ARM"
                OutputDir = "Win64"
            else:
                raise NotImplementedError()

            self.OutputDir = os.path.join(
                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)

            # compiled tools need to be added to path because antlr is referenced
            HostInfo = GetHostInfo()
            if TargetInfoArch == HostInfo.arch:
                # not cross compiling
                shell_env.insert_path(self.OutputDir)
            else:
                # cross compiling:
                # as the VfrCompile tool is needed in the build process, we need
                # to build one for the host system, then add the path to the
                # tools to the PATH environment variable
                shell_environment.CheckpointBuildVars()
                if HostInfo.arch == "x86" and HostInfo.bit == "64":
                    host_arch = "X64"
                    host_toolchain_arch = "amd64"
                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
                elif HostInfo.arch == "x86" and HostInfo.bit == "32":
                    host_arch = "IA32"
                    host_toolchain_arch = "x86"
                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
                elif HostInfo.arch == "ARM" and HostInfo.bit == "64":
                    host_arch = "AARCH64"
                    host_toolchain_arch = "amd64_arm64"
                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
                elif HostInfo.arch == "ARM" and HostInfo.bit == "32":
                    host_arch = "ARM"
                    host_toolchain_arch = "x86_arm"
                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
                else:
                    raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))

                interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
                interesting_keys.extend(
                    ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
                interesting_keys.extend(
                    ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
                vc_vars = QueryVcVariables(
                    interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())
                for key in vc_vars.keys():
                    logging.debug(f"Var - {key} = {vc_vars[key]}")
                    if key.lower() == 'path':
                        shell_env.set_path(vc_vars[key])
                    else:
                        shell_env.set_shell_var(key, vc_vars[key])

                # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
                # this script is BUILDING the base tools.
                shell_env.set_shell_var('HOST_ARCH', host_arch)
                shell_env.insert_path(TempOutputDir)

                # All set, build the tools for the host system.
                ret = RunCmd('nmake.exe', None,
                             workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
                if ret != 0:
                    raise Exception("Failed to build base tools for host system.")

                # Copy the output to a temp directory
                TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")
                if not os.path.exists(TempFolder):
                    os.makedirs(TempFolder)
                for file in os.listdir(TempOutputDir):
                    shutil.copy(os.path.join(TempOutputDir, file), TempFolder)

                # Clean up the build output
                ret = RunCmd('nmake.exe', 'cleanall',
                             workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))

                # Remove the entire TempOutputDir
                shutil.rmtree(TempOutputDir)

                shell_environment.RevertBuildVars()
                shell_env.insert_path(TempFolder)

            # # Update environment with required VC vars.
            interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
            interesting_keys.extend(
                ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
            interesting_keys.extend(
                ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
            vc_vars = QueryVcVariables(
                interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())
            for key in vc_vars.keys():
                logging.debug(f"Var - {key} = {vc_vars[key]}")
                if key.lower() == 'path':
                    shell_env.set_path(vc_vars[key])
                else:
                    shell_env.set_shell_var(key, vc_vars[key])

            # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
            # this script is BUILDING the base tools.
            shell_env.set_shell_var('HOST_ARCH', self.target_arch)

            # Actually build the tools.
            output_stream = edk2_logging.create_output_stream()
            ret = RunCmd('nmake.exe', None,
                         workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
            edk2_logging.remove_output_stream(output_stream)
            problems = edk2_logging.scan_compiler_output(output_stream)
            for level, problem in problems:
                logging.log(level, problem)
            if ret != 0:
                raise Exception("Failed to build.")

            self.WritePathEnvFile(self.OutputDir)
            return ret

        elif self.tool_chain_tag.lower().startswith("gcc"):
            # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
            # this script is BUILDING the base tools.
            HostInfo = GetHostInfo()
            prefix = None
            TargetInfoArch = None
            if self.target_arch is not None:
                shell_env.set_shell_var('HOST_ARCH', self.target_arch)

                if "AARCH64" in self.target_arch:
                    prefix = shell_env.get_shell_var("GCC5_AARCH64_PREFIX")
                    if prefix == None:
                        # now check for install dir.  If set then set the Prefix
                        install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL")

                        # make GCC5_AARCH64_PREFIX to align with tools_def.txt
                        prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")

                    shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
                    TargetInfoArch = "ARM"

                elif "ARM" in self.target_arch:
                    prefix = shell_env.get_shell_var("GCC5_ARM_PREFIX")
                    if prefix == None:
                        # now check for install dir.  If set then set the Prefix
                        install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")

                        # make GCC5_ARM_PREFIX to align with tools_def.txt
                        prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")

                    shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
                    TargetInfoArch = "ARM"

                else:
                    TargetInfoArch = "x86"
            else:
                self.target_arch = HostInfo.arch
                TargetInfoArch = HostInfo.arch
            # Otherwise, the built binary arch will be consistent with the host system

            # Added logic to support cross compilation scenarios
            if TargetInfoArch != HostInfo.arch:
                # this is defaulting to the version that comes with Ubuntu 20.04
                ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34")
                work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot())
                pack_name = f"util-linux-{ver}"
                unzip_dir = os.path.join(work_dir, pack_name)

                if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")):
                    os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz"))
                if os.path.isdir(unzip_dir):
                    shutil.rmtree(unzip_dir)

                # cross compiling, need to rebuild libuuid for the target
                ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir)
                if ret != 0:
                    raise Exception(f"Failed to download libuuid version {ver} - {ret}")

                ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir)
                if ret != 0:
                    raise Exception(f"Failed to untar the downloaded file {ret}")

                # configure the source to use the cross compiler
                pack_name = f"util-linux-{ver}"
                if "AARCH64" in self.target_arch:
                    ret = RunCmd("sh", f"./configure --host=aarch64-linux  -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
                elif "ARM" in self.target_arch:
                    ret = RunCmd("sh", f"./configure --host=arm-linux  -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
                if ret != 0:
                    raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}")

                ret = RunCmd("make", "", workingdir=unzip_dir)
                if ret != 0:
                    raise Exception(f"Failed to build the libuuid with our gcc {ret}")

                shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir)
                shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src"))

            ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
            if ret != 0:
                raise Exception("Failed to build.")

            cpu_count = self.GetCpuThreads()

            output_stream = edk2_logging.create_output_stream()
            ret = RunCmd("make", f"-C .  -j {cpu_count}", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
            edk2_logging.remove_output_stream(output_stream)
            problems = edk2_logging.scan_compiler_output(output_stream)
            for level, problem in problems:
                logging.log(level, problem)
            if ret != 0:
                raise Exception("Failed to build.")

            self.OutputDir = os.path.join(
                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin")

            self.WritePathEnvFile(self.OutputDir)
            return ret

        logging.critical("Tool Chain not supported")
        return -1

    def GetCpuThreads(self) -> int:
        ''' Function to return number of cpus. If error return 1'''
        cpus = 1
        try:
            cpus = multiprocessing.cpu_count()
        except:
            # from the internet there are cases where cpu_count is not implemented.
            # will handle error by just doing single proc build
            pass
        return cpus



def main():
    Edk2ToolsBuild().Invoke()


if __name__ == "__main__":
    main()