BaseTools: Adding cross compilation of BaseTool for Windows ARM/ARM64

This change adds the support of crossbuilding basetool for Windows ARM/
ARM64 systems, which will enable the generally available pipeline agents
to build binary tools and make releases as they see fit.

The EDK2 base tools build script is also updated to support cross
compilation using this script.

The crossbuilt binary output is tested on Windows ARM based hardware
systems.

Cc: Rebecca Cran <rebecca@bsdio.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Signed-off-by: Kun Qin <kun.qin@microsoft.com>

BaseTools/.gitignore

@@ -17,5 +17,6 @@ Source/C/VfrCompile/VfrTokens.h
 Source/C/bin/
 Source/C/libs/
 Bin/Win32
+Bin/Win64
 Lib
 BaseToolsBuild/

BaseTools/Edk2ToolsBuild.py

@@ -12,10 +12,11 @@ import sys
 import logging
 import argparse
 import multiprocessing
+import shutil
 from edk2toolext import edk2_logging
-from edk2toolext.environment import self_describing_environment
+from edk2toolext.environment import self_describing_environment, shell_environment
 from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
-from edk2toollib.utility_functions import RunCmd
+from edk2toollib.utility_functions import RunCmd, GetHostInfo
 from edk2toollib.windows.locate_tools import QueryVcVariables
 
 
@@ -26,8 +27,12 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
         ParserObj = argparse.ArgumentParser()
         ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",
                                help="Set the toolchain used to compile the build tools")
+        ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
+                               help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
+                               "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
         args = ParserObj.parse_args()
         self.tool_chain_tag = args.tct
+        self.target_arch = args.arch
 
     def GetWorkspaceRoot(self):
         ''' Return the workspace root for initializing the SDE '''
@@ -110,6 +115,103 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
         shell_env.set_shell_var("PYTHON_COMMAND", pc)
 
         if self.tool_chain_tag.lower().startswith("vs"):
+            if self.target_arch is None:
+                # Put a default as IA32
+                self.target_arch = "IA32"
+
+            if self.target_arch == "IA32":
+                VcToolChainArch = "x86"
+                TargetInfoArch = "x86"
+                OutputDir = "Win32"
+            elif self.target_arch == "ARM":
+                VcToolChainArch = "x86_arm"
+                TargetInfoArch = "ARM"
+                OutputDir = "Win32"
+            elif self.target_arch == "X64":
+                VcToolChainArch = "amd64"
+                TargetInfoArch = "x86"
+                OutputDir = "Win64"
+            elif self.target_arch == "AARCH64":
+                VcToolChainArch = "amd64_arm64"
+                TargetInfoArch = "ARM"
+                OutputDir = "Win64"
+            else:
+                raise NotImplementedError()
+
+            self.OutputDir = os.path.join(
+                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)
+
+            # compiled tools need to be added to path because antlr is referenced
+            HostInfo = GetHostInfo()
+            if TargetInfoArch == HostInfo.arch:
+                # not cross compiling
+                shell_env.insert_path(self.OutputDir)
+            else:
+                # cross compiling:
+                # as the VfrCompile tool is needed in the build process, we need
+                # to build one for the host system, then add the path to the
+                # tools to the PATH environment variable
+                shell_environment.CheckpointBuildVars()
+                if HostInfo.arch == "x86" and HostInfo.bit == "64":
+                    host_arch = "X64"
+                    host_toolchain_arch = "amd64"
+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
+                elif HostInfo.arch == "x86" and HostInfo.bit == "32":
+                    host_arch = "IA32"
+                    host_toolchain_arch = "x86"
+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
+                elif HostInfo.arch == "ARM" and HostInfo.bit == "64":
+                    host_arch = "AARCH64"
+                    host_toolchain_arch = "amd64_arm64"
+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
+                elif HostInfo.arch == "ARM" and HostInfo.bit == "32":
+                    host_arch = "ARM"
+                    host_toolchain_arch = "x86_arm"
+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
+                else:
+                    raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))
+
+                interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
+                interesting_keys.extend(
+                    ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
+                interesting_keys.extend(
+                    ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
+                vc_vars = QueryVcVariables(
+                    interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())
+                for key in vc_vars.keys():
+                    logging.debug(f"Var - {key} = {vc_vars[key]}")
+                    if key.lower() == 'path':
+                        shell_env.set_path(vc_vars[key])
+                    else:
+                        shell_env.set_shell_var(key, vc_vars[key])
+
+                # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
+                # this script is BUILDING the base tools.
+                shell_env.set_shell_var('HOST_ARCH', host_arch)
+                shell_env.insert_path(TempOutputDir)
+
+                # All set, build the tools for the host system.
+                ret = RunCmd('nmake.exe', None,
+                             workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
+                if ret != 0:
+                    raise Exception("Failed to build base tools for host system.")
+
+                # Copy the output to a temp directory
+                TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")
+                if not os.path.exists(TempFolder):
+                    os.makedirs(TempFolder)
+                for file in os.listdir(TempOutputDir):
+                    shutil.copy(os.path.join(TempOutputDir, file), TempFolder)
+
+                # Clean up the build output
+                ret = RunCmd('nmake.exe', 'cleanall',
+                             workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
+
+                # Remove the entire TempOutputDir
+                shutil.rmtree(TempOutputDir)
+
+                shell_environment.RevertBuildVars()
+                shell_env.insert_path(TempFolder)
 
             # # Update environment with required VC vars.
             interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
@@ -118,7 +220,7 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
             interesting_keys.extend(
                 ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
             vc_vars = QueryVcVariables(
-                interesting_keys, 'x86', vs_version=self.tool_chain_tag.lower())
+                interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())
             for key in vc_vars.keys():
                 logging.debug(f"Var - {key} = {vc_vars[key]}")
                 if key.lower() == 'path':
@@ -126,11 +228,9 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
                 else:
                     shell_env.set_shell_var(key, vc_vars[key])
 
-            self.OutputDir = os.path.join(
+            # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
-                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
+            # this script is BUILDING the base tools.
-
+            shell_env.set_shell_var('HOST_ARCH', self.target_arch)
-            # compiled tools need to be added to path because antlr is referenced
-            shell_env.insert_path(self.OutputDir)
 
             # Actually build the tools.
             output_stream = edk2_logging.create_output_stream()

BaseTools/Source/C/Makefile

@@ -4,7 +4,9 @@
 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
+!IFNDEF HOST_ARCH
 HOST_ARCH = IA32
+!ENDIF
 
 !INCLUDE Makefiles\ms.common
 

BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak

@@ -8,6 +8,15 @@ PCCTS_HOME=$(BASE_TOOLS_PATH)\Source\C\VfrCompile\Pccts
 ANTLR_SRC=$(PCCTS_HOME)\antlr
 PCCTS_H=$(PCCTS_HOME)\h
 
+!IFNDEF HOST_ARCH
+HOST_ARCH = IA32
+!ENDIF
+
+!IF "$(HOST_ARCH)"=="IA32" || "$(HOST_ARCH)"=="ARM"
+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win32
+!ELSE
+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win64
+!ENDIF
 
 # Support directories
 SET=$(PCCTS_HOME)\support\set
@@ -27,10 +36,10 @@ SUPPORT_OBJS = set.obj
 
 # Dependencies
 
-$(EDK_TOOLS_PATH)\Bin\Win32\antlr.exe: $(ANTLR_OBJS) $(SUPPORT_OBJS)
+$(SYS_BIN_PATH)\antlr.exe: $(ANTLR_OBJS) $(SUPPORT_OBJS)
     $(CC) $(CFLAGS) -Feantlr.exe $(ANTLR_OBJS) $(SUPPORT_OBJS)
-    -@if not exist $(EDK_TOOLS_PATH)\Bin\Win32 mkdir $(EDK_TOOLS_PATH)\Bin\Win32
+    -@if not exist $(SYS_BIN_PATH) mkdir $(SYS_BIN_PATH)
-		copy antlr.exe $(EDK_TOOLS_PATH)\Bin\Win32
+		copy antlr.exe $(SYS_BIN_PATH)
 
 
 antlr.obj: $(ANTLR_SRC)\antlr.c \

BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak

@@ -12,6 +12,15 @@ PCCTS_H=$(PCCTS_HOME)\h
 # Support directories
 SET=$(PCCTS_HOME)\support\set
 
+!IFNDEF HOST_ARCH
+HOST_ARCH = IA32
+!ENDIF
+
+!IF "$(HOST_ARCH)"=="IA32" || "$(HOST_ARCH)"=="ARM"
+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win32
+!ELSE
+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win64
+!ENDIF
 
 # Compiler stuff
 CC = cl
@@ -26,10 +35,10 @@ SUPPORT_OBJS = set.obj
 
 # Dependencies
 
-$(EDK_TOOLS_PATH)\Bin\Win32\dlg.exe: $(DLG_OBJS) $(SUPPORT_OBJS)
+$(SYS_BIN_PATH)\dlg.exe: $(DLG_OBJS) $(SUPPORT_OBJS)
     $(CC) $(CFLAGS) -Fedlg.exe $(DLG_OBJS) $(SUPPORT_OBJS)
-    -@if not exist $(EDK_TOOLS_PATH)\Bin\Win32 mkdir $(EDK_TOOLS_PATH)\Bin\Win32
+    -@if not exist $(SYS_BIN_PATH) mkdir $(SYS_BIN_PATH)
-		copy dlg.exe $(EDK_TOOLS_PATH)\Bin\Win32
+		copy dlg.exe $(SYS_BIN_PATH)
 
 dlg_p.obj: $(DLG_SRC)\dlg_p.c \
 					$(PCCTS_H)\antlr.h \