""" This tool is designed to facilitate downloading, patching, and building of library dependencies for the Jami daemon project on windows. MSBuild toolset and sdk versions can be supplied as parameters and will be sedded into vcxproj files accordingly. A package can be defined with a json like this: { "name": "mylibrary", "version": "76a5006623539a58262d33458a5605be096b3a10", "url": "https://git.example.com/gorblok/mylibrary/archive/__VERSION__.tar.gz", "deps": ["mydep"], "use_cmake" : true, "defines": ["SEGFAULTS=0", "MY_CMAKE_DEFINE=true"], "patches": ["some_patch.patch"], "win_patches": ["some_windows_line_ending_patch.patch"], "project_paths": ["mylibrary-static.vcxproj"], "with_env" : "10.0.16299.0", "custom_scripts": { "pre_build": [], "build": [], "post_build": [] } } """ import sys import os import subprocess import platform import argparse import json import re import zipfile import tarfile import multiprocessing import shutil import shlex import glob import time from datetime import timedelta import struct import importlib import logging import traceback import re import fileinput root_logger = logging.getLogger(__name__) log = None # project paths daemon_msvc_dir = os.path.dirname(os.path.realpath(__file__)) daemon_dir = os.path.dirname(os.path.dirname(daemon_msvc_dir)) daemon_build_dir = daemon_dir + r'\build' contrib_src_dir = daemon_dir + r'\contrib\src' contrib_build_dir = daemon_dir + r'\contrib\build' contrib_tmp_dir = daemon_dir + r'\contrib\tarballs' plugins_bin_dir = daemon_dir + r'\..\plugins\build' plugins_dir = daemon_dir + r'\..\plugins' # SCM wget_args = [ '--no-check-certificate', '--retry-connrefused', '--waitretry=1', '--read-timeout=20', '--timeout=15', '--tries=4'] git_apply_args = ['apply', '--reject', '--ignore-whitespace', '--whitespace=fix'] patch_args = ['-flp1', '-i'] # vs help win_sdk_default = '10.0.16299.0' win_toolset_default = '142' vs_where_path = os.path.join( os.environ['ProgramFiles(x86)'], 'Microsoft Visual Studio', 'Installer', 'vswhere.exe' ) host_is_64bit = (False, True)[platform.machine().endswith('64')] python_is_64bit = (False, True)[8 * struct.calcsize("P") == 64] def shellquote(s, windows=False): if not windows: return "'" + s.replace("'", "'\''") + "'" else: return '\"' + s + '\"' def getLatestVSVersion(): args = [ '-latest', '-products *', '-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64', '-property installationVersion' ] cmd = [vs_where_path] + args output = subprocess.check_output(' '.join(cmd)).decode('utf-8') if output: return output.splitlines()[0].split('.')[0] else: return def findVSLatestDir(): args = [ '-latest', '-products *', '-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64', '-property installationPath' ] cmd = [vs_where_path] + args output = subprocess.check_output( ' '.join(cmd)).decode( 'utf-8', errors='ignore') if output: return output.splitlines()[0] else: return def findMSBuild(): filename = 'MSBuild.exe' for root, _, files in os.walk(findVSLatestDir() + r'\\MSBuild'): if filename in files: return os.path.join(root, filename) def getVSEnv(arch='x64', platform='', version=''): env_cmd = 'set path=%path:"=% && ' + \ getVSEnvCmd(arch, platform, version) + ' && set' p = subprocess.Popen(env_cmd, shell=True, stdout=subprocess.PIPE) stdout, _ = p.communicate() out = stdout.decode('utf-8', errors='ignore').split("\r\n")[5:-1] return dict(s.split('=', 1) for s in out) def getCMakeGenerator(vs_version): if vs_version == '15': return '\"Visual Studio 15 2017 Win64\"' else: return '\"Visual Studio ' + vs_version + ' 2019\" -A x64' def getVSEnvCmd(arch='x64', platform='', version=''): vcEnvInit = [findVSLatestDir() + r'\VC\Auxiliary\Build\"vcvarsall.bat'] if platform != '': args = [arch, platform, version] else: args = [arch, version] if args: vcEnvInit.extend(args) vcEnvInit = 'call \"' + ' '.join(vcEnvInit) return vcEnvInit def make_plugin(pkg_info, force, sdk_version, toolset): pkg_name = pkg_info.get('name') version = pkg_info.get('version') extractLibs = pkg_info.get('extractLibs') plugin_path = (plugins_dir + "/" + pkg_name).replace("\\", "/") if (extractLibs): path = plugins_dir + r'\contrib\libs.tar.gz' with tarfile.open(path, 'r', encoding="utf8", errors='ignore') as tarball: tarball.extractall(plugins_dir + r'\contrib') pkg_build_uptodate = False pkg_ver_uptodate = False # attempt to get the current built version current_version = '' # check build file for current version build_file = plugins_bin_dir + r'\\.' + pkg_name if os.path.exists(build_file): if force: os.remove(build_file) else: pkg_build_uptodate = is_build_uptodate(pkg_name, build_file) with open(build_file, 'r+', encoding="utf8", errors='ignore') as f: current_version = f.read() if current_version == version: pkg_ver_uptodate = True for dep in pkg_info.get('deps', []): dep_build_dep = resolve(dep, False, sdk_version, toolset) if dep_build_dep: pkg_build_uptodate = False pkg_up_to_date = pkg_build_uptodate & pkg_ver_uptodate cmake_defines = "" for define in pkg_info.get('defines', []): cmake_defines += " -D" + define + " " if not pkg_up_to_date or current_version is None or force: root_logger.warning( "Building plugin with preferred sdk version %s and toolset %s", sdk_version, toolset) env_set = 'false' if pkg_info.get('with_env', '') == '' else 'true' sdk_to_use = sdk_version if env_set == 'false' else pkg_info.get( 'with_env', '') cmake_script = "cmake -DCMAKE_SYSTEM_VERSION=" + sdk_version + \ " -G " + getCMakeGenerator(getLatestVSVersion()) + cmake_defines + \ " -T " + toolset + \ " -S " + plugin_path + " -B " + plugin_path + "/msvc" root_logger.warning("Cmake generating vcxproj files") _ = getSHrunner().exec_batch(cmake_script) build(pkg_name, plugin_path, pkg_info.get('project_paths', []), pkg_info.get('custom_scripts', {}), env_set, sdk_to_use, toolset) track_build(pkg_name, version, True) def make_daemon(pkg_info, force, sdk_version, toolset): cmake_script = 'cmake -DCMAKE_CONFIGURATION_TYPES="ReleaseLib_win32" -DCMAKE_SYSTEM_VERSION=' + sdk_version + \ ' -DCMAKE_VS_PLATFORM_NAME="x64" -G ' + getCMakeGenerator(getLatestVSVersion( )) + ' -T $(DefaultPlatformToolset) -S ../../ -B ../../build' root_logger.warning("Cmake generating vcxproj files") result = getSHrunner().exec_batch(cmake_script) if result[0] != 0: sys.exit("Cmake Errors") for dep in pkg_info.get('deps', []): resolve(dep, False, sdk_version, toolset) root_logger.warning( "Building daemon with preferred sdk version %s and toolset %s", sdk_version, toolset) env_set = 'false' if pkg_info.get('with_env', '') == '' else 'true' sdk_to_use = sdk_version if env_set == 'false' else pkg_info.get( 'with_env', '') build('daemon', daemon_build_dir, pkg_info.get('project_paths', []), pkg_info.get('custom_scripts', {}), env_set, sdk_to_use, toolset, conf=pkg_info.get('configuration', 'Release')) def make(pkg_info, force, sdk_version, toolset, isPlugin): pkg_name = pkg_info.get('name') if pkg_name == 'daemon': return make_daemon(pkg_info, force, sdk_version, toolset) if isPlugin: return make_plugin(pkg_info, force, sdk_version, toolset) version = pkg_info.get('version') pkg_build_uptodate = False pkg_ver_uptodate = False # attempt to get the current built version current_version = '' # check build file for current version build_file = contrib_build_dir + r'\\.' + pkg_name if os.path.exists(build_file): if force: os.remove(build_file) else: pkg_build_uptodate = is_build_uptodate(pkg_name, build_file) with open(build_file, 'r+', encoding="utf8", errors='ignore') as f: current_version = f.read() if current_version == version: pkg_ver_uptodate = True for dep in pkg_info.get('deps', []): dep_build_dep = resolve(dep, False, sdk_version, toolset) if dep_build_dep: pkg_build_uptodate = False pkg_up_to_date = pkg_build_uptodate & pkg_ver_uptodate if not pkg_up_to_date or current_version is None or force: if current_version != '': log.debug(pkg_name + ' currently @: ' + current_version) if force: log.debug('Forcing fetch/patch/build for ' + pkg_name) should_fetch = not pkg_up_to_date pkg_build_path = contrib_build_dir + '\\' + pkg_name should_fetch &= not os.path.exists(pkg_build_path) if not pkg_up_to_date or force: if not force and not current_version is None: log.warning(pkg_name + ' is not up to date') if (should_fetch or force) and fetch_pkg( pkg_name, version, pkg_info['url'], force): apply(pkg_name, pkg_info.get('patches', []), pkg_info.get('win_patches', [])) env_set = 'false' if pkg_info.get('with_env', '') == '' else 'true' sdk_to_use = sdk_version if env_set == 'false' else pkg_info.get( 'with_env', '') # configure with cmake ? use_cmake = pkg_info.get('use_cmake', False) if use_cmake: cmake_defines = "" for define in pkg_info.get('defines', []): cmake_defines += " -D" + define if not pkg_up_to_date or current_version is None or force: cmake_conf_script = "cmake -G " + getCMakeGenerator(getLatestVSVersion( )) + cmake_defines + " -S '" + pkg_build_path + "' -B '" + pkg_build_path + "\\build'" log.debug("Configuring with Cmake") result = getSHrunner().exec_batch(cmake_conf_script) if result[0] != 0: log.error("Error configuring with CMake") exit(1) if build(pkg_name, contrib_build_dir + '\\' + pkg_name, pkg_info.get('project_paths', []), pkg_info.get('custom_scripts', {}), env_set, sdk_to_use, toolset, use_cmake=use_cmake): track_build(pkg_name, version, isPlugin) else: log.error("Couldn't build contrib " + pkg_name) exit(1) log.info(pkg_name + ' up to date') return True # did not need build log.info(pkg_name + ' already up to date') return False def fetch_pkg(pkg_name, version, url, force): version_replace = re.compile(re.escape('__VERSION__')) full_url = version_replace.sub(version, url) if not full_url: log.error(pkg_name + ' missing url in package configuration') return False archive_name = full_url[full_url.rfind("/") + 1:] archive_path = contrib_tmp_dir + '\\' + pkg_name + '_' + archive_name if not os.path.exists(archive_path): log.debug('Fetching ' + pkg_name + ' from: ' + full_url) args = [full_url, '-O', archive_path] args.extend(wget_args) dl_result = getSHrunner().exec_batch('wget', args) if dl_result[0] != 0: log.warning( 'wget failure. Using powershell Invoke-WebRequest instead') args = ['-Uri', full_url, '-OutFile', archive_path] dl_result = getSHrunner().exec_ps1('Invoke-WebRequest', args) return extract_archive(pkg_name, archive_name, archive_path) else: log.warning(archive_name + ' already exists in the tarball/archive directory') decomp_result = extract_archive(pkg_name, archive_name, archive_path) if not decomp_result and force: log.debug('Removing old tarball for ' + archive_name) getSHrunner().exec_batch('del', ['/s', '/q', archive_name]) return fetch_pkg(pkg_name, version, url, False) else: return True return False def remove_archive_if_needed(pkg_build_path, dirty_path): if os.path.exists(pkg_build_path): log.debug('Removing old package ' + pkg_build_path) getSHrunner().exec_batch('rmdir', ['/s', '/q', pkg_build_path]) elif os.path.exists(dirty_path): log.debug('Removing partial decompression ' + dirty_path) getSHrunner().exec_batch('rmdir', ['/s', '/q', dirty_path]) def extract_tar(pkg_build_path, name, path): with tarfile.open(path, 'r', encoding="utf8", errors='ignore') as tarball: tar_common_prefix = os.path.commonprefix(tarball.getnames()) dirty_path = contrib_build_dir + '\\' + tar_common_prefix remove_archive_if_needed(pkg_build_path, dirty_path) log.debug('Decompressing ' + name + ' to ' + pkg_build_path) tarball.extractall(contrib_build_dir) os.rename(contrib_build_dir + '\\' + tar_common_prefix, pkg_build_path) return True return False def extract_zip(pkg_build_path, name, path): with zipfile.ZipFile(path, 'r') as ziparchive: zip_common_prefix = os.path.commonprefix(ziparchive.namelist()) dirty_path = contrib_build_dir + '\\' + zip_common_prefix remove_archive_if_needed(pkg_build_path, dirty_path) log.debug('Decompressing ' + name + ' to ' + pkg_build_path) ziparchive.extractall(contrib_build_dir) os.rename(contrib_build_dir + '\\' + zip_common_prefix, pkg_build_path) return True return False def extract_archive(pkg_name, name, path): pkg_build_path = contrib_build_dir + '\\' + pkg_name if tarfile.is_tarfile(path): return extract_tar(pkg_build_path, name, path) elif zipfile.is_zipfile(path): return extract_zip(pkg_build_path, name, path) def apply_linux(patch_path): log.debug('applying linux patch ' + patch_path) args = [] args.extend(patch_args) args.append(patch_path) return getSHrunner().exec_sh('patch', args) def apply_windows(patch_path): log.debug('applying windows patch ' + patch_path) args = [] args.extend(git_apply_args) args.append(patch_path) return getSHrunner().exec_batch('git', args) def apply(pkg_name, patches, win_patches): log.debug('patching ' + pkg_name + '...') tmp_dir = os.getcwd() pkg_build_path = contrib_build_dir + '\\' + pkg_name if not os.path.exists(pkg_build_path): os.makedirs(pkg_build_path) os.chdir(pkg_build_path) base_sh_src_path = get_sh_path(contrib_src_dir) # 1. git patches (LF) for p in patches: patch_path = base_sh_src_path + '/' + pkg_name + '/' + p result = apply_linux(patch_path) if result[0]: log.error('Couldn\'t apply patch: ' + patch_path) exit(1) # 2. windows git patches (CR/LF) for wp in win_patches: patch_path = contrib_src_dir + '\\' + pkg_name + '\\' + wp result = apply_windows(patch_path) if result[0]: log.error('Couldn\'t apply patch: ' + patch_path) exit(1) os.chdir(tmp_dir) def get_pkg_file(pkg_name, isPlugin): if pkg_name == 'daemon': pkg_location = daemon_msvc_dir elif (isPlugin): pkg_location = plugins_dir + r'\\' + pkg_name else: pkg_location = daemon_dir + r'\contrib\src\\' + pkg_name pkg_json_file = pkg_location + r"\\package.json" if not os.path.exists(pkg_json_file): log.error("No package info for " + pkg_name) sys.exit(1) return pkg_json_file def resolve(pkg_name, force=False, sdk_version='', toolset='', isPlugin=False): pkg_json_file = get_pkg_file(pkg_name, isPlugin) with open(pkg_json_file, encoding="utf8", errors='ignore') as json_file: log.info('Resolving: ' + pkg_name) pkg_info = json.load(json_file) try: return make(pkg_info, force, sdk_version, toolset, isPlugin) except Exception as e: print(e) log.error('Make ' + pkg_name + ' failed!') sys.exit(1) def track_build(pkg_name, version, isPlugin): if isPlugin: build_file = plugins_bin_dir + '\\.' + pkg_name else: build_file = contrib_build_dir + '\\.' + pkg_name f = open(build_file, "w+", encoding="utf8", errors='ignore') f.write(version) f.close() def build(pkg_name, pkg_dir, project_paths, custom_scripts, with_env, sdk, toolset, arch='x64', conf='Release', use_cmake=False): getMSbuilder().set_msbuild_configuration(with_env, arch, conf, toolset) getMSbuilder().setup_vs_env(sdk) success = True build_operations = 0 tmp_dir = os.getcwd() os.chdir(pkg_dir) # pre_build custom step (CMake...) pre_build_scripts = custom_scripts.get("pre_build", []) if pre_build_scripts: log.debug('Pre_build phase') for script in pre_build_scripts: result = getSHrunner().exec_batch(script) success &= not result[0] build_operations += 1 # build custom step (nmake...) build_scripts = custom_scripts.get("build", []) if build_scripts: log.debug('Custom Build phase') for script in build_scripts: result = getSHrunner().exec_batch(script) success &= not result[0] build_operations += 1 # vcxproj files if project_paths: log.debug('Msbuild phase') for pp in project_paths: project_full_path = pkg_dir + '\\' + pp log.debug('Building: ' + pkg_name + " with sdk version " + sdk + " and toolset " + toolset) getMSbuilder().build(pkg_name, project_full_path, sdk, toolset) build_operations += 1 else: # build directly with cmake if use_cmake is True: log.debug('CMake build phase') cmake_build_script = "cmake --build '" + pkg_dir + \ "\\build' " + "--config " + conf result = getSHrunner().exec_batch(cmake_build_script) if result[0] != 0: log.error("Error building with CMake") exit(1) os.chdir(tmp_dir) # should cover header only, no cmake, etc ops = len(build_scripts) + len(project_paths) + len(pre_build_scripts) return success and build_operations == ops class Singleton: def __init__(self, decorated): self._decorated = decorated def instance(self): try: return self._instance except AttributeError: self._instance = self._decorated() return self._instance def __call__(self): raise TypeError('Singletons must be accessed through `instance()`.') def __instancecheck__(self, inst): return isinstance(inst, self._decorated) class ScriptType: ps1 = 1 cmd = 2 sh = 3 @Singleton class SHrunner(): def __init__(self): sys_path = (r'\Sysnative', r'\system32')[python_is_64bit] full_sys_path = os.path.expandvars('%systemroot%') + sys_path # powershell self.ps_path = full_sys_path + r'\WindowsPowerShell\v1.0\powershell.exe' if not os.path.exists(self.ps_path): log.error('Powershell not found at %s.' % self.ps_path) sys.exit(1) # bash if not os.environ.get('JENKINS_URL'): self.sh_path = full_sys_path + r'\bash.exe' else: self.sh_path = r'\"C:\Program Files\Git\git-bash.exe\"' if not os.path.exists(self.sh_path): log.warning('Bash not found at ' + self.sh_path) self.sh_path = shutil.which('bash.exe') if not os.path.exists(self.sh_path): log.error('No bash found') sys.exit(1) else: self.sh_path = shellquote(self.sh_path, windows=True) log.debug('Using alternate bash found at ' + self.sh_path) self.project_env_vars = { 'DAEMON_DIR': daemon_dir, 'CONTRIB_SRC_DIR': contrib_src_dir, 'CONTRIB_BUILD_DIR': contrib_build_dir, 'VCVARSALL_CMD': getVSEnvCmd(), 'CMAKE_GENERATOR': getCMakeGenerator(getLatestVSVersion()) } self.base_env_vars = self.project_env_vars.copy() self.base_env_vars.update(os.environ.copy()) self.vs_env_vars = {} def set_vs_env_vars(self, env_target): self.vs_env_vars = {} self.vs_env_vars = self.project_env_vars.copy() self.vs_env_vars.update(getVSEnv(version=env_target)) def exec_script(self, script_type=ScriptType.cmd, script=None, args=[]): if script_type is ScriptType.cmd: cmd = [script] if not args: cmd = shlex.split(script) elif script_type is ScriptType.ps1: cmd = [self.ps_path, '-ExecutionPolicy', 'ByPass', script] elif script_type is ScriptType.sh: cmd = [self.sh_path, '-c ', '\"' + script] if args: cmd.extend(args) if script_type is ScriptType.sh: cmd[-1] = cmd[-1] + '\"' cmd = " ".join(cmd) p = subprocess.Popen(cmd, shell=True, stderr=sys.stderr, stdout=sys.stdout, env=self.vs_env_vars if self.vs_env_vars else self.base_env_vars) rtrn, perr = p.communicate() rcode = p.returncode data = None if perr: data = json.dumps(perr.decode('utf-8', 'ignore')) else: data = rtrn return rcode, data def exec_batch(self, script=None, args=[]): return self.exec_script(ScriptType.cmd, script, args) def exec_ps1(self, script=None, args=[]): return self.exec_script(ScriptType.ps1, script, args) def exec_sh(self, script=None, args=[]): return self.exec_script(ScriptType.sh, script, args) def getSHrunner(): return SHrunner.instance() @Singleton class MSbuilder: def __init__(self): self.vsenv_done = False self.msbuild = findMSBuild() self.default_msbuild_args = [ '/nologo', '/verbosity:normal', '/maxcpucount:' + str(multiprocessing.cpu_count())] self.set_msbuild_configuration() def set_msbuild_configuration(self, with_env='false', arch='x64', configuration='Release', toolset=win_toolset_default): self.extra_msbuild_args = [ '/p:Platform=' + arch, '/p:Configuration=' + configuration, '/p:PlatformToolset=' + toolset, '/p:useenv=' + with_env ] @staticmethod def replace_vs_prop(filename, prop, val): p = re.compile(r'(?s)<' + prop + r'\s?.*?>(.*?)<\/' + prop + r'>') val = r'<' + prop + r'>' + val + r'' with fileinput.FileInput(filename, inplace=True) as file: for line in file: print(re.sub(p, val, line), end='') def build(self, pkg_name, proj_path, sdk_version, toolset): if not os.path.isfile(self.msbuild): raise IOError('msbuild.exe not found. path=' + self.msbuild) if os.environ.get('JENKINS_URL'): log.info("Jenkins Clear DebugInformationFormat") self.__class__.replace_vs_prop(proj_path, 'DebugInformationFormat', 'None') # force chosen sdk self.__class__.replace_vs_prop(proj_path, 'WindowsTargetPlatformVersion', sdk_version) # force chosen toolset self.__class__.replace_vs_prop(proj_path, 'PlatformToolset', toolset) args = [] args.extend(self.default_msbuild_args) args.extend(self.extra_msbuild_args) args.append(proj_path) result = getSHrunner().exec_batch(self.msbuild, args) if result[0] == 1: log.error("Build failed when building " + pkg_name) sys.exit(1) def setup_vs_env(self, env_target): if self.vsenv_done: log.debug('vs environment already initialized') return log.debug('Setting up vs environment') getSHrunner().set_vs_env_vars(env_target) self.vsenv_done = True def getMSbuilder(): return MSbuilder.instance() def parse_args(): ap = argparse.ArgumentParser(description="Windows Jami build tool") ap.add_argument( '-b', '--build', help='Build latest contrib') ap.add_argument( '-f', '--force', action='store_true', help='Force action') ap.add_argument( '-c', '--clean', help='Cleans out build directory for a contrib') ap.add_argument( '-d', '--debug', default='DEBUG', help='Sets the logging level') ap.add_argument( '-i', '--indent', action='store_true', help='Sets whether the logs are indented to stack frames') ap.add_argument( '-v', '--verbose', action='store_true', help='Sets whether the logs are verbose or not') ap.add_argument( '-p', '--purge', action='store_true', help='Cleans out contrib tarball directory') ap.add_argument( '-s', '--sdk', default=win_sdk_default, type=str, help='Use specified windows sdk version') ap.add_argument( '-t', '--toolset', default=win_toolset_default, type=str, help='Use specified platform toolset version') ap.add_argument( '-P', '--plugin', default=False, action='store_true', help="Defines if we're building a plugin" ) parsed_args = ap.parse_args() if parsed_args.toolset: if parsed_args.toolset[0] != 'v': parsed_args.toolset = 'v' + parsed_args.toolset return parsed_args def main(): start_time = time.time() parsed_args = parse_args() setup_logging(lvl=parsed_args.debug, verbose=parsed_args.verbose, do_indent=parsed_args.indent) if not host_is_64bit: log.error('These scripts will only run on a 64-bit Windows system for now!') sys.exit(1) if int(getLatestVSVersion()) < 15: log.error('These scripts require at least Visual Studio v15 2017!') sys.exit(1) if parsed_args.purge: if os.path.exists(contrib_tmp_dir): log.warning('Removing contrib tarballs ' + contrib_tmp_dir) getSHrunner().exec_batch( 'del', ['/s', '/f', '/q', contrib_tmp_dir + '\\*.tar.*']) getSHrunner().exec_batch( 'del', ['/s', '/f', '/q', contrib_tmp_dir + '\\*.tgz.*']) getSHrunner().exec_batch( 'del', ['/s', '/f', '/q', contrib_tmp_dir + '\\*.zip.*']) else: log.warning('No tarballs to remove') if parsed_args.clean: if os.path.exists(contrib_build_dir) and parsed_args.clean == 'all': log.warning('Removing contrib builds ' + contrib_build_dir) getSHrunner().exec_batch('rmdir', ['/s', '/q', contrib_build_dir]) else: pkg_json_file = get_pkg_file(parsed_args.clean) with open(pkg_json_file, encoding="utf8", errors='ignore') as json_file: pkg_info = json.load(json_file) if (not parsed_args.plugin): exclude_dirs = contrib_build_dir dir_to_clean = exclude_dirs + '\\' + pkg_info['name'] file_to_clean = exclude_dirs + '\\.' + pkg_info['name'] if os.path.exists(dir_to_clean) or os.path.exists( file_to_clean): log.warning('Removing build ' + dir_to_clean) getSHrunner().exec_batch( 'rmdir', ['/s', '/q', dir_to_clean]) getSHrunner().exec_batch( 'del', ['/s', '/f', '/q', file_to_clean]) else: log.warning('No builds to remove') else: exclude_dirs = plugins_bin_dir files_to_clean = [exclude_dirs + '\\.' + pkg_info['name']] for name in glob.iglob( exclude_dirs + '\\**\\' + pkg_info['name'] + ".jpl", recursive=True): files_to_clean.append(name) for name in files_to_clean: getSHrunner().exec_batch( 'del', ['/s', '/f', '/q', name]) getSHrunner().exec_batch( 'rmdir', ['/s', '/q', plugins_dir + "\\" + parsed_args.clean + "\\msvc"]) if parsed_args.build: if not os.path.exists(contrib_build_dir): os.makedirs(contrib_build_dir) log.info('Making: ' + parsed_args.build) resolve(parsed_args.build, parsed_args.force, parsed_args.sdk, parsed_args.toolset, parsed_args.plugin) log.info('Make done for: ' + parsed_args.build) log.debug("--- %s ---" % secondsToStr(time.time() - start_time)) def get_sh_path(path): driveless_path = path.replace(os.path.sep, '/')[3:] drive_letter = os.path.splitdrive(daemon_dir)[0][0].lower() wsl_drive_path = '/mnt/' + drive_letter + '/' no_echo = ' &> /dev/null' result = getSHrunner().exec_sh('pwd | grep ' + wsl_drive_path + no_echo) if result[0]: # using git bash return '/' + drive_letter + '/' + driveless_path # using wsl return wsl_drive_path + driveless_path def newest_file(root): file_list = [] for path, _, files in os.walk(root): for name in files: file_list.append(os.path.join(path, name)) latest_file = max(file_list, key=os.path.getmtime) return latest_file def is_build_uptodate(pkg_name, build_file): root = contrib_build_dir + '\\' + pkg_name file_list = [] for path, _, files in os.walk(root): for name in files: file_list.append(os.path.join(path, name)) if not file_list: return False latest_file = max(file_list, key=os.path.getmtime) t_mod = os.path.getmtime(latest_file) t_build = os.path.getmtime(build_file) return t_mod < t_build def secondsToStr(elapsed=None): return str(timedelta(seconds=elapsed)) class CustomAdapter(logging.LoggerAdapter): @staticmethod def indent(): indentation_level = len(traceback.extract_stack()) return indentation_level - 4 - 2 # Remove logging infrastructure frames def process(self, msg, kwargs): return '{i}{m}'.format(i=' ' * (self.indent()), m=msg), kwargs def setup_logging(lvl=logging.DEBUG, verbose=False, do_indent=False): format = '' if verbose: format = '[ %(levelname)-8s %(created).6f %(funcName)10s:%(lineno)4s ] ' fmt = format + '%(message)s' try: import coloredlogs coloredlogs.install( level=lvl, logger=root_logger, fmt=fmt, level_styles={ 'debug': {'color': 'blue'}, 'info': {'color': 'green'}, 'warn': {'color': 'yellow'}, 'error': {'color': 'red'}, 'critical': {'color': 'red', 'bold': True} }, field_styles={ 'asctime': {'color': 'magenta'}, 'created': {'color': 'magenta'}, 'levelname': {'color': 'cyan'}, 'funcName': {'color': 'black', 'bold': True}, 'lineno': {'color': 'black', 'bold': True} }) except ImportError: root_logger.setLevel(logging.DEBUG) logging.basicConfig(level=lvl, format=fmt) if do_indent: global log log = CustomAdapter(logging.getLogger(__name__), {}) else: log = logging.getLogger(__name__) if __name__ == '__main__': main()