Virtualization
/
meson
mirror of https://gitlab.com/qemu-project/meson.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5644 from bonzini/meson-exe-cmdline 

Show command line in `ninja -v` for `capture: true` custom targets and generators
This commit is contained in:
Jussi Pakkanen 2019-08-03 01:28:27 +03:00 committed by GitHub
parent cba23413c5 d34e532020
commit 2e6df380f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 118 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
mesonbuild/backend/backends.py
View File

@ -69,17 +69,13 @@ class TargetInstallData:
self.optional = optional self.optional = optional
class ExecutableSerialisation: class ExecutableSerialisation:
def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper, def __init__(self, cmd_args, env=None, exe_wrapper=None,
workdir, extra_paths, capture, needs_exe_wrapper: bool): workdir=None, extra_paths=None, capture=None):
self.name = name
self.fname = fname
self.cmd_args = cmd_args self.cmd_args = cmd_args
self.env = env self.env = env or {}
self.is_cross = is_cross
if exe_wrapper is not None: if exe_wrapper is not None:
assert(isinstance(exe_wrapper, dependencies.ExternalProgram)) assert(isinstance(exe_wrapper, dependencies.ExternalProgram))
self.exe_runner = exe_wrapper self.exe_runner = exe_wrapper
self.needs_exe_wrapper = needs_exe_wrapper
self.workdir = workdir self.workdir = workdir
self.extra_paths = extra_paths self.extra_paths = extra_paths
self.capture = capture self.capture = capture
@ -323,26 +319,61 @@ class Backend:
raise MesonException('Unknown data type in object list.') raise MesonException('Unknown data type in object list.')
return obj_list return obj_list
def serialize_executable(self, tname, exe, cmd_args, workdir, env=None, def as_meson_exe_cmdline(self, tname, exe, cmd_args, workdir=None,
extra_paths=None, capture=None): for_machine=MachineChoice.BUILD,
extra_bdeps=None, capture=None, force_serialize=False):
''' '''
Serialize an executable for running with a generator or a custom target Serialize an executable for running with a generator or a custom target
''' '''
import hashlib import hashlib
if env is None: machine = self.environment.machines[for_machine]
env = {} if machine.is_windows() or machine.is_cygwin():
if extra_paths is None: extra_paths = self.determine_windows_extra_paths(exe, extra_bdeps or [])
# The callee didn't check if we needed extra paths, so check it here else:
if mesonlib.is_windows() or mesonlib.is_cygwin(): extra_paths = []
extra_paths = self.determine_windows_extra_paths(exe, [])
else: if isinstance(exe, dependencies.ExternalProgram):
extra_paths = [] exe_cmd = exe.get_command()
# Can't just use exe.name here; it will likely be run more than once exe_for_machine = exe.for_machine
elif isinstance(exe, (build.BuildTarget, build.CustomTarget)):
exe_cmd = [self.get_target_filename_abs(exe)]
exe_for_machine = exe.for_machine
else:
exe_cmd = [exe]
exe_for_machine = MachineChoice.BUILD
is_cross_built = not self.environment.machines.matches_build_machine(exe_for_machine)
if is_cross_built and self.environment.need_exe_wrapper():
exe_wrapper = self.environment.get_exe_wrapper()
if not exe_wrapper.found():
msg = 'The exe_wrapper {!r} defined in the cross file is ' \
'needed by target {!r}, but was not found. Please ' \
'check the command and/or add it to PATH.'
raise MesonException(msg.format(exe_wrapper.name, tname))
else:
if exe_cmd[0].endswith('.jar'):
exe_cmd = ['java', '-jar'] + exe_cmd
elif exe_cmd[0].endswith('.exe') and not (mesonlib.is_windows() or mesonlib.is_cygwin()):
exe_cmd = ['mono'] + exe_cmd
exe_wrapper = None
force_serialize = force_serialize or extra_paths or workdir or \
exe_wrapper or any('\n' in c for c in cmd_args)
if not force_serialize:
if not capture:
return None
return (self.environment.get_build_command() +
['--internal', 'exe', '--capture', capture, '--'] + exe_cmd + cmd_args)
workdir = workdir or self.environment.get_build_dir()
env = {}
if isinstance(exe, (dependencies.ExternalProgram, if isinstance(exe, (dependencies.ExternalProgram,
build.BuildTarget, build.CustomTarget)): build.BuildTarget, build.CustomTarget)):
basename = exe.name basename = exe.name
else: else:
basename = os.path.basename(exe) basename = os.path.basename(exe)
# Can't just use exe.name here; it will likely be run more than once
# Take a digest of the cmd args, env, workdir, and capture. This avoids # Take a digest of the cmd args, env, workdir, and capture. This avoids
# collisions and also makes the name deterministic over regenerations # collisions and also makes the name deterministic over regenerations
# which avoids a rebuild by Ninja because the cmdline stays the same. # which avoids a rebuild by Ninja because the cmdline stays the same.
@ -352,31 +383,11 @@ class Backend:
scratch_file = 'meson_exe_{0}_{1}.dat'.format(basename, digest) scratch_file = 'meson_exe_{0}_{1}.dat'.format(basename, digest)
exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file) exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file)
with open(exe_data, 'wb') as f: with open(exe_data, 'wb') as f:
if isinstance(exe, dependencies.ExternalProgram): es = ExecutableSerialisation(exe_cmd + cmd_args, env,
exe_cmd = exe.get_command() exe_wrapper, workdir,
exe_for_machine = exe.for_machine extra_paths, capture)
elif isinstance(exe, (build.BuildTarget, build.CustomTarget)):
exe_cmd = [self.get_target_filename_abs(exe)]
exe_for_machine = exe.for_machine
else:
exe_cmd = [exe]
exe_for_machine = MachineChoice.BUILD
is_cross_built = not self.environment.machines.matches_build_machine(exe_for_machine)
if is_cross_built and self.environment.need_exe_wrapper():
exe_wrapper = self.environment.get_exe_wrapper()
if not exe_wrapper.found():
msg = 'The exe_wrapper {!r} defined in the cross file is ' \
'needed by target {!r}, but was not found. Please ' \
'check the command and/or add it to PATH.'
raise MesonException(msg.format(exe_wrapper.name, tname))
else:
exe_wrapper = None
es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env,
is_cross_built, exe_wrapper, workdir,
extra_paths, capture,
self.environment.need_exe_wrapper())
pickle.dump(es, f) pickle.dump(es, f)
return exe_data return self.environment.get_build_command() + ['--internal', 'exe', '--unpickle', exe_data]
def serialize_tests(self): def serialize_tests(self):
test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat')

56
mesonbuild/backend/ninjabackend.py
View File

@ -658,33 +658,13 @@ int dummy;
# Add a dependency on all the outputs of this target # Add a dependency on all the outputs of this target
for output in d.get_outputs(): for output in d.get_outputs():
elem.add_dep(os.path.join(self.get_target_dir(d), output)) elem.add_dep(os.path.join(self.get_target_dir(d), output))
serialize = False
extra_paths = [] meson_exe_cmd = self.as_meson_exe_cmdline(target.name, target.command[0], cmd[1:],
# If the target requires capturing stdout, then use the serialized for_machine=target.for_machine,
# executable wrapper to capture that output and save it to a file. extra_bdeps=target.get_transitive_build_target_deps(),
if target.capture: capture=ofilenames[0] if target.capture else None)
serialize = True if meson_exe_cmd:
# If the command line requires a newline, also use the wrapper, as cmd = meson_exe_cmd
# ninja does not support them in its build rule syntax.
if any('\n' in c for c in cmd):
serialize = True
# Windows doesn't have -rpath, so for EXEs that need DLLs built within
# the project, we need to set PATH so the DLLs are found. We use
# a serialized executable wrapper for that and check if the
# CustomTarget command needs extra paths first.
machine = self.environment.machines[target.for_machine]
if machine.is_windows() or machine.is_cygwin():
extra_bdeps = target.get_transitive_build_target_deps()
extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
if extra_paths:
serialize = True
if serialize:
exe_data = self.serialize_executable(target.name, target.command[0], cmd[1:],
# All targets are built from the build dir
self.environment.get_build_dir(),
extra_paths=extra_paths,
capture=ofilenames[0] if target.capture else None)
cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
cmd_type = 'meson_exe.py custom' cmd_type = 'meson_exe.py custom'
else: else:
cmd_type = 'custom' cmd_type = 'custom'
@ -1786,19 +1766,13 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
outfilelist = outfilelist[len(generator.outputs):] outfilelist = outfilelist[len(generator.outputs):]
args = self.replace_paths(target, args, override_subdir=subdir) args = self.replace_paths(target, args, override_subdir=subdir)
cmdlist = exe_arr + self.replace_extra_args(args, genlist) cmdlist = exe_arr + self.replace_extra_args(args, genlist)
if generator.capture: meson_exe_cmd = self.as_meson_exe_cmdline('generator ' + cmdlist[0],
exe_data = self.serialize_executable( cmdlist[0], cmdlist[1:],
'generator ' + cmdlist[0], capture=outfiles[0] if generator.capture else None)
cmdlist[0], if meson_exe_cmd:
cmdlist[1:], cmdlist = meson_exe_cmd
self.environment.get_build_dir(), abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
capture=outfiles[0] os.makedirs(abs_pdir, exist_ok=True)
)
cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
os.makedirs(abs_pdir, exist_ok=True)
else:
cmd = cmdlist
elem = NinjaBuildElement(self.all_outputs, outfiles, rulename, infilename) elem = NinjaBuildElement(self.all_outputs, outfiles, rulename, infilename)
elem.add_dep([self.get_target_filename(x) for x in generator.depends]) elem.add_dep([self.get_target_filename(x) for x in generator.depends])
@ -1813,7 +1787,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
elem.add_item('DESC', 'Generating source from {!r}.'.format(sole_output)) elem.add_item('DESC', 'Generating source from {!r}.'.format(sole_output))
if isinstance(exe, build.BuildTarget): if isinstance(exe, build.BuildTarget):
elem.add_dep(self.get_target_filename(exe)) elem.add_dep(self.get_target_filename(exe))
elem.add_item('COMMAND', cmd) elem.add_item('COMMAND', cmdlist)
self.add_build(elem) self.add_build(elem)
def scan_fortran_module_outputs(self, target): def scan_fortran_module_outputs(self, target):

43
mesonbuild/backend/vs2010backend.py
View File

@ -143,22 +143,24 @@ class Vs2010Backend(backends.Backend):
for x in args] for x in args]
args = [x.replace('\\', '/') for x in args] args = [x.replace('\\', '/') for x in args]
cmd = exe_arr + self.replace_extra_args(args, genlist) cmd = exe_arr + self.replace_extra_args(args, genlist)
if generator.capture: # Always use a wrapper because MSBuild eats random characters when
exe_data = self.serialize_executable( # there are many arguments.
'generator ' + cmd[0], tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
cmd[0], cmd = self.as_meson_exe_cmdline(
cmd[1:], 'generator ' + cmd[0],
self.environment.get_build_dir(), cmd[0],
capture=outfiles[0] cmd[1:],
) workdir=tdir_abs,
cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] capture=outfiles[0] if generator.capture else None,
abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) force_serialize=True
os.makedirs(abs_pdir, exist_ok=True) )
deps = cmd[-1:] + deps
abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
os.makedirs(abs_pdir, exist_ok=True)
cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename) cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename)
ET.SubElement(cbs, 'Command').text = ' '.join(self.quote_arguments(cmd)) ET.SubElement(cbs, 'Command').text = ' '.join(self.quote_arguments(cmd))
ET.SubElement(cbs, 'Outputs').text = ';'.join(outfiles) ET.SubElement(cbs, 'Outputs').text = ';'.join(outfiles)
if deps: ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps)
ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps)
return generator_output_files, custom_target_output_files, custom_target_include_dirs return generator_output_files, custom_target_output_files, custom_target_include_dirs
def generate(self, interp): def generate(self, interp):
@ -558,19 +560,18 @@ class Vs2010Backend(backends.Backend):
# there are many arguments. # there are many arguments.
tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
extra_bdeps = target.get_transitive_build_target_deps() extra_bdeps = target.get_transitive_build_target_deps()
extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps) wrapper_cmd = self.as_meson_exe_cmdline(target.name, target.command[0], cmd[1:],
exe_data = self.serialize_executable(target.name, target.command[0], cmd[1:], # All targets run from the target dir
# All targets run from the target dir workdir=tdir_abs,
tdir_abs, extra_bdeps=extra_bdeps,
extra_paths=extra_paths, capture=ofilenames[0] if target.capture else None,
capture=ofilenames[0] if target.capture else None) force_serialize=True)
wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
if target.build_always_stale: if target.build_always_stale:
# Use a nonexistent file to always consider the target out-of-date. # Use a nonexistent file to always consider the target out-of-date.
ofilenames += [self.nonexistent_file(os.path.join(self.environment.get_scratch_dir(), ofilenames += [self.nonexistent_file(os.path.join(self.environment.get_scratch_dir(),
'outofdate.file'))] 'outofdate.file'))]
self.add_custom_build(root, 'custom_target', ' '.join(self.quote_arguments(wrapper_cmd)), self.add_custom_build(root, 'custom_target', ' '.join(self.quote_arguments(wrapper_cmd)),
deps=[exe_data] + srcs + depend_files, outputs=ofilenames) deps=wrapper_cmd[-1:] + srcs + depend_files, outputs=ofilenames)
ET.SubElement(root, 'Import', Project=r'$(VCTargetsPath)\Microsoft.Cpp.targets') ET.SubElement(root, 'Import', Project=r'$(VCTargetsPath)\Microsoft.Cpp.targets')
self.generate_custom_generator_commands(target, root) self.generate_custom_generator_commands(target, root)
self.add_regen_dependency(root) self.add_regen_dependency(root)

57
mesonbuild/scripts/meson_exe.py
View File

@ -20,12 +20,14 @@ import platform
import subprocess import subprocess
from .. import mesonlib from .. import mesonlib
from ..backend.backends import ExecutableSerialisation
options = None options = None
def buildparser(): def buildparser():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?')
parser.add_argument('args', nargs='+') parser.add_argument('--unpickle')
parser.add_argument('--capture')
return parser return parser
def is_windows(): def is_windows():
@ -36,28 +38,14 @@ def is_cygwin():
platname = platform.system().lower() platname = platform.system().lower()
return 'cygwin' in platname return 'cygwin' in platname
def run_with_mono(fname):
if fname.endswith('.exe') and not (is_windows() or is_cygwin()):
return True
return False
def run_exe(exe): def run_exe(exe):
if exe.fname[0].endswith('.jar'): if exe.exe_runner:
cmd = ['java', '-jar'] + exe.fname if not exe.exe_runner.found():
elif not exe.is_cross and run_with_mono(exe.fname[0]): raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} with not-found '
cmd = ['mono'] + exe.fname 'wrapper {!r}'.format(exe.cmd_args[0], exe.exe_runner.get_path()))
cmd_args = exe.exe_runner.get_command() + exe.cmd_args
else: else:
if exe.is_cross and exe.needs_exe_wrapper: cmd_args = exe.cmd_args
if exe.exe_runner is None:
raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} '
'with no wrapper'.format(exe.name))
elif not exe.exe_runner.found():
raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} with not-found '
'wrapper {!r}'.format(exe.name, exe.exe_runner.get_path()))
else:
cmd = exe.exe_runner.get_command() + exe.fname
else:
cmd = exe.fname
child_env = os.environ.copy() child_env = os.environ.copy()
child_env.update(exe.env) child_env.update(exe.env)
if exe.extra_paths: if exe.extra_paths:
@ -73,7 +61,7 @@ def run_exe(exe):
else: else:
child_env['WINEPATH'] = wine_path child_env['WINEPATH'] = wine_path
p = subprocess.Popen(cmd + exe.cmd_args, env=child_env, cwd=exe.workdir, p = subprocess.Popen(cmd_args, env=child_env, cwd=exe.workdir,
close_fds=False, close_fds=False,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
@ -101,13 +89,22 @@ def run_exe(exe):
def run(args): def run(args):
global options global options
options = buildparser().parse_args(args) parser = buildparser()
if len(options.args) != 1: options, cmd_args = parser.parse_known_args(args)
print('Test runner for Meson. Do not run on your own, mmm\'kay?') # argparse supports double dash to separate options and positional arguments,
print(sys.argv[0] + ' [data file]') # but the user has to remove it manually.
exe_data_file = options.args[0] if cmd_args and cmd_args[0] == '--':
with open(exe_data_file, 'rb') as f: cmd_args = cmd_args[1:]
exe = pickle.load(f) if not options.unpickle and not cmd_args:
parser.error('either --unpickle or executable and arguments are required')
if options.unpickle:
if cmd_args or options.capture:
parser.error('no other arguments can be used with --unpickle')
with open(options.unpickle, 'rb') as f:
exe = pickle.load(f)
else:
exe = ExecutableSerialisation(cmd_args, capture=options.capture)
return run_exe(exe) return run_exe(exe)
if __name__ == '__main__': if __name__ == '__main__':

2
test cases/common/185 escape and unicode/file.py
View File

@ -6,5 +6,5 @@ import os
with open(sys.argv[1]) as fh: with open(sys.argv[1]) as fh:
content = fh.read().replace("{NAME}", sys.argv[2]) content = fh.read().replace("{NAME}", sys.argv[2])
with open(os.path.join(sys.argv[3]), 'w') as fh: with open(os.path.join(sys.argv[3]), 'w', errors='replace') as fh:
fh.write(content) fh.write(content)