Add depfile to configure_file()

In qemu, minikconf generates a depfile that meson could use to
automatically reconfigure on dependency change.

Note: someone clever can perhaps find a way to express this with a
ninja rule & depfile=. I didn't manage, so I wrote a simple depfile
parser.

docs/markdown/Reference-manual.md

@@ -260,6 +260,9 @@ These are all the supported keyword arguments:
   substitutions.
 - `copy` *(added 0.47.0)* as explained above, if specified Meson only
   copies the file from input to output.
+- `depfile` *(added 0.52.0)* is a dependency file that the command can write listing
+  all the additional files this target depends on. A change
+  in any one of these files triggers a reconfiguration.
 - `format` *(added 0.46.0)* the format of defines. It defaults to `meson`, and so substitutes
 `#mesondefine` statements and variables surrounded by `@` characters, you can also use `cmake`
 to replace `#cmakedefine` statements and variables with the `${variable}` syntax. Finally you can use

docs/markdown/snippets/configure_file_enhancements.md

@@ -1,3 +1,6 @@
 ## Enhancements to `configure_file()`
 
 `input:` now accepts multiple input file names for `command:`-configured file.
+
+`depfile:` keyword argument is now accepted. The dependency file can
+list all the additional files the configure target depends on.

mesonbuild/depfile.py

@@ -0,0 +1,85 @@
+# Copyright 2019 Red Hat, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+def parse(lines):
+    rules = []
+    targets = []
+    deps = []
+    in_deps = False
+    out = ''
+    for line in lines:
+        if not line.endswith('\n'):
+            line += '\n'
+        escape = None
+        for c in line:
+            if escape:
+                if escape == '$' and c != '$':
+                    out += '$'
+                if escape == '\\' and c == '\n':
+                    continue
+                out += c
+                escape = None
+                continue
+            if c == '\\' or c == '$':
+                escape = c
+                continue
+            elif c in (' ', '\n'):
+                if out != '':
+                    if in_deps:
+                        deps.append(out)
+                    else:
+                        targets.append(out)
+                out = ''
+                if c == '\n':
+                    rules.append((targets, deps))
+                    targets = []
+                    deps = []
+                    in_deps = False
+                continue
+            elif c == ':':
+                targets.append(out)
+                out = ''
+                in_deps = True
+                continue
+            out += c
+    return rules
+
+Target = collections.namedtuple('Target', ['deps'])
+
+class DepFile:
+    def __init__(self, lines):
+        rules = parse(lines)
+        depfile = {}
+        for (targets, deps) in rules:
+            for target in targets:
+                t = depfile.setdefault(target, Target(deps=set()))
+                for dep in deps:
+                    t.deps.add(dep)
+        self.depfile = depfile
+
+    def get_all_dependencies(self, target, visited=None):
+        deps = set()
+        if not visited:
+            visited = set()
+        if target in visited:
+            return set()
+        visited.add(target)
+        target = self.depfile.get(target)
+        if not target:
+            return set()
+        deps.update(target.deps)
+        for dep in target.deps:
+            deps.update(self.get_all_dependencies(dep, visited))
+        return deps

mesonbuild/interpreter.py

@@ -24,6 +24,7 @@ from . import mesonlib
 from .mesonlib import FileMode, MachineChoice, Popen_safe, listify, extract_as_list, has_path_sep
 from .dependencies import ExternalProgram
 from .dependencies import InternalDependency, Dependency, NotFoundDependency, DependencyException
+from .depfile import DepFile
 from .interpreterbase import InterpreterBase
 from .interpreterbase import check_stringlist, flatten, noPosargs, noKwargs, stringArgs, permittedKwargs, noArgsFlattening
 from .interpreterbase import InterpreterException, InvalidArguments, InvalidCode, SubdirDoneRequest
@@ -1960,6 +1961,7 @@ permitted_kwargs = {'add_global_arguments': {'language', 'native'},
                                        'configuration',
                                        'command',
                                        'copy',
+                                       'depfile',
                                        'install_dir',
                                        'install_mode',
                                        'capture',
@@ -3603,6 +3605,7 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
     @FeatureNewKwargs('configure_file', '0.46.0', ['format'])
     @FeatureNewKwargs('configure_file', '0.41.0', ['capture'])
     @FeatureNewKwargs('configure_file', '0.50.0', ['install'])
+    @FeatureNewKwargs('configure_file', '0.52.0', ['depfile'])
     @permittedKwargs(permitted_kwargs['configure_file'])
     def func_configure_file(self, node, args, kwargs):
         if len(args) > 0:
@@ -3648,6 +3651,13 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
         if output_format not in ('c', 'nasm'):
             raise InterpreterException('"format" possible values are "c" or "nasm".')
 
+        if 'depfile' in kwargs:
+            depfile = kwargs['depfile']
+            if not isinstance(depfile, str):
+                raise InterpreterException('depfile file name must be a string')
+        else:
+            depfile = None
+
         # Validate input
         inputs = self.source_strings_to_files(extract_as_list(kwargs, 'input'))
         inputs_abs = []
@@ -3665,6 +3675,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
             values = mesonlib.get_filenames_templates_dict(inputs_abs, None)
             outputs = mesonlib.substitute_values([output], values)
             output = outputs[0]
+            if depfile:
+                depfile = mesonlib.substitute_values([depfile], values)[0]
         ofile_rpath = os.path.join(self.subdir, output)
         if ofile_rpath in self.configure_file_outputs:
             mesonbuildfile = os.path.join(self.subdir, 'meson.build')
@@ -3716,6 +3728,9 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
             # that the command is run from is 'unspecified', so it could change.
             # Currently it's builddir/subdir for in_builddir else srcdir/subdir.
             values = mesonlib.get_filenames_templates_dict(inputs_abs, [ofile_abs])
+            if depfile:
+                depfile = os.path.join(self.environment.get_scratch_dir(), depfile)
+                values['@DEPFILE@'] = depfile
             # Substitute @INPUT@, @OUTPUT@, etc here.
             cmd = mesonlib.substitute_values(kwargs['command'], values)
             mlog.log('Configuring', mlog.bold(output), 'with command')
@@ -3731,6 +3746,15 @@ This will become a hard error in the future.''' % kwargs['input'], location=self
                 if inputs_abs:
                     shutil.copymode(inputs_abs[0], dst_tmp)
                 mesonlib.replace_if_different(ofile_abs, dst_tmp)
+            if depfile:
+                mlog.log('Reading depfile:', mlog.bold(depfile))
+                with open(depfile, 'r') as f:
+                    df = DepFile(f.readlines())
+                    deps = df.get_all_dependencies(ofile_fname)
+                    for dep in deps:
+                        if dep not in self.build_def_files:
+                            self.build_def_files.append(dep)
+
         elif 'copy' in kwargs:
             if len(inputs_abs) != 1:
                 raise InterpreterException('Exactly one input file must be given in copy mode')

run_unittests.py

@@ -40,6 +40,7 @@ from pathlib import (PurePath, Path)
 from distutils.dir_util import copy_tree
 
 import mesonbuild.mlog
+import mesonbuild.depfile
 import mesonbuild.compilers
 import mesonbuild.environment
 import mesonbuild.mesonlib
@@ -1119,6 +1120,34 @@ class InternalTests(unittest.TestCase):
             self.assertEqual(quote_arg(arg), expected)
             self.assertEqual(split_args(expected)[0], arg)
 
+    def test_depfile(self):
+        for (f, target, expdeps) in [
+                # empty, unknown target
+                ([''], 'unknown', set()),
+                # simple target & deps
+                (['meson/foo.o  : foo.c   foo.h'], 'meson/foo.o', set({'foo.c', 'foo.h'})),
+                (['meson/foo.o: foo.c foo.h'], 'foo.c', set()),
+                # get all deps
+                (['meson/foo.o: foo.c foo.h',
+                  'foo.c: gen.py'], 'meson/foo.o', set({'foo.c', 'foo.h', 'gen.py'})),
+                (['meson/foo.o: foo.c foo.h',
+                  'foo.c: gen.py'], 'foo.c', set({'gen.py'})),
+                # linue continuation, multiple targets
+                (['foo.o \\', 'foo.h: bar'], 'foo.h', set({'bar'})),
+                (['foo.o \\', 'foo.h: bar'], 'foo.o', set({'bar'})),
+                # \\ handling
+                (['foo: Program\\ F\\iles\\\\X'], 'foo', set({'Program Files\\X'})),
+                # $ handling
+                (['f$o.o: c/b'], 'f$o.o', set({'c/b'})),
+                (['f$$o.o: c/b'], 'f$o.o', set({'c/b'})),
+                # cycles
+                (['a: b', 'b: a'], 'a', set({'a', 'b'})),
+                (['a: b', 'b: a'], 'b', set({'a', 'b'})),
+        ]:
+            d = mesonbuild.depfile.DepFile(f)
+            deps = d.get_all_dependencies(target)
+            self.assertEqual(deps, expdeps)
+
 
 @unittest.skipIf(is_tarball(), 'Skipping because this is a tarball release')
 class DataTests(unittest.TestCase):

test cases/common/14 configure file/generator-deps.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import sys, os
+from pathlib import Path
+
+if len(sys.argv) != 3:
+    print("Wrong amount of parameters.")
+
+build_dir = Path(os.environ['MESON_BUILD_ROOT'])
+subdir = Path(os.environ['MESON_SUBDIR'])
+outputf = Path(sys.argv[1])
+
+with outputf.open('w') as ofile:
+    ofile.write("#define ZERO_RESULT 0\n")
+
+depf = Path(sys.argv[2])
+if not depf.exists():
+    with depf.open('w') as ofile:
+        ofile.write("{}: depfile\n".format(outputf.name))

test cases/common/14 configure file/meson.build

@@ -57,6 +57,17 @@ if ret.returncode() != 0
   error('Error running command: @0@\n@1@'.format(ret.stdout(), ret.stderr()))
 endif
 
+genscript2deps = '@0@/generator-deps.py'.format(meson.current_source_dir())
+ofile2deps = '@0@/config2deps.h'.format(meson.current_build_dir())
+outf = configure_file(
+  output : 'config2deps.h',
+  depfile : 'depfile.d',
+  command : [genprog, genscript2deps, ofile2deps, '@DEPFILE@'])
+ret = run_command(check_file, outf)
+if ret.returncode() != 0
+  error('Error running command: @0@\n@1@'.format(ret.stdout(), ret.stderr()))
+endif
+
 found_script = find_program('generator.py')
 # More configure_file tests in here
 subdir('subdir')