backends/ninja: write depscan input files to json

Currently, we write each file to the command line, but this can result in
situations where the number of files passed exceeds OS imposed command
line limits. For compilers, we solve this with response files. For
depscan I've chosen to use a JSON list instead. JSON has several
advantages in that it's standardized, there's a built-in python module
for it, and it's familiar. I've also chosen to always use the JSON file
instead of having a heuristic to decide between JSON and not JSON,
while there may be a small performance trade off here, keeping the
implementation simple with only one path is wort it.

Fixes #9129

mesonbuild/backend/ninjabackend.py

@@ -11,18 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import typing as T
-import os
-import re
-import pickle
-import shlex
-import subprocess
+
 from collections import OrderedDict
 from enum import Enum, unique
-import itertools
-from textwrap import dedent
-from pathlib import PurePath, Path
 from functools import lru_cache
+from pathlib import PurePath, Path
+from textwrap import dedent
+import itertools
+import json
+import os
+import pickle
+import re
+import shlex
+import subprocess
+import typing as T
 
 from . import backends
 from .. import modules
@@ -915,9 +917,16 @@ class NinjaBackend(backends.Backend):
         pickle_base = target.name + '.dat'
         pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/')
         pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/')
+        json_abs = os.path.join(self.get_target_private_dir_abs(target), f'{target.name}-deps.json').replace('\\', '/')
         rule_name = 'depscan'
         scan_sources = self.select_sources_to_scan(compiled_sources)
-        elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources)
+
+        # Dump the sources as a json list. This avoids potential probllems where
+        # the number of sources passed to depscan exceedes the limit imposed by
+        # the OS.
+        with open(json_abs, 'w', encoding='utf-8') as f:
+            json.dump(scan_sources, f)
+        elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, json_abs)
         elem.add_item('picklefile', pickle_file)
         scaninfo = TargetDependencyScannerInfo(self.get_target_private_dir(target), source2object)
         with open(pickle_abs, 'wb') as p:

mesonbuild/scripts/depscan.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
 import pathlib
 import pickle
 import re
-import os
 import sys
 import typing as T
 
@@ -194,8 +195,9 @@ class DependencyScanner:
         return 0
 
 def run(args: T.List[str]) -> int:
-    pickle_file = args[0]
-    outfile = args[1]
-    sources = args[2:]
+    assert len(args) == 3, 'got wrong number of arguments!'
+    pickle_file, outfile, jsonfile = args
+    with open(jsonfile, 'r', encoding='utf-8') as f:
+        sources = json.load(f)
     scanner = DependencyScanner(pickle_file, outfile, sources)
     return scanner.scan()