From c2c2be44ed644199a5a9832bf9ac34fc3ef6b486 Mon Sep 17 00:00:00 2001 From: Tom Weaver Date: Tue, 25 May 2021 12:47:16 +0100 Subject: [PATCH] [Dexter] Add DexDeclareFile command to Dexter DexDeclareFile allows test producers to write test files with .dex extensions that contain pure dexter commands. .dex file commands do not need to be commented out like they do when written inline within test source files. DexDeclareFile commands are declarative in behaviour, they state that any Dexter command seen from this point on will have its path attribute set to the path declared in the DexDeclareFile command. Differential Revision: https://reviews.llvm.org/D99651 --- debuginfo-tests/dexter/Commands.md | 18 + debuginfo-tests/dexter/d.diff | 463 ++++++++++++++++++ .../dexter/dex/command/ParseCommand.py | 41 +- .../dex/command/commands/DexDeclareFile.py | 31 ++ .../dexter/dex/tools/TestToolBase.py | 32 +- .../dexter/dex/tools/clang_opt_bisect/Tool.py | 5 +- debuginfo-tests/dexter/dex/tools/test/Tool.py | 6 +- .../commands/penalty/dex_declare_file.cpp | 17 + .../dex_and_source/commands.dex | 2 + .../dex_and_source/lit.local.cfg.py | 1 + .../dex_declare_file/dex_and_source/test.cfg | 0 .../dex_declare_file/dex_and_source/test.cpp | 15 + .../precompiled_binary/commands.dex | 18 + .../precompiled_binary/lit.local.cfg.py | 1 + .../precompiled_binary/test.cpp | 4 + .../dex_commands/commands.dex | 19 + .../lit.local.cfg.py | 1 + .../source/test.cpp | 4 + .../lit.local.cfg.py | 1 + .../source/test file.cpp | 4 + .../windows_noncanonical_path/test.cfg | 0 .../windows_noncanonical_path/test.dex | 17 + 22 files changed, 673 insertions(+), 27 deletions(-) create mode 100644 debuginfo-tests/dexter/d.diff create mode 100644 debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py create mode 100644 debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg create mode 100644 debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex diff --git a/debuginfo-tests/dexter/Commands.md b/debuginfo-tests/dexter/Commands.md index 5de685906c01..da14b8d59ba7 100644 --- a/debuginfo-tests/dexter/Commands.md +++ b/debuginfo-tests/dexter/Commands.md @@ -9,6 +9,7 @@ * [DexLimitSteps](Commands.md#DexLimitSteps) * [DexLabel](Commands.md#DexLabel) * [DexWatch](Commands.md#DexWatch) +* [DexDeclareFile](Commands.md#DexDeclareFile) --- ## DexExpectProgramState @@ -231,6 +232,23 @@ arithmetic operators to get offsets from labels: ### Heuristic This command does not contribute to the heuristic score. +---- +## DexDeclareFile + DexDeclareFile(declared_file) + + Args: + name (str): A declared file path for which all subsequent commands + will have their path attribute set too. + +### Description +Set the path attribute of all commands from this point in the test onwards. +The new path holds until the end of the test file or until a new DexDeclareFile +command is encountered. Used in conjunction with .dex files, DexDeclareFile can +be used to write your dexter commands in a separate test file avoiding inlined +Dexter commands mixed with test source. + +### Heuristic +This command does not contribute to the heuristic score. --- ## DexWatch diff --git a/debuginfo-tests/dexter/d.diff b/debuginfo-tests/dexter/d.diff new file mode 100644 index 000000000000..fef582c14625 --- /dev/null +++ b/debuginfo-tests/dexter/d.diff @@ -0,0 +1,463 @@ +diff --git a/debuginfo-tests/dexter/Commands.md b/debuginfo-tests/dexter/Commands.md +index 5de685906c01..da14b8d59ba7 100644 +--- a/debuginfo-tests/dexter/Commands.md ++++ b/debuginfo-tests/dexter/Commands.md +@@ -9,6 +9,7 @@ + * [DexLimitSteps](Commands.md#DexLimitSteps) + * [DexLabel](Commands.md#DexLabel) + * [DexWatch](Commands.md#DexWatch) ++* [DexDeclareFile](Commands.md#DexDeclareFile) + + --- + ## DexExpectProgramState +@@ -231,6 +232,23 @@ arithmetic operators to get offsets from labels: + ### Heuristic + This command does not contribute to the heuristic score. + ++---- ++## DexDeclareFile ++ DexDeclareFile(declared_file) ++ ++ Args: ++ name (str): A declared file path for which all subsequent commands ++ will have their path attribute set too. ++ ++### Description ++Set the path attribute of all commands from this point in the test onwards. ++The new path holds until the end of the test file or until a new DexDeclareFile ++command is encountered. Used in conjunction with .dex files, DexDeclareFile can ++be used to write your dexter commands in a separate test file avoiding inlined ++Dexter commands mixed with test source. ++ ++### Heuristic ++This command does not contribute to the heuristic score. + + --- + ## DexWatch +diff --git a/debuginfo-tests/dexter/dex/command/ParseCommand.py b/debuginfo-tests/dexter/dex/command/ParseCommand.py +index c9908ef4b399..81e5c6c117f0 100644 +--- a/debuginfo-tests/dexter/dex/command/ParseCommand.py ++++ b/debuginfo-tests/dexter/dex/command/ParseCommand.py +@@ -12,12 +12,13 @@ Python code being embedded within DExTer commands. + import os + import unittest + from copy import copy +- ++from pathlib import PurePath + from collections import defaultdict, OrderedDict + + from dex.utils.Exceptions import CommandParseError + + from dex.command.CommandBase import CommandBase ++from dex.command.commands.DexDeclareFile import DexDeclareFile + from dex.command.commands.DexExpectProgramState import DexExpectProgramState + from dex.command.commands.DexExpectStepKind import DexExpectStepKind + from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder +@@ -37,6 +38,7 @@ def _get_valid_commands(): + { name (str): command (class) } + """ + return { ++ DexDeclareFile.get_name() : DexDeclareFile, + DexExpectProgramState.get_name() : DexExpectProgramState, + DexExpectStepKind.get_name() : DexExpectStepKind, + DexExpectStepOrder.get_name() : DexExpectStepOrder, +@@ -209,6 +211,8 @@ def add_line_label(labels, label, cmd_path, cmd_lineno): + + def _find_all_commands_in_file(path, file_lines, valid_commands): + labels = {} # dict of {name: line}. ++ cmd_path = path ++ declared_files = set() + commands = defaultdict(dict) + paren_balance = 0 + region_start = TextPoint(0, 0) +@@ -253,7 +257,7 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): + valid_commands[command_name], + labels, + raw_text, +- path, ++ cmd_path, + cmd_point.get_lineno(), + ) + except SyntaxError as e: +@@ -271,6 +275,14 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): + else: + if type(command) is DexLabel: + add_line_label(labels, command, path, cmd_point.get_lineno()) ++ elif type(command) is DexDeclareFile: ++ cmd_path = command.declared_file ++ if not os.path.isabs(cmd_path): ++ source_dir = os.path.dirname(path) ++ cmd_path = os.path.join(source_dir, cmd_path) ++ # TODO: keep stored paths as PurePaths for 'longer'. ++ cmd_path = str(PurePath(cmd_path)) ++ declared_files.add(cmd_path) + assert (path, cmd_point) not in commands[command_name], ( + command_name, commands[command_name]) + commands[command_name][path, cmd_point] = command +@@ -281,32 +293,34 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): + err_point.char += len(command_name) + msg = "Unbalanced parenthesis starting here" + raise format_parse_err(msg, path, file_lines, err_point) +- return dict(commands) ++ return dict(commands), declared_files + +-def _find_all_commands(source_files): ++def _find_all_commands(test_files): + commands = defaultdict(dict) + valid_commands = _get_valid_commands() +- for source_file in source_files: +- with open(source_file) as fp: ++ new_source_files = set() ++ for test_file in test_files: ++ with open(test_file) as fp: + lines = fp.readlines() +- file_commands = _find_all_commands_in_file(source_file, lines, +- valid_commands) ++ file_commands, declared_files = _find_all_commands_in_file(test_file, ++ lines, valid_commands) + for command_name in file_commands: + commands[command_name].update(file_commands[command_name]) ++ new_source_files |= declared_files + +- return dict(commands) ++ return dict(commands), new_source_files + +-def get_command_infos(source_files): ++def get_command_infos(test_files): + with Timer('parsing commands'): + try: +- commands = _find_all_commands(source_files) ++ commands, new_source_files = _find_all_commands(test_files) + command_infos = OrderedDict() + for command_type in commands: + for command in commands[command_type].values(): + if command_type not in command_infos: + command_infos[command_type] = [] + command_infos[command_type].append(command) +- return OrderedDict(command_infos) ++ return OrderedDict(command_infos), new_source_files + except CommandParseError as e: + msg = 'parser error: {}({}): {}\n{}\n{}\n'.format( + e.filename, e.lineno, e.info, e.src, e.caret) +@@ -344,7 +358,8 @@ class TestParseCommand(unittest.TestCase): + Returns: + { cmd_name: { (path, line): command_obj } } + """ +- return _find_all_commands_in_file(__file__, lines, self.valid_commands) ++ cmds, declared_files = _find_all_commands_in_file(__file__, lines, self.valid_commands) ++ return cmds + + + def _find_all_mock_values_in_lines(self, lines): +diff --git a/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py +new file mode 100644 +index 000000000000..c40c854575d9 +--- /dev/null ++++ b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py +@@ -0,0 +1,31 @@ ++# DExTer : Debugging Experience Tester ++# ~~~~~~ ~ ~~ ~ ~~ ++# ++# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. ++# See https://llvm.org/LICENSE.txt for license information. ++# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ++"""Commmand sets the path for all following commands to 'declared_file'. ++""" ++ ++from pathlib import PurePath ++ ++from dex.command.CommandBase import CommandBase ++ ++ ++class DexDeclareFile(CommandBase): ++ def __init__(self, declared_file): ++ ++ if not isinstance(declared_file, str): ++ raise TypeError('invalid argument type') ++ ++ # Use PurePath to create a cannonical platform path. ++ # TODO: keep paths as PurePath objects for 'longer' ++ self.declared_file = str(PurePath(declared_file)) ++ super(DexDeclareFile, self).__init__() ++ ++ @staticmethod ++ def get_name(): ++ return __class__.__name__ ++ ++ def eval(self): ++ return self.declared_file +diff --git a/debuginfo-tests/dexter/dex/tools/TestToolBase.py b/debuginfo-tests/dexter/dex/tools/TestToolBase.py +index a2d8a90c005e..cfea497124b5 100644 +--- a/debuginfo-tests/dexter/dex/tools/TestToolBase.py ++++ b/debuginfo-tests/dexter/dex/tools/TestToolBase.py +@@ -100,26 +100,38 @@ class TestToolBase(ToolBase): + options.executable = os.path.join( + self.context.working_directory.path, 'tmp.exe') + ++ # Test files contain dexter commands. ++ options.test_files = [] ++ # Source files are to be compiled by the builder script and may also ++ # contains dexter commands. ++ options.source_files = [] + if os.path.isdir(options.test_path): +- + subdirs = sorted([ + r for r, _, f in os.walk(options.test_path) + if 'test.cfg' in f + ]) + + for subdir in subdirs: +- +- # TODO: read file extensions from the test.cfg file instead so +- # that this isn't just limited to C and C++. +- options.source_files = [ +- os.path.normcase(os.path.join(subdir, f)) +- for f in os.listdir(subdir) if any( +- f.endswith(ext) for ext in ['.c', '.cpp']) +- ] ++ for f in os.listdir(subdir): ++ # TODO: read file extensions from the test.cfg file instead so ++ # that this isn't just limited to C and C++. ++ file_path = os.path.normcase(os.path.join(subdir, f)) ++ if f.endswith('.cpp'): ++ options.source_files.append(file_path) ++ elif f.endswith('.c'): ++ options.source_files.append(file_path) ++ elif f.endswith('.dex'): ++ options.test_files.append(file_path) ++ # Source files can contain dexter commands too. ++ options.test_files = options.test_files + options.source_files + + self._run_test(self._get_test_name(subdir)) + else: +- options.source_files = [options.test_path] ++ # We're dealing with a direct file path to a test file. If the file is non ++ # .dex, then it must be a source file. ++ if not options.test_path.endswith('.dex'): ++ options.source_files = [options.test_path] ++ options.test_files = [options.test_path] + self._run_test(self._get_test_name(options.test_path)) + + return self._handle_results() +diff --git a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +index 6e936bd98a3c..c910d9c537ca 100644 +--- a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py ++++ b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +@@ -92,8 +92,9 @@ class Tool(TestToolBase): + executable_path=self.context.options.executable, + source_paths=self.context.options.source_files, + dexter_version=self.context.version) +- step_collection.commands = get_command_infos( +- self.context.options.source_files) ++ step_collection.commands, new_source_files = get_command_infos( ++ self.context.options.test_files) ++ self.context.options.source_files.extend(list(new_source_files)) + debugger_controller = DefaultController(self.context, step_collection) + return debugger_controller + +diff --git a/debuginfo-tests/dexter/dex/tools/test/Tool.py b/debuginfo-tests/dexter/dex/tools/test/Tool.py +index 43191fd44bd5..2d3ddce8f7b6 100644 +--- a/debuginfo-tests/dexter/dex/tools/test/Tool.py ++++ b/debuginfo-tests/dexter/dex/tools/test/Tool.py +@@ -138,8 +138,10 @@ class Tool(TestToolBase): + source_paths=self.context.options.source_files, + dexter_version=self.context.version) + +- step_collection.commands = get_command_infos( +- self.context.options.source_files) ++ step_collection.commands, new_source_files = get_command_infos( ++ self.context.options.test_files) ++ ++ self.context.options.source_files.extend(list(new_source_files)) + + if 'DexLimitSteps' in step_collection.commands: + debugger_controller = ConditionalController(self.context, step_collection) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp +new file mode 100644 +index 000000000000..7860ffd5dda4 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp +@@ -0,0 +1,17 @@ ++// Purpose: ++// Check that \DexDeclareFile causes a DexExpectWatchValue's to generate a ++// missing value penalty when the declared path is incorrect. ++// ++// UNSUPPORTED: system-darwin ++// ++// ++// RUN: not %dexter_regression_test -- %s | FileCheck %s ++// CHECK: dex_declare_file.cpp ++ ++int main() { ++ int result = 0; ++ return result; //DexLabel('return') ++} ++ ++// DexDeclareFile('this_file_does_not_exist.cpp') ++// DexExpectWatchValue('result', 0, on_line='return') +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex +new file mode 100644 +index 000000000000..bbad7db943bf +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex +@@ -0,0 +1,2 @@ ++DexDeclareFile('test.cpp') ++DexExpectWatchValue('result', 0, on_line=14) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py +new file mode 100644 +index 000000000000..159c376beedb +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.cpp'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg +new file mode 100644 +index 000000000000..e69de29bb2d1 +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp +new file mode 100644 +index 000000000000..5f1d50efe8d0 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp +@@ -0,0 +1,15 @@ ++// Purpose: ++// Check that \DexDeclareFile changes the path of all succeeding commands ++// to the file path it declares. Also check that dexter correctly accepts ++// files with .dex extensions. ++// ++// UNSUPPORTED: system-darwin ++// ++// ++// RUN: %dexter_regression_test -- %S | FileCheck %s ++// CHECK: dex_and_source ++ ++int main() { ++ int result = 0; ++ return result; ++} +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex +new file mode 100644 +index 000000000000..1aec2f8f3b64 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex +@@ -0,0 +1,18 @@ ++# Purpose: ++# Check that \DexDeclareFile's file declaration can reference source files ++# in a precompiled binary. ++# ++# UNSUPPORTED: system-darwin ++# ++# RUN: %clang %S/test.cpp -O0 -g -o %t ++# RUN: %dexter_regression_test --binary %t %s | FileCheck %s ++# CHECK: commands.dex ++# ++# test.cpp ++# 1. int main() { ++# 2. int result = 0; ++# 3. return result; ++# 4. } ++ ++DexDeclareFile('test.cpp') ++DexExpectWatchValue('result', 0, on_line=3) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py +new file mode 100644 +index 000000000000..e65498f23dde +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.dex'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp +new file mode 100644 +index 000000000000..4d3cc5846e66 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp +@@ -0,0 +1,4 @@ ++int main() { ++ int result = 0; ++ return result; ++} +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex +new file mode 100644 +index 000000000000..964c770d3325 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex +@@ -0,0 +1,19 @@ ++# Purpose: ++# Check that \DexDeclareFile's file declaration can reference source files ++# not included in the test directory ++# ++# UNSUPPORTED: system-darwin ++# ++# RUN: %clang %S/../source/test.cpp -O0 -g -o %t ++# RUN: %dexter_regression_test --binary %t %s | FileCheck %s ++# RUN: rm %t ++# CHECK: commands.dex ++# ++# test.cpp ++# 1. int main() { ++# 2. int result = 0; ++# 3. return result; ++# 4. } ++ ++DexDeclareFile('../source/test.cpp') ++DexExpectWatchValue('result', 0, on_line=3) +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py +new file mode 100644 +index 000000000000..e65498f23dde +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.dex'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp +new file mode 100644 +index 000000000000..4d3cc5846e66 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp +@@ -0,0 +1,4 @@ ++int main() { ++ int result = 0; ++ return result; ++} +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py +new file mode 100644 +index 000000000000..e65498f23dde +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py +@@ -0,0 +1 @@ ++config.suffixes = ['.dex'] +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp +new file mode 100644 +index 000000000000..f6dcd82e93e7 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp +@@ -0,0 +1,4 @@ ++int main(const int argc, const char * argv[]) { ++ int result = argc; ++ return result; ++} +\ No newline at end of file +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg +new file mode 100644 +index 000000000000..e69de29bb2d1 +diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex +new file mode 100644 +index 000000000000..d9c9b80044b6 +--- /dev/null ++++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex +@@ -0,0 +1,17 @@ ++# Purpose: ++# Check that non-canonical paths resolve correctly on Windows. ++# ++# REQUIRES: system-windows ++# ++# RUN: %clang "%S/source/test file.cpp" -O0 -g -o %t ++# RUN: %dexter_regression_test --binary %t %s | FileCheck %s ++# CHECK: test.dex ++# ++# ./source/test file.cpp ++# 1 int main(const int argc, const char * argv[]) { ++# 2 int result = argc; ++# 3 return result; ++# 4 } ++ ++DexDeclareFile('./sOuRce\\test filE.cpp') ++DexExpectWatchValue('result', 1, on_line=3) diff --git a/debuginfo-tests/dexter/dex/command/ParseCommand.py b/debuginfo-tests/dexter/dex/command/ParseCommand.py index c9908ef4b399..81e5c6c117f0 100644 --- a/debuginfo-tests/dexter/dex/command/ParseCommand.py +++ b/debuginfo-tests/dexter/dex/command/ParseCommand.py @@ -12,12 +12,13 @@ Python code being embedded within DExTer commands. import os import unittest from copy import copy - +from pathlib import PurePath from collections import defaultdict, OrderedDict from dex.utils.Exceptions import CommandParseError from dex.command.CommandBase import CommandBase +from dex.command.commands.DexDeclareFile import DexDeclareFile from dex.command.commands.DexExpectProgramState import DexExpectProgramState from dex.command.commands.DexExpectStepKind import DexExpectStepKind from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder @@ -37,6 +38,7 @@ def _get_valid_commands(): { name (str): command (class) } """ return { + DexDeclareFile.get_name() : DexDeclareFile, DexExpectProgramState.get_name() : DexExpectProgramState, DexExpectStepKind.get_name() : DexExpectStepKind, DexExpectStepOrder.get_name() : DexExpectStepOrder, @@ -209,6 +211,8 @@ def add_line_label(labels, label, cmd_path, cmd_lineno): def _find_all_commands_in_file(path, file_lines, valid_commands): labels = {} # dict of {name: line}. + cmd_path = path + declared_files = set() commands = defaultdict(dict) paren_balance = 0 region_start = TextPoint(0, 0) @@ -253,7 +257,7 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): valid_commands[command_name], labels, raw_text, - path, + cmd_path, cmd_point.get_lineno(), ) except SyntaxError as e: @@ -271,6 +275,14 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): else: if type(command) is DexLabel: add_line_label(labels, command, path, cmd_point.get_lineno()) + elif type(command) is DexDeclareFile: + cmd_path = command.declared_file + if not os.path.isabs(cmd_path): + source_dir = os.path.dirname(path) + cmd_path = os.path.join(source_dir, cmd_path) + # TODO: keep stored paths as PurePaths for 'longer'. + cmd_path = str(PurePath(cmd_path)) + declared_files.add(cmd_path) assert (path, cmd_point) not in commands[command_name], ( command_name, commands[command_name]) commands[command_name][path, cmd_point] = command @@ -281,32 +293,34 @@ def _find_all_commands_in_file(path, file_lines, valid_commands): err_point.char += len(command_name) msg = "Unbalanced parenthesis starting here" raise format_parse_err(msg, path, file_lines, err_point) - return dict(commands) + return dict(commands), declared_files -def _find_all_commands(source_files): +def _find_all_commands(test_files): commands = defaultdict(dict) valid_commands = _get_valid_commands() - for source_file in source_files: - with open(source_file) as fp: + new_source_files = set() + for test_file in test_files: + with open(test_file) as fp: lines = fp.readlines() - file_commands = _find_all_commands_in_file(source_file, lines, - valid_commands) + file_commands, declared_files = _find_all_commands_in_file(test_file, + lines, valid_commands) for command_name in file_commands: commands[command_name].update(file_commands[command_name]) + new_source_files |= declared_files - return dict(commands) + return dict(commands), new_source_files -def get_command_infos(source_files): +def get_command_infos(test_files): with Timer('parsing commands'): try: - commands = _find_all_commands(source_files) + commands, new_source_files = _find_all_commands(test_files) command_infos = OrderedDict() for command_type in commands: for command in commands[command_type].values(): if command_type not in command_infos: command_infos[command_type] = [] command_infos[command_type].append(command) - return OrderedDict(command_infos) + return OrderedDict(command_infos), new_source_files except CommandParseError as e: msg = 'parser error: {}({}): {}\n{}\n{}\n'.format( e.filename, e.lineno, e.info, e.src, e.caret) @@ -344,7 +358,8 @@ class TestParseCommand(unittest.TestCase): Returns: { cmd_name: { (path, line): command_obj } } """ - return _find_all_commands_in_file(__file__, lines, self.valid_commands) + cmds, declared_files = _find_all_commands_in_file(__file__, lines, self.valid_commands) + return cmds def _find_all_mock_values_in_lines(self, lines): diff --git a/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py new file mode 100644 index 000000000000..c40c854575d9 --- /dev/null +++ b/debuginfo-tests/dexter/dex/command/commands/DexDeclareFile.py @@ -0,0 +1,31 @@ +# DExTer : Debugging Experience Tester +# ~~~~~~ ~ ~~ ~ ~~ +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +"""Commmand sets the path for all following commands to 'declared_file'. +""" + +from pathlib import PurePath + +from dex.command.CommandBase import CommandBase + + +class DexDeclareFile(CommandBase): + def __init__(self, declared_file): + + if not isinstance(declared_file, str): + raise TypeError('invalid argument type') + + # Use PurePath to create a cannonical platform path. + # TODO: keep paths as PurePath objects for 'longer' + self.declared_file = str(PurePath(declared_file)) + super(DexDeclareFile, self).__init__() + + @staticmethod + def get_name(): + return __class__.__name__ + + def eval(self): + return self.declared_file diff --git a/debuginfo-tests/dexter/dex/tools/TestToolBase.py b/debuginfo-tests/dexter/dex/tools/TestToolBase.py index a2d8a90c005e..cfea497124b5 100644 --- a/debuginfo-tests/dexter/dex/tools/TestToolBase.py +++ b/debuginfo-tests/dexter/dex/tools/TestToolBase.py @@ -100,26 +100,38 @@ class TestToolBase(ToolBase): options.executable = os.path.join( self.context.working_directory.path, 'tmp.exe') + # Test files contain dexter commands. + options.test_files = [] + # Source files are to be compiled by the builder script and may also + # contains dexter commands. + options.source_files = [] if os.path.isdir(options.test_path): - subdirs = sorted([ r for r, _, f in os.walk(options.test_path) if 'test.cfg' in f ]) for subdir in subdirs: - - # TODO: read file extensions from the test.cfg file instead so - # that this isn't just limited to C and C++. - options.source_files = [ - os.path.normcase(os.path.join(subdir, f)) - for f in os.listdir(subdir) if any( - f.endswith(ext) for ext in ['.c', '.cpp']) - ] + for f in os.listdir(subdir): + # TODO: read file extensions from the test.cfg file instead so + # that this isn't just limited to C and C++. + file_path = os.path.normcase(os.path.join(subdir, f)) + if f.endswith('.cpp'): + options.source_files.append(file_path) + elif f.endswith('.c'): + options.source_files.append(file_path) + elif f.endswith('.dex'): + options.test_files.append(file_path) + # Source files can contain dexter commands too. + options.test_files = options.test_files + options.source_files self._run_test(self._get_test_name(subdir)) else: - options.source_files = [options.test_path] + # We're dealing with a direct file path to a test file. If the file is non + # .dex, then it must be a source file. + if not options.test_path.endswith('.dex'): + options.source_files = [options.test_path] + options.test_files = [options.test_path] self._run_test(self._get_test_name(options.test_path)) return self._handle_results() diff --git a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py index 6e936bd98a3c..c910d9c537ca 100644 --- a/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +++ b/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py @@ -92,8 +92,9 @@ class Tool(TestToolBase): executable_path=self.context.options.executable, source_paths=self.context.options.source_files, dexter_version=self.context.version) - step_collection.commands = get_command_infos( - self.context.options.source_files) + step_collection.commands, new_source_files = get_command_infos( + self.context.options.test_files) + self.context.options.source_files.extend(list(new_source_files)) debugger_controller = DefaultController(self.context, step_collection) return debugger_controller diff --git a/debuginfo-tests/dexter/dex/tools/test/Tool.py b/debuginfo-tests/dexter/dex/tools/test/Tool.py index 43191fd44bd5..2d3ddce8f7b6 100644 --- a/debuginfo-tests/dexter/dex/tools/test/Tool.py +++ b/debuginfo-tests/dexter/dex/tools/test/Tool.py @@ -138,8 +138,10 @@ class Tool(TestToolBase): source_paths=self.context.options.source_files, dexter_version=self.context.version) - step_collection.commands = get_command_infos( - self.context.options.source_files) + step_collection.commands, new_source_files = get_command_infos( + self.context.options.test_files) + + self.context.options.source_files.extend(list(new_source_files)) if 'DexLimitSteps' in step_collection.commands: debugger_controller = ConditionalController(self.context, step_collection) diff --git a/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp new file mode 100644 index 000000000000..7860ffd5dda4 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/penalty/dex_declare_file.cpp @@ -0,0 +1,17 @@ +// Purpose: +// Check that \DexDeclareFile causes a DexExpectWatchValue's to generate a +// missing value penalty when the declared path is incorrect. +// +// UNSUPPORTED: system-darwin +// +// +// RUN: not %dexter_regression_test -- %s | FileCheck %s +// CHECK: dex_declare_file.cpp + +int main() { + int result = 0; + return result; //DexLabel('return') +} + +// DexDeclareFile('this_file_does_not_exist.cpp') +// DexExpectWatchValue('result', 0, on_line='return') diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex new file mode 100644 index 000000000000..bbad7db943bf --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/commands.dex @@ -0,0 +1,2 @@ +DexDeclareFile('test.cpp') +DexExpectWatchValue('result', 0, on_line=14) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py new file mode 100644 index 000000000000..159c376beedb --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.cpp'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cfg new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp new file mode 100644 index 000000000000..5f1d50efe8d0 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/dex_and_source/test.cpp @@ -0,0 +1,15 @@ +// Purpose: +// Check that \DexDeclareFile changes the path of all succeeding commands +// to the file path it declares. Also check that dexter correctly accepts +// files with .dex extensions. +// +// UNSUPPORTED: system-darwin +// +// +// RUN: %dexter_regression_test -- %S | FileCheck %s +// CHECK: dex_and_source + +int main() { + int result = 0; + return result; +} diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex new file mode 100644 index 000000000000..1aec2f8f3b64 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/commands.dex @@ -0,0 +1,18 @@ +# Purpose: +# Check that \DexDeclareFile's file declaration can reference source files +# in a precompiled binary. +# +# UNSUPPORTED: system-darwin +# +# RUN: %clang %S/test.cpp -O0 -g -o %t +# RUN: %dexter_regression_test --binary %t %s | FileCheck %s +# CHECK: commands.dex +# +# test.cpp +# 1. int main() { +# 2. int result = 0; +# 3. return result; +# 4. } + +DexDeclareFile('test.cpp') +DexExpectWatchValue('result', 0, on_line=3) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py new file mode 100644 index 000000000000..e65498f23dde --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.dex'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp new file mode 100644 index 000000000000..4d3cc5846e66 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary/test.cpp @@ -0,0 +1,4 @@ +int main() { + int result = 0; + return result; +} diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex new file mode 100644 index 000000000000..964c770d3325 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/dex_commands/commands.dex @@ -0,0 +1,19 @@ +# Purpose: +# Check that \DexDeclareFile's file declaration can reference source files +# not included in the test directory +# +# UNSUPPORTED: system-darwin +# +# RUN: %clang %S/../source/test.cpp -O0 -g -o %t +# RUN: %dexter_regression_test --binary %t %s | FileCheck %s +# RUN: rm %t +# CHECK: commands.dex +# +# test.cpp +# 1. int main() { +# 2. int result = 0; +# 3. return result; +# 4. } + +DexDeclareFile('../source/test.cpp') +DexExpectWatchValue('result', 0, on_line=3) diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py new file mode 100644 index 000000000000..e65498f23dde --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.dex'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp new file mode 100644 index 000000000000..4d3cc5846e66 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/precompiled_binary_different_dir/source/test.cpp @@ -0,0 +1,4 @@ +int main() { + int result = 0; + return result; +} diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py new file mode 100644 index 000000000000..e65498f23dde --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/lit.local.cfg.py @@ -0,0 +1 @@ +config.suffixes = ['.dex'] diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp new file mode 100644 index 000000000000..f6dcd82e93e7 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/source/test file.cpp @@ -0,0 +1,4 @@ +int main(const int argc, const char * argv[]) { + int result = argc; + return result; +} \ No newline at end of file diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.cfg new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex new file mode 100644 index 000000000000..d9c9b80044b6 --- /dev/null +++ b/debuginfo-tests/dexter/feature_tests/commands/perfect/dex_declare_file/windows_noncanonical_path/test.dex @@ -0,0 +1,17 @@ +# Purpose: +# Check that non-canonical paths resolve correctly on Windows. +# +# REQUIRES: system-windows +# +# RUN: %clang "%S/source/test file.cpp" -O0 -g -o %t +# RUN: %dexter_regression_test --binary %t %s | FileCheck %s +# CHECK: test.dex +# +# ./source/test file.cpp +# 1 int main(const int argc, const char * argv[]) { +# 2 int result = argc; +# 3 return result; +# 4 } + +DexDeclareFile('./sOuRce\\test filE.cpp') +DexExpectWatchValue('result', 1, on_line=3)