Virtualization
/
meson
mirror of https://gitlab.com/qemu-project/meson.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4929 from mensinda/rwAddRmTgt 

rewriter: Add and remove targets
This commit is contained in:
Jussi Pakkanen 2019-02-19 00:08:58 +02:00 committed by GitHub
parent a9e63568fe 4b7b5a7185
commit 6e15bcc504
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 237 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
mesonbuild/ast/interpreter.py
View File

@ -15,12 +15,14 @@
# This class contains the basic functionality needed to run any interpreter
# or an interpreter-based tool.
from .visitor import AstVisitor
from .. import interpreterbase, mparser, mesonlib
from .. import environment
from ..interpreterbase import InvalidArguments, BreakRequest, ContinueRequest
import os, sys
from typing import List
class DontCareObject(interpreterbase.InterpreterObject):
pass
@ -44,10 +46,12 @@ ADD_SOURCE = 0
REMOVE_SOURCE = 1
class AstInterpreter(interpreterbase.InterpreterBase):
def __init__(self, source_root, subdir):
def __init__(self, source_root: str, subdir: str, visitors: List[AstVisitor] = []):
super().__init__(source_root, subdir)
self.visitors = visitors
self.visited_subdirs = {}
self.assignments = {}
self.reverse_assignment = {}
self.funcs.update({'project': self.func_do_nothing,
'test': self.func_do_nothing,
'benchmark': self.func_do_nothing,
@ -104,6 +108,11 @@ class AstInterpreter(interpreterbase.InterpreterBase):
def func_do_nothing(self, node, args, kwargs):
return True
def load_root_meson_file(self):
super().load_root_meson_file()
for i in self.visitors:
self.ast.accept(i)
def func_subdir(self, node, args, kwargs):
args = self.flatten_args(args)
if len(args) != 1 or not isinstance(args[0], str):
@ -134,6 +143,8 @@ class AstInterpreter(interpreterbase.InterpreterBase):
raise me
self.subdir = subdir
for i in self.visitors:
codeblock.accept(i)
self.evaluate_codeblock(codeblock)
self.subdir = prev_subdir
@ -148,6 +159,8 @@ class AstInterpreter(interpreterbase.InterpreterBase):
if node.var_name not in self.assignments:
self.assignments[node.var_name] = []
self.assignments[node.var_name] += [node.value] # Save a reference to the value node
if hasattr(node.value, 'ast_id'):
self.reverse_assignment[node.value.ast_id] = node
self.evaluate_statement(node.value) # Evaluate the value just in case
def evaluate_indexing(self, node):
@ -185,6 +198,8 @@ class AstInterpreter(interpreterbase.InterpreterBase):
def assignment(self, node):
assert(isinstance(node, mparser.AssignmentNode))
self.assignments[node.var_name] = [node.value] # Save a reference to the value node
if hasattr(node.value, 'ast_id'):
self.reverse_assignment[node.value.ast_id] = node
self.evaluate_statement(node.value) # Evaluate the value just in case
def flatten_args(self, args, include_unknown_args: bool = False):

10
mesonbuild/ast/introspection.py
View File

@ -34,8 +34,8 @@ class IntrospectionHelper:
class IntrospectionInterpreter(AstInterpreter):
# Interpreter to detect the options without a build directory
# Most of the code is stolen from interperter.Interpreter
def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None):
super().__init__(source_root, subdir)
def __init__(self, source_root, subdir, backend, visitors=[], cross_file=None, subproject='', subproject_dir='subprojects', env=None):
super().__init__(source_root, subdir, visitors=visitors)
options = IntrospectionHelper(cross_file)
self.cross_file = cross_file
@ -162,9 +162,9 @@ class IntrospectionInterpreter(AstInterpreter):
# Try to resolve the ID and append the node to the queue
id = curr.value
if id in self.assignments and self.assignments[id]:
node = self.assignments[id][0]
if isinstance(node, (mparser.ArrayNode, mparser.IdNode, mparser.FunctionNode)):
srcqueue += [node]
tmp_node = self.assignments[id][0]
if isinstance(tmp_node, (mparser.ArrayNode, mparser.IdNode, mparser.FunctionNode)):
srcqueue += [tmp_node]
if arg_node is None:
continue
elemetary_nodes = list(filter(lambda x: isinstance(x, (str, mparser.StringNode)), arg_node.arguments))

10
mesonbuild/mparser.py
View File

@ -358,7 +358,8 @@ class FunctionNode(BaseNode):
self.args = args
class AssignmentNode(BaseNode):
def __init__(self, lineno, colno, var_name, value):
def __init__(self, subdir, lineno, colno, var_name, value):
self.subdir = subdir
self.lineno = lineno
self.colno = colno
self.var_name = var_name
@ -366,7 +367,8 @@ class AssignmentNode(BaseNode):
self.value = value
class PlusAssignmentNode(BaseNode):
def __init__(self, lineno, colno, var_name, value):
def __init__(self, subdir, lineno, colno, var_name, value):
self.subdir = subdir
self.lineno = lineno
self.colno = colno
self.var_name = var_name
@ -522,13 +524,13 @@ class Parser:
value = self.e1()
if not isinstance(left, IdNode):
raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno)
return PlusAssignmentNode(left.lineno, left.colno, left.value, value)
return PlusAssignmentNode(left.subdir, left.lineno, left.colno, left.value, value)
elif self.accept('assign'):
value = self.e1()
if not isinstance(left, IdNode):
raise ParseException('Assignment target must be an id.',
self.getline(), left.lineno, left.colno)
return AssignmentNode(left.lineno, left.colno, left.value, value)
return AssignmentNode(left.subdir, left.lineno, left.colno, left.value, value)
elif self.accept('questionmark'):
if self.in_ternary:
raise ParseException('Nested ternary operators are not allowed.',

136
mesonbuild/rewriter.py
View File

@ -28,6 +28,7 @@ from mesonbuild.mesonlib import MesonException
from . import mlog, mparser, environment
from functools import wraps
from pprint import pprint
from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, IdNode, FunctionNode, StringNode
import json, os
class RewriterException(MesonException):
@ -251,8 +252,10 @@ rewriter_keys = {
},
'target': {
'target': (str, None, None),
'operation': (str, None, ['src_add', 'src_rm', 'info']),
'operation': (str, None, ['src_add', 'src_rm', 'tgt_rm', 'tgt_add', 'info']),
'sources': (list, [], None),
'subdir': (str, '', None),
'target_type': (str, 'executable', ['both_libraries', 'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library']),
'debug': (bool, False, None)
}
}
@ -292,9 +295,10 @@ rewriter_func_kwargs = {
class Rewriter:
def __init__(self, sourcedir: str, generator: str = 'ninja'):
self.sourcedir = sourcedir
self.interpreter = IntrospectionInterpreter(sourcedir, '', generator)
self.id_generator = AstIDGenerator()
self.interpreter = IntrospectionInterpreter(sourcedir, '', generator, visitors = [AstIDGenerator(), AstIndentationGenerator()])
self.modefied_nodes = []
self.to_remove_nodes = []
self.to_add_nodes = []
self.functions = {
'kwargs': self.process_kwargs,
'target': self.process_target,
@ -306,8 +310,6 @@ class Rewriter:
self.interpreter.analyze()
mlog.log(' -- Project:', mlog.bold(self.interpreter.project_data['descriptive_name']))
mlog.log(' -- Version:', mlog.cyan(self.interpreter.project_data['version']))
self.interpreter.ast.accept(AstIndentationGenerator())
self.interpreter.ast.accept(self.id_generator)
def add_info(self, cmd_type: str, cmd_id: str, data: dict):
if self.info_dump is None:
@ -456,11 +458,16 @@ class Rewriter:
if num_changed > 0 and node not in self.modefied_nodes:
self.modefied_nodes += [node]
def find_assignment_node(self, node: mparser) -> AssignmentNode:
if hasattr(node, 'ast_id') and node.ast_id in self.interpreter.reverse_assignment:
return self.interpreter.reverse_assignment[node.ast_id]
return None
@RequiredKeys(rewriter_keys['target'])
def process_target(self, cmd):
mlog.log('Processing target', mlog.bold(cmd['target']), 'operation', mlog.cyan(cmd['operation']))
target = self.find_target(cmd['target'])
if target is None:
if target is None and cmd['operation'] != 'tgt_add':
mlog.error('Unknown target "{}" --> skipping'.format(cmd['target']))
if cmd['debug']:
pprint(self.interpreter.targets)
@ -471,13 +478,13 @@ class Rewriter:
# Utility function to get a list of the sources from a node
def arg_list_from_node(n):
args = []
if isinstance(n, mparser.FunctionNode):
if isinstance(n, FunctionNode):
args = list(n.args.arguments)
if n.func_name in build_target_functions:
args.pop(0)
elif isinstance(n, mparser.ArrayNode):
elif isinstance(n, ArrayNode):
args = n.args.arguments
elif isinstance(n, mparser.ArgumentNode):
elif isinstance(n, ArgumentNode):
args = n.arguments
return args
@ -494,15 +501,15 @@ class Rewriter:
for i in cmd['sources']:
mlog.log(' -- Adding source', mlog.green(i), 'at',
mlog.yellow('{}:{}'.format(os.path.join(node.subdir, environment.build_filename), node.lineno)))
token = mparser.Token('string', node.subdir, 0, 0, 0, None, i)
to_append += [mparser.StringNode(token)]
token = Token('string', node.subdir, 0, 0, 0, None, i)
to_append += [StringNode(token)]
# Append to the AST at the right place
if isinstance(node, mparser.FunctionNode):
if isinstance(node, FunctionNode):
node.args.arguments += to_append
elif isinstance(node, mparser.ArrayNode):
elif isinstance(node, ArrayNode):
node.args.arguments += to_append
elif isinstance(node, mparser.ArgumentNode):
elif isinstance(node, ArgumentNode):
node.arguments += to_append
# Mark the node as modified
@ -514,7 +521,7 @@ class Rewriter:
def find_node(src):
for i in target['sources']:
for j in arg_list_from_node(i):
if isinstance(j, mparser.StringNode):
if isinstance(j, StringNode):
if j.value == src:
return i, j
return None, None
@ -528,11 +535,11 @@ class Rewriter:
# Remove the found string node from the argument list
arg_node = None
if isinstance(root, mparser.FunctionNode):
if isinstance(root, FunctionNode):
arg_node = root.args
if isinstance(root, mparser.ArrayNode):
if isinstance(root, ArrayNode):
arg_node = root.args
if isinstance(root, mparser.ArgumentNode):
if isinstance(root, ArgumentNode):
arg_node = root
assert(arg_node is not None)
mlog.log(' -- Removing source', mlog.green(i), 'from',
@ -543,12 +550,47 @@ class Rewriter:
if root not in self.modefied_nodes:
self.modefied_nodes += [root]
elif cmd['operation'] == 'tgt_add':
if target is not None:
mlog.error('Can not add target', mlog.bold(cmd['target']), 'because it already exists')
return
# Build src list
src_arg_node = ArgumentNode(Token('string', cmd['subdir'], 0, 0, 0, None, ''))
src_arr_node = ArrayNode(src_arg_node, 0, 0)
src_far_node = ArgumentNode(Token('string', cmd['subdir'], 0, 0, 0, None, ''))
src_fun_node = FunctionNode(cmd['subdir'], 0, 0, 'files', src_far_node)
src_ass_node = AssignmentNode(cmd['subdir'], 0, 0, '{}_src'.format(cmd['target']), src_fun_node)
src_arg_node.arguments = [StringNode(Token('string', cmd['subdir'], 0, 0, 0, None, x)) for x in cmd['sources']]
src_far_node.arguments = [src_arr_node]
# Build target
tgt_arg_node = ArgumentNode(Token('string', cmd['subdir'], 0, 0, 0, None, ''))
tgt_fun_node = FunctionNode(cmd['subdir'], 0, 0, cmd['target_type'], tgt_arg_node)
tgt_ass_node = AssignmentNode(cmd['subdir'], 0, 0, '{}_tgt'.format(cmd['target']), tgt_fun_node)
tgt_arg_node.arguments = [
StringNode(Token('string', cmd['subdir'], 0, 0, 0, None, cmd['target'])),
IdNode(Token('string', cmd['subdir'], 0, 0, 0, None, '{}_src'.format(cmd['target'])))
]
src_ass_node.accept(AstIndentationGenerator())
tgt_ass_node.accept(AstIndentationGenerator())
self.to_add_nodes += [src_ass_node, tgt_ass_node]
elif cmd['operation'] == 'tgt_rm':
to_remove = self.find_assignment_node(target['node'])
if to_remove is None:
to_remove = target['node']
self.to_remove_nodes += [to_remove]
mlog.log(' -- Removing target', mlog.green(cmd['target']), 'at',
mlog.yellow('{}:{}'.format(os.path.join(to_remove.subdir, environment.build_filename), to_remove.lineno)))
elif cmd['operation'] == 'info':
# List all sources in the target
src_list = []
for i in target['sources']:
for j in arg_list_from_node(i):
if isinstance(j, mparser.StringNode):
if isinstance(j, StringNode):
src_list += [j.value]
test_data = {
'name': target['name'],
@ -566,20 +608,29 @@ class Rewriter:
def apply_changes(self):
assert(all(hasattr(x, 'lineno') and hasattr(x, 'colno') and hasattr(x, 'subdir') for x in self.modefied_nodes))
assert(all(isinstance(x, (mparser.ArrayNode, mparser.FunctionNode)) for x in self.modefied_nodes))
assert(all(hasattr(x, 'lineno') and hasattr(x, 'colno') and hasattr(x, 'subdir') for x in self.to_remove_nodes))
assert(all(isinstance(x, (ArrayNode, FunctionNode)) for x in self.modefied_nodes))
assert(all(isinstance(x, (ArrayNode, AssignmentNode, FunctionNode)) for x in self.to_remove_nodes))
# Sort based on line and column in reversed order
work_nodes = list(sorted(self.modefied_nodes, key=lambda x: x.lineno * 1000 + x.colno, reverse=True))
work_nodes = [{'node': x, 'action': 'modify'} for x in self.modefied_nodes]
work_nodes += [{'node': x, 'action': 'rm'} for x in self.to_remove_nodes]
work_nodes = list(sorted(work_nodes, key=lambda x: x['node'].lineno * 1000 + x['node'].colno, reverse=True))
work_nodes += [{'node': x, 'action': 'add'} for x in self.to_add_nodes]
# Generating the new replacement string
str_list = []
for i in work_nodes:
printer = AstPrinter()
i.accept(printer)
printer.post_process()
new_data = ''
if i['action'] == 'modify' or i['action'] == 'add':
printer = AstPrinter()
i['node'].accept(printer)
printer.post_process()
new_data = printer.result.strip()
data = {
'file': os.path.join(i.subdir, environment.build_filename),
'str': printer.result.strip(),
'node': i
'file': os.path.join(i['node'].subdir, environment.build_filename),
'str': new_data,
'node': i['node'],
'action': i['action']
}
str_list += [data]
@ -590,6 +641,10 @@ class Rewriter:
continue
fpath = os.path.realpath(os.path.join(self.sourcedir, i['file']))
fdata = ''
# Create an empty file if it does not exist
if not os.path.exists(fpath):
with open(fpath, 'w'):
pass
with open(fpath, 'r') as fp:
fdata = fp.read()
@ -608,7 +663,7 @@ class Rewriter:
}
# Replace in source code
for i in str_list:
def remove_node(i):
offsets = files[i['file']]['offsets']
raw = files[i['file']]['raw']
node = i['node']
@ -616,10 +671,10 @@ class Rewriter:
col = node.colno
start = offsets[line] + col
end = start
if isinstance(node, mparser.ArrayNode):
if isinstance(node, ArrayNode):
if raw[end] != '[':
mlog.warning('Internal error: expected "[" at {}:{} but got "{}"'.format(line, col, raw[end]))
continue
return
counter = 1
while counter > 0:
end += 1
@ -628,7 +683,8 @@ class Rewriter:
elif raw[end] == ']':
counter -= 1
end += 1
elif isinstance(node, mparser.FunctionNode):
elif isinstance(node, FunctionNode):
while raw[end] != '(':
end += 1
end += 1
@ -640,8 +696,26 @@ class Rewriter:
elif raw[end] == ')':
counter -= 1
end += 1
# Only removal is supported for assignments
elif isinstance(node, AssignmentNode) and i['action'] == 'rm':
if isinstance(node.value, (ArrayNode, FunctionNode)):
remove_node({'file': i['file'], 'str': '', 'node': node.value, 'action': 'rm'})
raw = files[i['file']]['raw']
while raw[end] != '=':
end += 1
end += 1 # Handle the '='
while raw[end] in [' ', '\n', '\t']:
end += 1
raw = files[i['file']]['raw'] = raw[:start] + i['str'] + raw[end:]
for i in str_list:
if i['action'] in ['modify', 'rm']:
remove_node(i)
elif i['action'] in ['add']:
files[i['file']]['raw'] += i['str'] + '\n'
# Write the files back
for key, val in files.items():
mlog.log('Rewriting', mlog.yellow(key))

56
run_unittests.py
View File

@ -5188,6 +5188,62 @@ class RewriterTests(BasePlatformTests):
out = self.extract_test_data(out)
self.assertDictEqual(list(out['target'].values())[0], expected)
def test_target_remove(self):
self.prime('1 basic')
self.rewrite(self.builddir, os.path.join(self.builddir, 'rmTgt.json'))
out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
out = self.extract_test_data(out)
expected = {
'target': {
'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp']},
'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileB.cpp', 'fileC.cpp']},
'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp']},
'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp']},
}
}
self.assertDictEqual(out, expected)
def test_tatrget_add(self):
self.prime('1 basic')
self.rewrite(self.builddir, os.path.join(self.builddir, 'addTgt.json'))
out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
out = self.extract_test_data(out)
expected = {
'target': {
'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp']},
'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileB.cpp', 'fileC.cpp']},
'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp']},
'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp']},
'trivialprog10@sha': {'name': 'trivialprog10', 'sources': ['new1.cpp', 'new2.cpp']},
}
}
self.assertDictEqual(out, expected)
def test_target_remove_subdir(self):
self.prime('2 subdirs')
self.rewrite(self.builddir, os.path.join(self.builddir, 'rmTgt.json'))
out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
out = self.extract_test_data(out)
self.assertDictEqual(out, {})
def test_tatrget_add_subdir(self):
self.prime('2 subdirs')
self.rewrite(self.builddir, os.path.join(self.builddir, 'addTgt.json'))
out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
out = self.extract_test_data(out)
expected = {'name': 'something', 'sources': ['first.c', 'second.c']}
self.assertDictEqual(list(out['target'].values())[0], expected)
def test_kwargs_info(self):
self.prime('3 kwargs')
out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))

9
test cases/rewrite/1 basic/addTgt.json Normal file
View File

@ -0,0 +1,9 @@
[
{
"type": "target",
"target": "trivialprog10",
"operation": "tgt_add",
"sources": ["new1.cpp", "new2.cpp"],
"target_type": "shared_library"
}
]

5
test cases/rewrite/1 basic/info.json
View File

@ -43,5 +43,10 @@
"type": "target",
"target": "trivialprog9",
"operation": "info"
},
{
"type": "target",
"target": "trivialprog10",
"operation": "info"
}
]

2
test cases/rewrite/1 basic/meson.build
View File

@ -15,4 +15,4 @@ exe5 = executable('trivialprog5', [src2, 'main.cpp'])
exe6 = executable('trivialprog6', 'main.cpp', 'fileA.cpp')
exe7 = executable('trivialprog7', 'fileB.cpp', src1, 'fileC.cpp')
exe8 = executable('trivialprog8', src3)
exe9 = executable('trivialprog9', src4)
executable('trivialprog9', src4)

12
test cases/rewrite/1 basic/rmTgt.json Normal file
View File

@ -0,0 +1,12 @@
[
{
"type": "target",
"target": "trivialprog1",
"operation": "tgt_rm"
},
{
"type": "target",
"target": "trivialprog9",
"operation": "tgt_rm"
}
]

10
test cases/rewrite/2 subdirs/addTgt.json Normal file
View File

@ -0,0 +1,10 @@
[
{
"type": "target",
"target": "newLib",
"operation": "tgt_add",
"sources": ["new1.cpp", "new2.cpp"],
"target_type": "shared_library",
"subdir": "sub2"
}
]

5
test cases/rewrite/2 subdirs/info.json
View File

@ -3,5 +3,10 @@
"type": "target",
"target": "something",
"operation": "info"
},
{
"type": "target",
"target": "newLib",
"operation": "info"
}
]

7
test cases/rewrite/2 subdirs/rmTgt.json Normal file
View File

@ -0,0 +1,7 @@
[
{
"type": "target",
"target": "something",
"operation": "tgt_rm"
}
]