[LLDB] Add ScalarLiteralNode and literal parsing in DIL (#152308)

This patch introduces `ScalarLiteralNode` without any uses by other
nodes yet. It also includes lexing and parsing for integer and floating
point numbers.
This commit is contained in:
Ilia Kuklin
2025-08-27 16:28:11 +05:00
committed by GitHub
parent ab5d749df2
commit 51230d9e1e
15 changed files with 433 additions and 31 deletions

View File

@@ -15,7 +15,8 @@ postfix_expression = primary_expression
| postfix_expression "." id_expression
| postfix_expression "->" id_expression ;
primary_expression = id_expression
primary_expression = numeric_literal
| id_expression
| "(" expression ")" ;
id_expression = unqualified_id
@@ -31,6 +32,9 @@ identifier = ? C99 Identifier ? ;
integer_literal = ? Integer constant: hexademical, decimal, octal, binary ? ;
numeric_literal = ? Integer constant: hexademical, decimal, octal, binary ?
| ? Floating constant ? ;
register = "$" ? Register name ? ;
nested_name_specifier = type_name "::"

View File

@@ -21,7 +21,9 @@ enum class NodeKind {
eArraySubscriptNode,
eBitExtractionNode,
eErrorNode,
eFloatLiteralNode,
eIdentifierNode,
eIntegerLiteralNode,
eMemberOfNode,
eUnaryOpNode,
};
@@ -178,6 +180,52 @@ private:
int64_t m_last_index;
};
enum class IntegerTypeSuffix { None, Long, LongLong };
class IntegerLiteralNode : public ASTNode {
public:
IntegerLiteralNode(uint32_t location, llvm::APInt value, uint32_t radix,
bool is_unsigned, IntegerTypeSuffix type)
: ASTNode(location, NodeKind::eIntegerLiteralNode),
m_value(std::move(value)), m_radix(radix), m_is_unsigned(is_unsigned),
m_type(type) {}
llvm::Expected<lldb::ValueObjectSP> Accept(Visitor *v) const override;
const llvm::APInt &GetValue() const { return m_value; }
uint32_t GetRadix() const { return m_radix; }
bool IsUnsigned() const { return m_is_unsigned; }
IntegerTypeSuffix GetTypeSuffix() const { return m_type; }
static bool classof(const ASTNode *node) {
return node->GetKind() == NodeKind::eIntegerLiteralNode;
}
private:
llvm::APInt m_value;
uint32_t m_radix;
bool m_is_unsigned;
IntegerTypeSuffix m_type;
};
class FloatLiteralNode : public ASTNode {
public:
FloatLiteralNode(uint32_t location, llvm::APFloat value)
: ASTNode(location, NodeKind::eFloatLiteralNode),
m_value(std::move(value)) {}
llvm::Expected<lldb::ValueObjectSP> Accept(Visitor *v) const override;
const llvm::APFloat &GetValue() const { return m_value; }
static bool classof(const ASTNode *node) {
return node->GetKind() == NodeKind::eFloatLiteralNode;
}
private:
llvm::APFloat m_value;
};
/// This class contains one Visit method for each specialized type of
/// DIL AST node. The Visit methods are used to dispatch a DIL AST node to
/// the correct function in the DIL expression evaluator for evaluating that
@@ -195,6 +243,10 @@ public:
Visit(const ArraySubscriptNode *node) = 0;
virtual llvm::Expected<lldb::ValueObjectSP>
Visit(const BitFieldExtractionNode *node) = 0;
virtual llvm::Expected<lldb::ValueObjectSP>
Visit(const IntegerLiteralNode *node) = 0;
virtual llvm::Expected<lldb::ValueObjectSP>
Visit(const FloatLiteralNode *node) = 0;
};
} // namespace lldb_private::dil

View File

@@ -54,6 +54,15 @@ private:
Visit(const ArraySubscriptNode *node) override;
llvm::Expected<lldb::ValueObjectSP>
Visit(const BitFieldExtractionNode *node) override;
llvm::Expected<lldb::ValueObjectSP>
Visit(const IntegerLiteralNode *node) override;
llvm::Expected<lldb::ValueObjectSP>
Visit(const FloatLiteralNode *node) override;
llvm::Expected<CompilerType>
PickIntegerType(lldb::TypeSystemSP type_system,
std::shared_ptr<ExecutionContextScope> ctx,
const IntegerLiteralNode *literal);
// Used by the interpreter to create objects, perform casts, etc.
lldb::TargetSP m_target;

View File

@@ -28,12 +28,14 @@ public:
arrow,
coloncolon,
eof,
float_constant,
identifier,
integer_constant,
l_paren,
l_square,
minus,
numeric_constant,
period,
plus,
r_paren,
r_square,
star,

View File

@@ -96,6 +96,9 @@ private:
std::string ParseIdExpression();
std::string ParseUnqualifiedId();
std::optional<int64_t> ParseIntegerConstant();
ASTNodeUP ParseNumericLiteral();
ASTNodeUP ParseIntegerLiteral();
ASTNodeUP ParseFloatingPointLiteral();
void BailOut(const std::string &error, uint32_t loc, uint16_t err_len);

View File

@@ -37,4 +37,13 @@ BitFieldExtractionNode::Accept(Visitor *v) const {
return v->Visit(this);
}
llvm::Expected<lldb::ValueObjectSP>
IntegerLiteralNode::Accept(Visitor *v) const {
return v->Visit(this);
}
llvm::Expected<lldb::ValueObjectSP> FloatLiteralNode::Accept(Visitor *v) const {
return v->Visit(this);
}
} // namespace lldb_private::dil

View File

@@ -7,7 +7,9 @@
//===----------------------------------------------------------------------===//
#include "lldb/ValueObject/DILEval.h"
#include "lldb/Core/Module.h"
#include "lldb/Symbol/CompileUnit.h"
#include "lldb/Symbol/TypeSystem.h"
#include "lldb/Symbol/VariableList.h"
#include "lldb/Target/RegisterContext.h"
#include "lldb/ValueObject/DILAST.h"
@@ -497,4 +499,107 @@ Interpreter::Visit(const BitFieldExtractionNode *node) {
return child_valobj_sp;
}
static llvm::Expected<lldb::TypeSystemSP>
GetTypeSystemFromCU(std::shared_ptr<StackFrame> ctx) {
SymbolContext symbol_context =
ctx->GetSymbolContext(lldb::eSymbolContextCompUnit);
lldb::LanguageType language = symbol_context.comp_unit->GetLanguage();
symbol_context = ctx->GetSymbolContext(lldb::eSymbolContextModule);
return symbol_context.module_sp->GetTypeSystemForLanguage(language);
}
static CompilerType GetBasicType(lldb::TypeSystemSP type_system,
lldb::BasicType basic_type) {
if (type_system)
return type_system.get()->GetBasicTypeFromAST(basic_type);
return CompilerType();
}
llvm::Expected<CompilerType>
Interpreter::PickIntegerType(lldb::TypeSystemSP type_system,
std::shared_ptr<ExecutionContextScope> ctx,
const IntegerLiteralNode *literal) {
// Binary, Octal, Hexadecimal and literals with a U suffix are allowed to be
// an unsigned integer.
bool unsigned_is_allowed = literal->IsUnsigned() || literal->GetRadix() != 10;
llvm::APInt apint = literal->GetValue();
llvm::SmallVector<std::pair<lldb::BasicType, lldb::BasicType>, 3> candidates;
if (literal->GetTypeSuffix() <= IntegerTypeSuffix::None)
candidates.emplace_back(lldb::eBasicTypeInt,
unsigned_is_allowed ? lldb::eBasicTypeUnsignedInt
: lldb::eBasicTypeInvalid);
if (literal->GetTypeSuffix() <= IntegerTypeSuffix::Long)
candidates.emplace_back(lldb::eBasicTypeLong,
unsigned_is_allowed ? lldb::eBasicTypeUnsignedLong
: lldb::eBasicTypeInvalid);
candidates.emplace_back(lldb::eBasicTypeLongLong,
lldb::eBasicTypeUnsignedLongLong);
for (auto [signed_, unsigned_] : candidates) {
CompilerType signed_type = type_system->GetBasicTypeFromAST(signed_);
if (!signed_type)
continue;
llvm::Expected<uint64_t> size = signed_type.GetBitSize(ctx.get());
if (!size)
return size.takeError();
if (!literal->IsUnsigned() && apint.isIntN(*size - 1))
return signed_type;
if (unsigned_ != lldb::eBasicTypeInvalid && apint.isIntN(*size))
return type_system->GetBasicTypeFromAST(unsigned_);
}
return llvm::make_error<DILDiagnosticError>(
m_expr,
"integer literal is too large to be represented in any integer type",
literal->GetLocation());
}
llvm::Expected<lldb::ValueObjectSP>
Interpreter::Visit(const IntegerLiteralNode *node) {
llvm::Expected<lldb::TypeSystemSP> type_system =
GetTypeSystemFromCU(m_exe_ctx_scope);
if (!type_system)
return type_system.takeError();
llvm::Expected<CompilerType> type =
PickIntegerType(*type_system, m_exe_ctx_scope, node);
if (!type)
return type.takeError();
Scalar scalar = node->GetValue();
// APInt from StringRef::getAsInteger comes with just enough bitwidth to
// hold the value. This adjusts APInt bitwidth to match the compiler type.
llvm::Expected<uint64_t> type_bitsize =
type->GetBitSize(m_exe_ctx_scope.get());
if (!type_bitsize)
return type_bitsize.takeError();
scalar.TruncOrExtendTo(*type_bitsize, false);
return ValueObject::CreateValueObjectFromScalar(m_target, scalar, *type,
"result");
}
llvm::Expected<lldb::ValueObjectSP>
Interpreter::Visit(const FloatLiteralNode *node) {
llvm::Expected<lldb::TypeSystemSP> type_system =
GetTypeSystemFromCU(m_exe_ctx_scope);
if (!type_system)
return type_system.takeError();
bool isFloat =
&node->GetValue().getSemantics() == &llvm::APFloat::IEEEsingle();
lldb::BasicType basic_type =
isFloat ? lldb::eBasicTypeFloat : lldb::eBasicTypeDouble;
CompilerType type = GetBasicType(*type_system, basic_type);
if (!type)
return llvm::make_error<DILDiagnosticError>(
m_expr, "unable to create a const literal", node->GetLocation());
Scalar scalar = node->GetValue();
return ValueObject::CreateValueObjectFromScalar(m_target, scalar, type,
"result");
}
} // namespace lldb_private::dil

View File

@@ -28,18 +28,23 @@ llvm::StringRef Token::GetTokenName(Kind kind) {
return "coloncolon";
case Kind::eof:
return "eof";
case Kind::float_constant:
return "float_constant";
case Kind::identifier:
return "identifier";
case Kind::integer_constant:
return "integer_constant";
case Kind::l_paren:
return "l_paren";
case Kind::l_square:
return "l_square";
case Kind::minus:
return "minus";
case Kind::numeric_constant:
return "numeric_constant";
case Kind::period:
return "period";
return "l_square";
case Kind::plus:
return "plus";
case Kind::r_paren:
return "r_paren";
case Kind::r_square:
@@ -70,13 +75,32 @@ static std::optional<llvm::StringRef> IsWord(llvm::StringRef expr,
return candidate;
}
static bool IsNumberBodyChar(char ch) { return IsDigit(ch) || IsLetter(ch); }
static bool IsNumberBodyChar(char ch) {
return IsDigit(ch) || IsLetter(ch) || ch == '.';
}
static std::optional<llvm::StringRef> IsNumber(llvm::StringRef expr,
llvm::StringRef &remainder) {
if (IsDigit(remainder[0])) {
llvm::StringRef number = remainder.take_while(IsNumberBodyChar);
remainder = remainder.drop_front(number.size());
static std::optional<llvm::StringRef> IsNumber(llvm::StringRef &remainder,
bool &isFloat) {
llvm::StringRef tail = remainder;
llvm::StringRef body = tail.take_while(IsNumberBodyChar);
size_t dots = body.count('.');
if (dots > 1 || dots == body.size())
return std::nullopt;
if (IsDigit(body.front()) || (body[0] == '.' && IsDigit(body[1]))) {
isFloat = dots == 1;
tail = tail.drop_front(body.size());
bool isHex = body.contains_insensitive('x');
bool hasExp = !isHex && body.contains_insensitive('e');
bool hasHexExp = isHex && body.contains_insensitive('p');
if (hasExp || hasHexExp) {
isFloat = true; // This marks numbers like 0x1p1 and 1e1 as float
if (body.ends_with_insensitive("e") || body.ends_with_insensitive("p"))
if (tail.consume_front("+") || tail.consume_front("-"))
tail = tail.drop_while(IsNumberBodyChar);
}
size_t number_length = remainder.size() - tail.size();
llvm::StringRef number = remainder.take_front(number_length);
remainder = remainder.drop_front(number_length);
return number;
}
return std::nullopt;
@@ -106,18 +130,21 @@ llvm::Expected<Token> DILLexer::Lex(llvm::StringRef expr,
return Token(Token::eof, "", (uint32_t)expr.size());
uint32_t position = cur_pos - expr.begin();
std::optional<llvm::StringRef> maybe_number = IsNumber(expr, remainder);
if (maybe_number)
return Token(Token::numeric_constant, maybe_number->str(), position);
bool isFloat = false;
std::optional<llvm::StringRef> maybe_number = IsNumber(remainder, isFloat);
if (maybe_number) {
auto kind = isFloat ? Token::float_constant : Token::integer_constant;
return Token(kind, maybe_number->str(), position);
}
std::optional<llvm::StringRef> maybe_word = IsWord(expr, remainder);
if (maybe_word)
return Token(Token::identifier, maybe_word->str(), position);
constexpr std::pair<Token::Kind, const char *> operators[] = {
{Token::amp, "&"}, {Token::arrow, "->"}, {Token::coloncolon, "::"},
{Token::l_paren, "("}, {Token::l_square, "["}, {Token::minus, "-"},
{Token::period, "."}, {Token::r_paren, ")"}, {Token::r_square, "]"},
{Token::star, "*"},
{Token::amp, "&"}, {Token::arrow, "->"}, {Token::coloncolon, "::"},
{Token::l_paren, "("}, {Token::l_square, "["}, {Token::minus, "-"},
{Token::period, "."}, {Token::plus, "+"}, {Token::r_paren, ")"},
{Token::r_square, "]"}, {Token::star, "*"},
};
for (auto [kind, str] : operators) {
if (remainder.consume_front(str))

View File

@@ -179,10 +179,13 @@ ASTNodeUP DILParser::ParsePostfixExpression() {
// Parse a primary_expression.
//
// primary_expression:
// numeric_literal
// id_expression
// "(" expression ")"
//
ASTNodeUP DILParser::ParsePrimaryExpression() {
if (CurToken().IsOneOf({Token::integer_constant, Token::float_constant}))
return ParseNumericLiteral();
if (CurToken().IsOneOf(
{Token::coloncolon, Token::identifier, Token::l_paren})) {
// Save the source location for the diagnostics message.
@@ -346,6 +349,7 @@ void DILParser::BailOut(const std::string &error, uint32_t loc,
m_dil_lexer.ResetTokenIdx(m_dil_lexer.NumLexedTokens() - 1);
}
// FIXME: Remove this once subscript operator uses ScalarLiteralNode.
// Parse a integer_literal.
//
// integer_literal:
@@ -370,6 +374,69 @@ std::optional<int64_t> DILParser::ParseIntegerConstant() {
return std::nullopt;
}
// Parse a numeric_literal.
//
// numeric_literal:
// ? Token::integer_constant ?
// ? Token::floating_constant ?
//
ASTNodeUP DILParser::ParseNumericLiteral() {
ASTNodeUP numeric_constant;
if (CurToken().Is(Token::integer_constant))
numeric_constant = ParseIntegerLiteral();
else
numeric_constant = ParseFloatingPointLiteral();
if (!numeric_constant) {
BailOut(llvm::formatv("Failed to parse token as numeric-constant: {0}",
CurToken()),
CurToken().GetLocation(), CurToken().GetSpelling().length());
return std::make_unique<ErrorNode>();
}
m_dil_lexer.Advance();
return numeric_constant;
}
ASTNodeUP DILParser::ParseIntegerLiteral() {
Token token = CurToken();
auto spelling = token.GetSpelling();
llvm::StringRef spelling_ref = spelling;
auto radix = llvm::getAutoSenseRadix(spelling_ref);
IntegerTypeSuffix type = IntegerTypeSuffix::None;
bool is_unsigned = false;
if (spelling_ref.consume_back_insensitive("u"))
is_unsigned = true;
if (spelling_ref.consume_back_insensitive("ll"))
type = IntegerTypeSuffix::LongLong;
else if (spelling_ref.consume_back_insensitive("l"))
type = IntegerTypeSuffix::Long;
// Suffix 'u' can be only specified only once, before or after 'l'
if (!is_unsigned && spelling_ref.consume_back_insensitive("u"))
is_unsigned = true;
llvm::APInt raw_value;
if (!spelling_ref.getAsInteger(radix, raw_value))
return std::make_unique<IntegerLiteralNode>(token.GetLocation(), raw_value,
radix, is_unsigned, type);
return nullptr;
}
ASTNodeUP DILParser::ParseFloatingPointLiteral() {
Token token = CurToken();
auto spelling = token.GetSpelling();
llvm::StringRef spelling_ref = spelling;
llvm::APFloat raw_float(llvm::APFloat::IEEEdouble());
if (spelling_ref.consume_back_insensitive("f"))
raw_float = llvm::APFloat(llvm::APFloat::IEEEsingle());
auto StatusOrErr = raw_float.convertFromString(
spelling_ref, llvm::APFloat::rmNearestTiesToEven);
if (!errorToBool(StatusOrErr.takeError()))
return std::make_unique<FloatLiteralNode>(token.GetLocation(), raw_float);
return nullptr;
}
void DILParser::Expect(Token::Kind kind) {
if (CurToken().IsNot(kind)) {
BailOut(llvm::formatv("expected {0}, got: {1}", kind, CurToken()),

View File

@@ -66,14 +66,15 @@ class TestFrameVarDILArraySubscript(TestBase):
self.expect(
"frame var 'int_arr[1.0]'",
error=True,
substrs=["expected 'r_square', got: <'.'"],
substrs=["failed to parse integer constant: <'1.0' (float_constant)>"],
)
# Test accessing bits in scalar types.
self.expect_var_path("idx_1[0]", value="1")
self.expect_var_path("idx_1[1]", value="0")
self.expect_var_path("1[0]", value="1")
# Bit adcess not valid for a reference.
# Bit access not valid for a reference.
self.expect(
"frame var 'idx_1_ref[0]'",
error=True,
@@ -86,11 +87,6 @@ class TestFrameVarDILArraySubscript(TestBase):
error=True,
substrs=["failed to parse integer constant"],
)
self.expect(
"frame var '1[2]'",
error=True,
substrs=["Unexpected token"],
)
# Base should not be a pointer to void
self.expect(

View File

@@ -35,7 +35,7 @@ class TestFrameVarDILIndirection(TestBase):
self.expect(
"frame variable '*1'",
error=True,
substrs=["Unexpected token: <'1' (numeric_constant)>"],
substrs=["dereference failed: not a pointer, reference or array type"],
)
self.expect(
"frame variable '*val'",

View File

@@ -0,0 +1,3 @@
CXX_SOURCES := main.cpp
include Makefile.rules

View File

@@ -0,0 +1,92 @@
"""
Test DIL literals.
"""
import lldb
from lldbsuite.test.lldbtest import *
from lldbsuite.test.decorators import *
from lldbsuite.test import lldbutil
class TestFrameVarDILLiterals(TestBase):
NO_DEBUG_INFO_TESTCASE = True
def test_literals(self):
self.build()
(target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
)
self.runCmd("settings set target.experimental.use-DIL true")
# Check number literals parsing
self.expect_var_path("1.0", value="1", type="double")
self.expect_var_path("1.0f", value="1", type="float")
self.expect_var_path("0x1.2p+3f", value="9", type="float")
self.expect_var_path("1", value="1", type="int")
self.expect_var_path("1u", value="1", type="unsigned int")
self.expect_var_path("0b1l", value="1", type="long")
self.expect_var_path("01ul", value="1", type="unsigned long")
self.expect_var_path("01lu", value="1", type="unsigned long")
self.expect_var_path("0o1ll", value="1", type="long long")
self.expect_var_path("0x1ULL", value="1", type="unsigned long long")
self.expect_var_path("0x1llu", value="1", type="unsigned long long")
self.expect(
"frame var '1LLL'",
error=True,
substrs=["Failed to parse token as numeric-constant"],
)
self.expect(
"frame var '1ullu'",
error=True,
substrs=["Failed to parse token as numeric-constant"],
)
# Check integer literal type edge cases (dil::Interpreter::PickIntegerType)
frame = thread.GetFrameAtIndex(0)
v = frame.GetValueForVariablePath("argc")
# Creating an SBType from a BasicType still requires any value from the frame
int_size = v.GetType().GetBasicType(lldb.eBasicTypeInt).GetByteSize()
long_size = v.GetType().GetBasicType(lldb.eBasicTypeLong).GetByteSize()
longlong_size = v.GetType().GetBasicType(lldb.eBasicTypeLongLong).GetByteSize()
longlong_str = "0x" + "F" * longlong_size * 2
longlong_str = str(int(longlong_str, 16))
self.assert_literal_type(frame, longlong_str, lldb.eBasicTypeUnsignedLongLong)
toolong_str = "0x" + "F" * longlong_size * 2 + "F"
self.expect(
f"frame var '{toolong_str}'",
error=True,
substrs=[
"integer literal is too large to be represented in any integer type"
],
)
# These check only apply if adjacent types have different sizes
if int_size < long_size:
# For exmaple, 0xFFFFFFFF and 4294967295 will have different types
# even though the numeric value is the same
hex_str = "0x" + "F" * int_size * 2
dec_str = str(int(hex_str, 16))
self.assert_literal_type(frame, hex_str, lldb.eBasicTypeUnsignedInt)
self.assert_literal_type(frame, dec_str, lldb.eBasicTypeLong)
long_str = "0x" + "F" * int_size * 2 + "F"
ulong_str = long_str + "u"
self.assert_literal_type(frame, long_str, lldb.eBasicTypeLong)
self.assert_literal_type(frame, ulong_str, lldb.eBasicTypeUnsignedLong)
if long_size < longlong_size:
hex_str = "0x" + "F" * long_size * 2
dec_str = str(int(hex_str, 16))
self.assert_literal_type(frame, hex_str, lldb.eBasicTypeUnsignedLong)
self.assert_literal_type(frame, dec_str, lldb.eBasicTypeLongLong)
longlong_str = "0x" + "F" * long_size * 2 + "F"
ulonglong_str = longlong_str + "u"
self.assert_literal_type(frame, longlong_str, lldb.eBasicTypeLongLong)
self.assert_literal_type(
frame, ulonglong_str, lldb.eBasicTypeUnsignedLongLong
)
def assert_literal_type(self, frame, literal, expected_type):
value = frame.GetValueForVariablePath(literal)
basic_type = value.GetType().GetBasicType()
self.assertEqual(basic_type, expected_type)

View File

@@ -0,0 +1,3 @@
int main(int argc, char **argv) {
return 0; // Set a breakpoint here
}

View File

@@ -151,22 +151,32 @@ TEST(DILLexerTests, IdentifiersTest) {
Token token = lexer.GetCurrentToken();
EXPECT_TRUE(token.IsNot(Token::identifier));
EXPECT_TRUE(token.IsOneOf({Token::eof, Token::coloncolon, Token::l_paren,
Token::r_paren, Token::numeric_constant}));
Token::r_paren, Token::integer_constant}));
}
}
TEST(DILLexerTests, NumbersTest) {
// These strings should lex into number tokens.
std::vector<std::string> valid_numbers = {"123", "0x123", "0123", "0b101"};
std::vector<std::string> valid_integers = {"123", "0x123", "0123", "0b101"};
std::vector<std::string> valid_floats = {
"1.2", ".2", "2.f", "0x1.2", "0x.2", ".2e1f",
"2.e+1f", "0x1.f", "0x1.2P1", "0x1.p-1f", "0x1.2P+3f", "1E1",
"1E+1", "0x1p1", "0x1p+1", "0xf.fp1f"};
// The lexer can lex these strings, but they should not be numbers.
std::vector<std::string> invalid_numbers = {"", "x123", "b123"};
std::vector<std::string> invalid_numbers = {"", "x123", "b123", "a.b"};
for (auto &str : valid_numbers) {
for (auto &str : valid_integers) {
SCOPED_TRACE(str);
EXPECT_THAT_EXPECTED(ExtractTokenData(str),
llvm::HasValue(testing::ElementsAre(
testing::Pair(Token::numeric_constant, str))));
testing::Pair(Token::integer_constant, str))));
}
for (auto &str : valid_floats) {
SCOPED_TRACE(str);
EXPECT_THAT_EXPECTED(ExtractTokenData(str),
llvm::HasValue(testing::ElementsAre(
testing::Pair(Token::float_constant, str))));
}
// Verify that none of the invalid numbers come out as numeric tokens.
for (auto &str : invalid_numbers) {
@@ -175,7 +185,27 @@ TEST(DILLexerTests, NumbersTest) {
EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
DILLexer lexer(*maybe_lexer);
Token token = lexer.GetCurrentToken();
EXPECT_TRUE(token.IsNot(Token::numeric_constant));
EXPECT_TRUE(token.IsNot(Token::integer_constant));
EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
}
// Verify that '-' and '+' are not lexed if they're not part of a number
std::vector<std::string> expressions = {"1+e", "0x1+p", "1.1+e",
"1.1e1+e", "0x1.1p-1-p", "1e-1+e",
"1e1+e", "0x1p-1-p", "0xe+e"};
for (auto &str : expressions) {
SCOPED_TRACE(str);
llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
DILLexer lexer(*maybe_lexer);
Token token = lexer.GetCurrentToken();
EXPECT_TRUE(
token.IsOneOf({Token::integer_constant, Token::float_constant}));
lexer.Advance();
token = lexer.GetCurrentToken();
EXPECT_TRUE(token.IsOneOf({Token::plus, Token::minus}));
lexer.Advance();
token = lexer.GetCurrentToken();
EXPECT_TRUE(token.Is(Token::identifier));
}
}