diff --git a/shared/source/device_binary_format/yaml/yaml_parser.cpp b/shared/source/device_binary_format/yaml/yaml_parser.cpp index 0f5dca4550..830834adbb 100644 --- a/shared/source/device_binary_format/yaml/yaml_parser.cpp +++ b/shared/source/device_binary_format/yaml/yaml_parser.cpp @@ -275,10 +275,12 @@ bool tokenize(ConstStringRef text, LinesCache &outLines, TokensCache &outTokens, context.isParsingIdent = false; auto tokEnd = consumeNameIdentifier(text, context.pos); if (tokEnd != context.pos) { + auto tokenData = ConstStringRef(context.pos, tokEnd - context.pos); + tokenData = tokenData.trimEnd(isWhitespace); if (context.lineTraits.hasDictionaryEntry) { - outTokens.push_back(Token(ConstStringRef(context.pos, tokEnd - context.pos), Token::LiteralString)); + outTokens.push_back(Token(tokenData, Token::LiteralString)); } else { - outTokens.push_back(Token(ConstStringRef(context.pos, tokEnd - context.pos), Token::Identifier)); + outTokens.push_back(Token(tokenData, Token::Identifier)); } } else { tokEnd = consumeNumberOrSign(text, context.pos); diff --git a/shared/source/device_binary_format/yaml/yaml_parser.h b/shared/source/device_binary_format/yaml/yaml_parser.h index d00d93307e..fe31c50ec7 100644 --- a/shared/source/device_binary_format/yaml/yaml_parser.h +++ b/shared/source/device_binary_format/yaml/yaml_parser.h @@ -31,6 +31,16 @@ constexpr bool isWhitespace(char c) { } } +constexpr bool isSeparationWhitespace(char c) { + switch (c) { + default: + return false; + case ' ': + case '\t': + return true; + } +} + constexpr bool isLetter(char c) { return ((c >= 'a') & (c <= 'z')) || ((c >= 'A') & (c <= 'Z')); } @@ -103,7 +113,9 @@ constexpr const char *consumeNameIdentifier(ConstStringRef wholeText, const char auto it = parsePos + 1; while (it < parseEnd) { if (false == isNameIdentifierCharacter(*it)) { - break; + if (false == isSeparationWhitespace(*it)) { + break; + } } ++it; } diff --git a/shared/source/utilities/const_stringref.h b/shared/source/utilities/const_stringref.h index 77ce77c479..c12a8fff1f 100644 --- a/shared/source/utilities/const_stringref.h +++ b/shared/source/utilities/const_stringref.h @@ -81,6 +81,21 @@ class ConstStringRef { } } + template + constexpr ConstStringRef trimEnd(Predicate &&pred) const noexcept { + const char *end = ptr + len; + end--; + auto newLen = len; + while (true == pred(*end)) { + if (0u == newLen) { + break; + } + newLen--; + end--; + } + return ConstStringRef(this->ptr, newLen); + } + constexpr const char *data() const noexcept { return ptr; } diff --git a/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp b/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp index c31118b5a0..be543d262e 100644 --- a/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp +++ b/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp @@ -223,9 +223,10 @@ TEST(YamlConsumeNameIdentifier, GivenInvalidNameIdentifierBeginningCharacterThen TEST(YamlConsumeNameIdentifier, GivenNameIdentifierBeginningCharacterThenConsumeWholeNameIdentifier) { for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { - bool isNameIndentifierChar = NEO::Yaml::isNameIdentifierCharacter(static_cast(c)); + bool isNameOrSeparationWhitespaceIndentifierChar = NEO::Yaml::isNameIdentifierCharacter(static_cast(c)); + isNameOrSeparationWhitespaceIndentifierChar |= NEO::Yaml::isSeparationWhitespace(static_cast(c)); char nameIdentifierStr[] = {'A', static_cast(c)}; - auto expected = nameIdentifierStr + (isNameIndentifierChar ? 2 : 1); + auto expected = nameIdentifierStr + (isNameOrSeparationWhitespaceIndentifierChar ? 2 : 1); EXPECT_EQ(expected, NEO::Yaml::consumeNameIdentifier(ConstStringRef::fromArray(nameIdentifierStr), nameIdentifierStr)) << c; } } @@ -1028,6 +1029,34 @@ TEST(YamlTokenize, GivenInvalidNumericLiteralThenReturnError) { EXPECT_TRUE(warnings.empty()) << warnings; } +TEST(YamlTokenize, GivenSpaceSeparatedStringAsValueThenReadItCorrectly) { + ConstStringRef yaml = "\nbanana: space separated string\napple: space separated with spaces at the end \n"; + + NEO::Yaml::Token expectedTokens[] = { + Token{"\n", NEO::Yaml::Token::SingleCharacter}, + Token{"banana", NEO::Yaml::Token::Identifier}, + Token{":", NEO::Yaml::Token::SingleCharacter}, + Token{"space separated string", NEO::Yaml::Token::LiteralString}, + Token{"\n", NEO::Yaml::Token::SingleCharacter}, + Token{"apple", NEO::Yaml::Token::Identifier}, + Token{":", NEO::Yaml::Token::SingleCharacter}, + Token{"space separated with spaces at the end", NEO::Yaml::Token::LiteralString}, + Token{"\n", NEO::Yaml::Token::SingleCharacter}}; + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } +} + TEST(YamlNode, WhenConstructedThenSetsUpProperDefaults) { { NEO::Yaml::Node node; diff --git a/shared/test/unit_test/device_binary_format/zebin_decoder_tests.cpp b/shared/test/unit_test/device_binary_format/zebin_decoder_tests.cpp index 2ed5aadf78..65d0dad266 100644 --- a/shared/test/unit_test/device_binary_format/zebin_decoder_tests.cpp +++ b/shared/test/unit_test/device_binary_format/zebin_decoder_tests.cpp @@ -542,7 +542,7 @@ kernels_misc_info: address_qualifier: __global access_qualifier: NONE type_name: 'int*;8' - type_qualifiers: NONE + type_qualifiers: restrict const volatile - name: kernel2 args_info: - index: 0 @@ -596,13 +596,15 @@ kernels_misc_info: EXPECT_STREQ(kernel1ArgInfo1.addressQualifier.c_str(), "__global"); EXPECT_STREQ(kernel1ArgInfo1.accessQualifier.c_str(), "NONE"); EXPECT_STREQ(kernel1ArgInfo1.type.c_str(), "int*"); - EXPECT_STREQ(kernel1ArgInfo1.typeQualifiers.c_str(), "NONE"); + EXPECT_STREQ(kernel1ArgInfo1.typeQualifiers.c_str(), "restrict const volatile"); const auto &kernel1ArgTraits1 = kernel1Info->kernelDescriptor.payloadMappings.explicitArgs.at(0).getTraits(); EXPECT_EQ(KernelArgMetadata::AccessNone, kernel1ArgTraits1.accessQualifier); EXPECT_EQ(KernelArgMetadata::AddrGlobal, kernel1ArgTraits1.addressQualifier); KernelArgMetadata::TypeQualifiers qual = {}; - qual.unknownQual = true; + qual.restrictQual = true; + qual.constQual = true; + qual.volatileQual = true; EXPECT_EQ(qual.packed, kernel1ArgTraits1.typeQualifiers.packed); EXPECT_EQ(4u, kernel2Info->kernelDescriptor.explicitArgsExtendedMetadata.size()); diff --git a/shared/test/unit_test/utilities/const_stringref_tests.cpp b/shared/test/unit_test/utilities/const_stringref_tests.cpp index 4eca78e247..426601060c 100644 --- a/shared/test/unit_test/utilities/const_stringref_tests.cpp +++ b/shared/test/unit_test/utilities/const_stringref_tests.cpp @@ -251,3 +251,19 @@ TEST(ConstStringStartsWithConstString, GivenInvalidPrefixThenReturnsFalse) { EXPECT_FALSE(str.startsWith(ConstStringRef("some diff"))); EXPECT_FALSE(str.startsWith(ConstStringRef("ome text"))); } + +TEST(ConstStringRefTrimEnd, givenTrimEndFunctionWithPredicateThenReturnTrimmedConstStringRefAccordingToPredicate) { + auto predicateIsNumber = [](char c) { + return c >= '0' && c <= '9'; + }; + ConstStringRef str = "some 10 text1024"; + auto trimmed = str.trimEnd(predicateIsNumber); + EXPECT_EQ(trimmed, ConstStringRef("some 10 text")); +} + +TEST(ConstStringRefTrimEnd, givenTrimEndFunctionWithPredicateThatReturnsTrueForAllElementsThenReturnEmptyConstStringRef) { + auto predicate = [](char c) { return true; }; + ConstStringRef str = "some text"; + auto trimmed = str.trimEnd(predicate); + EXPECT_EQ(0u, trimmed.length()); +}