intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-21 01:04:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add inline collection support in yaml parser

Browse Source 
Added support for inline collection in following syntax: [1, 2, 3]

Signed-off-by: Krystian Chmielewski <krystian.chmielewski@intel.com>
This commit is contained in:
Krystian Chmielewski
2021-10-25 15:50:14 +00:00
committed by Compute-Runtime-Automation
parent 6769df2f5d
commit 174c1dfe64
3 changed files with 173 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
shared/source/device_binary_format/yaml/yaml_parser.cpp
View File

@@ -199,12 +199,34 @@ bool tokenize(ConstStringRef text, LinesCache &outLines, TokensCache &outTokens,
context.isParsingIdent = false;
break;
}
case '[':
if (false == std::regex_search(context.pos, inlineCollectionRegex)) {
outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, inlineCollectionYamlErrorMsg.data());
return false;
}
context.lineTraits.hasInlineDataMarkers = true;
outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::CollectionBeg));
++context.pos;
break;
case ']':
if (false == context.lineTraits.hasInlineDataMarkers) {
outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, inlineCollectionYamlErrorMsg.data());
return false;
}
outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::CollectionEnd));
++context.pos;
break;
case ',':
if (false == context.lineTraits.hasInlineDataMarkers) {
outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, inlineCollectionYamlErrorMsg.data());
return false;
}
outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::SingleCharacter));
++context.pos;
break;
case '{':
case '}':
case '[':
case ']':
case ',':
outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, "NEO::Yaml : Inline collections are not supported yet");
outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, "NEO::Yaml : Inline dictionaries are not supported");
return false;
case ':':
context.lineTraits.hasDictionaryEntry = true;
@@ -329,29 +351,45 @@ bool buildTree(const LinesCache &lines, const TokensCache &tokens, NodesCache &o
}
}
if (lines[lineId].traits.hasInlineDataMarkers) {
outErrReason = "Inline collections are not supported yet\n";
return false;
} else {
if (Line::LineType::DictionaryEntry == lines[lineId].lineType) {
auto numTokensInLine = lines[lineId].last - lines[lineId].first + 1;
outNodes.rbegin()->key = lines[lineId].first;
UNRECOVERABLE_IF(numTokensInLine < 3); // at least key, : and \n
if (('#' != tokens[lines[lineId].first + 2]) && ('\n' != tokens[lines[lineId].first + 2])) {
outNodes.rbegin()->value = lines[lineId].first + 2;
}
} else {
auto numTokensInLine = lines[lineId].last - lines[lineId].first + 1;
(void)numTokensInLine;
UNRECOVERABLE_IF(numTokensInLine < 2); // at least : - and \n
UNRECOVERABLE_IF(Line::LineType::ListEntry != lines[lineId].lineType);
UNRECOVERABLE_IF('-' != tokens[lines[lineId].first]);
if (('#' != tokens[lines[lineId].first + 1]) && ('\n' != tokens[lines[lineId].first + 1])) {
outNodes.rbegin()->value = lines[lineId].first + 1;
if (Line::LineType::DictionaryEntry == lines[lineId].lineType) {
auto numTokensInLine = lines[lineId].last - lines[lineId].first + 1;
outNodes.rbegin()->key = lines[lineId].first;
UNRECOVERABLE_IF(numTokensInLine < 3); // at least key, : and \n
if (lines[lineId].traits.hasInlineDataMarkers) {
auto collectionBeg = lines[lineId].first + 2;
auto collectionEnd = lines[lineId].last - 1;
UNRECOVERABLE_IF(tokens[collectionBeg].traits.type != Token::Type::CollectionBeg || tokens[collectionEnd].traits.type != Token::Type::CollectionEnd);
auto &parentNode = *outNodes.rbegin();
Node *lastAddedNode = nullptr;
for (auto currTokenId = collectionBeg + 1; currTokenId < collectionEnd; currTokenId += 2) {
auto tokenType = tokens[currTokenId].traits.type;
UNRECOVERABLE_IF(tokenType != Token::Type::LiteralNumber && tokenType != Token::Type::LiteralString);
if (lastAddedNode == nullptr) {
lastAddedNode = &addNode(outNodes, parentNode);
} else {
lastAddedNode = &addNode(outNodes, *lastAddedNode, parentNode);
}
lastAddedNode->indent = currLineIndent + 1;
lastAddedNode->value = currTokenId;
}
nesting.push_back(parentNode.id);
} else if (('#' != tokens[lines[lineId].first + 2]) && ('\n' != tokens[lines[lineId].first + 2])) {
outNodes.rbegin()->value = lines[lineId].first + 2;
}
} else {
auto numTokensInLine = lines[lineId].last - lines[lineId].first + 1;
(void)numTokensInLine;
UNRECOVERABLE_IF(numTokensInLine < 2); // at least : - and \n
UNRECOVERABLE_IF(Line::LineType::ListEntry != lines[lineId].lineType);
UNRECOVERABLE_IF('-' != tokens[lines[lineId].first]);
if (('#' != tokens[lines[lineId].first + 1]) && ('\n' != tokens[lines[lineId].first + 1])) {
outNodes.rbegin()->value = lines[lineId].first + 1;
}
++lineId;
}
++lineId;
}
while (false == nesting.empty()) {

8
shared/source/device_binary_format/yaml/yaml_parser.h
View File

@@ -12,6 +12,7 @@
#include "shared/source/utilities/stackvec.h"
#include <iterator>
#include <regex>
#include <string>
namespace NEO {
@@ -148,7 +149,9 @@ struct Token {
SingleCharacter,
Comment,
FileSectionBeg,
FileSectionEnd };
FileSectionEnd,
CollectionBeg,
CollectionEnd };
constexpr Token(ConstStringRef tokData, Type tokType) {
pos = tokData.begin();
@@ -253,6 +256,9 @@ using LinesCache = StackVec<Line, 512>;
std::string constructYamlError(size_t lineNumber, const char *lineBeg, const char *parsePos, const char *reason = nullptr);
static std::regex inlineCollectionRegex(R"regex(^\[(\s*(\d|\w)+,?)+\]\s*\n)regex");
constexpr ConstStringRef inlineCollectionYamlErrorMsg = "NEO::Yaml : Inline collection is not in valid regex format - ^\\[(\\s*(\\d|\\w)+,?)+\\]\\s*\\n";
bool tokenize(ConstStringRef text, LinesCache &outLines, TokensCache &outTokens, std::string &outErrReason, std::string &outWarning);
using NodeId = uint32_t;