diff --git a/level_zero/core/source/module/module_imp.cpp b/level_zero/core/source/module/module_imp.cpp index d66cfe53ae..2ca71554c7 100644 --- a/level_zero/core/source/module/module_imp.cpp +++ b/level_zero/core/source/module/module_imp.cpp @@ -30,8 +30,8 @@ namespace L0 { namespace BuildOptions { -ConstStringRef optDisable = "-ze-opt-disable"; -ConstStringRef greaterThan4GbRequired = "-ze-opt-greater-than-4GB-buffer-required"; +NEO::ConstStringRef optDisable = "-ze-opt-disable"; +NEO::ConstStringRef greaterThan4GbRequired = "-ze-opt-greater-than-4GB-buffer-required"; } // namespace BuildOptions ModuleTranslationUnit::ModuleTranslationUnit(L0::Device *device) @@ -121,7 +121,7 @@ bool ModuleTranslationUnit::createFromNativeBinary(const char *input, size_t inp std::string decodeErrors; std::string decodeWarnings; ArrayRef archive(reinterpret_cast(input), inputSize); - auto singleDeviceBinary = unpackSingleDeviceBinary(archive, ConstStringRef(productAbbreviation, strlen(productAbbreviation)), targetDevice, + auto singleDeviceBinary = unpackSingleDeviceBinary(archive, NEO::ConstStringRef(productAbbreviation, strlen(productAbbreviation)), targetDevice, decodeErrors, decodeWarnings); if (decodeWarnings.empty() == false) { NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr, "%s\n", decodeWarnings.c_str()); @@ -555,7 +555,7 @@ ze_result_t ModuleImp::performDynamicLink(uint32_t numModules, return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; } -bool moveBuildOption(std::string &dstOptionsSet, std::string &srcOptionSet, ConstStringRef dstOptionName, ConstStringRef srcOptionName) { +bool moveBuildOption(std::string &dstOptionsSet, std::string &srcOptionSet, NEO::ConstStringRef dstOptionName, NEO::ConstStringRef srcOptionName) { auto optInSrcPos = srcOptionSet.find(srcOptionName.begin()); if (std::string::npos == optInSrcPos) { return false; diff --git a/level_zero/core/source/module/module_imp.h b/level_zero/core/source/module/module_imp.h index 92029b654f..391d6a4011 100644 --- a/level_zero/core/source/module/module_imp.h +++ b/level_zero/core/source/module/module_imp.h @@ -117,6 +117,6 @@ struct ModuleImp : public Module { bool isFullyLinked = false; }; -bool moveBuildOption(std::string &dstOptionsSet, std::string &srcOptionSet, ConstStringRef dstOptionName, ConstStringRef srcOptionName); +bool moveBuildOption(std::string &dstOptionsSet, std::string &srcOptionSet, NEO::ConstStringRef dstOptionName, NEO::ConstStringRef srcOptionName); } // namespace L0 diff --git a/opencl/source/built_ins/builtins_dispatch_builder.h b/opencl/source/built_ins/builtins_dispatch_builder.h index bb1bdc20b6..695a452dff 100644 --- a/opencl/source/built_ins/builtins_dispatch_builder.h +++ b/opencl/source/built_ins/builtins_dispatch_builder.h @@ -59,7 +59,7 @@ class BuiltinDispatchInfoBuilder { virtual ~BuiltinDispatchInfoBuilder() = default; template - void populate(Device &device, EBuiltInOps::Type operation, const char *options, KernelsDescArgsT &&... desc); + void populate(Device &device, EBuiltInOps::Type operation, ConstStringRef options, KernelsDescArgsT &&... desc); virtual bool buildDispatchInfos(MultiDispatchInfo &multiDispatchInfo, const BuiltinOpParams &operationParams) const { return false; diff --git a/opencl/source/built_ins/populate_built_ins.inl b/opencl/source/built_ins/populate_built_ins.inl index 81f4fcdc31..91f9446ed9 100644 --- a/opencl/source/built_ins/populate_built_ins.inl +++ b/opencl/source/built_ins/populate_built_ins.inl @@ -7,10 +7,10 @@ namespace NEO { template -void BuiltinDispatchInfoBuilder::populate(Device &device, EBuiltInOps::Type op, const char *options, KernelsDescArgsT &&... desc) { +void BuiltinDispatchInfoBuilder::populate(Device &device, EBuiltInOps::Type op, ConstStringRef options, KernelsDescArgsT &&... desc) { auto src = kernelsLib.getBuiltinsLib().getBuiltinCode(op, BuiltinCode::ECodeType::Any, device); prog.reset(BuiltinsLib::createProgramFromCode(src, device).release()); - prog->build(0, nullptr, options, nullptr, nullptr, kernelsLib.isCacheingEnabled()); + prog->build(0, nullptr, options.data(), nullptr, nullptr, kernelsLib.isCacheingEnabled()); grabKernels(std::forward(desc)...); } } // namespace NEO diff --git a/opencl/source/program/compile.cpp b/opencl/source/program/compile.cpp index 1c2545df3b..aae3e6f13d 100644 --- a/opencl/source/program/compile.cpp +++ b/opencl/source/program/compile.cpp @@ -82,7 +82,7 @@ cl_int Program::compile( options = (buildOptions != nullptr) ? buildOptions : ""; for (const auto &optionString : {CompilerOptions::gtpinRera, CompilerOptions::greaterThan4gbBuffersRequired}) { - size_t pos = options.find(optionString); + size_t pos = options.find(optionString.data()); if (pos != std::string::npos) { options.erase(pos, optionString.length()); CompilerOptions::concatenateAppend(internalOptions, optionString); diff --git a/opencl/source/program/link.cpp b/opencl/source/program/link.cpp index fe8f60cab5..2685467520 100644 --- a/opencl/source/program/link.cpp +++ b/opencl/source/program/link.cpp @@ -69,7 +69,7 @@ cl_int Program::link( options = (buildOptions != nullptr) ? buildOptions : ""; for (const auto &optionString : {CompilerOptions::gtpinRera, CompilerOptions::greaterThan4gbBuffersRequired}) { - size_t pos = options.find(optionString); + size_t pos = options.find(optionString.data()); if (pos != std::string::npos) { options.erase(pos, optionString.length()); CompilerOptions::concatenateAppend(internalOptions, optionString); diff --git a/opencl/test/unit_test/api/cl_get_kernel_arg_info_tests.inl b/opencl/test/unit_test/api/cl_get_kernel_arg_info_tests.inl index 43ee3688d4..47dca0c675 100644 --- a/opencl/test/unit_test/api/cl_get_kernel_arg_info_tests.inl +++ b/opencl/test/unit_test/api/cl_get_kernel_arg_info_tests.inl @@ -54,7 +54,7 @@ TEST_F(clGetKernelArgInfoTests, GivenValidParamsWhenGettingKernelArgInfoThenSucc pProgram, 1, &testedClDevice, - CompilerOptions::argInfo, + CompilerOptions::argInfo.data(), nullptr, nullptr); diff --git a/opencl/test/unit_test/api/cl_link_program_tests.inl b/opencl/test/unit_test/api/cl_link_program_tests.inl index b53edb79aa..a37b645f35 100644 --- a/opencl/test/unit_test/api/cl_link_program_tests.inl +++ b/opencl/test/unit_test/api/cl_link_program_tests.inl @@ -126,7 +126,7 @@ TEST_F(clLinkProgramTests, GivenCreateLibraryOptionWhenLinkingProgramThenSuccess pContext, 1, &testedClDevice, - CompilerOptions::createLibrary, + CompilerOptions::createLibrary.data(), 1, &program, nullptr, diff --git a/opencl/test/unit_test/built_ins/built_in_tests.cpp b/opencl/test/unit_test/built_ins/built_in_tests.cpp index 88bed8d1b4..d79619f6d6 100644 --- a/opencl/test/unit_test/built_ins/built_in_tests.cpp +++ b/opencl/test/unit_test/built_ins/built_in_tests.cpp @@ -1425,10 +1425,10 @@ TEST_F(BuiltInTests, createProgramFromCodeInternalOptionsFor32Bit) { ASSERT_NE(nullptr, program.get()); auto builtinInternalOptions = program->getInternalOptions(); - auto it = builtinInternalOptions.find(NEO::CompilerOptions::arch32bit); + auto it = builtinInternalOptions.find(NEO::CompilerOptions::arch32bit.data()); EXPECT_EQ(std::string::npos, it); - it = builtinInternalOptions.find(NEO::CompilerOptions::greaterThan4gbBuffersRequired); + it = builtinInternalOptions.find(NEO::CompilerOptions::greaterThan4gbBuffersRequired.data()); if (is32bit || pDevice->areSharedSystemAllocationsAllowed()) { EXPECT_NE(std::string::npos, it); } else { diff --git a/opencl/test/unit_test/fixtures/simple_arg_kernel_fixture.h b/opencl/test/unit_test/fixtures/simple_arg_kernel_fixture.h index ebb4275b11..b356c95d3f 100644 --- a/opencl/test/unit_test/fixtures/simple_arg_kernel_fixture.h +++ b/opencl/test/unit_test/fixtures/simple_arg_kernel_fixture.h @@ -262,7 +262,7 @@ class SimpleKernelStatelessFixture : public ProgramFixture { retVal = pProgram->build( 1, &deviceId, - CompilerOptions::greaterThan4gbBuffersRequired, + CompilerOptions::greaterThan4gbBuffersRequired.data(), nullptr, nullptr, false); diff --git a/opencl/test/unit_test/mocks/mock_compilers.cpp b/opencl/test/unit_test/mocks/mock_compilers.cpp index 6f0ac7cc12..3dba815580 100644 --- a/opencl/test/unit_test/mocks/mock_compilers.cpp +++ b/opencl/test/unit_test/mocks/mock_compilers.cpp @@ -411,7 +411,7 @@ void translate(bool usingIgc, CIF::Builtins::BufferSimple *src, CIF::Builtins::B options->GetSizeRaw()) { std::string opts(options->GetMemory(), options->GetMemory() + options->GetSize()); // handle special option "-create-library" - just erase it - size_t pos = opts.find(CompilerOptions::createLibrary, 0); + size_t pos = opts.find(CompilerOptions::createLibrary.data(), 0); if (pos != std::string::npos) { opts.erase(pos, CompilerOptions::createLibrary.length()); } diff --git a/opencl/test/unit_test/offline_compiler/ocloc_fatbinary_tests.cpp b/opencl/test/unit_test/offline_compiler/ocloc_fatbinary_tests.cpp index 9b81caa2d3..cb7f51e3dc 100644 --- a/opencl/test/unit_test/offline_compiler/ocloc_fatbinary_tests.cpp +++ b/opencl/test/unit_test/offline_compiler/ocloc_fatbinary_tests.cpp @@ -75,7 +75,7 @@ TEST(OclocFatBinaryAsProductId, GivenEnabledPlatformNameThenReturnsProperPlatfor auto names = NEO::toProductNames(platforms); for (size_t i = 0; i < platforms.size(); ++i) { auto idByName = NEO::asProductId(names[i], platforms); - EXPECT_EQ(idByName, platforms[i]) << names[i] << " : " << platforms[i] << " != " << idByName; + EXPECT_EQ(idByName, platforms[i]) << names[i].data() << " : " << platforms[i] << " != " << idByName; } } @@ -85,7 +85,7 @@ TEST(OclocFatBinaryAsProductId, GivenDisabledPlatformNameThenReturnsUnknownPlatf platforms.clear(); for (size_t i = 0; i < platforms.size(); ++i) { auto idByName = NEO::asProductId(names[i], platforms); - EXPECT_EQ(IGFX_UNKNOWN, platforms[i]) << names[i] << " : IGFX_UNKNOWN != " << idByName; + EXPECT_EQ(IGFX_UNKNOWN, platforms[i]) << names[i].data() << " : IGFX_UNKNOWN != " << idByName; } } diff --git a/opencl/test/unit_test/offline_compiler/offline_compiler_tests.cpp b/opencl/test/unit_test/offline_compiler/offline_compiler_tests.cpp index b9bd805401..46637942a2 100644 --- a/opencl/test/unit_test/offline_compiler/offline_compiler_tests.cpp +++ b/opencl/test/unit_test/offline_compiler/offline_compiler_tests.cpp @@ -564,7 +564,7 @@ TEST(OfflineCompilerTest, givenStatelessToStatefullOptimizationEnabledWhenDebugS mockOfflineCompiler.parseDebugSettings(); std::string internalOptions = mockOfflineCompiler.internalOptions; - size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg); + size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg.data()); EXPECT_NE(std::string::npos, found); } @@ -576,7 +576,7 @@ TEST(OfflineCompilerTest, givenStatelessToStatefullOptimizationEnabledWhenDebugS mockOfflineCompiler.parseDebugSettings(); std::string internalOptions = mockOfflineCompiler.internalOptions; - size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg); + size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg.data()); EXPECT_NE(std::string::npos, found); } @@ -588,7 +588,7 @@ TEST(OfflineCompilerTest, givenStatelessToStatefullOptimizationDisableddWhenDevi mockOfflineCompiler.parseDebugSettings(); std::string internalOptions = mockOfflineCompiler.internalOptions; - size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg); + size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg.data()); EXPECT_EQ(std::string::npos, found); } @@ -600,7 +600,7 @@ TEST(OfflineCompilerTest, givenStatelessToStatefullOptimizationEnabledWhenDevice mockOfflineCompiler.parseDebugSettings(); std::string internalOptions = mockOfflineCompiler.internalOptions; - size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg); + size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg.data()); EXPECT_NE(std::string::npos, found); } @@ -613,7 +613,7 @@ TEST(OfflineCompilerTest, givenStatelessToStatefullOptimizationDisabledWhenDevic mockOfflineCompiler.parseDebugSettings(); std::string internalOptions = mockOfflineCompiler.internalOptions; - size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg); + size_t found = internalOptions.find(NEO::CompilerOptions::hasBufferOffsetArg.data()); EXPECT_EQ(std::string::npos, found); } @@ -1076,7 +1076,7 @@ TEST(OfflineCompilerTest, givenInputOptionsAndOclockOptionsFileWithForceStosOptW mockOfflineCompiler->build(); auto &internalOptions = mockOfflineCompiler->internalOptions; - size_t found = internalOptions.find(NEO::CompilerOptions::greaterThan4gbBuffersRequired); + size_t found = internalOptions.find(NEO::CompilerOptions::greaterThan4gbBuffersRequired.data()); EXPECT_EQ(std::string::npos, found); } diff --git a/opencl/test/unit_test/program/program_tests.cpp b/opencl/test/unit_test/program/program_tests.cpp index 80c70d3a8d..7ca25bfd69 100644 --- a/opencl/test/unit_test/program/program_tests.cpp +++ b/opencl/test/unit_test/program/program_tests.cpp @@ -850,13 +850,13 @@ TEST_P(ProgramFromSourceTest, CreateWithSource_Build_Options_Duplicate) { retVal = pProgram->build(0, nullptr, nullptr, nullptr, nullptr, false); EXPECT_EQ(CL_SUCCESS, retVal); - retVal = pProgram->build(0, nullptr, CompilerOptions::fastRelaxedMath, nullptr, nullptr, false); + retVal = pProgram->build(0, nullptr, CompilerOptions::fastRelaxedMath.data(), nullptr, nullptr, false); EXPECT_EQ(CL_SUCCESS, retVal); - retVal = pProgram->build(0, nullptr, CompilerOptions::fastRelaxedMath, nullptr, nullptr, false); + retVal = pProgram->build(0, nullptr, CompilerOptions::fastRelaxedMath.data(), nullptr, nullptr, false); EXPECT_EQ(CL_SUCCESS, retVal); - retVal = pProgram->build(0, nullptr, CompilerOptions::finiteMathOnly, nullptr, nullptr, false); + retVal = pProgram->build(0, nullptr, CompilerOptions::finiteMathOnly.data(), nullptr, nullptr, false); EXPECT_EQ(CL_SUCCESS, retVal); retVal = pProgram->build(0, nullptr, nullptr, nullptr, nullptr, false); @@ -945,7 +945,7 @@ TEST_P(ProgramFromSourceTest, GivenDifferentCommpilerOptionsWhenBuildingProgramT Callback::watch(kernel1); EXPECT_NE(nullptr, kernel1); - retVal = pProgram->build(0, nullptr, CompilerOptions::fastRelaxedMath, nullptr, nullptr, true); + retVal = pProgram->build(0, nullptr, CompilerOptions::fastRelaxedMath.data(), nullptr, nullptr, true); EXPECT_EQ(CL_SUCCESS, retVal); auto hash2 = pProgram->getCachedFileName(); auto kernel2 = pProgram->getKernelInfo("CopyBuffer"); @@ -954,7 +954,7 @@ TEST_P(ProgramFromSourceTest, GivenDifferentCommpilerOptionsWhenBuildingProgramT Callback::unwatch(kernel1); Callback::watch(kernel2); - retVal = pProgram->build(0, nullptr, CompilerOptions::finiteMathOnly, nullptr, nullptr, true); + retVal = pProgram->build(0, nullptr, CompilerOptions::finiteMathOnly.data(), nullptr, nullptr, true); EXPECT_EQ(CL_SUCCESS, retVal); auto hash3 = pProgram->getCachedFileName(); auto kernel3 = pProgram->getKernelInfo("CopyBuffer"); @@ -1103,7 +1103,7 @@ TEST_P(ProgramFromSourceTest, GivenFlagsWhenCompilingProgramThenBuildOptionsHave program->sourceCode = "__kernel mock() {}"; // Ask to build created program without NEO::CompilerOptions::gtpinRera and NEO::CompilerOptions::greaterThan4gbBuffersRequired flags. - cl_int retVal = program->compile(0, nullptr, CompilerOptions::fastRelaxedMath, 0, nullptr, nullptr, nullptr, nullptr); + cl_int retVal = program->compile(0, nullptr, CompilerOptions::fastRelaxedMath.data(), 0, nullptr, nullptr, nullptr, nullptr); EXPECT_EQ(CL_SUCCESS, retVal); // Check build options that were applied @@ -1314,7 +1314,7 @@ TEST_P(ProgramFromSourceTest, GivenInvalidOptionsWhenCreatingLibraryThenCorrectE EXPECT_EQ(CL_SUCCESS, retVal); // create library successfully - retVal = pProgram->link(0, nullptr, CompilerOptions::createLibrary, 1, &program, nullptr, nullptr); + retVal = pProgram->link(0, nullptr, CompilerOptions::createLibrary.data(), 1, &program, nullptr, nullptr); EXPECT_EQ(CL_SUCCESS, retVal); // fail library creation - any link error (here caused by specifying unrecognized option) @@ -1329,7 +1329,7 @@ TEST_P(ProgramFromSourceTest, GivenInvalidOptionsWhenCreatingLibraryThenCorrectE failingProgram->setDevice(&device->getDevice()); // fail library creation - CompilerInterface cannot be obtained - retVal = failingProgram->link(0, nullptr, CompilerOptions::createLibrary, 1, &program, nullptr, nullptr); + retVal = failingProgram->link(0, nullptr, CompilerOptions::createLibrary.data(), 1, &program, nullptr, nullptr); EXPECT_EQ(CL_OUT_OF_HOST_MEMORY, retVal); std::swap(rootDeviceEnvironment, executionEnvironment->rootDeviceEnvironments[device->getRootDeviceIndex()]); } @@ -2006,7 +2006,7 @@ TEST_F(ProgramTests, GivenGtpinReraFlagWhenBuildingProgramThenCorrectOptionsAreS program->createdFrom = Program::CreatedFrom::SOURCE; // Ask to build created program without NEO::CompilerOptions::gtpinRera flag. - cl_int retVal = program->build(0, nullptr, CompilerOptions::fastRelaxedMath, nullptr, nullptr, false); + cl_int retVal = program->build(0, nullptr, CompilerOptions::fastRelaxedMath.data(), nullptr, nullptr, false); EXPECT_EQ(CL_SUCCESS, retVal); // Check build options that were applied @@ -2188,7 +2188,7 @@ TEST_F(Program32BitTests, givenDeviceWithForce32BitAddressingOnWhenProgramIsCrea MockProgram program(*pDevice->getExecutionEnvironment(), pContext, false, pDevice); auto &internalOptions = program.getInternalOptions(); std::string s1 = internalOptions; - size_t pos = s1.find(NEO::CompilerOptions::arch32bit); + size_t pos = s1.find(NEO::CompilerOptions::arch32bit.data()); if (is64bit) { EXPECT_NE(pos, std::string::npos); } else { @@ -2657,7 +2657,7 @@ TEST_F(ProgramTests, givenProgramWhenInternalOptionsArePassedWithInvalidValuesTh EXPECT_EQ(expectedOutput, program.getInternalOptions()); program.isOptionValueValidOverride = true; - buildOptions = CompilerOptions::gtpinRera; + buildOptions = std::string(CompilerOptions::gtpinRera); program.getInternalOptions().erase(); program.extractInternalOptions(buildOptions); EXPECT_EQ(expectedOutput, program.getInternalOptions()); @@ -2691,7 +2691,7 @@ TEST_F(ProgramTests, givenProgramWhenBuiltThenAdditionalOptionsAreApplied) { TEST_F(ProgramTests, WhenProgramIsCreatedThenItsDeviceIsProperlySet) { auto wasValidClDeviceUsed = [](MockProgram &program) -> bool { - return (program.getInternalOptions().find(CompilerOptions::arch32bit) != std::string::npos); + return (program.getInternalOptions().find(CompilerOptions::arch32bit.data()) != std::string::npos); }; MockExecutionEnvironment executionEnvironment; diff --git a/opencl/test/unit_test/program/program_with_kernel_debug_tests.cpp b/opencl/test/unit_test/program/program_with_kernel_debug_tests.cpp index 99bbf3ae3d..b1f1eadab9 100644 --- a/opencl/test/unit_test/program/program_with_kernel_debug_tests.cpp +++ b/opencl/test/unit_test/program/program_with_kernel_debug_tests.cpp @@ -267,7 +267,7 @@ TEST_F(ProgramWithKernelDebuggingTest, givenEnabledKernelDebugWhenProgramIsLinke TEST_F(ProgramWithKernelDebuggingTest, givenProgramWithKernelDebugEnabledWhenBuiltThenPatchTokenAllocateSipSurfaceHasSizeGreaterThanZero) { if (pDevice->getHardwareInfo().platform.eRenderCoreFamily >= IGFX_GEN9_CORE) { - retVal = pProgram->build(1, &device, CompilerOptions::debugKernelEnable, nullptr, nullptr, false); + retVal = pProgram->build(1, &device, CompilerOptions::debugKernelEnable.data(), nullptr, nullptr, false); EXPECT_EQ(CL_SUCCESS, retVal); auto kernelInfo = pProgram->getKernelInfo("CopyBuffer"); diff --git a/shared/source/compiler_interface/compiler_options/compiler_options_base.h b/shared/source/compiler_interface/compiler_options/compiler_options_base.h index 0e80ed444f..15e1c4efa3 100644 --- a/shared/source/compiler_interface/compiler_options/compiler_options_base.h +++ b/shared/source/compiler_interface/compiler_options/compiler_options_base.h @@ -137,6 +137,16 @@ class ConstConcatenation { size_t length = 0U; }; +template +bool operator==(const ConstStringRef &lhs, const ConstConcatenation &rhs) { + return lhs == rhs.operator ConstStringRef(); +} + +template +bool operator==(const ConstConcatenation &lhs, const ConstStringRef &rhs) { + return rhs == lhs; +} + bool contains(const char *options, ConstStringRef optionToFind); bool contains(const std::string &options, ConstStringRef optionToFind); diff --git a/shared/source/device_binary_format/CMakeLists.txt b/shared/source/device_binary_format/CMakeLists.txt index 564f7a78e1..185ad9bbc9 100644 --- a/shared/source/device_binary_format/CMakeLists.txt +++ b/shared/source/device_binary_format/CMakeLists.txt @@ -16,18 +16,20 @@ set(NEO_DEVICE_BINARY_FORMAT ${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_patchtokens.cpp ${CMAKE_CURRENT_SOURCE_DIR}/device_binary_formats.cpp ${CMAKE_CURRENT_SOURCE_DIR}/device_binary_formats.h - ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_decoder.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_decoder.h - ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_dumper.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_dumper.h - ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_validator.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_validator.h ${CMAKE_CURRENT_SOURCE_DIR}/elf/elf.h ${CMAKE_CURRENT_SOURCE_DIR}/elf/elf_decoder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elf/elf_decoder.h ${CMAKE_CURRENT_SOURCE_DIR}/elf/elf_encoder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elf/elf_encoder.h ${CMAKE_CURRENT_SOURCE_DIR}/elf/ocl_elf.h + ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_decoder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_decoder.h + ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_dumper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_dumper.h + ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_validator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_validator.h + ${CMAKE_CURRENT_SOURCE_DIR}/yaml/yaml_parser.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/yaml/yaml_parser.h ) set_property(GLOBAL PROPERTY NEO_DEVICE_BINARY_FORMAT ${NEO_DEVICE_BINARY_FORMAT}) diff --git a/shared/source/device_binary_format/ar/ar_decoder.cpp b/shared/source/device_binary_format/ar/ar_decoder.cpp index c3c4f95d29..31dc0e6a67 100644 --- a/shared/source/device_binary_format/ar/ar_decoder.cpp +++ b/shared/source/device_binary_format/ar/ar_decoder.cpp @@ -31,7 +31,7 @@ Ar decodeAr(const ArrayRef binary, std::string &outErrReason, std return {}; } - if (ConstStringRef(fileEntryHeader->trailingMagic) != arFileEntryTrailingMagic) { + if (ConstStringRef::fromArray(fileEntryHeader->trailingMagic) != arFileEntryTrailingMagic) { outWarnings.append("File entry header with identifier '" + std::string(fileEntryHeader->identifier, sizeof(fileEntryHeader->identifier)) + "' has invalid header trailing string"); } diff --git a/shared/source/device_binary_format/yaml/yaml_parser.cpp b/shared/source/device_binary_format/yaml/yaml_parser.cpp new file mode 100644 index 0000000000..7e5931a713 --- /dev/null +++ b/shared/source/device_binary_format/yaml/yaml_parser.cpp @@ -0,0 +1,391 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/device_binary_format/yaml/yaml_parser.h" + +namespace NEO { + +namespace Yaml { + +std::string constructYamlError(size_t lineNumber, const char *lineBeg, const char *parsePos, const char *reason) { + auto ret = "NEO::Yaml : Could not parse line : [" + std::to_string(lineNumber) + "] : [" + ConstStringRef(lineBeg, parsePos - lineBeg + 1).str() + "] <-- parser position on error"; + if (nullptr != reason) { + ret += ". Reason : "; + ret.append(reason); + } + ret += "\n"; + return ret; +} + +inline Node &addNode(NodesCache &outNodes, Node &parent) { + parent.firstChildId = static_cast(outNodes.size()); + parent.lastChildId = static_cast(outNodes.size()); + outNodes.resize(outNodes.size() + 1); + auto &curr = *outNodes.rbegin(); + curr.id = parent.lastChildId; + curr.parentId = parent.id; + ++parent.numChildren; + return curr; +} + +inline Node &addNode(NodesCache &outNodes, Node &prevSibling, Node &parent) { + prevSibling.nextSiblingId = static_cast(outNodes.size()); + outNodes.resize(outNodes.size() + 1); + auto &curr = *outNodes.rbegin(); + curr.id = prevSibling.nextSiblingId; + curr.parentId = parent.id; + parent.lastChildId = curr.id; + ++parent.numChildren; + return curr; +} + +struct TokenizerContext { + TokenizerContext(ConstStringRef text) + : pos(text.begin()), + end(text.end()), + lineBeginPos(text.begin()) { + lineTraits.reset(); + } + + const char *pos = nullptr; + const char *const end = nullptr; + + uint32_t lineIndent = 0U; + TokenId lineBegin = 0U; + const char *lineBeginPos = nullptr; + bool isParsingIdent = false; + Line::LineTraits lineTraits; +}; + +bool tokenizeEndLine(ConstStringRef text, LinesCache &outLines, TokensCache &outTokens, std::string &outErrReason, std::string &outWarning, TokenizerContext &context) { + TokenId lineEnd = static_cast(outTokens.size()); + outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::SingleCharacter)); + auto lineBegToken = outTokens[context.lineBegin]; + Line::LineType lineType = Line::LineType::Empty; + if (lineEnd != context.lineBegin) { + switch (lineBegToken.traits.type) { + default: + outErrReason = constructYamlError(outLines.size(), lineBegToken.pos, context.pos, "Internal error - undefined line type"); + return false; + case Token::SingleCharacter: + switch (lineBegToken.traits.character0) { + default: + outErrReason = constructYamlError(outLines.size(), lineBegToken.pos, context.pos, (std::string("Unhandled keyword character : ") + lineBegToken.traits.character0).c_str()); + return false; + case '#': + lineType = Line::LineType::Comment; + break; + case '-': + lineType = Line::LineType::ListEntry; + break; + } + break; + case Token::Identifier: + lineType = Line::LineType::DictionaryEntry; + break; + case Token::FileSectionBeg: + lineType = Line::LineType::FileSection; + break; + case Token::FileSectionEnd: + lineType = Line::LineType::FileSection; + break; + } + } + outLines.push_back(Line{lineType, static_cast(context.lineIndent), context.lineBegin, lineEnd, context.lineTraits}); + ++context.pos; + + context.lineIndent = 0U; + context.lineBegin = static_cast(outTokens.size()); + context.lineBeginPos = context.pos; + context.isParsingIdent = true; + context.lineTraits.reset(); + return true; +} + +bool tokenize(ConstStringRef text, LinesCache &outLines, TokensCache &outTokens, std::string &outErrReason, std::string &outWarning) { + if (text.empty()) { + outWarning.append("NEO::Yaml : input text is empty\n"); + return true; + } + + TokenizerContext context{text}; + context.isParsingIdent = true; + + while (context.pos < context.end) { + switch (context.pos[0]) { + case ' ': + context.lineIndent += context.isParsingIdent ? 1 : 0; + ++context.pos; + break; + case '\t': + if (context.isParsingIdent) { + context.lineIndent += 4U; + outWarning.append("NEO::Yaml : Tabs used as indent at line : " + std::to_string(outLines.size()) + "\n"); + } + ++context.pos; + break; + case '\r': + case '\0': + ++context.pos; + break; + case '#': { + context.isParsingIdent = false; + outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::SingleCharacter)); + auto commentIt = context.pos + 1; + while (commentIt < context.end) { + if ('\n' == commentIt[0]) { + break; + } + ++commentIt; + } + if (context.pos + 1 != commentIt) { + outTokens.push_back(Token(ConstStringRef(context.pos + 1, commentIt - (context.pos + 1)), Token::Comment)); + } + context.pos = commentIt; + break; + } + case '\n': { + if (false == tokenizeEndLine(text, outLines, outTokens, outErrReason, outWarning, context)) { + return false; + } + } break; + case '\"': + case '\'': { + context.isParsingIdent = false; + auto parseTokEnd = consumeStringLiteral(text, context.pos); + if (parseTokEnd == context.pos) { + outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, "Underminated string"); + return false; + } + outTokens.push_back(Token(ConstStringRef(context.pos, parseTokEnd - context.pos), Token::LiteralString)); + context.pos = parseTokEnd; + break; + } + case '-': { + ConstStringRef fileSectionMarker("---"); + if ((context.isParsingIdent) && isMatched(text, context.pos, fileSectionMarker)) { + outTokens.push_back(Token(ConstStringRef(context.pos, fileSectionMarker.size()), Token::FileSectionBeg)); + context.pos += fileSectionMarker.size(); + } else { + auto tokEnd = consumeNumberOrSign(text, context.pos); + if (tokEnd > context.pos + 1) { + outTokens.push_back(Token(ConstStringRef(context.pos, tokEnd - context.pos), Token::LiteralNumber)); + } else { + outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::SingleCharacter)); + } + + context.pos = tokEnd; + } + context.isParsingIdent = false; + break; + } + case '.': { + ConstStringRef fileSectionMarker("..."); + if ((context.isParsingIdent) && isMatched(text, context.pos, fileSectionMarker)) { + outTokens.push_back(Token(ConstStringRef(context.pos, fileSectionMarker.size()), Token::FileSectionEnd)); + context.pos += fileSectionMarker.size(); + } else { + outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::SingleCharacter)); + ++context.pos; + } + context.isParsingIdent = false; + break; + } + case '{': + case '}': + case '[': + case ']': + case ',': + outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, "NEO::Yaml : Inline collections are not supported yet"); + return false; + case ':': + context.lineTraits.hasDictionaryEntry = true; + context.isParsingIdent = false; + outTokens.push_back(Token(ConstStringRef(context.pos, 1), Token::SingleCharacter)); + ++context.pos; + break; + default: { + context.isParsingIdent = false; + auto tokEnd = consumeNameIdentifier(text, context.pos); + if (tokEnd != context.pos) { + if (context.lineTraits.hasDictionaryEntry) { + outTokens.push_back(Token(ConstStringRef(context.pos, tokEnd - context.pos), Token::LiteralString)); + } else { + outTokens.push_back(Token(ConstStringRef(context.pos, tokEnd - context.pos), Token::Identifier)); + } + } else { + tokEnd = consumeNumberOrSign(text, context.pos); + if (tokEnd > context.pos) { + outTokens.push_back(Token(ConstStringRef(context.pos, tokEnd - context.pos), Token::LiteralNumber)); + } else { + outErrReason = constructYamlError(outLines.size(), context.lineBeginPos, context.pos, "Invalid numeric literal"); + return false; + } + } + + context.pos = tokEnd; + break; + } + } + } + + if (outTokens.empty()) { + outWarning.append("NEO::Yaml : text tokenized to 0 tokens\n"); + } else { + if ('\n' != *outTokens.rbegin()) { + outWarning.append("NEO::Yaml : text does not end with newline\n"); + tokenizeEndLine(text, outLines, outTokens, outErrReason, outWarning, context); + } + } + return true; +} + +void finalizeNode(NodeId nodeId, const TokensCache &tokens, NodesCache &outNodes, std::string &outErrReason, std::string &outWarning) { + auto &node = outNodes[nodeId]; + if (invalidTokenId != node.key) { + return; + } + if (invalidTokenId == node.value) { + return; + } + auto valueTokenIt = node.value + 1; + auto colon = invalidTokenId; + while ('\n' != tokens[valueTokenIt]) { + if (':' == tokens[valueTokenIt]) { + colon = valueTokenIt; + } + ++valueTokenIt; + } + UNRECOVERABLE_IF((colon == invalidTokenId) || (colon + 1 == valueTokenIt)); + UNRECOVERABLE_IF(invalidNodeID == node.lastChildId) + outNodes[node.lastChildId].nextSiblingId = static_cast(outNodes.size()); + + outNodes.resize(outNodes.size() + 1); + auto &newNode = *outNodes.rbegin(); + newNode.id = static_cast(outNodes.size() - 1); + newNode.parentId = nodeId; + node.lastChildId = outNodes.rbegin()->id; + newNode.key = node.value; + newNode.value = colon + 1; + + node.value = invalidTokenId; + ++node.numChildren; +} + +bool buildTree(const LinesCache &lines, const TokensCache &tokens, NodesCache &outNodes, std::string &outErrReason, std::string &outWarning) { + StackVec nesting; + size_t lineId = 0U; + outNodes.resize(1); + outNodes.rbegin()->id = 0U; + outNodes.rbegin()->firstChildId = 1U; + outNodes.rbegin()->lastChildId = 1U; + nesting.resize(1); // root + + while (lineId < lines.size()) { + if (isUnused(lines[lineId].lineType)) { + ++lineId; + continue; + } + + auto currLineIndent = lines[lineId].indent; + if (currLineIndent == outNodes.rbegin()->indent) { + auto &prev = *outNodes.rbegin(); + auto &parent = outNodes[*nesting.rbegin()]; + auto &curr = addNode(outNodes, prev, parent); + curr.indent = currLineIndent; + } else if (currLineIndent > outNodes.rbegin()->indent) { + auto &parent = *outNodes.rbegin(); + auto &curr = addNode(outNodes, parent); + curr.indent = currLineIndent; + nesting.push_back(parent.id); + } else { + while (currLineIndent < outNodes[*nesting.rbegin()].indent) { + finalizeNode(*nesting.rbegin(), tokens, outNodes, outErrReason, outWarning); + UNRECOVERABLE_IF(nesting.empty()); + nesting.pop_back(); + } + bool hasInvalidIndent = (currLineIndent != outNodes[*nesting.rbegin()].indent); + if (hasInvalidIndent) { + outErrReason = constructYamlError(lineId, tokens[lines[lineId].first].pos, tokens[lines[lineId].first].pos + 1, "Invalid indentation"); + return false; + } else { + auto &prev = outNodes[*nesting.rbegin()]; + auto &parent = outNodes[prev.parentId]; + auto &curr = addNode(outNodes, prev, parent); + curr.indent = currLineIndent; + } + } + + if (lines[lineId].traits.hasInlineDataMarkers) { + outErrReason = "Inline collections are not supported yet\n"; + return false; + } else { + if (Line::LineType::DictionaryEntry == lines[lineId].lineType) { + auto numTokensInLine = lines[lineId].last - lines[lineId].first + 1; + outNodes.rbegin()->key = lines[lineId].first; + UNRECOVERABLE_IF(numTokensInLine < 3); // at least key, : and \n + if (('#' != tokens[lines[lineId].first + 2]) && ('\n' != tokens[lines[lineId].first + 2])) { + outNodes.rbegin()->value = lines[lineId].first + 2; + } + } else { + auto numTokensInLine = lines[lineId].last - lines[lineId].first + 1; + (void)numTokensInLine; + UNRECOVERABLE_IF(numTokensInLine < 2); // at least : - and \n + UNRECOVERABLE_IF(Line::LineType::ListEntry != lines[lineId].lineType); + UNRECOVERABLE_IF('-' != tokens[lines[lineId].first]); + if (('#' != tokens[lines[lineId].first + 1]) && ('\n' != tokens[lines[lineId].first + 1])) { + outNodes.rbegin()->value = lines[lineId].first + 1; + } + } + ++lineId; + } + } + + while (false == nesting.empty()) { + finalizeNode(*nesting.rbegin(), tokens, outNodes, outErrReason, outWarning); + nesting.pop_back(); + } + + if (1U == outNodes.size()) { + outWarning.append("NEO::Yaml : Text has no data\n"); + outNodes.clear(); + return true; + } + return true; +} + +DebugNode *buildDebugNodes(NEO::Yaml::NodeId rootId, const NEO::Yaml::NodesCache &nodes, const NEO::Yaml::TokensCache &tokens) { + DebugNode *curr = new DebugNode; + auto &src = nodes[rootId]; + curr->src = &src; + if (src.key != NEO::Yaml::invalidTokenId) { + curr->key = tokens[src.key].cstrref(); + } + if (src.value != NEO::Yaml::invalidTokenId) { + curr->value = tokens[src.value].cstrref(); + } + + auto childId = src.firstChildId; + while (NEO::Yaml::invalidNodeID != childId) { + curr->children.push_back(buildDebugNodes(childId, nodes, tokens)); + (*curr->children.rbegin())->parent = curr; + childId = nodes[childId].nextSiblingId; + } + return curr; +} + +DebugNode *YamlParser::buildDebugNodes(const Node &parent) const { + return NEO::Yaml::buildDebugNodes(parent.id, nodes, tokens); +} + +DebugNode *YamlParser::buildDebugNodes() const { + return (false == empty()) ? NEO::Yaml::buildDebugNodes(0U, nodes, tokens) : nullptr; +} + +} // namespace Yaml + +} // namespace NEO diff --git a/shared/source/device_binary_format/yaml/yaml_parser.h b/shared/source/device_binary_format/yaml/yaml_parser.h new file mode 100644 index 0000000000..075fed08bc --- /dev/null +++ b/shared/source/device_binary_format/yaml/yaml_parser.h @@ -0,0 +1,603 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/utilities/arrayref.h" +#include "shared/source/utilities/const_stringref.h" +#include "shared/source/utilities/stackvec.h" + +#include +#include + +namespace NEO { + +namespace Yaml { + +constexpr bool isWhitespace(char c) { + switch (c) { + default: + return false; + case ' ': + case '\t': + case '\r': + case '\n': + return true; + } +} + +constexpr bool isLetter(char c) { + return ((c >= 'a') & (c <= 'z')) | ((c >= 'A') & (c <= 'Z')); +} + +constexpr bool isNumber(char c) { + return ((c >= '0') & (c <= '9')); +} + +constexpr bool isAlphaNumeric(char c) { + return isLetter(c) | isNumber(c); +} + +constexpr bool isNameIdentifierCharacter(char c) { + return isAlphaNumeric(c) | ('_' == c); +} + +constexpr bool isNameIdentifierBeginningCharacter(char c) { + return isLetter(c) | ('_' == c); +} + +constexpr bool isSign(char c) { + return ('+' == c) | ('-' == c); +} + +inline bool isSpecificNameIdentifier(ConstStringRef wholeText, const char *parsePos, ConstStringRef pattern) { + UNRECOVERABLE_IF(parsePos < wholeText.begin()); + bool hasEnoughText = (reinterpret_cast(parsePos) + pattern.size() <= reinterpret_cast(wholeText.end())); + bool isEnd = (reinterpret_cast(parsePos) + pattern.size() == reinterpret_cast(wholeText.end())); + bool matched = hasEnoughText && + ((pattern == ConstStringRef(parsePos, pattern.size())) & (isEnd || (false == isNameIdentifierCharacter(parsePos[pattern.size()])))); + return matched; +} + +inline bool isMatched(ConstStringRef wholeText, const char *parsePos, ConstStringRef text) { + UNRECOVERABLE_IF(parsePos < wholeText.begin()); + bool hasEnoughText = (reinterpret_cast(parsePos) + text.size() <= reinterpret_cast(wholeText.end())); + return hasEnoughText && (text == ConstStringRef(parsePos, text.size())); +} + +constexpr const char *consumeNumberOrSign(ConstStringRef wholeText, const char *parsePos, bool allowSign = true) { + UNRECOVERABLE_IF(parsePos < wholeText.begin()); + UNRECOVERABLE_IF(parsePos == wholeText.end()); + auto parseEnd = wholeText.end(); + if (isNumber(*parsePos)) { + auto it = parsePos + 1; + while (it < parseEnd) { + if (false == (isNumber(*it) | ('.' == *it))) { + break; + } + ++it; + } + if (it < parseEnd) { + return isLetter(*it) ? parsePos : it; + } else { + return it; + } + } else if (isSign(*parsePos) && allowSign) { + if (parsePos + 1 < wholeText.end()) { + return consumeNumberOrSign(wholeText, parsePos + 1, false); + } else { + return parsePos + 1; + } + } + return parsePos; +} + +constexpr const char *consumeNameIdentifier(ConstStringRef wholeText, const char *parsePos) { + auto parseEnd = wholeText.end(); + if (isNameIdentifierBeginningCharacter(*parsePos)) { + auto it = parsePos + 1; + while (it < parseEnd) { + if (false == isNameIdentifierCharacter(*it)) { + break; + } + ++it; + } + return it; + } + return parsePos; +} + +constexpr const char *consumeStringLiteral(ConstStringRef wholeText, const char *parsePos) { + auto stringLiteralBeg = *parsePos; + switch (stringLiteralBeg) { + default: + return parsePos; + case '\'': + break; + case '\"': + break; + } + auto parseEnd = wholeText.end(); + auto it = parsePos + 1; + while (it < parseEnd) { + if (stringLiteralBeg == it[0]) { + if (it[-1] != '\\') { // allow escape characters + break; + } + } + ++it; + } + if (it == parseEnd) { + return parsePos; // unterminated literal + } + return it + 1; +} + +using TokenId = uint32_t; + +static constexpr TokenId invalidTokenId = std::numeric_limits::max(); + +struct Token { + enum Type : uint8_t { Identifier, + LiteralString, + LiteralNumber, + SingleCharacter, + Comment, + FileSectionBeg, + FileSectionEnd }; + + constexpr Token(ConstStringRef tokData, Type tokType) { + pos = tokData.begin(); + len = static_cast(tokData.length()); + traits.type = tokType; + traits.character0 = tokData[0]; + } + + const char *pos = nullptr; + uint32_t len = 0U; + struct { + Type type = Token::Type::Identifier; + char character0 = 0U; + } traits; + + constexpr ConstStringRef cstrref() const { + return ConstStringRef(pos, len); + } +}; + +static_assert(sizeof(Token) <= 16, ""); + +constexpr bool operator==(Token token, ConstStringRef matcher) { + return (matcher[0] == token.traits.character0) && (ConstStringRef(token.pos + 1, token.len - 1) == ConstStringRef(matcher.begin() + 1, matcher.size() - 1)); +} + +constexpr bool operator==(ConstStringRef matcher, Token token) { + return (token == matcher); +} + +constexpr bool operator!=(Token token, ConstStringRef matcher) { + return (false == (token == matcher)); +} + +constexpr bool operator!=(ConstStringRef matcher, Token token) { + return token != matcher; +} + +constexpr bool operator==(Token token, char matcher) { + return ((matcher == token.traits.character0) & (token.len = 1U)); +} + +constexpr bool operator==(char matcher, Token token) { + return token == matcher; +} + +constexpr bool operator!=(Token token, char matcher) { + return (false == (token == matcher)); +} + +constexpr bool operator!=(char matcher, Token token) { + return token != matcher; +} + +struct Line { + enum class LineType : uint8_t { Empty, + Comment, + FileSection, + DictionaryEntry, + ListEntry }; + TokenId first = 0U; + TokenId last = 0U; // note : NOT past last (aka end) + + uint16_t indent = 0U; + LineType lineType = LineType::Empty; + struct LineTraits { + union { + struct { + bool hasInlineDataMarkers : 1; + bool hasDictionaryEntry : 1; + }; + uint8_t packed; + }; + void reset() { + this->packed = 0U; + } + constexpr LineTraits() : packed(0) { + } + } traits; + + constexpr Line(LineType lineType, uint16_t indent, TokenId first, TokenId last, LineTraits traits) + : first(first), last(last), indent(indent), lineType(lineType), traits{} { + this->traits = traits; + } +}; +static_assert(sizeof(Line) == 12, ""); + +using TokensCache = StackVec; +using LinesCache = StackVec; + +std::string constructYamlError(size_t lineNumber, const char *lineBeg, const char *parsePos, const char *reason = nullptr); + +bool tokenize(ConstStringRef text, LinesCache &outLines, TokensCache &outTokens, std::string &outErrReason, std::string &outWarning); + +using NodeId = uint16_t; +static constexpr NodeId invalidNodeID = std::numeric_limits::max(); + +struct Node { + TokenId key = invalidTokenId; + TokenId value = invalidTokenId; + uint32_t indent = 0; + NodeId id = invalidNodeID; + NodeId parentId = invalidNodeID; + NodeId firstChildId = invalidNodeID; + NodeId lastChildId = invalidNodeID; + NodeId nextSiblingId = invalidNodeID; + uint16_t numChildren = 0U; + + Node() = default; + + explicit Node(uint32_t indent) : indent(indent) { + } +}; +static_assert(sizeof(Node) == 24, ""); +using NodesCache = StackVec; + +constexpr bool isUnused(Line::LineType lineType) { + switch (lineType) { + default: + return false; + case Line::LineType::Empty: + return true; + case Line::LineType::Comment: + return true; + case Line::LineType::FileSection: + return true; + } +} + +bool buildTree(const LinesCache &lines, const TokensCache &tokens, NodesCache &outNodes, std::string &outErrReason, std::string &outWarning); + +inline const Node *findChildByKey(const Node &parent, const NodesCache &allNodes, const TokensCache &allTokens, const ConstStringRef key) { + auto childId = parent.firstChildId; + while (invalidNodeID != childId) { + if ((invalidTokenId != allNodes[childId].key) && (key == allTokens[allNodes[childId].key])) { + break; + } + childId = allNodes[childId].nextSiblingId; + } + + return (invalidNodeID != childId) ? &allNodes[childId] : nullptr; +} + +inline const Node *getFirstChild(const Node &parent, const NodesCache &allNodes) { + auto childId = parent.firstChildId; + if (invalidNodeID == childId) { + return nullptr; + } + + return &allNodes[childId]; +} + +inline const Node *getLastChild(const Node &parent, const NodesCache &allNodes) { + auto childId = parent.lastChildId; + if (invalidNodeID == childId) { + return nullptr; + } + + return &allNodes[childId]; +} + +struct ConstSiblingsFwdIterator : std::iterator { + ConstSiblingsFwdIterator(NodeId currId, const NodesCache *allNodes) + : allNodes(allNodes), currId(currId) { + } + + ConstSiblingsFwdIterator(const ConstSiblingsFwdIterator &rhs) + : allNodes(rhs.allNodes), currId(rhs.currId) { + } + + ConstSiblingsFwdIterator &operator=(const ConstSiblingsFwdIterator &rhs) { + allNodes = rhs.allNodes; + currId = rhs.currId; + return *this; + } + + bool operator==(const ConstSiblingsFwdIterator &rhs) const { + return (allNodes == rhs.allNodes) & (currId == rhs.currId); + } + + bool operator!=(const ConstSiblingsFwdIterator &rhs) const { + return false == (*this == rhs); + } + + const Node &operator*() { + return (*allNodes)[currId]; + } + + const Node *operator->() { + return &(*allNodes)[currId]; + } + + ConstSiblingsFwdIterator &operator++() { + if (invalidNodeID != currId) { + currId = (*allNodes)[currId].nextSiblingId; + } + return *this; + } + + ConstSiblingsFwdIterator operator++(int) { + auto nextId = currId; + if (invalidNodeID != currId) { + nextId = (*allNodes)[currId].nextSiblingId; + } + auto prevId = currId; + currId = nextId; + return ConstSiblingsFwdIterator(prevId, allNodes); + } + + protected: + const NodesCache *allNodes = nullptr; + NodeId currId = invalidNodeID; +}; + +struct ConstChildrenRange { + ConstChildrenRange(const Node &first, const NodesCache &allNodes) + : allNodes(allNodes), firstId(first.id) { + } + + ConstChildrenRange(const NodeId firstId, const NodesCache &allNodes) + : allNodes(allNodes), firstId(firstId) { + } + + ConstSiblingsFwdIterator begin() const { + return ConstSiblingsFwdIterator(firstId, &allNodes); + } + + ConstSiblingsFwdIterator end() const { + return ConstSiblingsFwdIterator(invalidNodeID, &allNodes); + } + + protected: + const NodesCache &allNodes; + const NodeId firstId = invalidNodeID; +}; + +struct DebugNode { + ~DebugNode() { + for (auto c : children) { + delete c; + } + } + + ConstStringRef key; + std::vector children; + ConstStringRef value; + DebugNode *parent = nullptr; + const Node *src = nullptr; +}; + +DebugNode *buildDebugNodes(NEO::Yaml::NodeId rootId, const NEO::Yaml::NodesCache &nodes, const NEO::Yaml::TokensCache &tokens); + +struct YamlParser { + YamlParser() { + } + + bool parse(const ConstStringRef text, std::string &outErrReason, std::string &outWarning) { + auto success = NEO::Yaml::tokenize(text, lines, tokens, outErrReason, outWarning); + success = success && NEO::Yaml::buildTree(lines, tokens, nodes, outErrReason, outWarning); + if (false == success) { + nodes.clear(); + } + return success; + } + + bool empty() const { + return (0U == nodes.size()); + } + + const Node *getRoot() { + return &nodes[0]; + } + + ConstStringRef readKey(const Node &node) const { + return (invalidTokenId != node.key) ? tokens[node.key].cstrref() : ""; + } + + ConstStringRef readValue(const Node &node) const { + return (invalidTokenId != node.value) ? tokens[node.value].cstrref() : ""; + } + + const Token *getValueToken(const Node &node) const { + return (invalidTokenId != node.value) ? &tokens[node.value] : nullptr; + } + + template + bool readValueChecked(const Node &node, T &outValue) const; + + ConstStringRef readValueNoQuotes(const Node &node) const { + if (invalidTokenId == node.value) { + return ""; + } + auto &tok = tokens[node.value]; + if (Token::Type::LiteralString != tok.traits.type) { + return tok.cstrref(); + } + if ((tok.traits.character0 != '\'') && (tok.traits.character0 != '\"')) { + return tok.cstrref(); + } + return ConstStringRef(tok.pos + 1, tok.len - 2); + } + + ConstChildrenRange createChildrenRange(const Node &parent) const { + return ConstChildrenRange(nodes[parent.firstChildId], nodes); + } + + const Node *findNodeWithKeyDfs(const ConstStringRef key) const { + for (auto &node : nodes) { + if (readKey(node) == key) { + return &node; + } + } + return nullptr; + } + + const Node *getChild(const Node &parent, const ConstStringRef key) const { + return findChildByKey(parent, nodes, tokens, key); + } + + DebugNode *buildDebugNodes(const Node &parent) const; + DebugNode *buildDebugNodes() const; + + protected: + TokensCache tokens; + LinesCache lines; + NodesCache nodes; +}; + +template <> +inline bool YamlParser::readValueChecked(const Node &node, uint64_t &outValue) const { + if (invalidTokenId == node.value) { + return false; + } + const auto &token = tokens[node.value]; + if (Token::Type::LiteralNumber != token.traits.type) { + return false; + } + StackVec nullTerminated{token.pos, token.pos + token.len}; + nullTerminated.push_back('\0'); + outValue = atoll(nullTerminated.begin()); + return true; +} + +template <> +inline bool YamlParser::readValueChecked(const Node &node, uint32_t &outValue) const { + uint64_t uint64V = 0U; + bool validValue = readValueChecked(node, uint64V); + validValue &= uint64V <= std::numeric_limits::max(); + outValue = static_cast(uint64V); + return validValue; +} + +template <> +inline bool YamlParser::readValueChecked(const Node &node, uint16_t &outValue) const { + uint64_t uint64V = 0U; + bool validValue = readValueChecked(node, uint64V); + validValue &= uint64V <= std::numeric_limits::max(); + outValue = static_cast(uint64V); + return validValue; +} + +template <> +inline bool YamlParser::readValueChecked(const Node &node, uint8_t &outValue) const { + uint64_t uint64V = 0U; + bool validValue = readValueChecked(node, uint64V); + validValue &= uint64V <= std::numeric_limits::max(); + outValue = static_cast(uint64V); + return validValue; +} + +template <> +inline bool YamlParser::readValueChecked(const Node &node, bool &outValue) const { + if (invalidTokenId == node.value) { + return false; + } + const auto &token = tokens[node.value]; + if (Token::Type::LiteralString != token.traits.type) { + return false; + } + + // valid values : y/n yes/no true/false on/off (case insesitive) + if (token.len > 5) { + return false; + } + + switch (token.traits.character0) { + default: + return false; + case 'y': + case 'Y': { + outValue = true; + switch (token.len) { + default: + return false; + case 1: + return true; + case 3: + return equalsCaseInsesitive(ConstStringRef("es"), ConstStringRef(token.cstrref().begin() + 1, 2)); + } + break; + } + case 'n': + case 'N': { + outValue = false; + switch (token.len) { + default: + return false; + case 1: + return true; + case 2: + return ((token.cstrref()[1] == 'o') | (token.cstrref()[1] == 'O')); + } + break; + } + case 't': + case 'T': { + outValue = true; + if (token.len != 4) { + return false; + } + return equalsCaseInsesitive(ConstStringRef("rue"), ConstStringRef(token.cstrref().begin() + 1, 3)); + } + case 'f': + case 'F': { + outValue = false; + if (token.len != 5) { + return false; + } + return equalsCaseInsesitive(ConstStringRef("alse"), ConstStringRef(token.cstrref().begin() + 1, 4)); + } + case 'o': + case 'O': { + switch (token.len) { + default: + return false; + case 2: + outValue = true; + return ((token.cstrref()[1] == 'n') | (token.cstrref()[1] == 'N')); + case 3: + outValue = false; + return equalsCaseInsesitive(ConstStringRef("ff"), ConstStringRef(token.cstrref().begin() + 1, 2)); + } + break; + } + } + + return true; +} + +} // namespace Yaml + +} // namespace NEO diff --git a/shared/source/utilities/const_stringref.h b/shared/source/utilities/const_stringref.h index 0d62d5e5d0..efaaa109b7 100644 --- a/shared/source/utilities/const_stringref.h +++ b/shared/source/utilities/const_stringref.h @@ -8,8 +8,12 @@ #pragma once #include +#include +#include #include +namespace NEO { + constexpr size_t constLength(const char *string) { if (nullptr == string) { return 0U; @@ -34,25 +38,61 @@ class ConstStringRef { return *this; } - template - constexpr ConstStringRef(const char (&array)[Length]) noexcept - : ptr(array), len(((Length > 0) && (array[Length - 1] == '\0')) ? (Length - 1) : Length) { + constexpr ConstStringRef(const char &c) noexcept + : ptr(&c), len(1) { + } + + constexpr ConstStringRef(const char *const ptr) noexcept + : ptr(ptr), len(constLength(ptr)) { } constexpr ConstStringRef(const char *const ptr, const size_t length) noexcept : ptr(ptr), len(length) { } + template + static constexpr ConstStringRef fromArray(const char (&array)[Length]) noexcept { + return ConstStringRef(array, Length); + } + ConstStringRef(const std::string &str) noexcept : ptr(str.data()), len(str.length()) { } + constexpr ConstStringRef substr(int offset, int len) const noexcept { + if (len >= 0) { + return ConstStringRef(this->ptr + offset, len); + } else { + return ConstStringRef(this->ptr + offset, this->len + len - offset); + } + } + + constexpr ConstStringRef substr(int offset) const noexcept { + return ConstStringRef(this->ptr + offset, this->len - offset); + } + + constexpr ConstStringRef truncated(int len) const noexcept { + if (len >= 0) { + return ConstStringRef(this->ptr, len); + } else { + return ConstStringRef(this->ptr, this->len + len); + } + } + constexpr const char *data() const noexcept { return ptr; } - constexpr operator const char *() const noexcept { - return ptr; + constexpr char operator[](size_t pos) const noexcept { + return ptr[pos]; + } + + constexpr char operator[](int pos) const noexcept { + return ptr[pos]; + } + + explicit operator std::string() const { + return str(); } std::string str() const { @@ -98,6 +138,8 @@ class ConstStringRef { } protected: + ConstStringRef(std::nullptr_t) = delete; + const char *ptr = nullptr; size_t len = 0U; }; @@ -116,26 +158,28 @@ constexpr bool equals(const ConstStringRef &lhs, const ConstStringRef &rhs) { return true; } -template -constexpr bool equals(const ConstStringRef &lhs, const T *rhs) { - return equals(lhs, ConstStringRef(rhs, constLength(rhs))); -} +constexpr bool equals(const ConstStringRef &lhs, const char *rhs) { + for (size_t i = 0, e = lhs.size(); i < e; ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + if ((rhs[i] == '\0') && (i + 1 < e)) { + return false; + } + } -inline bool equals(const ConstStringRef &lhs, const std::string &rhs) { - return equals(lhs, ConstStringRef(rhs.data(), rhs.length())); + return true; } constexpr bool operator==(const ConstStringRef &lhs, const ConstStringRef &rhs) { return equals(lhs, rhs); } -template -constexpr bool operator==(const ConstStringRef &lhs, const RhsT &rhs) { +constexpr bool operator==(const ConstStringRef &lhs, const char *rhs) { return equals(lhs, rhs); } -template -constexpr bool operator==(const LhsT &lhs, const ConstStringRef &rhs) { +constexpr bool operator==(const char *lhs, const ConstStringRef &rhs) { return equals(rhs, lhs); } @@ -143,12 +187,28 @@ constexpr bool operator!=(const ConstStringRef &lhs, const ConstStringRef &rhs) return false == equals(lhs, rhs); } -template -constexpr bool operator!=(const ConstStringRef &lhs, const RhsT &rhs) { - return false == (lhs == rhs); +constexpr bool operator!=(const ConstStringRef &lhs, const char *rhs) { + return false == equals(lhs, rhs); } -template -constexpr bool operator!=(const LhsT &lhs, const ConstStringRef &rhs) { - return rhs != lhs; +constexpr bool operator!=(const char *lhs, const ConstStringRef &rhs) { + return false == equals(rhs, lhs); } + +constexpr bool equalsCaseInsesitive(const ConstStringRef &lhs, const ConstStringRef &rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + + constexpr auto caseDiff = 'a' - 'A'; + for (size_t i = 0, e = lhs.size(); i < e; ++i) { + + if ((lhs[i] != rhs[i]) & (lhs[i] + caseDiff != rhs[i]) & (lhs[i] != rhs[i] + caseDiff)) { + return false; + } + } + + return true; +} + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/utilities/stackvec.h b/shared/source/utilities/stackvec.h index ebbb3a0ef2..37b4bd50d3 100644 --- a/shared/source/utilities/stackvec.h +++ b/shared/source/utilities/stackvec.h @@ -205,6 +205,18 @@ class StackVec { ++onStackSize; } + void pop_back() { // NOLINT + if (usesDynamicMem()) { + dynamicMem->pop_back(); + return; + } + + UNRECOVERABLE_IF(0 == onStackSize); + + clearStackObjects(onStackSize - 1, 1U); + --onStackSize; + } + DataType &operator[](std::size_t idx) { if (usesDynamicMem()) { return (*dynamicMem)[idx]; diff --git a/shared/test/unit_test/compiler_interface/compiler_options_tests.cpp b/shared/test/unit_test/compiler_interface/compiler_options_tests.cpp index 5943933ed2..3a1b243c3a 100644 --- a/shared/test/unit_test/compiler_interface/compiler_options_tests.cpp +++ b/shared/test/unit_test/compiler_interface/compiler_options_tests.cpp @@ -23,23 +23,25 @@ TEST(CompilerOptions, WhenConcatenateIsCalledThenUsesSpaceAsSeparator) { auto expected = (std::string(NEO::CompilerOptions::optDisable) + " " + NEO::CompilerOptions::finiteMathOnly.data()); EXPECT_STREQ(expected.c_str(), concatenated.c_str()); - constexpr ConstStringRef toConcatenate[] = {"a", "b", "c"}; + constexpr NEO::ConstStringRef toConcatenate[] = {"a", "b", "c"}; constexpr ConstConcatenation constConcatenationSpecificSize(toConcatenate); constexpr ConstConcatenation<> constConcatenationDefaultSize(toConcatenate); - EXPECT_TRUE(ConstStringRef("a b c") == constConcatenationSpecificSize); - EXPECT_TRUE(ConstStringRef("a b c") == constConcatenationDefaultSize); + EXPECT_TRUE(NEO::ConstStringRef("a b c") == constConcatenationSpecificSize); + EXPECT_TRUE(NEO::ConstStringRef("a b c") == constConcatenationDefaultSize); + EXPECT_TRUE(constConcatenationSpecificSize == NEO::ConstStringRef("a b c")); + EXPECT_TRUE(constConcatenationDefaultSize == NEO::ConstStringRef("a b c")); } TEST(CompilerOptions, WhenConcatenateAppendIsCalledThenAddsSpaceAsSeparatorOnlyIfMissing) { using namespace NEO::CompilerOptions; std::string concatenated = NEO::CompilerOptions::optDisable.data(); concatenateAppend(concatenated, NEO::CompilerOptions::finiteMathOnly); - auto expected = (std::string(NEO::CompilerOptions::optDisable) + " " + NEO::CompilerOptions::finiteMathOnly.data()); + auto expected = (NEO::CompilerOptions::optDisable.str() + " " + NEO::CompilerOptions::finiteMathOnly.data()); EXPECT_STREQ(expected.c_str(), concatenated.c_str()); concatenated += " "; concatenateAppend(concatenated, NEO::CompilerOptions::fastRelaxedMath); expected += " "; - expected += NEO::CompilerOptions::fastRelaxedMath; + expected += NEO::CompilerOptions::fastRelaxedMath.data(); EXPECT_STREQ(expected.c_str(), concatenated.c_str()); } diff --git a/shared/test/unit_test/device_binary_format/CMakeLists.txt b/shared/test/unit_test/device_binary_format/CMakeLists.txt index 0b0dd08730..66835f74b1 100644 --- a/shared/test/unit_test/device_binary_format/CMakeLists.txt +++ b/shared/test/unit_test/device_binary_format/CMakeLists.txt @@ -18,4 +18,6 @@ target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_dumper_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_tests.h ${CMAKE_CURRENT_SOURCE_DIR}/patchtokens_validator_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/yaml/yaml_parser_tests.cpp ) + diff --git a/shared/test/unit_test/device_binary_format/ar/ar_decoder_tests.cpp b/shared/test/unit_test/device_binary_format/ar/ar_decoder_tests.cpp index bbbba2faec..18f5f89b9c 100644 --- a/shared/test/unit_test/device_binary_format/ar/ar_decoder_tests.cpp +++ b/shared/test/unit_test/device_binary_format/ar/ar_decoder_tests.cpp @@ -119,7 +119,7 @@ TEST(ArDecoderDecodeAr, GivenValidArThenDecodingSucceeds) { EXPECT_TRUE(ar.longFileNamesEntry.fileName.empty()); ASSERT_EQ(1U, ar.files.size()); EXPECT_EQ(reinterpret_cast(arStorage.data() + arMagic.size()), ar.files[0].fullHeader); - EXPECT_EQ("a", ar.files[0].fileName); + EXPECT_EQ(NEO::ConstStringRef("a"), ar.files[0].fileName); EXPECT_EQ(arStorage.data() + arMagic.size() + sizeof(ArFileEntryHeader), ar.files[0].fileData.begin()); EXPECT_EQ(8U, ar.files[0].fileData.size()); } @@ -139,8 +139,8 @@ TEST(ArDecoderDecodeAr, GivenArWhenFileEntryHeaderHasEmptyIdentifierThenDecoding EXPECT_EQ(nullptr, ar.magic); EXPECT_EQ(0U, ar.files.size()); EXPECT_EQ(nullptr, ar.longFileNamesEntry.fullHeader); - EXPECT_TRUE(decodeWarnings.empty()); - EXPECT_FALSE(decodeErrors.empty()); + EXPECT_TRUE(decodeWarnings.empty()) << decodeWarnings; + EXPECT_FALSE(decodeErrors.empty()) << decodeErrors; EXPECT_STREQ("Corrupt AR archive - file entry does not have identifier : '/ '", decodeErrors.c_str()); } diff --git a/shared/test/unit_test/device_binary_format/ar/ar_encoder_tests.cpp b/shared/test/unit_test/device_binary_format/ar/ar_encoder_tests.cpp index f0dfdbc4b5..46a183f218 100644 --- a/shared/test/unit_test/device_binary_format/ar/ar_encoder_tests.cpp +++ b/shared/test/unit_test/device_binary_format/ar/ar_encoder_tests.cpp @@ -15,16 +15,17 @@ #include using namespace NEO::Ar; +using namespace NEO; TEST(ArFileEntryHeader, GivenDefaultArFileEntryHeaderThenSectionAreProperlyPopulated) { ArFileEntryHeader header = {}; - EXPECT_EQ(ConstStringRef("/ ", 16), ConstStringRef(header.identifier)); - EXPECT_EQ(ConstStringRef("0 ", 12), ConstStringRef(header.fileModificationTimestamp)); - EXPECT_EQ(ConstStringRef("0 ", 6), ConstStringRef(header.ownerId)); - EXPECT_EQ(ConstStringRef("0 ", 6), ConstStringRef(header.groupId)); - EXPECT_EQ(ConstStringRef("644 ", 8), ConstStringRef(header.fileMode)); - EXPECT_EQ(ConstStringRef("0 ", 10), ConstStringRef(header.fileSizeInBytes)); - EXPECT_EQ(ConstStringRef("\x60\x0A", 2), ConstStringRef(header.trailingMagic)); + EXPECT_EQ(ConstStringRef("/ ", 16), ConstStringRef::fromArray(header.identifier)); + EXPECT_EQ(ConstStringRef("0 ", 12), ConstStringRef::fromArray(header.fileModificationTimestamp)); + EXPECT_EQ(ConstStringRef("0 ", 6), ConstStringRef::fromArray(header.ownerId)); + EXPECT_EQ(ConstStringRef("0 ", 6), ConstStringRef::fromArray(header.groupId)); + EXPECT_EQ(ConstStringRef("644 ", 8), ConstStringRef::fromArray(header.fileMode)); + EXPECT_EQ(ConstStringRef("0 ", 10), ConstStringRef::fromArray(header.fileSizeInBytes)); + EXPECT_EQ(ConstStringRef("\x60\x0A", 2), ConstStringRef::fromArray(header.trailingMagic)); } TEST(ArEncoder, GivenTooLongIdentifierThenAppendingFileFails) { @@ -47,13 +48,13 @@ TEST(ArEncoder, GivenEmptyArThenEncodedFileConsistsOfOnlyArMagic) { } TEST(ArEncoder, GivenValidFileEntriesThenAppendingFileSucceeds) { - std::string fileName = "file1.txt"; + ConstStringRef fileName = "file1.txt"; const uint8_t fileData[18] = "23571113171923293"; ArEncoder encoder; auto returnedSection = encoder.appendFileEntry(fileName, fileData); ASSERT_NE(nullptr, returnedSection); ArFileEntryHeader expectedSection; - memcpy_s(expectedSection.identifier, sizeof(expectedSection.identifier), fileName.c_str(), fileName.size()); + memcpy_s(expectedSection.identifier, sizeof(expectedSection.identifier), fileName.data(), fileName.size()); expectedSection.identifier[fileName.size()] = '/'; expectedSection.fileSizeInBytes[0] = '1'; expectedSection.fileSizeInBytes[1] = '8'; @@ -69,8 +70,8 @@ TEST(ArEncoder, GivenValidFileEntriesThenAppendingFileSucceeds) { } TEST(ArEncoder, GivenValidTwoFileEntriesWith2byteUnalignedDataThenPaddingIsImplicitlyAdded) { - std::string fileName0 = "a"; - std::string fileName1 = "b"; + ConstStringRef fileName0 = "a"; + ConstStringRef fileName1 = "b"; const uint8_t data0[7] = "123456"; const uint8_t data1[16] = "9ABCDEFGHIJKLMN"; ArEncoder encoder; @@ -106,9 +107,9 @@ TEST(ArEncoder, GivenValidTwoFileEntriesWith2byteUnalignedDataThenPaddingIsImpli } TEST(ArEncoder, GivenValidTwoFileEntriesWhen8BytePaddingIsRequestedThenPaddingFileEntriesAreAddedWhenNeeded) { - std::string fileName0 = "a"; - std::string fileName1 = "b"; - std::string fileName2 = "c"; + ConstStringRef fileName0 = "a"; + ConstStringRef fileName1 = "b"; + ConstStringRef fileName2 = "c"; const uint8_t data0[4] = "123"; // will require padding before const uint8_t data1[8] = "9ABCDEF"; // won't require padding before const uint8_t data2[16] = "9ABCDEF"; // will require padding before diff --git a/shared/test/unit_test/device_binary_format/device_binary_formats_tests.cpp b/shared/test/unit_test/device_binary_format/device_binary_formats_tests.cpp index e310c666e8..3b9814c42f 100644 --- a/shared/test/unit_test/device_binary_format/device_binary_formats_tests.cpp +++ b/shared/test/unit_test/device_binary_format/device_binary_formats_tests.cpp @@ -43,7 +43,7 @@ TEST(IsAnyDeviceBinaryFormat, GivenArFormatThenReturnsTrue) { TEST(UnpackSingleDeviceBinary, GivenUnknownBinaryThenReturnError) { const uint8_t data[] = "none of known formats"; - ConstStringRef requestedProductAbbreviation = "unk"; + auto requestedProductAbbreviation = "unk"; NEO::TargetDevice requestedTargetDevice; std::string outErrors; std::string outWarnings; @@ -62,7 +62,7 @@ TEST(UnpackSingleDeviceBinary, GivenUnknownBinaryThenReturnError) { TEST(UnpackSingleDeviceBinary, GivenPatchtoknsBinaryThenReturnSelf) { PatchTokensTestData::ValidEmptyProgram patchtokensProgram; - ConstStringRef requestedProductAbbreviation = "unk"; + auto requestedProductAbbreviation = "unk"; NEO::TargetDevice requestedTargetDevice; requestedTargetDevice.coreFamily = static_cast(patchtokensProgram.header->Device); requestedTargetDevice.stepping = patchtokensProgram.header->SteppingId; @@ -91,7 +91,7 @@ TEST(UnpackSingleDeviceBinary, GivenOclElfBinaryThenReturnPatchtokensBinary) { elfEnc.getElfFileHeader().type = NEO::Elf::ET_OPENCL_EXECUTABLE; elfEnc.appendSection(NEO::Elf::SHT_OPENCL_DEV_BINARY, NEO::Elf::SectionNamesOpenCl::deviceBinary, patchtokensProgram.storage); - ConstStringRef requestedProductAbbreviation = "unk"; + auto requestedProductAbbreviation = "unk"; NEO::TargetDevice requestedTargetDevice; requestedTargetDevice.coreFamily = static_cast(patchtokensProgram.header->Device); requestedTargetDevice.stepping = patchtokensProgram.header->SteppingId; @@ -130,7 +130,7 @@ TEST(UnpackSingleDeviceBinary, GivenArBinaryWithOclElfThenReturnPatchtokensBinar std::string outWarnings; auto elfData = elfEnc.encode(); - std::string requiredProduct = NEO::hardwarePrefix[productFamily]; + auto requiredProduct = NEO::hardwarePrefix[productFamily]; std::string requiredStepping = std::to_string(patchtokensProgram.header->SteppingId); std::string requiredPointerSize = (patchtokensProgram.header->GPUPointerSizeInBytes == 4) ? "32" : "64"; NEO::Ar::ArEncoder arEnc(true); diff --git a/shared/test/unit_test/device_binary_format/elf/elf_encoder_tests.cpp b/shared/test/unit_test/device_binary_format/elf/elf_encoder_tests.cpp index e6fbc23bba..0c8ca3b81a 100644 --- a/shared/test/unit_test/device_binary_format/elf/elf_encoder_tests.cpp +++ b/shared/test/unit_test/device_binary_format/elf/elf_encoder_tests.cpp @@ -442,9 +442,9 @@ TEST(ElfEncoder, WhenAppendingEmptySectionNameThenAlwaysReturn0AsOffset) { TEST(ElfEncoder, WhenAppendingSectionNameThenEmplacedStringIsAlwaysNullterminated) { ElfEncoder elfEncoder64(true, true); - auto strOffset = elfEncoder64.appendSectionName(ConstStringRef("abc", 2)); - auto strOffset2 = elfEncoder64.appendSectionName(ConstStringRef("de", 3)); - auto strOffset3 = elfEncoder64.appendSectionName(ConstStringRef("g")); + auto strOffset = elfEncoder64.appendSectionName(NEO::ConstStringRef("abc", 2)); + auto strOffset2 = elfEncoder64.appendSectionName(NEO::ConstStringRef("de", 3)); + auto strOffset3 = elfEncoder64.appendSectionName(NEO::ConstStringRef("g")); elfEncoder64.appendSection(SHT_NOBITS, "my_name_is_important", {}); EXPECT_EQ(strOffset + 3, strOffset2); EXPECT_EQ(strOffset2 + 3, strOffset3); diff --git a/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp b/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp new file mode 100644 index 0000000000..14802d48ae --- /dev/null +++ b/shared/test/unit_test/device_binary_format/yaml/yaml_parser_tests.cpp @@ -0,0 +1,2682 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/device_binary_format/yaml/yaml_parser.h" + +#include "test.h" + +#include +#include +#include + +using namespace NEO::Yaml; +using namespace NEO; + +TEST(YamlIsWhitespace, GivenCharThenReturnsTrueOnlyWhenCharIsWhitespace) { + std::set whitespaces{' ', '\t', '\r', '\n'}; + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = whitespaces.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isWhitespace(static_cast(c))) << static_cast(c); + } +} + +template +struct IteratorAsValue : std::iterator { + IteratorAsValue(const T &v) : value(v) {} + T operator*() const { return value; } + IteratorAsValue &operator++() { + ++value; + return *this; + } + bool operator==(const IteratorAsValue &rhs) const { return this->value == rhs.value; } + bool operator!=(const IteratorAsValue &rhs) const { return this->value != rhs.value; } + bool operator<(const IteratorAsValue &rhs) const { + return this->value < rhs.value; + } + T value; +}; + +TEST(YamlIsLetter, GivenCharThenReturnsTrueOnlyWhenCharIsLetter) { + std::set validChars{}; + using It = IteratorAsValue; + validChars.insert(It{'a'}, ++It{'z'}); + validChars.insert(It{'A'}, ++It{'Z'}); + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = validChars.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isLetter(static_cast(c))) << static_cast(c); + } +} + +TEST(YamlIsNumber, GivenCharThenReturnsTrueOnlyWhenCharIsNumber) { + std::set validChars{}; + using It = IteratorAsValue; + validChars.insert(It{'0'}, ++It{'9'}); + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = validChars.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isNumber(static_cast(c))) << static_cast(c); + } +} + +TEST(YamlIsAlphaNumeric, GivenCharThenReturnsTrueOnlyWhenCharIsNumberOrLetter) { + std::set validChars{}; + using It = IteratorAsValue; + validChars.insert(It{'a'}, ++It{'z'}); + validChars.insert(It{'A'}, ++It{'Z'}); + validChars.insert(It{'0'}, ++It{'9'}); + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = validChars.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isAlphaNumeric(static_cast(c))) << static_cast(c); + } +} + +TEST(YamlIsNameIdentifierCharacter, GivenCharThenReturnsTrueOnlyWhenCharIsNumberOrLetterOrUnderscore) { + std::set validChars{}; + using It = IteratorAsValue; + validChars.insert(It{'a'}, ++It{'z'}); + validChars.insert(It{'A'}, ++It{'Z'}); + validChars.insert(It{'0'}, ++It{'9'}); + validChars.insert('_'); + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = validChars.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isNameIdentifierCharacter(static_cast(c))) << static_cast(c); + } +} + +TEST(YamlIsNameBeginningCharacter, GivenCharThenReturnsTrueOnlyWhenCharIsLetterOrUnderscore) { + std::set validChars{}; + using It = IteratorAsValue; + validChars.insert(It{'a'}, ++It{'z'}); + validChars.insert(It{'A'}, ++It{'Z'}); + validChars.insert('_'); + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = validChars.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isNameIdentifierBeginningCharacter(static_cast(c))) << static_cast(c); + } +} + +TEST(YamlIsSign, GivenCharThenReturnsTrueOnlyWhenCharIsPlusOrMinus) { + std::set validChars{}; + using It = IteratorAsValue; + validChars.insert('+'); + validChars.insert('-'); + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool expected = validChars.count(static_cast(c)) > 0; + EXPECT_EQ(expected, NEO::Yaml::isSign(static_cast(c))) << static_cast(c); + } +} + +TEST(YamlIsSpecificNameIdentifier, WhenTextIsEmptyThenReturnFalse) { + ConstStringRef text = "a"; + EXPECT_TRUE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin(), "a")); + + ConstStringRef empty; + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(empty, empty.begin(), "a")); +} + +TEST(YamlIsSpecificNameIdentifier, WhenTextIsTooShortThenReturnFalse) { + ConstStringRef text = "abc"; + EXPECT_TRUE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 1, "bc")); + + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin(), "bcd")); +} + +TEST(YamlIsSpecificNameIdentifier, WhenIdetifierIsSubstringThenReturnFalse) { + ConstStringRef text = " bcD efg ij_ kl5"; + EXPECT_TRUE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 1, "bcD")); + EXPECT_TRUE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 5, "efg")); + EXPECT_TRUE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 9, "ij_")); + EXPECT_TRUE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 13, "kl5")); + + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 1, "bc")); + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 1, "b")); + + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 5, "ef")); + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 5, "e")); + + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 9, "ij")); + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 9, "i")); + + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 13, "kl")); + EXPECT_FALSE(NEO::Yaml::isSpecificNameIdentifier(text, text.begin() + 13, "k")); +} + +TEST(YamlIsMatched, WhenTextIsEmptyThenReturnFalse) { + ConstStringRef text = "a"; + EXPECT_TRUE(NEO::Yaml::isMatched(text, text.begin(), "a")); + + ConstStringRef empty; + EXPECT_FALSE(NEO::Yaml::isMatched(empty, empty.begin(), "a")); +} + +TEST(YamlIsMatched, WhenTextIsTooShortThenReturnFalse) { + ConstStringRef text = "abc"; + EXPECT_TRUE(NEO::Yaml::isMatched(text, text.begin() + 1, "bc")); + + EXPECT_FALSE(NEO::Yaml::isMatched(text, text.begin(), "bcd")); +} + +TEST(YamlConsumeNumberOrSign, GivenValidNumberOrSignThenReturnProperEndingPosition) { + ConstStringRef plus5 = "a+5"; + ConstStringRef minus7 = "b -7 ["; + ConstStringRef plus = "c+"; + ConstStringRef minus = "d- "; + ConstStringRef num957 = "e957"; + ConstStringRef num9dot57 = "f9.57"; + EXPECT_EQ(plus5.begin() + 3, NEO::Yaml::consumeNumberOrSign(plus5, plus5.begin() + 1)); + EXPECT_EQ(minus7.begin() + 5, NEO::Yaml::consumeNumberOrSign(minus7, minus7.begin() + 3)); + EXPECT_EQ(plus.begin() + 2, NEO::Yaml::consumeNumberOrSign(plus, plus.begin() + 1)); + EXPECT_EQ(minus.begin() + 2, NEO::Yaml::consumeNumberOrSign(minus, minus.begin() + 1)); + EXPECT_EQ(num957.begin() + 4, NEO::Yaml::consumeNumberOrSign(num957, num957.begin() + 1)); + EXPECT_EQ(num9dot57.begin() + 5, NEO::Yaml::consumeNumberOrSign(num9dot57, num9dot57.begin() + 1)); +} + +TEST(YamlConsumeNumberOrSign, GivenInvalidCharacterThenReturnCurrentParsePosition) { + ConstStringRef plus5 = "a+5"; + ConstStringRef minus7 = "b -7 ["; + ConstStringRef plus = "c+"; + ConstStringRef minus = "d- "; + ConstStringRef num957 = "e957"; + ConstStringRef num9dot57 = "f9.57"; + ConstStringRef num9dot57X = "f9.57X"; + ConstStringRef plusPlusSeven = "++7"; + + EXPECT_EQ(plus5.begin(), NEO::Yaml::consumeNumberOrSign(plus5, plus5.begin())); + EXPECT_EQ(minus7.begin(), NEO::Yaml::consumeNumberOrSign(minus7, minus7.begin())); + EXPECT_EQ(plus.begin(), NEO::Yaml::consumeNumberOrSign(plus, plus.begin())); + EXPECT_EQ(minus.begin(), NEO::Yaml::consumeNumberOrSign(minus, minus.begin())); + EXPECT_EQ(num957.begin(), NEO::Yaml::consumeNumberOrSign(num957, num957.begin())); + EXPECT_EQ(num9dot57.begin(), NEO::Yaml::consumeNumberOrSign(num9dot57, num9dot57.begin())); + EXPECT_EQ(num9dot57X.begin() + 1, NEO::Yaml::consumeNumberOrSign(num9dot57X, num9dot57X.begin() + 1)); + EXPECT_EQ(plusPlusSeven.begin() + 1, NEO::Yaml::consumeNumberOrSign(plusPlusSeven, plusPlusSeven.begin())); + + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool isSignOrNumber = NEO::Yaml::isSign(static_cast(c)) | NEO::Yaml::isNumber(static_cast(c)); + char numberStr[] = {static_cast(c), '\0'}; + auto expected = numberStr + (isSignOrNumber ? 1 : 0); + EXPECT_EQ(expected, NEO::Yaml::consumeNumberOrSign(ConstStringRef::fromArray(numberStr), numberStr)) << c; + } +} + +TEST(YamlConsumeNameIdentifier, GivenInvalidNameIdentifierBeginningCharacterThenReturnCurrentParsePosition) { + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool isNameIndentifierBeginChar = NEO::Yaml::isNameIdentifierBeginningCharacter(static_cast(c)); + char nameIdentifierStr[] = {static_cast(c), '\0'}; + auto expected = nameIdentifierStr + (isNameIndentifierBeginChar ? 1 : 0); + EXPECT_EQ(expected, NEO::Yaml::consumeNameIdentifier(ConstStringRef::fromArray(nameIdentifierStr), nameIdentifierStr)) << c; + } +} + +TEST(YamlConsumeNameIdentifier, GivenNameIdentifierBeginningCharacterThenConsumeWholeNameIdentifier) { + for (int c = std::numeric_limits::min(); c <= std::numeric_limits::max(); ++c) { + bool isNameIndentifierChar = NEO::Yaml::isNameIdentifierCharacter(static_cast(c)); + char nameIdentifierStr[] = {'A', static_cast(c)}; + auto expected = nameIdentifierStr + (isNameIndentifierChar ? 2 : 1); + EXPECT_EQ(expected, NEO::Yaml::consumeNameIdentifier(ConstStringRef::fromArray(nameIdentifierStr), nameIdentifierStr)) << c; + } +} + +TEST(YamlConsumeStringLiteral, GivenQuotedStringThenConsumeUntilEndingMarkIsMet) { + ConstStringRef notQuoted = "a+5"; + ConstStringRef singleQuote = "\'abc de fg\'ijkl"; + ConstStringRef doubleQuote = "\"abc de fg\"ijkl"; + ConstStringRef escapeSingleQuote = "\'abc de\\\' fg\'ijkl"; + ConstStringRef escapeDoubleQuote = "\"abc de\\\" fg\"ijkl"; + ConstStringRef mixedQuoteOuterDouble = "\"abc de\' fg\"ijkl"; + ConstStringRef mixedQuoteOuterSingle = "\'abc de\" fg\'ijkl"; + ConstStringRef unterminatedSingleQuote = "\'abc de"; + ConstStringRef unterminatedDoubleQuote = "\"abc de"; + + EXPECT_EQ(notQuoted.begin(), NEO::Yaml::consumeStringLiteral(notQuoted, notQuoted.begin())) << notQuoted.data(); + EXPECT_EQ(singleQuote.begin() + 11, NEO::Yaml::consumeStringLiteral(singleQuote, singleQuote.begin())) << singleQuote.data(); + EXPECT_EQ(doubleQuote.begin() + 11, NEO::Yaml::consumeStringLiteral(doubleQuote, doubleQuote.begin())) << doubleQuote.data(); + EXPECT_EQ(escapeSingleQuote.begin() + 13, NEO::Yaml::consumeStringLiteral(escapeSingleQuote, escapeSingleQuote.begin())) << escapeSingleQuote.data(); + EXPECT_EQ(escapeDoubleQuote.begin() + 13, NEO::Yaml::consumeStringLiteral(escapeDoubleQuote, escapeDoubleQuote.begin())) << escapeDoubleQuote.data(); + EXPECT_EQ(mixedQuoteOuterDouble.begin() + 12, NEO::Yaml::consumeStringLiteral(mixedQuoteOuterDouble, mixedQuoteOuterDouble.begin())) << mixedQuoteOuterDouble.data(); + EXPECT_EQ(mixedQuoteOuterSingle.begin() + 12, NEO::Yaml::consumeStringLiteral(mixedQuoteOuterSingle, mixedQuoteOuterSingle.begin())) << mixedQuoteOuterSingle.data(); + EXPECT_EQ(unterminatedSingleQuote.begin(), NEO::Yaml::consumeStringLiteral(unterminatedSingleQuote, unterminatedSingleQuote.begin())) << unterminatedSingleQuote.data(); + EXPECT_EQ(unterminatedDoubleQuote.begin(), NEO::Yaml::consumeStringLiteral(unterminatedDoubleQuote, unterminatedDoubleQuote.begin())) << unterminatedDoubleQuote.data(); +} + +TEST(YamlToken, WhenConstructedThenSetsUpProperDefaults) { + ConstStringRef str = "\"some string\""; + ConstStringRef identifier = "someIdentifier"; + NEO::Yaml::Token tokenString{str, NEO::Yaml::Token::LiteralString}; + NEO::Yaml::Token tokenNameIdentifier{identifier, NEO::Yaml::Token::Identifier}; + + EXPECT_EQ(str.begin(), tokenString.pos); + EXPECT_EQ(str.length(), tokenString.len); + EXPECT_EQ(str[0], tokenString.traits.character0); + EXPECT_EQ(NEO::Yaml::Token::LiteralString, tokenString.traits.type); + EXPECT_EQ(str, tokenString.cstrref()); + + EXPECT_EQ(identifier.begin(), tokenNameIdentifier.pos); + EXPECT_EQ(identifier.length(), tokenNameIdentifier.len); + EXPECT_EQ(identifier[0], tokenNameIdentifier.traits.character0); + EXPECT_EQ(NEO::Yaml::Token::Identifier, tokenNameIdentifier.traits.type); + EXPECT_EQ(identifier, tokenNameIdentifier.cstrref()); +} + +TEST(YamlTokenMatchers, WhenTokenIsComparedAgainstTextThenReturnTrueOnlyIfMatched) { + NEO::Yaml::Token tokenString{"abc", NEO::Yaml::Token::Identifier}; + EXPECT_TRUE(ConstStringRef("abc") == tokenString); + EXPECT_TRUE(tokenString == ConstStringRef("abc")); + EXPECT_FALSE(ConstStringRef("abd") == tokenString); + EXPECT_FALSE(tokenString == ConstStringRef("abd")); + EXPECT_TRUE(ConstStringRef("abd") != tokenString); + EXPECT_TRUE(tokenString != ConstStringRef("abd")); +} + +TEST(YamlTokenMatchers, WhenTokenIsComparedAgainstCharacterThenReturnTrueOnlyIfMatched) { + NEO::Yaml::Token tokenString{"+", NEO::Yaml::Token::Identifier}; + EXPECT_TRUE('+' == tokenString); + EXPECT_TRUE(tokenString == '+'); + EXPECT_FALSE('-' == tokenString); + EXPECT_FALSE(tokenString == '-'); + EXPECT_TRUE('-' != tokenString); + EXPECT_TRUE(tokenString != '-'); +} + +TEST(YamlLineTraits, WhenResetThenFlagsGetCleared) { + NEO::Yaml::Line::LineTraits traits; + traits.hasDictionaryEntry = true; + traits.hasInlineDataMarkers = true; + EXPECT_NE(0U, traits.packed); + + traits.reset(); + EXPECT_EQ(0U, traits.packed); + EXPECT_FALSE(traits.hasDictionaryEntry); + EXPECT_FALSE(traits.hasInlineDataMarkers); +} + +TEST(YamlLine, WhenConstructedThenSetsUpProperDefaults) { + NEO::Yaml::Line::LineTraits emptyTraits; + emptyTraits.reset(); + NEO::Yaml::Line commentLine{NEO::Yaml::Line::LineType::Comment, 4, 6, 8, emptyTraits}; + + NEO::Yaml::Line::LineTraits dictionaryEntryTraits; + dictionaryEntryTraits.reset(); + dictionaryEntryTraits.hasDictionaryEntry = true; + NEO::Yaml::Line dictEntryLine{NEO::Yaml::Line::LineType::DictionaryEntry, 2, 128, 146, dictionaryEntryTraits}; + + EXPECT_EQ(4U, commentLine.indent); + EXPECT_EQ(6U, commentLine.first); + EXPECT_EQ(8U, commentLine.last); + EXPECT_EQ(NEO::Yaml::Line::LineType::Comment, commentLine.lineType); + EXPECT_EQ(emptyTraits.packed, commentLine.traits.packed); + + EXPECT_EQ(2U, dictEntryLine.indent); + EXPECT_EQ(128U, dictEntryLine.first); + EXPECT_EQ(146U, dictEntryLine.last); + EXPECT_EQ(NEO::Yaml::Line::LineType::DictionaryEntry, dictEntryLine.lineType); + EXPECT_EQ(dictionaryEntryTraits.packed, dictEntryLine.traits.packed); +} + +TEST(YamlConstructParseError, WhenConstructingErrorThenTakesLineNumberAndContextIntoAccount) { + ConstStringRef line{"Not really a yaml"}; + EXPECT_STREQ("NEO::Yaml : Could not parse line : [146] : [Not r] <-- parser position on error\n", NEO::Yaml::constructYamlError(146, line.begin(), line.begin() + 4).c_str()); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [146] : [Not r] <-- parser position on error. Reason : Yup, not a yaml\n", NEO::Yaml::constructYamlError(146, line.begin(), line.begin() + 4, "Yup, not a yaml").c_str()); +} + +TEST(YamlTokenize, GivenEmptyInputStringThenEmitsWarning) { + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize("", lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(lines.empty()); + EXPECT_TRUE(tokens.empty()); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_STREQ("NEO::Yaml : input text is empty\n", warnings.c_str()); + + warnings.clear(); + success = NEO::Yaml::tokenize(" ", lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(lines.empty()); + EXPECT_TRUE(tokens.empty()); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_STREQ("NEO::Yaml : text tokenized to 0 tokens\n", warnings.c_str()); + + warnings.clear(); + success = NEO::Yaml::tokenize("\r", lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(lines.empty()); + EXPECT_TRUE(tokens.empty()); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_STREQ("NEO::Yaml : text tokenized to 0 tokens\n", warnings.c_str()); + + warnings.clear(); + success = NEO::Yaml::tokenize(ConstStringRef::fromArray("\0"), lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(lines.empty()); + EXPECT_TRUE(tokens.empty()); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_STREQ("NEO::Yaml : text tokenized to 0 tokens\n", warnings.c_str()); +} + +TEST(YamlTokenize, GivenTabsAsIndentThenEmitsWarning) { + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize("\t", lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(lines.empty()); + EXPECT_TRUE(tokens.empty()); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_STREQ("NEO::Yaml : Tabs used as indent at line : 0\nNEO::Yaml : text tokenized to 0 tokens\n", warnings.c_str()); +} + +TEST(YamlTokenize, WhenTextDoesNotEndWithNewlineThenEmitsWarning) { + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize("aaaaa", lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_EQ(1U, lines.size()); + EXPECT_EQ(2U, tokens.size()); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_STREQ("NEO::Yaml : text does not end with newline\n", warnings.c_str()); +} + +TEST(YamlTokenize, WhenTextIsNotPartOfACollectionThenEmitsError) { + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize("7\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [7\n] <-- parser position on error. Reason : Internal error - undefined line type\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + success = NEO::Yaml::tokenize(": a\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [: a\n] <-- parser position on error. Reason : Unhandled keyword character : :\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; +} + +TEST(YamlTokenize, GivenInlineCollectionThenReturnErrorAsUnsupported) { + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize("[\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [[] <-- parser position on error. Reason : NEO::Yaml : Inline collections are not supported yet\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + success = NEO::Yaml::tokenize("]\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : []] <-- parser position on error. Reason : NEO::Yaml : Inline collections are not supported yet\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + success = NEO::Yaml::tokenize("{\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [{] <-- parser position on error. Reason : NEO::Yaml : Inline collections are not supported yet\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + success = NEO::Yaml::tokenize("}\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [}] <-- parser position on error. Reason : NEO::Yaml : Inline collections are not supported yet\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + success = NEO::Yaml::tokenize("a , b\n", lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [a ,] <-- parser position on error. Reason : NEO::Yaml : Inline collections are not supported yet\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; +} + +namespace NEO { +namespace Yaml { + +bool operator==(const NEO::Yaml::Line &lhs, const NEO::Yaml::Line &rhs) { + bool matched = (lhs.first == rhs.first); + matched &= (lhs.last == rhs.last); + matched &= (lhs.indent == rhs.indent); + matched &= (lhs.lineType == rhs.lineType); + matched &= (lhs.traits.packed == rhs.traits.packed); + return matched; +} + +bool operator==(const NEO::Yaml::Token &lhs, const NEO::Yaml::Token &rhs) { + bool matched = (lhs.cstrref() == rhs); + matched &= (lhs.traits.type == rhs.traits.type); + return matched; +} + +} // namespace Yaml +} // namespace NEO + +TEST(YamlTokenize, GivenMultilineDictionaryThenTokenizeAllEntries) { + ConstStringRef yaml = + R"===( + apple : red + banana : yellow + orange : orange +)==="; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 0 + + // line 1 + Token{"apple", NEO::Yaml::Token::Identifier}, // token 1 + Token{":", NEO::Yaml::Token::SingleCharacter}, // token 2 + Token{"red", NEO::Yaml::Token::LiteralString}, // token 3 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 4 + + // line 2 + Token{"banana", NEO::Yaml::Token::Identifier}, // token 5 + Token{":", NEO::Yaml::Token::SingleCharacter}, // token 6 + Token{"yellow", NEO::Yaml::Token::LiteralString}, // token 7 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 8 + + // line 3 + Token{"orange", NEO::Yaml::Token::Identifier}, // token 9 + Token{":", NEO::Yaml::Token::SingleCharacter}, // token 10 + Token{"orange", NEO::Yaml::Token::LiteralString}, // token 11 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 12 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::DictionaryEntry, 4, 1, 4, {}}, + Line{NEO::Yaml::Line::LineType::DictionaryEntry, 4, 5, 8, {}}, + Line{NEO::Yaml::Line::LineType::DictionaryEntry, 4, 9, 12, {}}, + }; + expectedLines[1].traits.hasDictionaryEntry = true; + expectedLines[2].traits.hasDictionaryEntry = true; + expectedLines[3].traits.hasDictionaryEntry = true; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenMultilineListThenTokenizeAllEntries) { + ConstStringRef yaml = + R"===( + - apple + - banana + - orange +)==="; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 0 + + // line 1 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 1 + Token{"apple", NEO::Yaml::Token::Identifier}, // token 2 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 3 + + // line 2 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 4 + Token{"banana", NEO::Yaml::Token::Identifier}, // token 5 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 6 + + // line 3 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 7 + Token{"orange", NEO::Yaml::Token::Identifier}, // token 8 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 9 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 3, 1, 3, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 3, 4, 6, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 3, 7, 9, {}}, + }; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenCommentThenTokenizeTillEndOfLineAsOneToken) { + ConstStringRef yaml = "orange : green # needs more sun\n"; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"orange", NEO::Yaml::Token::Identifier}, // token 0 + Token{":", NEO::Yaml::Token::SingleCharacter}, // token 1 + Token{"green", NEO::Yaml::Token::LiteralString}, // token 2 + Token{"#", NEO::Yaml::Token::SingleCharacter}, // token 3 + Token{" needs more sun", NEO::Yaml::Token::Comment}, // token 4 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 5 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::DictionaryEntry, 0, 0, 5, {}}, + }; + + expectedLines[0].traits.hasDictionaryEntry = true; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } + + ConstStringRef yamlCommentTillEnd = ConstStringRef(yaml.data(), 25); + tokens.clear(); + lines.clear(); + success = NEO::Yaml::tokenize(yamlCommentTillEnd, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_FALSE(warnings.empty()); + + expectedTokens[4].len -= static_cast(yaml.length() - yamlCommentTillEnd.length() - expectedTokens[5].len); + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size() - 1; ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenEmptyCommentMarkerThenDontCreateEmptyComment) { + ConstStringRef yaml = "orange : green #\n"; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"orange", NEO::Yaml::Token::Identifier}, // token 0 + Token{":", NEO::Yaml::Token::SingleCharacter}, // token 1 + Token{"green", NEO::Yaml::Token::LiteralString}, // token 2 + Token{"#", NEO::Yaml::Token::SingleCharacter}, // token 3 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 4 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::DictionaryEntry, 0, 0, 4, {}}, + }; + + expectedLines[0].traits.hasDictionaryEntry = true; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenCommentAtTheBeginningOfTheLineThenMarkWholeLineAsComment) { + ConstStringRef yaml = "#orange : green\n"; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"#", NEO::Yaml::Token::SingleCharacter}, // token 0 + Token{"orange : green", NEO::Yaml::Token::Comment}, // token 1 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 2 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::Comment, 0, 0, 2, {}}, + }; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenFileSectionMarkersThenTokenizesThemProperly) { + ConstStringRef yaml = + R"===( +--- + - banana +... +)==="; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 0 + + // line 1 + Token{"---", NEO::Yaml::Token::FileSectionBeg}, // token 1 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 2 + + // line 2 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 3 + Token{"banana", NEO::Yaml::Token::Identifier}, // token 4 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 5 + + // line 3 + Token{"...", NEO::Yaml::Token::FileSectionEnd}, // token 6 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 7 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::FileSection, 0, 1, 2, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 1, 3, 5, {}}, + Line{NEO::Yaml::Line::LineType::FileSection, 0, 6, 7, {}}, + }; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenInvalidFileSectionMarkersThenTreatThemAsText) { + ConstStringRef yaml = "- .\n"; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 1 + Token{".", NEO::Yaml::Token::SingleCharacter}, // token 2 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 3 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 0, 2, {}}, + }; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + yaml = "..\n"; + success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [..\n] <-- parser position on error. Reason : Unhandled keyword character : .\n", errors.c_str()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; +} + +TEST(YamlTokenize, GivenStringLiteralsThenTokenizesThemProperly) { + ConstStringRef yaml = + R"===( +- "abc defg ijk" +- '23 57' +- 'needs to "match"' +- "match 'needs to'" +)==="; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 0 + + // line 1 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 1 + Token{"\"abc defg ijk\"", NEO::Yaml::Token::LiteralString}, // token 2 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 3 + + // line 2 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 4 + Token{"\'23 57\'", NEO::Yaml::Token::LiteralString}, // token 5 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 6 + + // line 3 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 7 + Token{"\'needs to \"match\"\'", NEO::Yaml::Token::LiteralString}, // token 8 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 9 + + // line 4 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 10 + Token{"\"match \'needs to\'\"", NEO::Yaml::Token::LiteralString}, // token 11 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 12 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 1, 3, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 4, 6, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 7, 9, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 10, 12, {}}, + }; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenUnterminatedStringLiteralsThenReturnsError) { + ConstStringRef yaml = "\"aaaa"; + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [\"] <-- parser position on error. Reason : Underminated string\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; +} + +TEST(YamlTokenize, GivenNumericLiteralsThenTokenizesThemProperly) { + ConstStringRef yaml = + R"===( +- -5 +- +7 +- 136 +- 95.8 +- +14.7 +- -63.1 +- 0 +)==="; + + NEO::Yaml::Token expectedTokens[] = { + // line 0 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 0 + + // line 1 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 1 + Token{"-5", NEO::Yaml::Token::LiteralNumber}, // token 2 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 3 + + // line 2 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 4 + Token{"+7", NEO::Yaml::Token::LiteralNumber}, // token 5 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 6 + + // line 3 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 7 + Token{"136", NEO::Yaml::Token::LiteralNumber}, // token 8 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 9 + + // line 4 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 10 + Token{"95.8", NEO::Yaml::Token::LiteralNumber}, // token 11 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 12 + + // line 5 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 13 + Token{"+14.7", NEO::Yaml::Token::LiteralNumber}, // token 14 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 15 + + // line 6 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 16 + Token{"-63.1", NEO::Yaml::Token::LiteralNumber}, // token 17 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 18 + + // line 7 + Token{"-", NEO::Yaml::Token::SingleCharacter}, // token 19 + Token{"0", NEO::Yaml::Token::LiteralNumber}, // token 20 + Token{"\n", NEO::Yaml::Token::SingleCharacter}, // token 21 + }; + + NEO::Yaml::Line expectedLines[] = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 1, 3, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 4, 6, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 7, 9, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 10, 12, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 13, 15, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 16, 18, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 19, 21, {}}, + }; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(warnings.empty()) << warnings; + + ASSERT_EQ(sizeof(expectedTokens) / sizeof(expectedTokens[0]), tokens.size()); + for (size_t i = 0; i < tokens.size(); ++i) { + EXPECT_EQ(expectedTokens[i], tokens[i]) << i; + } + + ASSERT_EQ(sizeof(expectedLines) / sizeof(expectedLines[0]), lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + EXPECT_EQ(expectedLines[i], lines[i]) << i; + } +} + +TEST(YamlTokenize, GivenInvalidNumericLiteralThenReturnError) { + ConstStringRef yaml = "@5\n"; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [0] : [@] <-- parser position on error. Reason : Invalid numeric literal\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()) << warnings; +} + +TEST(YamlNode, WhenConstructedThenSetsUpProperDefaults) { + { + NEO::Yaml::Node node; + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.id); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.lastChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.parentId); + EXPECT_EQ(0U, node.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, node.key); + EXPECT_EQ(NEO::Yaml::invalidTokenId, node.value); + EXPECT_EQ(0U, node.numChildren); + } + + { + NEO::Yaml::Node node(7U); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.id); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.lastChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, node.parentId); + EXPECT_EQ(7U, node.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, node.key); + EXPECT_EQ(NEO::Yaml::invalidTokenId, node.value); + EXPECT_EQ(0U, node.numChildren); + } +} + +TEST(YamlLineTypeIsUnused, WhenLineContaintsNoDataThenReturnTrue) { + EXPECT_TRUE(NEO::Yaml::isUnused(NEO::Yaml::Line::LineType::Empty)); + EXPECT_TRUE(NEO::Yaml::isUnused(NEO::Yaml::Line::LineType::Comment)); + EXPECT_TRUE(NEO::Yaml::isUnused(NEO::Yaml::Line::LineType::FileSection)); +} + +TEST(YamlLineTypeIsUnused, WhenLineContaintsDataThenReturnFalse) { + EXPECT_FALSE(NEO::Yaml::isUnused(NEO::Yaml::Line::LineType::DictionaryEntry)); + EXPECT_FALSE(NEO::Yaml::isUnused(NEO::Yaml::Line::LineType::ListEntry)); +} + +TEST(YamlBuildTree, GivenEmptyInputThenEmitsWarning) { + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + NEO::Yaml::NodesCache tree; + std::string errors, warnings; + bool success = NEO::Yaml::buildTree(lines, tokens, tree, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(tree.empty()); + + EXPECT_STREQ("NEO::Yaml : Text has no data\n", warnings.c_str()); +} + +TEST(YamlBuildTree, GivenEmptyLinesThenSkipsThem) { + NEO::Yaml::LinesCache lines = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::Comment, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::FileSection, 0, 0, 0, {}}, + }; + NEO::Yaml::TokensCache tokens; + NEO::Yaml::NodesCache tree; + std::string errors, warnings; + bool success = NEO::Yaml::buildTree(lines, tokens, tree, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(errors.empty()) << errors; + EXPECT_TRUE(tree.empty()); + + EXPECT_STREQ("NEO::Yaml : Text has no data\n", warnings.c_str()); +} + +TEST(YamlBuildTree, GivenInlineCollectionsThenReturnsFalseAsUnsupported) { + NEO::Yaml::LinesCache lines = { + Line{NEO::Yaml::Line::LineType::Empty, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::ListEntry, 0, 0, 0, {}}, + Line{NEO::Yaml::Line::LineType::FileSection, 0, 0, 0, {}}, + }; + lines[1].traits.hasInlineDataMarkers = true; + NEO::Yaml::TokensCache tokens; + NEO::Yaml::NodesCache tree; + std::string errors, warnings; + bool success = NEO::Yaml::buildTree(lines, tokens, tree, errors, warnings); + EXPECT_FALSE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_STREQ("Inline collections are not supported yet\n", errors.c_str()); +} + +template +auto at(ContainerT &container, IndexT index) -> decltype(std::declval()[0]) & { + if (index >= container.size()) { + throw std::out_of_range(std::to_string(index) + " >= " + std::to_string(container.size())); + } + return container[index]; +} + +TEST(YamlBuildTree, GivenListThenProperlyCreatesChildNodes) { + ConstStringRef yaml = + R"===( + - apple + - banana + - orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + ASSERT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + ASSERT_EQ(4U, treeNodes.size()); + + auto &list = *treeNodes.begin(); + EXPECT_GT(treeNodes.size(), list.id); + EXPECT_EQ(0U, list.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, list.key); + EXPECT_EQ(NEO::Yaml::invalidTokenId, list.value); + EXPECT_EQ(3U, list.numChildren); + EXPECT_EQ(NEO::Yaml::invalidNodeID, list.parentId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, list.nextSiblingId); + + ASSERT_GT(treeNodes.size(), list.firstChildId); + ASSERT_GT(treeNodes.size(), list.lastChildId); + + auto &apple = treeNodes[list.firstChildId]; + ASSERT_GT(treeNodes.size(), apple.nextSiblingId); + auto &banana = treeNodes[apple.nextSiblingId]; + auto &orange = treeNodes[list.lastChildId]; + + EXPECT_EQ(3U, apple.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, apple.key); + EXPECT_GT(tokens.size(), apple.value); + EXPECT_EQ("apple", tokens[apple.value].cstrref()); + EXPECT_EQ(0U, apple.numChildren); + EXPECT_EQ(list.id, apple.parentId); + EXPECT_EQ(banana.id, apple.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, apple.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, apple.lastChildId); + + EXPECT_EQ(3U, banana.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, banana.key); + EXPECT_GT(tokens.size(), banana.value); + EXPECT_EQ("banana", tokens[banana.value].cstrref()); + EXPECT_EQ(0U, banana.numChildren); + EXPECT_EQ(list.id, banana.parentId); + EXPECT_EQ(orange.id, banana.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.lastChildId); + + EXPECT_EQ(3U, orange.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, orange.key); + EXPECT_GT(tokens.size(), orange.value); + EXPECT_EQ("orange", tokens[orange.value].cstrref()); + EXPECT_EQ(0U, orange.numChildren); + EXPECT_EQ(list.id, orange.parentId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, orange.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.lastChildId); +} + +TEST(YamlBuildTree, GivenDictionaryThenProperlyCreatesChildNodes) { + ConstStringRef yaml = + R"===( + apple : red + banana : yellow + orange : orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + ASSERT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + ASSERT_EQ(4U, treeNodes.size()); + + auto &dictionary = *treeNodes.begin(); + EXPECT_GT(treeNodes.size(), dictionary.id); + EXPECT_EQ(0U, dictionary.indent); + EXPECT_EQ(NEO::Yaml::invalidTokenId, dictionary.key); + EXPECT_EQ(NEO::Yaml::invalidTokenId, dictionary.value); + EXPECT_EQ(3U, dictionary.numChildren); + EXPECT_EQ(NEO::Yaml::invalidNodeID, dictionary.parentId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, dictionary.nextSiblingId); + + ASSERT_GT(treeNodes.size(), dictionary.firstChildId); + ASSERT_GT(treeNodes.size(), dictionary.lastChildId); + + auto &apple = treeNodes[dictionary.firstChildId]; + ASSERT_GT(treeNodes.size(), apple.nextSiblingId); + auto &banana = treeNodes[apple.nextSiblingId]; + auto &orange = treeNodes[dictionary.lastChildId]; + + EXPECT_EQ(4U, apple.indent); + EXPECT_GT(tokens.size(), apple.key); + EXPECT_EQ("apple", tokens[apple.key].cstrref()); + EXPECT_GT(tokens.size(), apple.value); + EXPECT_EQ("red", tokens[apple.value].cstrref()); + EXPECT_EQ(0U, apple.numChildren); + EXPECT_EQ(dictionary.id, apple.parentId); + EXPECT_EQ(banana.id, apple.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, apple.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, apple.lastChildId); + + EXPECT_EQ(4U, banana.indent); + EXPECT_GT(tokens.size(), banana.key); + EXPECT_EQ("banana", tokens[banana.key].cstrref()); + EXPECT_GT(tokens.size(), banana.value); + EXPECT_EQ("yellow", tokens[banana.value].cstrref()); + EXPECT_EQ(0U, banana.numChildren); + EXPECT_EQ(dictionary.id, banana.parentId); + EXPECT_EQ(orange.id, banana.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.lastChildId); + + EXPECT_EQ(4U, orange.indent); + EXPECT_GT(tokens.size(), orange.key); + EXPECT_EQ("orange", tokens[orange.key].cstrref()); + EXPECT_GT(tokens.size(), orange.value); + EXPECT_EQ("orange", tokens[orange.value].cstrref()); + EXPECT_EQ(0U, orange.numChildren); + EXPECT_EQ(dictionary.id, orange.parentId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, orange.nextSiblingId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.firstChildId); + EXPECT_EQ(NEO::Yaml::invalidNodeID, banana.lastChildId); +} + +TEST(YamlBuildTree, GivenNestedCollectionsThenProperlyCreatesChildNodes) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + ASSERT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + ASSERT_EQ(13U, treeNodes.size()); + + auto &rootNode = *treeNodes.begin(); + ASSERT_EQ(3U, rootNode.numChildren); + auto &banana = at(treeNodes, rootNode.firstChildId); + auto &apple = at(treeNodes, banana.nextSiblingId); + auto &orange = at(treeNodes, apple.nextSiblingId); + auto &appleRed = at(treeNodes, apple.firstChildId); + auto &appleGreen = at(treeNodes, appleRed.nextSiblingId); + auto &appleDict = at(treeNodes, appleGreen.nextSiblingId); + auto &appleTypes = at(treeNodes, appleDict.firstChildId); + auto &appleTypesRome = at(treeNodes, appleTypes.firstChildId); + auto &appleTypesCameo = at(treeNodes, appleTypesRome.nextSiblingId); + auto &appleFlavors = at(treeNodes, appleTypes.nextSiblingId); + auto &appleFlavorsSweet = at(treeNodes, appleFlavors.firstChildId); + auto &appleFlavorsBitter = at(treeNodes, appleFlavorsSweet.nextSiblingId); + + EXPECT_STREQ("banana", at(tokens, banana.key).cstrref().str().c_str()); + EXPECT_STREQ("yellow", at(tokens, banana.value).cstrref().str().c_str()); + + EXPECT_STREQ("orange", at(tokens, orange.key).cstrref().str().c_str()); + EXPECT_STREQ("orange", at(tokens, orange.value).cstrref().str().c_str()); + + EXPECT_STREQ("apple", at(tokens, apple.key).cstrref().str().c_str()); + EXPECT_STREQ("red", at(tokens, appleRed.value).cstrref().str().c_str()); + EXPECT_STREQ("green", at(tokens, appleGreen.value).cstrref().str().c_str()); + EXPECT_EQ(NEO::Yaml::invalidTokenId, appleDict.key); + EXPECT_STREQ("types", at(tokens, appleTypes.key).cstrref().str().c_str()); + EXPECT_STREQ("rome", at(tokens, appleTypesRome.value).cstrref().str().c_str()); + EXPECT_STREQ("cameo", at(tokens, appleTypesCameo.value).cstrref().str().c_str()); + EXPECT_STREQ("flavors", at(tokens, appleFlavors.key).cstrref().str().c_str()); + EXPECT_STREQ("sweet", at(tokens, appleFlavorsSweet.value).cstrref().str().c_str()); + EXPECT_STREQ("bitter", at(tokens, appleFlavorsBitter.value).cstrref().str().c_str()); +} + +TEST(YamlBuildTree, WhenTabsAreUsedAfterIndentWasParsedThenTreatThemAsSeparators) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + ASSERT_TRUE(success); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + ASSERT_EQ(13U, treeNodes.size()); + + auto &rootNode = *treeNodes.begin(); + ASSERT_EQ(3U, rootNode.numChildren); + auto &banana = at(treeNodes, rootNode.firstChildId); + auto &apple = at(treeNodes, banana.nextSiblingId); + auto &orange = at(treeNodes, apple.nextSiblingId); + auto &appleRed = at(treeNodes, apple.firstChildId); + auto &appleGreen = at(treeNodes, appleRed.nextSiblingId); + auto &appleDict = at(treeNodes, appleGreen.nextSiblingId); + auto &appleTypes = at(treeNodes, appleDict.firstChildId); + auto &appleTypesRome = at(treeNodes, appleTypes.firstChildId); + auto &appleTypesCameo = at(treeNodes, appleTypesRome.nextSiblingId); + auto &appleFlavors = at(treeNodes, appleTypes.nextSiblingId); + auto &appleFlavorsSweet = at(treeNodes, appleFlavors.firstChildId); + auto &appleFlavorsBitter = at(treeNodes, appleFlavorsSweet.nextSiblingId); + + EXPECT_STREQ("banana", at(tokens, banana.key).cstrref().str().c_str()); + EXPECT_STREQ("yellow", at(tokens, banana.value).cstrref().str().c_str()); + + EXPECT_STREQ("orange", at(tokens, orange.key).cstrref().str().c_str()); + EXPECT_STREQ("orange", at(tokens, orange.value).cstrref().str().c_str()); + + EXPECT_STREQ("apple", at(tokens, apple.key).cstrref().str().c_str()); + EXPECT_STREQ("red", at(tokens, appleRed.value).cstrref().str().c_str()); + EXPECT_STREQ("green", at(tokens, appleGreen.value).cstrref().str().c_str()); + EXPECT_EQ(NEO::Yaml::invalidTokenId, appleDict.key); + EXPECT_STREQ("types", at(tokens, appleTypes.key).cstrref().str().c_str()); + EXPECT_STREQ("rome", at(tokens, appleTypesRome.value).cstrref().str().c_str()); + EXPECT_STREQ("cameo", at(tokens, appleTypesCameo.value).cstrref().str().c_str()); + EXPECT_STREQ("flavors", at(tokens, appleFlavors.key).cstrref().str().c_str()); + EXPECT_STREQ("sweet", at(tokens, appleFlavorsSweet.value).cstrref().str().c_str()); + EXPECT_STREQ("bitter", at(tokens, appleFlavorsBitter.value).cstrref().str().c_str()); +} + +TEST(YamlBuildTree, GivenInvalidIndentationThenBuldingOfTreeFails) { + ConstStringRef invalidListIndentYaml = + R"===( + - red + - green + - blue +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(invalidListIndentYaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [2] : [- ] <-- parser position on error. Reason : Invalid indentation\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()); + + ConstStringRef invalidDictIndentYaml = + R"===( + colors: + - red + flavors : sweet +)==="; + + lines.clear(); + tokens.clear(); + warnings.clear(); + errors.clear(); + success = NEO::Yaml::tokenize(invalidDictIndentYaml, lines, tokens, errors, warnings); + EXPECT_TRUE(success); + + treeNodes.clear(); + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_FALSE(success); + EXPECT_STREQ("NEO::Yaml : Could not parse line : [3] : [fl] <-- parser position on error. Reason : Invalid indentation\n", errors.c_str()); + EXPECT_TRUE(warnings.empty()); +} + +TEST(YamlTreeFindChildByKey, WhenChildWithKeyDoesNotExistThenReturnsNull) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + + auto grapes = NEO::Yaml::findChildByKey(*treeNodes.begin(), treeNodes, tokens, "grapes"); + auto apple = NEO::Yaml::findChildByKey(*treeNodes.begin(), treeNodes, tokens, "apple"); + EXPECT_STREQ("apple", at(tokens, apple->key).cstrref().str().c_str()); + EXPECT_EQ(nullptr, grapes); + ASSERT_NE(nullptr, apple); + auto appleTraits = NEO::Yaml::findChildByKey(*apple, treeNodes, tokens, "traits"); + EXPECT_EQ(nullptr, appleTraits); +} + +TEST(YamlTreeFindChildByKey, WhenChildWithKeyDoesExistThenReturnsIt) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + + auto apple = NEO::Yaml::findChildByKey(*treeNodes.begin(), treeNodes, tokens, "apple"); + ASSERT_NE(nullptr, apple); + EXPECT_STREQ("apple", at(tokens, apple->key).cstrref().str().c_str()); +} + +TEST(YamlBuildTree, GivenCommentInDictionaryEntryThenDonTreatItAsValue) { + ConstStringRef yaml = + R"===( + apple : # red + - green + - yellow +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + ASSERT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + auto apple = NEO::Yaml::findChildByKey(*treeNodes.begin(), treeNodes, tokens, "apple"); + ASSERT_NE(nullptr, apple); + EXPECT_EQ(NEO::Yaml::invalidTokenId, apple->value); + ASSERT_EQ(2U, apple->numChildren); +} + +TEST(YamlBuildTree, GivenCommentInListEntryThenDonTreatItAsValue) { + ConstStringRef yaml = + R"===( + - #green + - yellow +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + ASSERT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + EXPECT_TRUE(warnings.empty()) << warnings; + EXPECT_TRUE(errors.empty()) << errors; + + auto root = &*treeNodes.begin(); + ASSERT_NE(nullptr, root); + EXPECT_EQ(NEO::Yaml::invalidTokenId, root->value); + ASSERT_EQ(2U, root->numChildren); + + auto appleFirstEntry = NEO::Yaml::getFirstChild(*root, treeNodes); + EXPECT_EQ(NEO::Yaml::invalidTokenId, appleFirstEntry->value); +} + +TEST(YamlTreeGetFirstChild, WhenChildExistsThenReturnsIt) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : + - yellow + - green +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + + auto apple = NEO::Yaml::getFirstChild(*treeNodes.begin(), treeNodes); + ASSERT_NE(nullptr, apple); + auto appleRed = NEO::Yaml::getFirstChild(*apple, treeNodes); + ASSERT_NE(nullptr, appleRed); + auto appleRedChild = NEO::Yaml::getFirstChild(*appleRed, treeNodes); + EXPECT_EQ(nullptr, appleRedChild); + + EXPECT_STREQ("apple", at(tokens, apple->key).cstrref().str().c_str()); + EXPECT_STREQ("red", at(tokens, appleRed->value).cstrref().str().c_str()); + EXPECT_EQ(apple->id, appleRed->parentId); +} + +TEST(YamlTreeGetLastChild, WhenChildExistsThenReturnsIt) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : + - yellow + - green +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + + auto banana = NEO::Yaml::getLastChild(*treeNodes.begin(), treeNodes); + ASSERT_NE(nullptr, banana); + auto bananaGreen = NEO::Yaml::getLastChild(*banana, treeNodes); + ASSERT_NE(nullptr, bananaGreen); + auto bananaGreenChild = NEO::Yaml::getLastChild(*bananaGreen, treeNodes); + EXPECT_EQ(nullptr, bananaGreenChild); + + EXPECT_STREQ("banana", at(tokens, banana->key).cstrref().str().c_str()); + EXPECT_STREQ("green", at(tokens, bananaGreen->value).cstrref().str().c_str()); + EXPECT_EQ(banana->id, bananaGreen->parentId); +} + +TEST(YamlTreeConstSiblingsFwdIterator, WhenConstructedThenSetsUpProperMembers) { + struct WhiteBoxConstSiblingsFwdIterator : ConstSiblingsFwdIterator { + using ConstSiblingsFwdIterator::allNodes; + using ConstSiblingsFwdIterator::ConstSiblingsFwdIterator; + using ConstSiblingsFwdIterator::currId; + }; + + NEO::Yaml::NodesCache nodesCache; + WhiteBoxConstSiblingsFwdIterator it(1U, &nodesCache); + EXPECT_EQ(&nodesCache, it.allNodes); + EXPECT_EQ(1U, it.currId); + + WhiteBoxConstSiblingsFwdIterator it2{it}; + EXPECT_EQ(&nodesCache, it2.allNodes); + EXPECT_EQ(1U, it2.currId); + + WhiteBoxConstSiblingsFwdIterator it3(7U, &nodesCache + 1); + it3 = it; + EXPECT_EQ(&nodesCache, it2.allNodes); + EXPECT_EQ(1U, it2.currId); +} + +TEST(YamlTreeConstSiblingsFwdIterator, WhenComparisonOperatorsAreUsedThenComparesMembers) { + NEO::Yaml::NodesCache nodesCache; + nodesCache.resize(2); + ConstSiblingsFwdIterator first{0U, &nodesCache}; + ConstSiblingsFwdIterator second{1U, &nodesCache}; + ConstSiblingsFwdIterator firstAgain{0U, &nodesCache}; + EXPECT_TRUE(first == first); + EXPECT_TRUE(second == second); + EXPECT_TRUE(firstAgain == firstAgain); + EXPECT_TRUE(first == firstAgain); + EXPECT_TRUE(firstAgain == first); + EXPECT_FALSE(first == second); + EXPECT_FALSE(second == first); + EXPECT_FALSE(firstAgain == second); + EXPECT_FALSE(second == firstAgain); + + EXPECT_FALSE(first != first); + EXPECT_FALSE(second != second); + EXPECT_FALSE(firstAgain != firstAgain); + EXPECT_FALSE(first != firstAgain); + EXPECT_FALSE(firstAgain != first); + EXPECT_TRUE(first != second); + EXPECT_TRUE(second != first); + EXPECT_TRUE(firstAgain != second); + EXPECT_TRUE(second != firstAgain); +} + +TEST(YamlTreeConstSiblingsFwdIterator, WhenDereferenceOperatorsAreUsedThenReturnsReferenceToProperNode) { + NEO::Yaml::NodesCache nodesCache; + nodesCache.resize(2); + ConstSiblingsFwdIterator first{0U, &nodesCache}; + ConstSiblingsFwdIterator second{1U, &nodesCache}; + EXPECT_EQ(nodesCache.begin(), &*first); + EXPECT_EQ(nodesCache.begin() + 1, &*second); + + nodesCache.begin()->id = 7U; + (nodesCache.begin() + 1)->id = 13U; + EXPECT_EQ(7U, first->id); + EXPECT_EQ(13U, second->id); +} + +TEST(YamlTreeConstSiblingsFwdIterator, WhenIncrementOperatorsAreUsedThenAdvancesInTheCollection) { + NEO::Yaml::NodesCache nodesCache; + nodesCache.resize(2); + nodesCache[0].key = 3; + nodesCache[1].key = 7; + + ConstSiblingsFwdIterator end{NEO::Yaml::invalidNodeID, &nodesCache}; + { + ConstSiblingsFwdIterator it{0U, &nodesCache}; + auto it2 = it++; + ASSERT_NE(end, it2); + EXPECT_EQ(3U, it2->key); + EXPECT_EQ(end, it); + auto it3 = it++; + EXPECT_EQ(end, it); + EXPECT_EQ(end, it3); + } + + { + ConstSiblingsFwdIterator it{0U, &nodesCache}; + auto next = ++it; + EXPECT_EQ(end, next); + EXPECT_EQ(end, it); + auto further = ++it; + EXPECT_EQ(end, further); + EXPECT_EQ(end, it); + } + + nodesCache[0].nextSiblingId = 1U; + { + ConstSiblingsFwdIterator it{0U, &nodesCache}; + auto it2 = it++; + ASSERT_NE(end, it); + ASSERT_NE(end, it2); + EXPECT_EQ(3U, it2->key); + EXPECT_EQ(7U, it->key); + auto it3 = it++; + ASSERT_NE(end, it2); + ASSERT_NE(end, it3); + EXPECT_EQ(3U, it2->key); + EXPECT_EQ(7U, it3->key); + EXPECT_EQ(end, it); + } + + { + ConstSiblingsFwdIterator it{0U, &nodesCache}; + auto next = ++it; + ASSERT_NE(end, next); + ASSERT_NE(end, it); + EXPECT_EQ(7U, it->key); + EXPECT_EQ(7U, next->key); + auto third = ++next; + ASSERT_NE(end, it); + EXPECT_EQ(7U, it->key); + EXPECT_EQ(end, next); + EXPECT_EQ(end, third); + } +} + +TEST(YamlTreeConstChildrenRange, WhenConstructedThenSetsUpProperMembers) { + NEO::Yaml::NodesCache nodesCache; + nodesCache.resize(2); + nodesCache[0].id = 0U; + nodesCache[0].key = 7U; + ConstChildrenRange rangeByNode{*nodesCache.begin(), nodesCache}; + ConstChildrenRange rangeByNodeId{0U, nodesCache}; + + { + auto beg = rangeByNode.begin(); + auto end = rangeByNode.end(); + EXPECT_NE(beg, end); + EXPECT_EQ(7U, beg->key); + EXPECT_EQ(end, ++beg); + } + + { + auto beg = rangeByNodeId.begin(); + auto end = rangeByNodeId.end(); + EXPECT_NE(beg, end); + EXPECT_EQ(7U, beg->key); + EXPECT_EQ(end, ++beg); + } +} + +TEST(YamlTreeDebugNode, WhenConstructedThenSetsUpProperMembers) { + NEO::Yaml::DebugNode debugNode; + EXPECT_TRUE(debugNode.key.empty()); + EXPECT_TRUE(debugNode.children.empty()); + EXPECT_TRUE(debugNode.value.empty()); + EXPECT_EQ(nullptr, debugNode.parent); + EXPECT_EQ(nullptr, debugNode.src); +} + +TEST(YamlTreeBuildDebugNodes, GivenTreeNodesThenBuildsDebugFriendlyRepresentation) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string warnings; + std::string errors; + bool success = NEO::Yaml::tokenize(yaml, lines, tokens, errors, warnings); + + NEO::Yaml::NodesCache treeNodes; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, errors, warnings); + EXPECT_TRUE(success); + + auto rootDebugNode = NEO::Yaml::buildDebugNodes(treeNodes.begin()->id, treeNodes, tokens); + EXPECT_TRUE(rootDebugNode->key.empty()); + EXPECT_TRUE(rootDebugNode->value.empty()); + EXPECT_EQ(nullptr, rootDebugNode->parent); + ASSERT_EQ(3U, rootDebugNode->children.size()); + + EXPECT_STREQ("banana", rootDebugNode->children[0]->key.str().c_str()); + EXPECT_STREQ("yellow", rootDebugNode->children[0]->value.str().c_str()); + EXPECT_STREQ("apple", rootDebugNode->children[1]->key.str().c_str()); + EXPECT_EQ(3U, rootDebugNode->children[1]->children.size()); + EXPECT_STREQ("orange", rootDebugNode->children[2]->key.str().c_str()); + EXPECT_STREQ("orange", rootDebugNode->children[2]->value.str().c_str()); + + delete rootDebugNode; +} + +TEST(YamlParser, WhenConstructedThenSetsUpProperDefaults) { + NEO::Yaml::YamlParser parser; + EXPECT_TRUE(parser.empty()); + EXPECT_EQ(nullptr, parser.buildDebugNodes()); +} + +TEST(YamlParserParse, WhenTokenizerFailsThenParserPropagatesTheError) { + ConstStringRef yaml = "\"aaaa"; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_FALSE(success); + EXPECT_TRUE(parser.empty()); + + std::string tokenizerErrors; + std::string tokenizerWarnings; + NEO::Yaml::TokensCache tokens; + NEO::Yaml::LinesCache lines; + success = NEO::Yaml::tokenize(yaml, lines, tokens, tokenizerErrors, tokenizerWarnings); + EXPECT_FALSE(success); + + EXPECT_STREQ(tokenizerErrors.c_str(), parserErrors.c_str()); + EXPECT_STREQ(tokenizerWarnings.c_str(), parserWarnings.c_str()); +} + +TEST(YamlParserParse, WhenTreeBuildingFailsThenParserPropagatesTheError) { + ConstStringRef yaml = + R"===( + - red + - green + - blue +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_FALSE(success); + EXPECT_TRUE(parser.empty()); + + NEO::Yaml::LinesCache lines; + NEO::Yaml::TokensCache tokens; + std::string tokenizerWarnings; + std::string tokenizerErrors; + success = NEO::Yaml::tokenize(yaml, lines, tokens, tokenizerErrors, tokenizerWarnings); + EXPECT_TRUE(success); + + NEO::Yaml::NodesCache treeNodes; + std::string treeBuildingWarnings; + std::string treeBuildingErrors; + success = NEO::Yaml::buildTree(lines, tokens, treeNodes, treeBuildingErrors, treeBuildingWarnings); + EXPECT_FALSE(success); + EXPECT_STREQ(treeBuildingErrors.c_str(), parserErrors.c_str()); + EXPECT_STREQ(treeBuildingWarnings.c_str(), parserWarnings.c_str()); +} + +TEST(YamlParserParse, GivenValidYamlThenParsesItCorrectly) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : + - yellow + - green +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + EXPECT_FALSE(parser.empty()); + EXPECT_TRUE(parserErrors.empty()); + EXPECT_TRUE(parserWarnings.empty()); +} + +TEST(YamlParser, WhenGetRootIsCalledThenReturnsFirstNode) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : + - yellow + - green +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + ASSERT_NE(nullptr, parser.getRoot()); + EXPECT_EQ(0U, parser.getRoot()->id); +} + +TEST(YamlParser, WhenGetChildIsCalledThenSearchesForChildNodeByKey) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + ASSERT_NE(nullptr, parser.getRoot()); + EXPECT_EQ(0U, parser.getRoot()->id); + + auto apple = parser.getChild(*parser.getRoot(), "apple"); + ASSERT_NE(nullptr, apple); + EXPECT_EQ(3U, apple->numChildren); + EXPECT_NE(parser.getRoot(), apple); +} + +TEST(YamlParser, GivenNodeWhenReadKeyIsCalledThenReturnsStringRepresentationOfKey) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : + - yellow + - green +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto banana = parser.getChild(*parser.getRoot(), "banana"); + ASSERT_NE(nullptr, banana); + EXPECT_STREQ("banana", parser.readKey(*banana).str().c_str()); +} + +TEST(YamlParser, GivenNodeWhenReadValueIsCalledThenReturnsStringRepresentationOfKey) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : yellow +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto banana = parser.getChild(*parser.getRoot(), "banana"); + ASSERT_NE(nullptr, banana); + EXPECT_STREQ("yellow", parser.readValue(*banana).str().c_str()); + + auto apple = parser.getChild(*parser.getRoot(), "apple"); + EXPECT_STREQ("", parser.readValue(*apple).str().c_str()); +} + +TEST(YamlParser, GivenNodeWhenGetValueTokenIsCalledThenReturnsTokenRepresetingTheValue) { + ConstStringRef yaml = + R"===( + apple : + - red + - green + banana : yellow +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto apple = parser.getChild(*parser.getRoot(), "apple"); + ASSERT_NE(nullptr, apple); + EXPECT_EQ(nullptr, parser.getValueToken(*apple)); + + auto banana = parser.getChild(*parser.getRoot(), "banana"); + ASSERT_NE(nullptr, banana); + auto bananaValueToken = parser.getValueToken(*banana); + ASSERT_NE(nullptr, bananaValueToken); + EXPECT_EQ('y', bananaValueToken->traits.character0); +} + +TEST(YamlParser, WhenFindNodeWithKeyDfsIsCalledThenSearchesForFirstMatchUsingDfs) { + ConstStringRef yaml = + R"===( + apple : + color : red + taste : bitter + banana : + color : yellow + taste : sweet + color : green +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto appleColor = parser.findNodeWithKeyDfs("color"); + ASSERT_NE(nullptr, appleColor); + EXPECT_STREQ("color", parser.readKey(*appleColor).str().c_str()); + EXPECT_STREQ("red", parser.readValue(*appleColor).str().c_str()); + + EXPECT_EQ(nullptr, parser.findNodeWithKeyDfs("colors")); +} + +TEST(YamlParser, GivenNodeWhenCreateChildrenRangeIsCalledThenCreatesProperRange) { + ConstStringRef yaml = + R"===( + apple : + color : red + taste : bitter + banana : + color : yellow + taste : sweet + color : green +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto root = parser.getRoot(); + auto childrenRange = parser.createChildrenRange(*root); + auto it = childrenRange.begin(); + ASSERT_NE(childrenRange.end(), it); + EXPECT_STREQ("apple", parser.readKey(*it).str().c_str()); + ++it; + ASSERT_NE(childrenRange.end(), it); + EXPECT_STREQ("banana", parser.readKey(*it).str().c_str()); + ++it; + ASSERT_NE(childrenRange.end(), it); + EXPECT_STREQ("color", parser.readKey(*it).str().c_str()); + ++it; + EXPECT_EQ(childrenRange.end(), it); +} + +TEST(YamlParser, WhenBuildDebugNodesIsCalledThenBuildsDebugFriendlyRepresentation) { + ConstStringRef yaml = + R"===( + banana : yellow + apple : + - red + - green + - + types : + - rome + - cameo + flavors : + - sweet + - bitter + orange : orange +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto rootDebugNode = parser.buildDebugNodes(); + EXPECT_TRUE(rootDebugNode->key.empty()); + EXPECT_TRUE(rootDebugNode->value.empty()); + EXPECT_EQ(nullptr, rootDebugNode->parent); + ASSERT_EQ(3U, rootDebugNode->children.size()); + + EXPECT_STREQ("banana", rootDebugNode->children[0]->key.str().c_str()); + EXPECT_STREQ("yellow", rootDebugNode->children[0]->value.str().c_str()); + EXPECT_STREQ("apple", rootDebugNode->children[1]->key.str().c_str()); + EXPECT_EQ(3U, rootDebugNode->children[1]->children.size()); + EXPECT_STREQ("orange", rootDebugNode->children[2]->key.str().c_str()); + EXPECT_STREQ("orange", rootDebugNode->children[2]->value.str().c_str()); + + delete rootDebugNode; + + auto flavors = parser.findNodeWithKeyDfs("flavors"); + ASSERT_NE(nullptr, flavors); + auto flavorsDebugNode = parser.buildDebugNodes(*flavors); + ASSERT_NE(nullptr, flavorsDebugNode); + ASSERT_EQ(2U, flavorsDebugNode->children.size()); + EXPECT_STREQ("flavors", flavorsDebugNode->key.str().c_str()); + EXPECT_TRUE(flavorsDebugNode->value.empty()); + EXPECT_STREQ("sweet", flavorsDebugNode->children[0]->value.str().c_str()); + EXPECT_STREQ("bitter", flavorsDebugNode->children[1]->value.str().c_str()); + + delete flavorsDebugNode; +} + +TEST(YamlParserReadValueNoQuotes, GivenStringWithQuotesThenReturnSubstring) { + ConstStringRef yaml = R"===( +banana : "yellow color" +apple : 'red color' +)==="; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto banana = parser.findNodeWithKeyDfs("banana"); + ASSERT_NE(nullptr, banana); + EXPECT_STREQ("\"yellow color\"", parser.readValue(*banana).str().c_str()); + EXPECT_STREQ("yellow color", parser.readValueNoQuotes(*banana).str().c_str()); + + auto apple = parser.findNodeWithKeyDfs("apple"); + ASSERT_NE(nullptr, apple); + EXPECT_STREQ("\'red color\'", parser.readValue(*apple).str().c_str()); + EXPECT_STREQ("red color", parser.readValueNoQuotes(*apple).str().c_str()); +} + +TEST(YamlParserReadValueNoQuotes, GivenStringWithoutQuotesThenReturnStringAsIs) { + ConstStringRef yaml = R"===( +banana : yellow +apple : + - red +)==="; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto banana = parser.findNodeWithKeyDfs("banana"); + ASSERT_NE(nullptr, banana); + EXPECT_STREQ("yellow", parser.readValue(*banana).str().c_str()); + EXPECT_STREQ("yellow", parser.readValueNoQuotes(*banana).str().c_str()); + + auto apple = parser.findNodeWithKeyDfs("apple"); + ASSERT_NE(nullptr, apple); + EXPECT_STREQ("", parser.readValueNoQuotes(*apple).str().c_str()); +} + +TEST(YamlParserReadValueNoQuotes, GivenNonStringValueThenReturnTextRepresentationAsIs) { + ConstStringRef yaml = R"===( +banana : 5 +)==="; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto banana = parser.findNodeWithKeyDfs("banana"); + ASSERT_NE(nullptr, banana); + EXPECT_STREQ("5", parser.readValueNoQuotes(*banana).str().c_str()); +} + +TEST(YamlParserReadValueCheckedUint64, GivenNodeWithoutASingleValueThenReturnFalse) { + ConstStringRef yaml = R"===( +list : + - a + - b +)==="; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + auto notIntNode = parser.getChild(*parser.getRoot(), "list"); + ASSERT_NE(notIntNode, nullptr); + uint64_t readUint64 = 0; + auto readSuccess = parser.readValueChecked(*notIntNode, readUint64); + EXPECT_FALSE(readSuccess); +} + +TEST(YamlParserReadValueCheckedUint64, GivenNonIntegerValueThenReturnFalse) { + ConstStringRef yaml = "not_integer : five"; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + auto notIntNode = parser.getChild(*parser.getRoot(), "not_integer"); + ASSERT_NE(notIntNode, nullptr); + uint64_t readUint64 = 0; + auto readSuccess = parser.readValueChecked(*notIntNode, readUint64); + EXPECT_FALSE(readSuccess); +} + +TEST(YamlParserReadValueCheckedUint64, GivenIntegerThenParsesItCorrectly) { + ConstStringRef yaml = "integer64 : 6294967295"; + uint64_t expectedUint64 = 6294967295; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + auto int64Node = parser.getChild(*parser.getRoot(), "integer64"); + ASSERT_NE(int64Node, nullptr); + uint64_t readUint64 = 0; + auto readSuccess = parser.readValueChecked(*int64Node, readUint64); + EXPECT_TRUE(readSuccess); + EXPECT_EQ(expectedUint64, readUint64); +} + +TEST(YamlParserReadValueCheckedUint32, GivenIntegerThenParsesItCorrectly) { + ConstStringRef yaml = R"===( +integer64 : 6294967295 +integer32 : 294967295 +)==="; + uint32_t expectedUint32 = 294967295U; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto int64Node = parser.getChild(*parser.getRoot(), "integer64"); + ASSERT_NE(int64Node, nullptr); + uint32_t readUint32 = 0; + auto readSuccess = parser.readValueChecked(*int64Node, readUint32); + EXPECT_FALSE(readSuccess); + + auto int32Node = parser.getChild(*parser.getRoot(), "integer32"); + ASSERT_NE(int32Node, nullptr); + readUint32 = 0; + readSuccess = parser.readValueChecked(*int32Node, readUint32); + EXPECT_TRUE(readSuccess); + EXPECT_EQ(expectedUint32, readUint32); +} + +TEST(YamlParserReadValueCheckedUint16, GivenIntegerThenParsesItCorrectly) { + ConstStringRef yaml = R"===( +integer32 : 294967295 +integer16 : 65530 +)==="; + + uint16_t expectedUint16 = 65530U; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto int32Node = parser.getChild(*parser.getRoot(), "integer32"); + ASSERT_NE(int32Node, nullptr); + uint16_t readUint16 = 0; + auto readSuccess = parser.readValueChecked(*int32Node, readUint16); + EXPECT_FALSE(readSuccess); + + auto int16Node = parser.getChild(*parser.getRoot(), "integer16"); + ASSERT_NE(int16Node, nullptr); + readSuccess = parser.readValueChecked(*int16Node, readUint16); + EXPECT_TRUE(readSuccess); + EXPECT_EQ(expectedUint16, readUint16); +} + +TEST(YamlParserReadValueCheckedUint8, GivenIntegerThenParsesItCorrectly) { + ConstStringRef yaml = R"===( +integer16 : 65530 +integer8 : 250 +)==="; + + uint8_t expectedUint8 = 250U; + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + auto int16Node = parser.getChild(*parser.getRoot(), "integer16"); + ASSERT_NE(int16Node, nullptr); + uint8_t readUint8 = 0; + auto readSuccess = parser.readValueChecked(*int16Node, readUint8); + EXPECT_FALSE(readSuccess); + + auto int8Node = parser.getChild(*parser.getRoot(), "integer8"); + ASSERT_NE(int8Node, nullptr); + readSuccess = parser.readValueChecked(*int8Node, readUint8); + EXPECT_TRUE(readSuccess); + EXPECT_EQ(expectedUint8, readUint8); +} + +TEST(YamlParserReadValueCheckedBool, GivenFlagsThenParsesThemCorrectly) { + ConstStringRef yaml = R"===( +ones_that_are_true : + y : y + Y : y + + yes : yes + Yes : Yes + YES : YES + + true : true + True : True + TRUE : TRUE + + on : on + ON : ON + +ones_that_are_false : + n : n + N : N + + no : no + No : No + NO : NO + + false : false + False : False + FALSE : FALSE + + off : off + Off : Off + OFF : OFF +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + uint32_t expectedNumberOfPositiveCases = 10; + uint32_t expectedNumberOfNegativeCases = 11; + + auto onesThatAreTrue = parser.getChild(*parser.getRoot(), "ones_that_are_true"); + auto onesThatAreFalse = parser.getChild(*parser.getRoot(), "ones_that_are_false"); + ASSERT_NE(nullptr, onesThatAreTrue); + ASSERT_NE(nullptr, onesThatAreFalse); + + EXPECT_EQ(expectedNumberOfPositiveCases, onesThatAreTrue->numChildren); + EXPECT_EQ(expectedNumberOfNegativeCases, onesThatAreFalse->numChildren); + + for (auto &node : parser.createChildrenRange(*onesThatAreTrue)) { + bool value = false; + bool readSuccess = false; + readSuccess = parser.readValueChecked(node, value); + EXPECT_TRUE(readSuccess); + EXPECT_TRUE(value) << parser.readValue(node).str(); + } + + for (auto &node : parser.createChildrenRange(*onesThatAreFalse)) { + bool value = false; + bool readSuccess = false; + readSuccess = parser.readValueChecked(node, value); + EXPECT_TRUE(readSuccess); + EXPECT_FALSE(value) << parser.readValue(node).str(); + } +} + +TEST(YamlParserReadValueCheckedBool, GivenNonFlagEntriesThenFailParsing) { + ConstStringRef yaml = R"===( +ones_that_look_like_true : + one : 1 + + long_yesssss : long_yesssss + + yess : yess + Yos : Yos + + tru : tru + truA : truA + + o : o + og : og + +ones_that_look_like_false : + zero : 0 + + None : None + ni : ni + + fals : fals + falsA : falsA + + of : of + ofo : ofo + + alse : alse + +not_really_a_bool: + - imalist + - true +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + + uint32_t expectedNumberOfPositiveCases = 8; + uint32_t expectedNumberOfNegativeCases = 8; + + auto onesThatLookLikeTrue = parser.getChild(*parser.getRoot(), "ones_that_look_like_true"); + auto onesThatLookLikeFalse = parser.getChild(*parser.getRoot(), "ones_that_look_like_false"); + auto notReallyABool = parser.getChild(*parser.getRoot(), "not_really_a_bool"); + ASSERT_NE(nullptr, onesThatLookLikeTrue); + ASSERT_NE(nullptr, onesThatLookLikeFalse); + ASSERT_NE(nullptr, notReallyABool); + + EXPECT_EQ(expectedNumberOfPositiveCases, onesThatLookLikeTrue->numChildren); + EXPECT_EQ(expectedNumberOfNegativeCases, onesThatLookLikeFalse->numChildren); + + for (auto &node : parser.createChildrenRange(*onesThatLookLikeTrue)) { + bool value = false; + bool readSuccess = false; + readSuccess = parser.readValueChecked(node, value); + EXPECT_FALSE(readSuccess) << parser.readValue(node).str(); + } + + for (auto &node : parser.createChildrenRange(*onesThatLookLikeFalse)) { + bool value = false; + bool readSuccess = false; + readSuccess = parser.readValueChecked(node, value); + EXPECT_FALSE(readSuccess) << parser.readValue(node).str(); + } + + { + bool value = false; + bool readSuccess = false; + readSuccess = parser.readValueChecked(*notReallyABool, value); + EXPECT_FALSE(readSuccess) << parser.readValue(*notReallyABool).str(); + } +} + +TEST(YamlParser, GivenSimpleZebinThenParsesItCorrectly) { + ConstStringRef yaml = R"===(--- +kernels: + - name: k + execution_env: + actual_kernel_start_offset: 0 + grf_count: 128 + has_no_stateless_write: true + simd_size: 32 + subgroup_independent_forward_progress: false + payload_arguments: + - arg_type: global_id_offset + offset: 0 + size: 12 + - arg_type: local_size + offset: 12 + size: 12 + - arg_type: arg_bypointer + offset: 0 + size: 0 + arg_index: 0 + addrmode: stateful + addrspace: global + access_type: readwrite + - arg_type: arg_bypointer + offset: 32 + size: 8 + arg_index: 0 + addrmode: stateless + addrspace: global + access_type: readwrite + per_thread_payload_arguments: + - arg_type: local_id + offset: 0 + size: 192 + binding_table_indexes: + - bti_value: 0 + arg_index: 0 +... +)==="; + + std::string parserErrors; + std::string parserWarnings; + NEO::Yaml::YamlParser parser; + bool success = parser.parse(yaml, parserErrors, parserWarnings); + EXPECT_TRUE(success); + EXPECT_TRUE(parserErrors.empty()) << parserErrors; + EXPECT_TRUE(parserWarnings.empty()) << parserWarnings; + + auto debugNodes = parser.buildDebugNodes(); + ASSERT_EQ(1U, debugNodes->children.size()); + delete debugNodes; + + const NEO::Yaml::Node *ndKernels = parser.findNodeWithKeyDfs("kernels"); + ASSERT_NE(nullptr, ndKernels); + ASSERT_EQ(1U, ndKernels->numChildren); + const NEO::Yaml::Node *ndKernel0 = &*parser.createChildrenRange(*ndKernels).begin(); + ASSERT_NE(nullptr, ndKernel0); + const NEO::Yaml::Node *ndName = parser.getChild(*ndKernel0, "name"); + const NEO::Yaml::Node *ndExecutionEnv = parser.getChild(*ndKernel0, "execution_env"); + const NEO::Yaml::Node *ndPayloadArgs = parser.getChild(*ndKernel0, "payload_arguments"); + const NEO::Yaml::Node *ndPerThreadPayloadArgs = parser.getChild(*ndKernel0, "per_thread_payload_arguments"); + const NEO::Yaml::Node *ndBtis = parser.getChild(*ndKernel0, "binding_table_indexes"); + ASSERT_NE(nullptr, ndName); + ASSERT_NE(nullptr, ndExecutionEnv); + ASSERT_NE(nullptr, ndPayloadArgs); + ASSERT_NE(nullptr, ndPerThreadPayloadArgs); + ASSERT_NE(nullptr, ndBtis); + EXPECT_STREQ("k", parser.readValue(*ndName).str().c_str()); + { // exec env + auto ndActualKernelStartOffset = parser.getChild(*ndExecutionEnv, "actual_kernel_start_offset"); + auto ndGrfCount = parser.getChild(*ndExecutionEnv, "grf_count"); + auto ndHasNoStatelessWrite = parser.getChild(*ndExecutionEnv, "has_no_stateless_write"); + auto ndSimdSize = parser.getChild(*ndExecutionEnv, "simd_size"); + auto ndSubgroupIfp = parser.getChild(*ndExecutionEnv, "subgroup_independent_forward_progress"); + ASSERT_NE(nullptr, ndActualKernelStartOffset); + ASSERT_NE(nullptr, ndGrfCount); + ASSERT_NE(nullptr, ndHasNoStatelessWrite); + ASSERT_NE(nullptr, ndSimdSize); + ASSERT_NE(nullptr, ndSubgroupIfp); + uint32_t actualKernelStartOffset; + uint32_t grfCount; + bool hasNoStatelessWrite; + uint32_t simdSize; + bool subgroupIfp; + EXPECT_TRUE(parser.readValueChecked(*ndActualKernelStartOffset, actualKernelStartOffset)); + EXPECT_TRUE(parser.readValueChecked(*ndGrfCount, grfCount)); + EXPECT_TRUE(parser.readValueChecked(*ndHasNoStatelessWrite, hasNoStatelessWrite)); + EXPECT_TRUE(parser.readValueChecked(*ndSimdSize, simdSize)); + EXPECT_TRUE(parser.readValueChecked(*ndSubgroupIfp, subgroupIfp)); + EXPECT_EQ(0U, actualKernelStartOffset); + EXPECT_EQ(128U, grfCount); + EXPECT_TRUE(hasNoStatelessWrite); + EXPECT_EQ(32U, simdSize); + EXPECT_FALSE(subgroupIfp); + } + + { // payload arguments + ASSERT_EQ(4U, ndPayloadArgs->numChildren); + auto argIt = parser.createChildrenRange(*ndPayloadArgs).begin(); + auto *ndArg0 = &*(argIt++); + auto *ndArg1 = &*(argIt++); + auto *ndArg2 = &*(argIt++); + auto *ndArg3 = &*argIt; + ASSERT_NE(nullptr, ndArg0); + ASSERT_NE(nullptr, ndArg1); + ASSERT_NE(nullptr, ndArg2); + ASSERT_NE(nullptr, ndArg3); + + { // arg0 + auto ndArgType = parser.getChild(*ndArg0, "arg_type"); + auto ndArgOffset = parser.getChild(*ndArg0, "offset"); + auto ndArgSize = parser.getChild(*ndArg0, "size"); + ASSERT_NE(nullptr, ndArgType); + ASSERT_NE(nullptr, ndArgOffset); + ASSERT_NE(nullptr, ndArgSize); + uint32_t argOffset = 0U; + uint32_t argSize = 0U; + EXPECT_TRUE(parser.readValueChecked(*ndArgOffset, argOffset)); + EXPECT_TRUE(parser.readValueChecked(*ndArgSize, argSize)); + EXPECT_STREQ("global_id_offset", parser.readValue(*ndArgType).str().c_str()); + EXPECT_EQ(0U, argOffset); + EXPECT_EQ(12U, argSize); + } + + { // arg1 + auto ndArgType = parser.getChild(*ndArg1, "arg_type"); + auto ndArgOffset = parser.getChild(*ndArg1, "offset"); + auto ndArgSize = parser.getChild(*ndArg1, "size"); + ASSERT_NE(nullptr, ndArgType); + ASSERT_NE(nullptr, ndArgOffset); + ASSERT_NE(nullptr, ndArgSize); + uint32_t argOffset = 0U; + uint32_t argSize = 0U; + EXPECT_TRUE(parser.readValueChecked(*ndArgOffset, argOffset)); + EXPECT_TRUE(parser.readValueChecked(*ndArgSize, argSize)); + EXPECT_STREQ("local_size", parser.readValue(*ndArgType).str().c_str()); + EXPECT_EQ(12U, argOffset); + EXPECT_EQ(12U, argSize); + } + + { // arg2 + auto ndArgType = parser.getChild(*ndArg2, "arg_type"); + auto ndArgOffset = parser.getChild(*ndArg2, "offset"); + auto ndArgSize = parser.getChild(*ndArg2, "size"); + auto ndArgIndex = parser.getChild(*ndArg2, "arg_index"); + auto ndAddrMode = parser.getChild(*ndArg2, "addrmode"); + auto ndAddrSpace = parser.getChild(*ndArg2, "addrspace"); + auto ndAccessType = parser.getChild(*ndArg2, "access_type"); + ASSERT_NE(nullptr, ndArgType); + ASSERT_NE(nullptr, ndArgOffset); + ASSERT_NE(nullptr, ndArgSize); + ASSERT_NE(nullptr, ndArgIndex); + ASSERT_NE(nullptr, ndAddrMode); + ASSERT_NE(nullptr, ndAddrSpace); + ASSERT_NE(nullptr, ndAccessType); + uint32_t argOffset = 0U; + uint32_t argSize = 0U; + uint32_t argIndex = 0U; + EXPECT_TRUE(parser.readValueChecked(*ndArgOffset, argOffset)); + EXPECT_TRUE(parser.readValueChecked(*ndArgSize, argSize)); + EXPECT_TRUE(parser.readValueChecked(*ndArgIndex, argIndex)); + EXPECT_STREQ("arg_bypointer", parser.readValue(*ndArgType).str().c_str()); + EXPECT_STREQ("stateful", parser.readValue(*ndAddrMode).str().c_str()); + EXPECT_STREQ("global", parser.readValue(*ndAddrSpace).str().c_str()); + EXPECT_STREQ("readwrite", parser.readValue(*ndAccessType).str().c_str()); + EXPECT_EQ(0U, argOffset); + EXPECT_EQ(0U, argSize); + EXPECT_EQ(0U, argIndex); + } + + { // arg3 + auto ndArgType = parser.getChild(*ndArg3, "arg_type"); + auto ndArgOffset = parser.getChild(*ndArg3, "offset"); + auto ndArgSize = parser.getChild(*ndArg3, "size"); + auto ndArgIndex = parser.getChild(*ndArg3, "arg_index"); + auto ndAddrMode = parser.getChild(*ndArg3, "addrmode"); + auto ndAddrSpace = parser.getChild(*ndArg3, "addrspace"); + auto ndAccessType = parser.getChild(*ndArg3, "access_type"); + ASSERT_NE(nullptr, ndArgType); + ASSERT_NE(nullptr, ndArgOffset); + ASSERT_NE(nullptr, ndArgSize); + ASSERT_NE(nullptr, ndArgIndex); + ASSERT_NE(nullptr, ndAddrMode); + ASSERT_NE(nullptr, ndAddrSpace); + ASSERT_NE(nullptr, ndAccessType); + uint32_t argOffset = 0U; + uint32_t argSize = 0U; + uint32_t argIndex = 0U; + EXPECT_TRUE(parser.readValueChecked(*ndArgOffset, argOffset)); + EXPECT_TRUE(parser.readValueChecked(*ndArgSize, argSize)); + EXPECT_TRUE(parser.readValueChecked(*ndArgIndex, argIndex)); + EXPECT_STREQ("arg_bypointer", parser.readValue(*ndArgType).str().c_str()); + EXPECT_STREQ("stateless", parser.readValue(*ndAddrMode).str().c_str()); + EXPECT_STREQ("global", parser.readValue(*ndAddrSpace).str().c_str()); + EXPECT_STREQ("readwrite", parser.readValue(*ndAccessType).str().c_str()); + EXPECT_EQ(32U, argOffset); + EXPECT_EQ(8U, argSize); + EXPECT_EQ(0U, argIndex); + } + } + + { // per-thread payload arguments + ASSERT_EQ(1U, ndPerThreadPayloadArgs->numChildren); + auto argIt = parser.createChildrenRange(*ndPerThreadPayloadArgs).begin(); + auto *ndArg0 = &*argIt; + ASSERT_NE(nullptr, ndArg0); + + { // arg0 + auto ndArgType = parser.getChild(*ndArg0, "arg_type"); + auto ndArgOffset = parser.getChild(*ndArg0, "offset"); + auto ndArgSize = parser.getChild(*ndArg0, "size"); + ASSERT_NE(nullptr, ndArgType); + ASSERT_NE(nullptr, ndArgOffset); + ASSERT_NE(nullptr, ndArgSize); + uint32_t argOffset = 0U; + uint32_t argSize = 0U; + EXPECT_TRUE(parser.readValueChecked(*ndArgOffset, argOffset)); + EXPECT_TRUE(parser.readValueChecked(*ndArgSize, argSize)); + EXPECT_STREQ("local_id", parser.readValue(*ndArgType).str().c_str()); + EXPECT_EQ(0U, argOffset); + EXPECT_EQ(192U, argSize); + } + } + + { // binding_table_indexes + ASSERT_EQ(1U, ndBtis->numChildren); + auto argIt = parser.createChildrenRange(*ndBtis).begin(); + auto *ndBti0 = &*argIt; + ASSERT_NE(nullptr, ndBti0); + + { // bti0 + auto ndBtiValue = parser.getChild(*ndBti0, "bti_value"); + auto ndArgIndex = parser.getChild(*ndBti0, "arg_index"); + ASSERT_NE(nullptr, ndBtiValue); + ASSERT_NE(nullptr, ndArgIndex); + uint32_t btiValue = 0U; + uint32_t argIndex = 0U; + EXPECT_TRUE(parser.readValueChecked(*ndBtiValue, btiValue)); + EXPECT_TRUE(parser.readValueChecked(*ndArgIndex, argIndex)); + EXPECT_EQ(0U, btiValue); + EXPECT_EQ(0U, argIndex); + } + } +} \ No newline at end of file diff --git a/shared/test/unit_test/utilities/const_stringref_tests.cpp b/shared/test/unit_test/utilities/const_stringref_tests.cpp index 7221c5cbea..634b724626 100644 --- a/shared/test/unit_test/utilities/const_stringref_tests.cpp +++ b/shared/test/unit_test/utilities/const_stringref_tests.cpp @@ -9,10 +9,12 @@ #include "gtest/gtest.h" -TEST(ConstStringRef, WhenCreatingFromConstantArrayThenIsConstexpr) { - static constexpr ConstStringRef str0("some_text"); - static_assert(9 == str0.length(), ""); - static_assert(9 == str0.size(), ""); +using namespace NEO; + +TEST(ConstStringRef, WhenCreatingFromConstantArrayThenIsConstexprAndContainsAllArrayElements) { + static constexpr ConstStringRef str0 = ConstStringRef::fromArray("some_text"); + static_assert(10U == str0.length(), ""); + static_assert(10U == str0.size(), ""); static_assert(false == str0.empty(), ""); static_assert('s' == str0[0], ""); static_assert('o' == str0[1], ""); @@ -32,7 +34,7 @@ TEST(ConstStringRef, WhenCreatingFromConstantArrayThenIsConstexpr) { static_assert('e' == str1[1], ""); static_assert('c' == str1[2], ""); - static_assert('s' == static_cast(str1)[0], ""); + static_assert('s' == str1.data()[0], ""); static_assert('s' == *str1.begin(), ""); static_assert(3 == str1.end() - str1.begin(), ""); static_assert(str1.begin() == str1.data(), ""); @@ -42,7 +44,7 @@ TEST(ConstStringRef, WhenCreatingFromConstantArrayThenIsConstexpr) { static_assert(0U == strEmpty.size(), ""); static_assert(strEmpty.empty(), ""); - static_assert(9 == str0.length(), ""); + static_assert(10 == str0.length(), ""); static_assert(str0 == str0, ""); static_assert(str1 != str0, ""); @@ -102,13 +104,88 @@ TEST(ConstStringRef, WhenCopyAsignedThenIdenticalAsOrigin) { TEST(ConstStringRef, WhenCheckingForInclusionThenDoesNotReadOutOfBounds) { static constexpr ConstStringRef str1("Text", 2); ConstStringRef substr1("Tex"); - EXPECT_FALSE(str1.contains(substr1)); + EXPECT_FALSE(str1.contains(substr1.data())); static constexpr ConstStringRef str2("AabAac"); ConstStringRef substr2("Aac"); - EXPECT_TRUE(str2.contains(substr2)); + EXPECT_TRUE(str2.contains(substr2.data())); static constexpr ConstStringRef str3("AabAac"); ConstStringRef substr3("Aacd"); - EXPECT_FALSE(str3.contains(substr3)); + EXPECT_FALSE(str3.contains(substr3.data())); +} + +TEST(ConstStringRef, WhenCreatingFromStringThenUsesUnderlyingDataAndLength) { + std::string src = "abc"; + ConstStringRef fromCopy = src; + EXPECT_EQ(src.data(), fromCopy.begin()); + EXPECT_EQ(src.data() + src.size(), fromCopy.end()); + ConstStringRef fromMove = std::move(src); + EXPECT_EQ(fromCopy.begin(), fromMove.begin()); + EXPECT_EQ(fromCopy.end(), fromMove.end()); +} + +TEST(ConstStringRef, WhenCreatingFromCStringThenImplicitlyCalculatesLength) { + const char *src = "text"; + ConstStringRef fromCString = src; + EXPECT_EQ(src, fromCString.begin()); + EXPECT_EQ(4U, fromCString.size()); +} + +TEST(ConstStringRefSubstr, GivenPositiveLengthThenCountFromLeft) { + ConstStringRef fromCString = "some text"; + ConstStringRef substr = fromCString.substr(2, 2); + EXPECT_EQ(fromCString.data() + 2, substr.begin()); + EXPECT_EQ(fromCString.data() + 4, substr.end()); + EXPECT_EQ(2U, substr.length()); +} + +TEST(ConstStringRefSubstr, GivenNegativeLengthThenCountFromRight) { + ConstStringRef fromCString = "some text"; + ConstStringRef substr = fromCString.substr(2, -2); + EXPECT_EQ(fromCString.data() + 2, substr.begin()); + EXPECT_EQ(fromCString.data() + fromCString.length() - 2, substr.end()); + EXPECT_EQ(5U, substr.length()); +} + +TEST(ConstStringRefSubstr, GivenOnlyPositionThenReturnRemainingPartOfString) { + ConstStringRef fromCString = "some text"; + ConstStringRef substr = fromCString.substr(2); + EXPECT_EQ(fromCString.data() + 2, substr.begin()); + EXPECT_EQ(fromCString.end(), substr.end()); + EXPECT_EQ(fromCString.length() - 2, substr.length()); +} + +TEST(ConstStringRefTruncated, GivenPositiveLengthThenCountFromLeft) { + ConstStringRef fromCString = "some text"; + ConstStringRef substr = fromCString.truncated(2); + EXPECT_EQ(fromCString.begin(), substr.begin()); + EXPECT_EQ(fromCString.begin() + 2, substr.end()); + EXPECT_EQ(2U, substr.length()); +} + +TEST(ConstStringRefTruncated, GivenNegativeLengthThenCountFromRight) { + ConstStringRef fromCString = "some text"; + ConstStringRef substr = fromCString.truncated(-2); + EXPECT_EQ(fromCString.begin(), substr.begin()); + EXPECT_EQ(fromCString.data() + fromCString.length() - 2, substr.end()); + EXPECT_EQ(7U, substr.length()); +} + +TEST(ConstStringRefEqualsCaseInsesitive, WhenSizesDifferReturnFalse) { + ConstStringRef lhs = ConstStringRef::fromArray("\0"); + ConstStringRef rhs = ConstStringRef::fromArray("\0\0"); + EXPECT_FALSE(equalsCaseInsesitive(lhs, rhs)); +} + +TEST(ConstStringRefEqualsCaseInsesitive, WhenStringsDontMatchThenReturnFalse) { + EXPECT_FALSE(equalsCaseInsesitive(ConstStringRef("abc"), ConstStringRef("abd"))); +} + +TEST(ConstStringRefEqualsCaseInsesitive, WhenStringsIdenticalThenReturnTrue) { + EXPECT_TRUE(equalsCaseInsesitive(ConstStringRef("abc"), ConstStringRef("abc"))); +} + +TEST(ConstStringRefEqualsCaseInsesitive, WhenStringsDifferOnlyByCaseThenReturnTrue) { + EXPECT_TRUE(equalsCaseInsesitive(ConstStringRef("aBc"), ConstStringRef("Abc"))); } diff --git a/shared/test/unit_test/utilities/containers_tests.cpp b/shared/test/unit_test/utilities/containers_tests.cpp index 7453e24f41..4cc6d119f5 100644 --- a/shared/test/unit_test/utilities/containers_tests.cpp +++ b/shared/test/unit_test/utilities/containers_tests.cpp @@ -1252,6 +1252,49 @@ TEST(StackVec, PushBack) { ASSERT_FALSE(contains(&v, &*v.begin())); } +TEST(StackVecPopBack, GivenNonEmptyStackVecThenRemoveSingleElementFromTheEnd) { + using Type = uint32_t; + StackVec v; + ASSERT_EQ(2U, v.capacity()); + v.push_back(3); + EXPECT_EQ(1U, v.size()); + EXPECT_EQ(3U, *v.rbegin()); + v.push_back(5); + EXPECT_EQ(2U, v.size()); + EXPECT_EQ(5U, *v.rbegin()); + + v.pop_back(); + EXPECT_EQ(1U, v.size()); + EXPECT_EQ(3U, *v.rbegin()); + v.pop_back(); + EXPECT_EQ(0U, v.size()); + + v.push_back(3); + EXPECT_EQ(1U, v.size()); + EXPECT_EQ(3U, *v.rbegin()); + v.push_back(5); + EXPECT_EQ(2U, v.size()); + EXPECT_EQ(5U, *v.rbegin()); + v.push_back(7); + EXPECT_EQ(3U, v.size()); + EXPECT_EQ(7U, *v.rbegin()); + v.push_back(11); + EXPECT_EQ(4U, v.size()); + EXPECT_EQ(11U, *v.rbegin()); + + v.pop_back(); + EXPECT_EQ(3U, v.size()); + EXPECT_EQ(7U, *v.rbegin()); + v.pop_back(); + EXPECT_EQ(2U, v.size()); + EXPECT_EQ(5U, *v.rbegin()); + v.pop_back(); + EXPECT_EQ(1U, v.size()); + EXPECT_EQ(3U, *v.rbegin()); + v.pop_back(); + EXPECT_EQ(0U, v.size()); +} + TEST(StackVec, Reserve) { using Type = uint32_t; StackVec v;