diff --git a/opencl/test/unit_test/offline_compiler/CMakeLists.txt b/opencl/test/unit_test/offline_compiler/CMakeLists.txt index d63a1341a2..136f68da07 100644 --- a/opencl/test/unit_test/offline_compiler/CMakeLists.txt +++ b/opencl/test/unit_test/offline_compiler/CMakeLists.txt @@ -18,6 +18,8 @@ set(IGDRCL_SRCS_cloc set(IGDRCL_SRCS_offline_compiler_mock ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_decoder.h ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_encoder.h + ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_zebin_decoder.h + ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_zebin_encoder.h ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_iga_wrapper.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_argument_helper.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_multi_command.h @@ -58,6 +60,7 @@ set(IGDRCL_SRCS_offline_compiler_tests ${CMAKE_CURRENT_SOURCE_DIR}/decoder/encoder_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/decoder/iga_wrapper_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/decoder/iga_wrapper_tests.h + ${CMAKE_CURRENT_SOURCE_DIR}/decoder/zebin_manipulator_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/default_cache_config_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/environment.h ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp diff --git a/opencl/test/unit_test/offline_compiler/decoder/mock/mock_iga_wrapper.h b/opencl/test/unit_test/offline_compiler/decoder/mock/mock_iga_wrapper.h index 68f90c9dbe..be0856d673 100644 --- a/opencl/test/unit_test/offline_compiler/decoder/mock/mock_iga_wrapper.h +++ b/opencl/test/unit_test/offline_compiler/decoder/mock/mock_iga_wrapper.h @@ -27,9 +27,11 @@ struct MockIgaWrapper : public IgaWrapper { } void setGfxCore(GFXCORE_FAMILY core) override { + setGfxCoreWasCalled = true; } void setProductFamily(PRODUCT_FAMILY product) override { + setProductFamilyWasCalled = true; } bool isKnownPlatform() const override { @@ -48,4 +50,6 @@ struct MockIgaWrapper : public IgaWrapper { bool disasmWasCalled = false; bool asmWasCalled = false; bool isKnownPlatformReturnValue = false; + bool setProductFamilyWasCalled = false; + bool setGfxCoreWasCalled = false; }; diff --git a/opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_decoder.h b/opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_decoder.h new file mode 100644 index 0000000000..3580bdf94d --- /dev/null +++ b/opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_decoder.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/offline_compiler/source/decoder/zebin_manipulator.h" +#include "shared/source/device_binary_format/elf/elf_decoder.h" + +template +struct MockZebinDecoder : public NEO::ZebinManipulator::ZebinDecoder { + using Base = NEO::ZebinManipulator::ZebinDecoder; + using ErrorCode = NEO::ZebinManipulator::ErrorCode; + using ElfT = NEO::Elf::Elf; + using Base::arguments; + using Base::decodeZebin; + using Base::dumpKernelData; + using Base::dumpSectionInfo; + using Base::iga; + using Base::printHelp; + + MockZebinDecoder(OclocArgHelper *argHelper) : Base::ZebinDecoder(argHelper) { + auto iga = std::make_unique(); + mockIga = iga.get(); + this->iga = std::move(iga); + }; + + ErrorCode decodeZebin(ArrayRef zebin, ElfT &outElf) override { + if (callBaseDecodeZebin) { + return Base::decodeZebin(zebin, outElf); + } + return returnValueDecodeZebin; + } + + std::vector getIntelGTNotes(ElfT &elf) override { + getIntelGTNotesCalled = true; + if (callBaseGetIntelGTNotes) { + return Base::getIntelGTNotes(elf); + } + return returnValueGetIntelGTNotes; + } + + std::vector dumpElfSections(ElfT &elf) override { + return returnValueDumpElfSections; + } + + void dumpSectionInfo(const std::vector §ionInfos) override { + return; + } + + bool callBaseGetIntelGTNotes = false; + bool callBaseDecodeZebin = false; + ErrorCode returnValueDecodeZebin = NEO::OclocErrorCode::SUCCESS; + std::vector returnValueDumpElfSections; + std::vector returnValueGetIntelGTNotes; + bool getIntelGTNotesCalled = false; + MockIgaWrapper *mockIga; +}; \ No newline at end of file diff --git a/opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_encoder.h b/opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_encoder.h new file mode 100644 index 0000000000..21b60b831a --- /dev/null +++ b/opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_encoder.h @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/offline_compiler/source/decoder/zebin_manipulator.h" +#include "shared/source/device_binary_format/elf/elf_encoder.h" + +#include "opencl/test/unit_test/offline_compiler/decoder/mock/mock_iga_wrapper.h" + +template +struct MockZebinEncoder : public NEO::ZebinManipulator::ZebinEncoder { + using Base = NEO::ZebinManipulator::ZebinEncoder; + using ErrorCode = NEO::ZebinManipulator::ErrorCode; + using SectionInfo = NEO::ZebinManipulator::SectionInfo; + using ElfEncoderT = NEO::Elf::ElfEncoder; + using Base::appendKernel; + using Base::appendRel; + using Base::appendRela; + using Base::appendSymtab; + using Base::iga; + using Base::parseSymbols; + using Base::printHelp; + + MockZebinEncoder(OclocArgHelper *argHelper) : Base::ZebinEncoder(argHelper) { + auto iga = std::make_unique(); + mockIga = iga.get(); + this->iga = std::move(iga); + retValGetIntelGTNotes.resize(1); + retValGetIntelGTNotes[0].type = NEO::Elf::IntelGTSectionType::ProductFamily; + retValGetIntelGTNotes[0].data = ArrayRef::fromAny(&productFamily, 1); + } + + ErrorCode loadSectionsInfo(std::vector §ionInfos) override { + if (callBaseLoadSectionsInfo) { + return Base::loadSectionsInfo(sectionInfos); + } + return retValLoadSectionsInfo; + } + + ErrorCode checkIfAllFilesExist(const std::vector §ionInfos) override { + if (callBaseCheckIfAllFilesExist) { + return Base::checkIfAllFilesExist(sectionInfos); + } + return retValCheckIfAllFilesExist; + } + + std::vector getIntelGTNotesSection(const std::vector §ionInfos) override { + if (callBaseGetIntelGTNotesSection) { + return Base::getIntelGTNotesSection(sectionInfos); + } + return {}; + } + + std::vector getIntelGTNotes(const std::vector &intelGtNotesSection) override { + if (callBaseGetIntelGTNotes) { + return Base::getIntelGTNotes(intelGtNotesSection); + } + return retValGetIntelGTNotes; + } + + ErrorCode appendSections(ElfEncoderT &encoder, const std::vector §ionInfos) override { + return retValAppendSections; + } + + std::string parseKernelAssembly(ArrayRef kernelAssembly) override { + parseKernelAssemblyCalled = true; + if (callBaseParseKernelAssembly) { + return Base::parseKernelAssembly(kernelAssembly); + } + return {}; + } + + MockIgaWrapper *mockIga; + std::vector retValGetIntelGTNotes; + ErrorCode retValLoadSectionsInfo = NEO::OclocErrorCode::SUCCESS; + ErrorCode retValCheckIfAllFilesExist = NEO::OclocErrorCode::SUCCESS; + ErrorCode retValAppendSections = NEO::OclocErrorCode::SUCCESS; + bool callBaseGetIntelGTNotesSection = false; + bool callBaseGetIntelGTNotes = false; + bool callBaseLoadSectionsInfo = false; + bool callBaseParseKernelAssembly = false; + bool callBaseCheckIfAllFilesExist = false; + bool parseKernelAssemblyCalled = false; + PRODUCT_FAMILY productFamily = PRODUCT_FAMILY::IGFX_DG2; +}; \ No newline at end of file diff --git a/opencl/test/unit_test/offline_compiler/decoder/zebin_manipulator_tests.cpp b/opencl/test/unit_test/offline_compiler/decoder/zebin_manipulator_tests.cpp new file mode 100644 index 0000000000..e31cac714a --- /dev/null +++ b/opencl/test/unit_test/offline_compiler/decoder/zebin_manipulator_tests.cpp @@ -0,0 +1,616 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/offline_compiler/source/decoder/zebin_manipulator.h" +#include "shared/offline_compiler/source/ocloc_api.h" +#include "shared/offline_compiler/source/ocloc_error_code.h" +#include "shared/source/device_binary_format/elf/elf.h" +#include "shared/source/device_binary_format/elf/elf_encoder.h" +#include "shared/test/common/mocks/mock_modules_zebin.h" +#include "shared/test/common/test_macros/test.h" + +#include "opencl/test/unit_test/offline_compiler/decoder/mock/mock_iga_wrapper.h" +#include "opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_decoder.h" +#include "opencl/test/unit_test/offline_compiler/decoder/mock/mock_zebin_encoder.h" +#include "opencl/test/unit_test/offline_compiler/mock/mock_argument_helper.h" + +#include "igfxfmid.h" + +#include + +template +struct MockZebin { + MockZebin() { + NEO::Elf::ElfEncoder encoder; + encoder.getElfFileHeader().machine = NEO::Elf::ELF_MACHINE::EM_INTELGT; + encoder.getElfFileHeader().type = NEO::Elf::ELF_TYPE_ZEBIN::ET_ZEBIN_EXE; + + uint8_t kernelData[] = {0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38}; + encoder.appendSection(NEO::Elf::SHT_PROGBITS, NEO::Elf::SectionsNamesZebin::textPrefix.str() + "exit_kernel", {kernelData, 8}); + + uint8_t dataGlobal[] = {0x00, 0x01, 0x02, 0x03, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; + encoder.appendSection(NEO::Elf::SHT_PROGBITS, NEO::Elf::SectionsNamesZebin::dataGlobal, {dataGlobal, 12}); + + uint8_t debugInfo[] = {0x10, 0x11, 0x12, 0x13, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; + encoder.appendSection(NEO::Elf::SHT_PROGBITS, NEO::Elf::SectionsNamesZebin::debugInfo, {debugInfo, 12}); + + uint8_t spvData[] = {0x20, 0x21, 0x22, 0x23}; + encoder.appendSection(NEO::Elf::SHT_ZEBIN_SPIRV, NEO::Elf::SectionsNamesZebin::spv, {spvData, 4}); + + NEO::ConstStringRef buildOptions = "-ze-intel-allow-zebin"; + encoder.appendSection(NEO::Elf::SHT_ZEBIN_MISC, NEO::Elf::SectionsNamesZebin::buildOptions, + {reinterpret_cast(buildOptions.data()), buildOptions.size()}); + + std::string zeInfo = R"===(' +--- +version: '1.15' +kernels: + - name: exit_kernel + execution_env: + grf_count: 128 + simd_size: 16 +... +)==="; + encoder.appendSection(NEO::Elf::SHT_ZEBIN_ZEINFO, NEO::Elf::SectionsNamesZebin::zeInfo, + {reinterpret_cast(zeInfo.data()), zeInfo.size()}); + + NEO::Elf::ZebinTargetFlags flags; + flags.packed = 0U; + auto intelGTNotes = ZebinTestData::createIntelGTNoteSection(IGFX_DG2, IGFX_XE_HPG_CORE, flags, versionToString({1, 15})); + encoder.appendSection(NEO::Elf::SHT_NOTE, NEO::Elf::SectionsNamesZebin::noteIntelGT, + {intelGTNotes.data(), intelGTNotes.size()}); + + NEO::Elf::ElfRel dataRelocation; + dataRelocation.offset = 0x4; + dataRelocation.setRelocationType(NEO::Elf::RELOC_TYPE_ZEBIN::R_ZE_SYM_ADDR); + dataRelocation.setSymbolTableIndex(2); + auto &dataRelSec = encoder.appendSection(NEO::Elf::SHT_REL, NEO::Elf::SpecialSectionNames::relPrefix.str() + NEO::Elf::SectionsNamesZebin::dataGlobal.str(), + {reinterpret_cast(&dataRelocation), sizeof(dataRelocation)}); + dataRelSec.link = 10; + dataRelSec.info = 2; + + NEO::Elf::ElfRela debugRelocation; + debugRelocation.offset = 0x4; + debugRelocation.setRelocationType(NEO::Elf::RELOC_TYPE_ZEBIN::R_ZE_SYM_ADDR); + debugRelocation.setSymbolTableIndex(1); + auto &debugDataRelaSec = encoder.appendSection(NEO::Elf::SHT_RELA, NEO::Elf::SpecialSectionNames::relaPrefix.str() + NEO::Elf::SectionsNamesZebin::debugInfo.str(), + {reinterpret_cast(&debugRelocation), sizeof(debugRelocation)}); + debugDataRelaSec.link = 10; + debugDataRelaSec.info = 3; + + NEO::Elf::ElfSymbolEntry symbols[3] = {}; + symbols[1].shndx = 1; + symbols[1].value = 0x0; + symbols[1].name = encoder.appendSectionName("exit_kernel"); + symbols[1].setBinding(NEO::Elf::SYMBOL_TABLE_BIND::STB_LOCAL); + symbols[1].setType(NEO::Elf::SYMBOL_TABLE_TYPE::STT_FUNC); + + symbols[2].shndx = 2; + symbols[2].value = 0x0; + symbols[2].name = encoder.appendSectionName("var_x"); + symbols[2].setBinding(NEO::Elf::SYMBOL_TABLE_BIND::STB_GLOBAL); + symbols[2].setType(NEO::Elf::SYMBOL_TABLE_TYPE::STT_OBJECT); + + auto &symtabSec = encoder.appendSection(NEO::Elf::SHT_SYMTAB, NEO::Elf::SectionsNamesZebin::symtab, + {reinterpret_cast(symbols), sizeof(symbols)}); + symtabSec.link = 11; + symtabSec.info = 2; + storage = encoder.encode(); + } + std::vector storage; +}; + +TEST(ZebinManipulatorTests, GivenValidZebinWhenItIsDisassembledAndAssembledBackThenItIsSameBinary) { + MockZebin mockZebin32; + MockZebin mockZebin64; + for (const auto zebin : {&mockZebin32.storage, &mockZebin64.storage}) { + // Disassemble + uint32_t disasmNumOutputs; + uint8_t **disasmDataOutputs; + uint64_t *disasmLenOutputs; + char **disasmNameOutputs; + std::array disasmArgs = { + "ocloc", + "disasm", + "-file", + "zebin.bin", + "-skip-asm-translation"}; + + uint32_t numSources = 1; + const uint8_t *dataSources[1] = {zebin->data()}; + const uint64_t lenSources[1] = {zebin->size()}; + const char *nameSources[1] = {"zebin.bin"}; + + int retVal = oclocInvoke(static_cast(disasmArgs.size()), disasmArgs.data(), + numSources, dataSources, lenSources, nameSources, + 0, nullptr, nullptr, nullptr, + &disasmNumOutputs, &disasmDataOutputs, &disasmLenOutputs, &disasmNameOutputs); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + + // Assemble + std::array asmArgs = { + "ocloc", + "asm", + "-file", + "zebin.bin"}; + + uint32_t asmNumOutputs; + uint8_t **asmDataOutputs; + uint64_t *asmLenOutputs; + char **asmNameOutputs; + + retVal = oclocInvoke(static_cast(asmArgs.size()), asmArgs.data(), + disasmNumOutputs, const_cast(disasmDataOutputs), disasmLenOutputs, (const char **)disasmNameOutputs, + 0, nullptr, nullptr, nullptr, + &asmNumOutputs, &asmDataOutputs, &asmLenOutputs, &asmNameOutputs); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + + // Check + ArrayRef rebuiltZebin(asmDataOutputs[0], static_cast(asmLenOutputs[0])); + EXPECT_EQ(zebin->size(), rebuiltZebin.size()); + if (zebin->size() == rebuiltZebin.size()) { + EXPECT_EQ(0, std::memcmp(zebin->data(), rebuiltZebin.begin(), zebin->size())); + } + + oclocFreeOutput(&disasmNumOutputs, &disasmDataOutputs, &disasmLenOutputs, &disasmNameOutputs); + oclocFreeOutput(&asmNumOutputs, &asmDataOutputs, &asmLenOutputs, &asmNameOutputs); + } +}; + +struct ZebinManipulatorValidateArgumentsFixture { + ZebinManipulatorValidateArgumentsFixture() : argHelper(filesMap) {} + void setUp() {} + void tearDown() {} + MockOclocArgHelper::FilesMap filesMap; + MockOclocArgHelper argHelper; + MockIgaWrapper iga; + NEO::ZebinManipulator::Arguments arguments; +}; + +using ZebinManipulatorValidateInputTests = Test; +TEST_F(ZebinManipulatorValidateInputTests, GivenValidInputWhenValidatingInputThenPopulateArgumentsCorrectly) { + std::vector args = {"ocloc", + "asm/disasm", + "-file", + "zebin.bin", + "-device", + "dg2", + "-dump", + "./dump/", + "-q", + "-skip-asm-translation"}; + + auto retVal = NEO::ZebinManipulator::validateInput(args, &iga, &argHelper, arguments); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + + EXPECT_EQ("zebin.bin", arguments.binaryFile); + EXPECT_EQ("./dump/", arguments.pathToDump); + EXPECT_TRUE(iga.setProductFamilyWasCalled); + EXPECT_TRUE(argHelper.getPrinterRef().isSuppressed()); + EXPECT_TRUE(arguments.skipIGAdisassembly); +} + +TEST_F(ZebinManipulatorValidateInputTests, GivenHelpArgumentWhenValidatingInputThenShowHelpIsSet) { + std::vector args = {"ocloc", + "asm/disasm", + "--help"}; + auto retVal = NEO::ZebinManipulator::validateInput(args, &iga, &argHelper, arguments); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_TRUE(arguments.showHelp); +} + +TEST_F(ZebinManipulatorValidateInputTests, GivenInvalidInputWhenValidatingInputThenReturnsError) { + std::vector args = {"ocloc", + "asm/disasm", + "-unknown_arg"}; + + testing::internal::CaptureStdout(); + auto retVal = NEO::ZebinManipulator::validateInput(args, &iga, &argHelper, arguments); + const auto output{testing::internal::GetCapturedStdout()}; + + EXPECT_EQ(NEO::OclocErrorCode::INVALID_COMMAND_LINE, retVal); + EXPECT_EQ("Unknown argument -unknown_arg\n", output); +} + +TEST_F(ZebinManipulatorValidateInputTests, GivenMissingFileWhenValidatingInputThenReturnsError) { + std::vector args = {"ocloc", + "asm/disasm", + "-dump", + "./dump/"}; + + testing::internal::CaptureStdout(); + auto retVal = NEO::ZebinManipulator::validateInput(args, &iga, &argHelper, arguments); + const auto output{testing::internal::GetCapturedStdout()}; + + EXPECT_EQ(NEO::OclocErrorCode::INVALID_COMMAND_LINE, retVal); + EXPECT_EQ("Error: Missing -file argument\n", output); +} + +TEST_F(ZebinManipulatorValidateInputTests, GivenMissingSecondPartOfTheArgumentWhenValidatingInputThenReturnsError) { + std::vector args = {"ocloc", + "asm/disasm", + "-arg"}; + for (const auto halfArg : {"-file", "-device", "-dump"}) { + args[2] = halfArg; + testing::internal::CaptureStdout(); + auto retVal = NEO::ZebinManipulator::validateInput(args, &iga, &argHelper, arguments); + const auto output{testing::internal::GetCapturedStdout()}; + + EXPECT_EQ(NEO::OclocErrorCode::INVALID_COMMAND_LINE, retVal); + const auto expectedOutput = "Unknown argument " + std::string(halfArg) + "\n"; + EXPECT_EQ(expectedOutput, output); + } +} + +TEST_F(ZebinManipulatorValidateInputTests, GivenValidArgsButDumpNotSpecifiedWhenValidatingInputThenReturnSuccessAndSetDumpToDefaultAndPrintWarning) { + std::vector args = {"ocloc", + "asm/disasm", + "-file", + "binary.bin"}; + testing::internal::CaptureStdout(); + auto retVal = NEO::ZebinManipulator::validateInput(args, &iga, &argHelper, arguments); + const auto output{testing::internal::GetCapturedStdout()}; + + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_EQ("Warning: Path to dump -dump not specified. Using \"./dump/\" as dump folder.\n", output); +} + +TEST(ZebinManipulatorTests, GivenIntelGTNotesWithProductFamilyWhenParsingIntelGTNoteSectionsForDeviceThenIgaProductFamilyIsSet) { + PRODUCT_FAMILY productFamily = PRODUCT_FAMILY::IGFX_DG2; + std::vector intelGTnotes; + intelGTnotes.resize(1); + intelGTnotes[0].type = NEO::Elf::IntelGTSectionType::ProductFamily; + intelGTnotes[0].data = ArrayRef::fromAny(&productFamily, 1); + + auto iga = std::make_unique(); + auto retVal = NEO::ZebinManipulator::parseIntelGTNotesSectionForDevice(intelGTnotes, iga.get()); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_TRUE(iga->setProductFamilyWasCalled); +} + +TEST(ZebinManipulatorTests, GivenIntelGTNotesWithGfxCoreFamilyWhenParsingIntelGTNoteSectionsForDeviceThenIgaGfxCoreIsSet) { + GFXCORE_FAMILY gfxCore = GFXCORE_FAMILY::IGFX_XE_HPG_CORE; + std::vector intelGTnotes; + intelGTnotes.resize(1); + intelGTnotes[0].type = NEO::Elf::IntelGTSectionType::GfxCore; + intelGTnotes[0].data = ArrayRef::fromAny(&gfxCore, 1); + + auto iga = std::make_unique(); + auto retVal = NEO::ZebinManipulator::parseIntelGTNotesSectionForDevice(intelGTnotes, iga.get()); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_TRUE(iga->setGfxCoreWasCalled); +} + +TEST(ZebinManipulatorTests, GivenIntelGTNotesWithoutProductFamilyOrGfxCoreFamilyEntryWhenParsingIntelGTNoteSectionsForDeviceThenReturnError) { + std::vector intelGTnotes; + + auto iga = std::make_unique(); + auto retVal = NEO::ZebinManipulator::parseIntelGTNotesSectionForDevice(intelGTnotes, iga.get()); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_DEVICE, retVal); +} + +TEST(ZebinManipulatorTests, GivenNonZebinBinaryWhenGetBinaryFormatForDisassembleThenReturnPatchTokensFormat) { + MockOclocArgHelper::FilesMap files; + files.insert({"binary.bin", "000000000000000"}); + MockOclocArgHelper argHelper(files); + + auto format = NEO::ZebinManipulator::getBinaryFormatForDisassemble(&argHelper, {"ocloc", "disasm", "-file", "binary.bin"}); + EXPECT_EQ(NEO::ZebinManipulator::BinaryFormats::PatchTokens, format); +} + +TEST(ZebinManipulatorTests, GivenEmptySectionsInfoWhenCheckingIfIs64BitZebinThenReturnFalse) { + MockOclocArgHelper::FilesMap files; + files.insert({"sections.txt", ""}); + MockOclocArgHelper argHelper(files); + auto retVal = NEO::ZebinManipulator::is64BitZebin(&argHelper, "sections.txt"); + EXPECT_FALSE(retVal); +} + +TEST(ZebinManipulatorTests, GivenInvalidSectionsInfoWhenCheckingIfIs64BitZebinThenReturnFalse) { + MockOclocArgHelper::FilesMap files; + files.insert({"sections.txt", "ElfType"}); + MockOclocArgHelper argHelper(files); + auto retVal = NEO::ZebinManipulator::is64BitZebin(&argHelper, "sections.txt"); + EXPECT_FALSE(retVal); +} + +template +struct ZebinDecoderFixture { + ZebinDecoderFixture() : argHelper(filesMap), decoder(&argHelper) { + argHelper.messagePrinter = MessagePrinter(true); + }; + void setUp() {} + void tearDown() {} + std::string getOutput() { return argHelper.messagePrinter.getLog().str(); } + MockOclocArgHelper::FilesMap filesMap; + MockOclocArgHelper argHelper; + MockZebinDecoder decoder; +}; + +using ZebinDecoderTests = Test>; +TEST_F(ZebinDecoderTests, GivenErrorWhenDecodingZebinWhenDecodeThenErrorIsReturned) { + decoder.returnValueDecodeZebin = NEO::OclocErrorCode::INVALID_FILE; + const auto retVal = decoder.decode(); + EXPECT_EQ(decoder.returnValueDecodeZebin, retVal); + EXPECT_EQ("Error while decoding zebin.\n", getOutput()); +} + +TEST_F(ZebinDecoderTests, GivenNoIntelGTNotesWhenDecodeThenErrorIsReturned) { + decoder.returnValueGetIntelGTNotes = {}; + const auto retVal = decoder.decode(); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_EQ("Error missing or invalid Intel GT Notes section.\n", getOutput()); +} + +TEST_F(ZebinDecoderTests, GivenInvalidIntelGTNotesWhenDecodeThenErrorIsReturned) { + decoder.returnValueGetIntelGTNotes = {}; + decoder.returnValueGetIntelGTNotes.resize(1); + const std::string zebinVersion = "1.0"; + decoder.returnValueGetIntelGTNotes[0].type = NEO::Elf::IntelGTSectionType::ZebinVersion; + decoder.returnValueGetIntelGTNotes[0].data = ArrayRef::fromAny(zebinVersion.data(), zebinVersion.length()); + + const auto retVal = decoder.decode(); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_DEVICE, retVal); + EXPECT_EQ("Error while parsing Intel GT Notes section for device.\n", getOutput()); +} + +TEST_F(ZebinDecoderTests, GivenElfWithInvalidIntelGTNotesWhenGetIntelGTNotesThenEmptyVectorIsReturned) { + NEO::Elf::ElfEncoder elfEncoder; + NEO::Elf::ZebinTargetFlags flags; + flags.packed = 0U; + const uint8_t intelGTNotes[8] = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7}; + elfEncoder.appendSection(NEO::Elf::SHT_NOTE, NEO::Elf::SectionsNamesZebin::noteIntelGT, + {intelGTNotes, 8U}); + auto elfBinary = elfEncoder.encode(); + ASSERT_FALSE(elfBinary.empty()); + std::string errors, warnings; + auto elf = NEO::Elf::decodeElf(elfBinary, errors, warnings); + ASSERT_TRUE(errors.empty()) << errors; + ASSERT_TRUE(warnings.empty()) << warnings; + + decoder.callBaseGetIntelGTNotes = true; + auto retVal = decoder.getIntelGTNotes(elf); + EXPECT_TRUE(retVal.empty()); + EXPECT_FALSE(getOutput().empty()); +} + +TEST_F(ZebinDecoderTests, GivenElfWithValidIntelGTNotesWhenGetIntelGTNotesThenIntelGTNotesAreReturned) { + NEO::Elf::ElfEncoder elfEncoder; + NEO::Elf::ZebinTargetFlags flags; + flags.packed = 0U; + auto intelGTNotes = ZebinTestData::createIntelGTNoteSection(IGFX_DG2, IGFX_XE_HPG_CORE, flags, versionToString({1, 15})); + elfEncoder.appendSection(NEO::Elf::SHT_NOTE, NEO::Elf::SectionsNamesZebin::noteIntelGT, + {intelGTNotes.data(), intelGTNotes.size()}); + auto elfBinary = elfEncoder.encode(); + ASSERT_FALSE(elfBinary.empty()); + std::string errors, warnings; + auto elf = NEO::Elf::decodeElf(elfBinary, errors, warnings); + ASSERT_TRUE(errors.empty()) << errors; + ASSERT_TRUE(warnings.empty()) << warnings; + + decoder.callBaseGetIntelGTNotes = true; + auto retVal = decoder.getIntelGTNotes(elf); + EXPECT_FALSE(retVal.empty()); + EXPECT_TRUE(getOutput().empty()); +} + +TEST_F(ZebinDecoderTests, GivenSkipIGADisassemblyWhenDecodeThenDoNotParseIntelGTNotes) { + decoder.arguments.skipIGAdisassembly = true; + + const auto retVal = decoder.decode(); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_FALSE(decoder.callBaseDecodeZebin); +} + +TEST_F(ZebinDecoderTests, GivenNoFailsWhenDecodeThenSuccessIsReturned) { + decoder.returnValueGetIntelGTNotes = {}; + decoder.returnValueGetIntelGTNotes.resize(1); + const PRODUCT_FAMILY productFamily = PRODUCT_FAMILY::IGFX_DG2; + decoder.returnValueGetIntelGTNotes[0].type = NEO::Elf::IntelGTSectionType::ProductFamily; + decoder.returnValueGetIntelGTNotes[0].data = ArrayRef::fromAny(&productFamily, 1U); + + const auto retVal = decoder.decode(); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); +} + +TEST_F(ZebinDecoderTests, GivenInvalidZebinWhenDecodeZebinThenErrorIsReturned) { + decoder.callBaseDecodeZebin = true; + uint8_t invalidFile[16] = {0}; + NEO::Elf::Elf elf; + const auto retVal = decoder.decodeZebin(ArrayRef::fromAny(invalidFile, 16), elf); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_EQ("decodeElf error: Invalid or missing ELF header\n", getOutput()); +} + +TEST_F(ZebinDecoderTests, GivenSkipIgaDisassemblyWhenDumpKernelDataThenDumpAsBinary) { + argHelper.interceptOutput = true; + decoder.arguments.skipIGAdisassembly = true; + const uint8_t kernel[16] = {0}; + decoder.dumpKernelData(".text.kernel", {kernel, 16}); + EXPECT_FALSE(decoder.mockIga->disasmWasCalled); + auto dump = argHelper.interceptedFiles.find(".text.kernel"); + ASSERT_NE(dump, argHelper.interceptedFiles.end()); + EXPECT_EQ(16U, dump->second.size()); +} + +TEST_F(ZebinDecoderTests, GivenFailOnTryDissasembleGenISAWhenDumpKernelDataThenDumpAsBinary) { + argHelper.interceptOutput = true; + decoder.mockIga->asmToReturn = ""; + const uint8_t kernel[16] = {0}; + decoder.dumpKernelData(".text.kernel", {kernel, 16}); + EXPECT_TRUE(decoder.mockIga->disasmWasCalled); + auto dump = argHelper.interceptedFiles.find(".text.kernel"); + ASSERT_NE(dump, argHelper.interceptedFiles.end()); + EXPECT_EQ(16U, dump->second.size()); +} + +TEST_F(ZebinDecoderTests, GivenSuccessfulDissasembleGenIsaWhenDumpKernelDataThenDumpAsAssembly) { + argHelper.interceptOutput = true; + std::string asmToReturn = "assembly"; + decoder.mockIga->asmToReturn = asmToReturn; + const uint8_t kernel[16] = {0}; + decoder.dumpKernelData(".text.kernel", {kernel, 16}); + EXPECT_TRUE(decoder.mockIga->disasmWasCalled); + auto dump = argHelper.interceptedFiles.find(".text.kernel.asm"); + ASSERT_NE(dump, argHelper.interceptedFiles.end()); + EXPECT_EQ(asmToReturn, dump->second); +} + +TEST_F(ZebinDecoderTests, WhenPrintHelpIsCalledThenHelpIsPrinted) { + decoder.printHelp(); + EXPECT_FALSE(getOutput().empty()); +} + +template +struct ZebinEncoderFixture { + ZebinEncoderFixture() : argHelper(filesMap), encoder(&argHelper) { + argHelper.messagePrinter = MessagePrinter(true); + }; + void setUp() {} + void tearDown() {} + std::string getOutput() { return argHelper.messagePrinter.getLog().str(); } + MockOclocArgHelper::FilesMap filesMap; + MockOclocArgHelper argHelper; + MockZebinEncoder encoder; +}; + +using ZebinEncoderTests = Test>; +TEST_F(ZebinEncoderTests, GivenErrorOnLoadSectionsInfoWhenEncodeThenErrorIsReturned) { + encoder.retValLoadSectionsInfo = NEO::OclocErrorCode::INVALID_FILE; + auto retVal = encoder.encode(); + EXPECT_EQ(encoder.retValLoadSectionsInfo, retVal); + EXPECT_EQ("Error while loading sections file.\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenErrorOnCheckIfAllFilesExistWhenEncodeThenErrorIsReturned) { + encoder.retValCheckIfAllFilesExist = NEO::OclocErrorCode::INVALID_FILE; + auto retVal = encoder.encode(); + EXPECT_EQ(encoder.retValCheckIfAllFilesExist, retVal); + EXPECT_EQ("Error: Missing one or more section files.\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenMissingIntelGTNotesWhenEncodeThenErrorIsReturned) { + encoder.retValGetIntelGTNotes = {}; + auto retVal = encoder.encode(); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_DEVICE, retVal); + EXPECT_EQ("Error while parsing Intel GT Notes section for device.\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenErrorOnAppendingSectionsWhenEncodeThenErrorIsReturned) { + encoder.retValAppendSections = NEO::OclocErrorCode::INVALID_FILE; + auto retVal = encoder.encode(); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_EQ("Error while appending elf sections.\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenNoIntelGTNotesWhenGetIntelGTNotesSectionThenEmptyVectorIsReturned) { + std::vector sectionInfos = {{".note.notIntelGT", NEO::Elf::SHT_NOTE}}; + encoder.callBaseGetIntelGTNotesSection = true; + auto retVal = encoder.getIntelGTNotesSection(sectionInfos); + EXPECT_TRUE(retVal.empty()); +} + +TEST_F(ZebinEncoderTests, GivenInvalidSectionsInfoFileWhenLoadSectionsInfoThenErrorIsReturned) { + filesMap.insert({NEO::ZebinManipulator::sectionsInfoFilename.str(), ""}); + encoder.callBaseLoadSectionsInfo = true; + std::vector sectionInfos; + auto retVal = encoder.loadSectionsInfo(sectionInfos); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_TRUE(sectionInfos.empty()); +} + +TEST_F(ZebinEncoderTests, GivenInvalidIntelGTNotesSectionWhenGetIntelGTNotesThenEmptyVectorIsReturned) { + encoder.callBaseGetIntelGTNotes = true; + std::vector invalidNotes = {0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7}; + auto retVal = encoder.getIntelGTNotes(invalidNotes); + EXPECT_TRUE(retVal.empty()); +} + +TEST_F(ZebinEncoderTests, GivenInvalidSymtabFileWhenAppendSymtabThenErrorIsReturned) { + MockElfEncoder elfEncoder; + NEO::ZebinManipulator::SectionInfo sectionInfo; + sectionInfo.name = ".symtab"; + sectionInfo.type = NEO::Elf::SHT_SYMTAB; + std::unordered_map secNameToId; + auto retVal = encoder.appendSymtab(elfEncoder, sectionInfo, 0U, secNameToId); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_EQ("Error: Empty symtab file: .symtab\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenInvalidRelFileWhenAppendRelThenErrorIsReturned) { + MockElfEncoder elfEncoder; + NEO::ZebinManipulator::SectionInfo sectionInfo; + sectionInfo.name = ".rel.text"; + sectionInfo.type = NEO::Elf::SHT_REL; + std::unordered_map secNameToId; + auto retVal = encoder.appendRel(elfEncoder, sectionInfo, 1U, 2U); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_EQ("Error: Empty relocations file: .rel.text\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenInvalidRelaFileWhenAppendRelaThenErrorIsReturned) { + MockElfEncoder elfEncoder; + NEO::ZebinManipulator::SectionInfo sectionInfo; + sectionInfo.name = ".rela.text"; + sectionInfo.type = NEO::Elf::SHT_RELA; + std::unordered_map secNameToId; + auto retVal = encoder.appendRela(elfEncoder, sectionInfo, 1U, 2U); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); + EXPECT_EQ("Error: Empty relocations file: .rela.text\n", getOutput()); +} + +TEST_F(ZebinEncoderTests, GivenAsmFileWhenAppendKernelThenTranslateItToBinaryFile) { + MockElfEncoder elfEncoder; + NEO::ZebinManipulator::SectionInfo sectionInfo; + sectionInfo.name = ".text.kernel"; + sectionInfo.type = NEO::Elf::SHT_PROGBITS; + filesMap.insert({sectionInfo.name + ".asm", "assembly"}); + + auto retVal = encoder.appendKernel(elfEncoder, sectionInfo); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_TRUE(encoder.parseKernelAssemblyCalled); +} + +TEST_F(ZebinEncoderTests, GivenAsmFileMissingWhenAppendKernelThenUseBinaryFile) { + MockElfEncoder elfEncoder; + NEO::ZebinManipulator::SectionInfo sectionInfo; + sectionInfo.name = ".text.kernel"; + sectionInfo.type = NEO::Elf::SHT_PROGBITS; + filesMap.insert({sectionInfo.name, "kernelData"}); + auto retVal = encoder.appendKernel(elfEncoder, sectionInfo); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_FALSE(encoder.parseKernelAssemblyCalled); +} + +TEST_F(ZebinEncoderTests, GivenSuccessfulAssemblyGenISAWhenParseKernelAssemblyThenAssemblyIsReturned) { + encoder.mockIga->binaryToReturn = "1234"; + encoder.callBaseParseKernelAssembly = true; + auto retVal = encoder.parseKernelAssembly({}); + EXPECT_EQ(encoder.mockIga->binaryToReturn, retVal); +} + +TEST_F(ZebinEncoderTests, GivenUnsuccessfulAssemblyGenISAWhenParseKernelAssemblyThenEmptyStringIsReturned) { + encoder.mockIga->binaryToReturn = ""; + encoder.callBaseParseKernelAssembly = true; + auto retVal = encoder.parseKernelAssembly({}); + EXPECT_TRUE(retVal.empty()); +} + +TEST_F(ZebinEncoderTests, GivenMissingFileWhenCheckIfAllFilesExistThenErrorIsReturned) { + encoder.callBaseCheckIfAllFilesExist = true; + std::vector sectionInfos; + sectionInfos.resize(1); + sectionInfos[0].name = ".text.kernel"; + sectionInfos[0].type = NEO::Elf::SHT_PROGBITS; + auto retVal = encoder.checkIfAllFilesExist(sectionInfos); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_FILE, retVal); +} + +TEST_F(ZebinEncoderTests, WhenPrintHelpIsCalledThenHelpIsPrinted) { + encoder.printHelp(); + EXPECT_FALSE(getOutput().empty()); +} diff --git a/opencl/test/unit_test/offline_compiler/mock/mock_argument_helper.h b/opencl/test/unit_test/offline_compiler/mock/mock_argument_helper.h index 420b7969fe..1e84fe60de 100644 --- a/opencl/test/unit_test/offline_compiler/mock/mock_argument_helper.h +++ b/opencl/test/unit_test/offline_compiler/mock/mock_argument_helper.h @@ -23,6 +23,7 @@ class MockOclocArgHelper : public OclocArgHelper { using OclocArgHelper::hasOutput; using OclocArgHelper::headers; using OclocArgHelper::inputs; + using OclocArgHelper::messagePrinter; using OclocArgHelper::findSourceFile; @@ -64,6 +65,11 @@ class MockOclocArgHelper : public OclocArgHelper { if (shouldReturnEmptyVectorOfStrings) { lines.clear(); + } else { + auto ss = std::stringstream(filesMap[filename]); + for (std::string line; std::getline(ss, line);) { + lines.push_back(line); + } } } diff --git a/shared/offline_compiler/source/CMakeLists.txt b/shared/offline_compiler/source/CMakeLists.txt index 299e964e5e..56770c755a 100644 --- a/shared/offline_compiler/source/CMakeLists.txt +++ b/shared/offline_compiler/source/CMakeLists.txt @@ -58,6 +58,7 @@ set(CLOC_LIB_SRCS_LIB ${NEO_SHARED_DIRECTORY}/helpers/product_config_helper.cpp ${NEO_SHARED_DIRECTORY}/helpers/product_config_helper.h ${NEO_SHARED_DIRECTORY}/os_interface/os_library.h + ${NEO_SHARED_DIRECTORY}/utilities/directory.h ${NEO_SHARED_DIRECTORY}/utilities/io_functions.cpp ${NEO_SHARED_DIRECTORY}/utilities/io_functions.h ${OCLOC_DIRECTORY}/source/default_cache_config.cpp @@ -69,6 +70,8 @@ set(CLOC_LIB_SRCS_LIB ${OCLOC_DIRECTORY}/source/decoder/helper.h ${OCLOC_DIRECTORY}/source/decoder/iga_wrapper.h ${OCLOC_DIRECTORY}/source/decoder/translate_platform_base.h + ${OCLOC_DIRECTORY}/source/decoder/zebin_manipulator.cpp + ${OCLOC_DIRECTORY}/source/decoder/zebin_manipulator.h ${OCLOC_DIRECTORY}/source/multi_command.cpp ${OCLOC_DIRECTORY}/source/multi_command.h ${OCLOC_DIRECTORY}/source/ocloc_api.cpp @@ -119,6 +122,7 @@ if(WIN32) ${NEO_SHARED_DIRECTORY}/os_interface/windows/os_inc.h ${NEO_SHARED_DIRECTORY}/os_interface/windows/os_library_win.cpp ${NEO_SHARED_DIRECTORY}/os_interface/windows/os_library_win.h + ${NEO_SOURCE_DIR}/shared/source/utilities/windows/directory.cpp ) else() list(APPEND CLOC_LIB_SRCS_LIB @@ -128,6 +132,7 @@ else() ${NEO_SHARED_DIRECTORY}/os_interface/linux/os_library_linux.h ${NEO_SHARED_DIRECTORY}/os_interface/linux/sys_calls_linux.cpp ${OCLOC_DIRECTORY}/source/linux/os_library_ocloc_helper.cpp + ${NEO_SOURCE_DIR}/shared/source/utilities/linux/directory.cpp ) endif() diff --git a/shared/offline_compiler/source/decoder/zebin_manipulator.cpp b/shared/offline_compiler/source/decoder/zebin_manipulator.cpp new file mode 100644 index 0000000000..d70c75f893 --- /dev/null +++ b/shared/offline_compiler/source/decoder/zebin_manipulator.cpp @@ -0,0 +1,689 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "zebin_manipulator.h" + +#include "shared/offline_compiler/source/decoder/iga_wrapper.h" +#include "shared/offline_compiler/source/ocloc_arg_helper.h" +#include "shared/offline_compiler/source/ocloc_error_code.h" +#include "shared/source/device_binary_format/device_binary_formats.h" +#include "shared/source/device_binary_format/elf/elf.h" +#include "shared/source/device_binary_format/elf/elf_decoder.h" +#include "shared/source/device_binary_format/elf/elf_encoder.h" +#include "shared/source/device_binary_format/elf/zebin_elf.h" +#include "shared/source/device_binary_format/zebin_decoder.h" + +#include + +namespace NEO::ZebinManipulator { + +ErrorCode parseIntelGTNotesSectionForDevice(const std::vector &intelGTNotes, IgaWrapper *iga) { + size_t productFamilyNoteId = std::numeric_limits::max(); + size_t gfxCoreNoteId = std::numeric_limits::max(); + for (size_t i = 0; i < intelGTNotes.size(); i++) { + if (intelGTNotes[i].type == Elf::IntelGTSectionType::ProductFamily) { + productFamilyNoteId = i; + } else if (intelGTNotes[i].type == Elf::IntelGTSectionType::GfxCore) { + gfxCoreNoteId = i; + } + } + + if (productFamilyNoteId != std::numeric_limits::max()) { + UNRECOVERABLE_IF(sizeof(PRODUCT_FAMILY) != intelGTNotes[productFamilyNoteId].data.size()); + auto productFamily = *reinterpret_cast(intelGTNotes[productFamilyNoteId].data.begin()); + iga->setProductFamily(productFamily); + return OclocErrorCode::SUCCESS; + } else if (gfxCoreNoteId != std::numeric_limits::max()) { + UNRECOVERABLE_IF(sizeof(GFXCORE_FAMILY) != intelGTNotes[gfxCoreNoteId].data.size()); + auto gfxCore = *reinterpret_cast(intelGTNotes[gfxCoreNoteId].data.begin()); + iga->setGfxCore(gfxCore); + return OclocErrorCode::SUCCESS; + } + + return OclocErrorCode::INVALID_DEVICE; +} + +ErrorCode validateInput(const std::vector &args, IgaWrapper *iga, OclocArgHelper *argHelper, Arguments &outArguments) { + for (size_t argIndex = 2; argIndex < args.size(); ++argIndex) { + const auto &currArg = args[argIndex]; + const bool hasMoreArgs = (argIndex + 1 < args.size()); + if ("-file" == currArg && hasMoreArgs) { + outArguments.binaryFile = args[++argIndex]; + } else if ("-device" == currArg && hasMoreArgs) { + iga->setProductFamily(getProductFamilyFromDeviceName(args[++argIndex])); + } else if ("-dump" == currArg && hasMoreArgs) { + outArguments.pathToDump = args[++argIndex]; + addSlash(outArguments.pathToDump); + } else if ("--help" == currArg) { + outArguments.showHelp = true; + return OclocErrorCode::SUCCESS; + } else if ("-q" == currArg) { + argHelper->getPrinterRef() = MessagePrinter(true); + iga->setMessagePrinter(argHelper->getPrinterRef()); + } else if ("-skip-asm-translation" == currArg) { + outArguments.skipIGAdisassembly = true; + } else { + argHelper->printf("Unknown argument %s\n", currArg.c_str()); + return OclocErrorCode::INVALID_COMMAND_LINE; + } + } + + if (outArguments.binaryFile.empty()) { + argHelper->printf("Error: Missing -file argument\n"); + return OclocErrorCode::INVALID_COMMAND_LINE; + } + + if (outArguments.pathToDump.empty()) { + argHelper->printf("Warning: Path to dump -dump not specified. Using \"./dump/\" as dump folder.\n"); + outArguments.pathToDump = "dump/"; + } + + return OclocErrorCode::SUCCESS; +} + +bool is64BitZebin(OclocArgHelper *argHelper, const std::string §ionsInfoFilepath) { + std::vector lines; + argHelper->readFileToVectorOfStrings(sectionsInfoFilepath, lines); + if (lines.empty()) { + return false; + } + + auto ss = std::stringstream(lines[0]); + std::vector elfTypeStringSplit; + while (ss.good()) { + auto &element = elfTypeStringSplit.emplace_back(); + std::getline(ss, element, ' '); + } + return elfTypeStringSplit.size() == 2 && std::stoi(elfTypeStringSplit[1]) == 64; +} + +BinaryFormats getBinaryFormatForAssemble(OclocArgHelper *argHelper, const std::vector &args) { + auto it = std::find(args.begin(), args.end(), "-dump"); + std::string dump = (it != args.end() && (it + 1) != args.end()) ? *(it + 1) : "dump/"; + auto sectionsInfoFilepath = dump + ZebinManipulator::sectionsInfoFilename.str(); + const bool usesZebin = argHelper->fileExists(sectionsInfoFilepath); + if (usesZebin) { + return ZebinManipulator::is64BitZebin(argHelper, sectionsInfoFilepath) ? BinaryFormats::Zebin64b : BinaryFormats::Zebin32b; + } + return BinaryFormats::PatchTokens; +} + +BinaryFormats getBinaryFormatForDisassemble(OclocArgHelper *argHelper, const std::vector &args) { + auto it = std::find(args.begin(), args.end(), "-file"); + if (it != args.end() && (it + 1) != args.end()) { + auto file = argHelper->readBinaryFile(*(it + 1)); + auto fileRef = ArrayRef::fromAny(file.data(), file.size()); + if (NEO::isDeviceBinaryFormat(fileRef)) { + auto numBits = Elf::getElfNumBits(fileRef); + return numBits == Elf::EI_CLASS_64 ? BinaryFormats::Zebin64b : BinaryFormats::Zebin32b; + } + } + return BinaryFormats::PatchTokens; +} + +template ZebinDecoder::ZebinDecoder(OclocArgHelper *argHelper); +template ZebinDecoder::ZebinDecoder(OclocArgHelper *argHelper); +template +ZebinDecoder::ZebinDecoder(OclocArgHelper *argHelper) : argHelper(argHelper), + iga(new IgaWrapper) { + iga->setMessagePrinter(argHelper->getPrinterRef()); +} + +template ZebinDecoder::~ZebinDecoder(); +template ZebinDecoder::~ZebinDecoder(); +template +ZebinDecoder::~ZebinDecoder() {} + +template ErrorCode ZebinDecoder::decode(); +template ErrorCode ZebinDecoder::decode(); +template +ErrorCode ZebinDecoder::decode() { + auto zebinBinary = argHelper->readBinaryFile(this->arguments.binaryFile); + ArrayRef zebinBinaryRef = {reinterpret_cast(zebinBinary.data()), + zebinBinary.size()}; + + ElfT elf; + ErrorCode retVal = decodeZebin(ArrayRef::fromAny(zebinBinary.data(), zebinBinary.size()), elf); + if (retVal != OclocErrorCode::SUCCESS) { + argHelper->printf("Error while decoding zebin.\n"); + return retVal; + } + + if (false == arguments.skipIGAdisassembly) { + auto intelGTNotes = getIntelGTNotes(elf); + if (intelGTNotes.empty()) { + argHelper->printf("Error missing or invalid Intel GT Notes section.\n"); + return OclocErrorCode::INVALID_FILE; + } + + retVal = parseIntelGTNotesSectionForDevice(intelGTNotes, iga.get()); + if (retVal != OclocErrorCode::SUCCESS) { + argHelper->printf("Error while parsing Intel GT Notes section for device.\n"); + return retVal; + } + } + + auto sectionsInfo = dumpElfSections(elf); + dumpSectionInfo(sectionsInfo); + + return OclocErrorCode::SUCCESS; +} + +template ErrorCode ZebinDecoder::validateInput(const std::vector &args); +template ErrorCode ZebinDecoder::validateInput(const std::vector &args); +template +ErrorCode ZebinDecoder::validateInput(const std::vector &args) { + return ZebinManipulator::validateInput(args, iga.get(), argHelper, arguments); +} + +template void ZebinDecoder::printHelp(); +template void ZebinDecoder::printHelp(); +template +void ZebinDecoder::printHelp() { + argHelper->printf(R"===(Disassembles Zebin. +Output of such operation is a set of files that can be later used to reassemble back. +Symbols and relocations are translated into human readable format. Kernels are translated +into assembly. File named "sections.txt" is created which describes zebin sections. + +Usage: ocloc disasm -file [-dump ] [-device ] [-skip-asm-translation] + -file Input file to be disassembled. + + -dump Optional. Path for files representing decoded binary. Default is './dump'. + + -device Optional. Target device of input binary. + + -skip-asm-translation Optional. Skips parsing intelGTNotes for device and skips kernel + translation to assembly. + + --help Print this usage message. +)==="); +} + +template +void ZebinDecoder::dump(ConstStringRef name, ArrayRef data) { + auto outPath = arguments.pathToDump + name.str(); + argHelper->saveOutput(outPath, data.begin(), data.size()); +} + +template +void ZebinDecoder::dumpKernelData(ConstStringRef name, ArrayRef data) { + std::string disassembledKernel; + if (false == arguments.skipIGAdisassembly && + iga->tryDisassembleGenISA(data.begin(), static_cast(data.size()), disassembledKernel)) { + dump(name.str() + ".asm", {reinterpret_cast(disassembledKernel.data()), disassembledKernel.length()}); + } else { + dump(name, data); + } +} + +template +void ZebinDecoder::dumpSymtab(ElfT &elf, ArrayRef symtabData) { + ArrayRef symbols(reinterpret_cast(symtabData.begin()), + symtabData.size() / sizeof(ElfSymT)); + + std::stringstream symbolsFile; + symbolsFile << "Id, Name, Section, Value, Type, Visibility, Binding\n"; + int symbolID = 0; + for (auto &symbol : symbols) { + auto symbolName = elf.getSymbolName(symbol.name); + if (symbolName.empty()) { + symbolName = "UNDEF"; + } + auto sectionName = elf.getSectionName(symbol.shndx); + if (sectionName.empty()) { + sectionName = "UNDEF"; + } + + symbolsFile << std::to_string(symbolID++) << ", " + << symbolName << ", " + << sectionName << ", " + << std::to_string(symbol.value) << ", " + << std::to_string(symbol.getType()) << ", " + << std::to_string(symbol.getVisibility()) << ", " + << std::to_string(symbol.getBinding()) << "\n"; + } + auto symbolsFileStr = symbolsFile.str(); + dump(Elf::SectionsNamesZebin::symtab, ArrayRef::fromAny(symbolsFileStr.data(), symbolsFileStr.size())); +} + +template +std::vector ZebinDecoder::dumpElfSections(ElfT &elf) { + std::vector sectionInfos; + for (size_t secId = 1U; secId < elf.sectionHeaders.size(); secId++) { + auto &[header, data] = elf.sectionHeaders[secId]; + auto sectionName = elf.getSectionName(static_cast(secId)); + if (header->type == Elf::SHT_PROGBITS && + ConstStringRef(sectionName).startsWith(Elf::SectionsNamesZebin::textPrefix)) { + dumpKernelData(sectionName, data); + } else if (header->type == Elf::SHT_SYMTAB) { + dumpSymtab(elf, data); + } else if (header->type == Elf::SHT_REL) { + dumpRel(sectionName, data); + } else if (header->type == Elf::SHT_RELA) { + dumpRela(sectionName, data); + } else if (header->type == Elf::SHT_STRTAB) { + continue; + } else { + dump(sectionName, data); + } + sectionInfos.push_back({sectionName, header->type}); + } + return sectionInfos; +} + +template +void ZebinDecoder::dumpSectionInfo(const std::vector §ionInfos) { + std::stringstream sectionsInfoStr; + sectionsInfoStr << "ElfType " << (numBits == Elf::EI_CLASS_64 ? "64b" : "32b") << std::endl; + sectionsInfoStr << "Section name, Section type" << std::endl; + for (const auto §ionInfo : sectionInfos) { + sectionsInfoStr << sectionInfo.name << ", " << std::to_string(sectionInfo.type) << std::endl; + } + auto sectionInfoStr = sectionsInfoStr.str(); + dump(sectionsInfoFilename, ArrayRef::fromAny(sectionInfoStr.data(), sectionInfoStr.size())); +} + +template +ErrorCode ZebinDecoder::decodeZebin(ArrayRef zebin, ElfT &outElf) { + std::string errors, warnings; + outElf = Elf::decodeElf(zebin, errors, warnings); + + if (false == errors.empty()) { + argHelper->printf("decodeElf error: %s\n", errors.c_str()); + return OclocErrorCode::INVALID_FILE; + } + + return OclocErrorCode::SUCCESS; +} + +template +std::vector ZebinDecoder::getIntelGTNotes(ElfT &elf) { + std::vector intelGTNotes; + std::string errors, warnings; + NEO::getIntelGTNotes(elf, intelGTNotes, errors, warnings); + if (false == errors.empty()) { + argHelper->printf("Error when reading intelGTNotes: %s\n", errors.c_str()); + } + return intelGTNotes; +} + +template +void ZebinDecoder::dumpRel(ConstStringRef name, ArrayRef data) { + ArrayRef relocs = {reinterpret_cast(data.begin()), + data.size() / sizeof(ElfRelT)}; + std::stringstream relocsFile; + relocsFile << "Offset, Type, SymbolIdx\n"; + for (auto &reloc : relocs) { + relocsFile << std::to_string(reloc.offset) << ", " + << std::to_string(reloc.getRelocationType()) << ", " + << std::to_string(reloc.getSymbolTableIndex()) << "\n"; + } + auto relocsFileStr = relocsFile.str(); + dump(name, ArrayRef::fromAny(relocsFileStr.data(), relocsFileStr.length())); +} + +template +void ZebinDecoder::dumpRela(ConstStringRef name, ArrayRef data) { + ArrayRef relocs = {reinterpret_cast(data.begin()), + data.size() / sizeof(ElfRelaT)}; + std::stringstream relocsFile; + relocsFile << "Offset, Type, SymbolIdx, Addend\n"; + for (auto &reloc : relocs) { + relocsFile << std::to_string(reloc.offset) << ", " + << std::to_string(reloc.getRelocationType()) << ", " + << std::to_string(reloc.getSymbolTableIndex()) << ", " + << std::to_string(reloc.addend) << "\n"; + } + auto relocsFileStr = relocsFile.str(); + dump(name, ArrayRef::fromAny(relocsFileStr.data(), relocsFileStr.length())); +} + +template ZebinEncoder::ZebinEncoder(OclocArgHelper *argHelper); +template ZebinEncoder::ZebinEncoder(OclocArgHelper *argHelper); +template +ZebinEncoder::ZebinEncoder(OclocArgHelper *argHelper) : argHelper(argHelper), iga(new IgaWrapper) { + iga->setMessagePrinter(argHelper->getPrinterRef()); +} + +template ZebinEncoder::~ZebinEncoder(); +template ZebinEncoder::~ZebinEncoder(); +template +ZebinEncoder::~ZebinEncoder() {} + +template ErrorCode ZebinEncoder::encode(); +template ErrorCode ZebinEncoder::encode(); +template +ErrorCode ZebinEncoder::encode() { + ErrorCode retVal = OclocErrorCode::SUCCESS; + + std::vector sectionInfos; + retVal = loadSectionsInfo(sectionInfos); + if (retVal != OclocErrorCode::SUCCESS) { + argHelper->printf("Error while loading sections file.\n"); + return retVal; + } + + retVal = checkIfAllFilesExist(sectionInfos); + if (retVal != OclocErrorCode::SUCCESS) { + argHelper->printf("Error: Missing one or more section files.\n"); + return retVal; + } + + auto intelGTNotesSectionData = getIntelGTNotesSection(sectionInfos); + auto intelGTNotes = getIntelGTNotes(intelGTNotesSectionData); + retVal = parseIntelGTNotesSectionForDevice(intelGTNotes, iga.get()); + if (retVal != OclocErrorCode::SUCCESS) { + argHelper->printf("Error while parsing Intel GT Notes section for device.\n"); + return retVal; + } + + ElfEncoderT elfEncoder; + elfEncoder.getElfFileHeader().machine = Elf::ELF_MACHINE::EM_INTELGT; + elfEncoder.getElfFileHeader().type = Elf::ELF_TYPE_ZEBIN::ET_ZEBIN_EXE; + + retVal = appendSections(elfEncoder, sectionInfos); + if (retVal != OclocErrorCode::SUCCESS) { + argHelper->printf("Error while appending elf sections.\n"); + return retVal; + } + + auto zebin = elfEncoder.encode(); + argHelper->saveOutput(getFilePath(arguments.binaryFile), zebin.data(), zebin.size()); + + return OclocErrorCode::SUCCESS; +} + +template ErrorCode ZebinEncoder::validateInput(const std::vector &args); +template ErrorCode ZebinEncoder::validateInput(const std::vector &args); +template +ErrorCode ZebinEncoder::validateInput(const std::vector &args) { + return ZebinManipulator::validateInput(args, iga.get(), argHelper, arguments); +} + +template void ZebinEncoder::printHelp(); +template void ZebinEncoder::printHelp(); +template +void ZebinEncoder::printHelp() { + argHelper->printf(R"===(Assembles Zebin from input files. +It's expected that input files were previously generated by 'ocloc disasm' +command or are compatible with 'ocloc disasm' output (especially in terms of +file naming scheme). + +Usage: ocloc asm -file [-dump ] [-device ] [-skip-asm-translation] + -file Name of the newly assembled zebin. + + -dump Optional. Path to the input directory containing disassembled binary. + Default is './dump'. + + -device Optional. Target device of input binary. + + --help Print this usage message. +)==="); +} + +template +std::vector ZebinEncoder::getIntelGTNotesSection(const std::vector §ionInfos) { + bool containsIntelGTNoteSection = false; + for (auto §ionInfo : sectionInfos) { + if (sectionInfo.type == Elf::SHT_NOTE && + sectionInfo.name == Elf::SectionsNamesZebin::noteIntelGT) { + containsIntelGTNoteSection = true; + break; + } + } + if (false == containsIntelGTNoteSection) { + return {}; + } + + return argHelper->readBinaryFile(getFilePath(Elf::SectionsNamesZebin::noteIntelGT.data())); +} + +template +std::vector ZebinEncoder::getIntelGTNotes(const std::vector &intelGtNotesSection) { + std::vector intelGTNotes; + std::string errors, warnings; + auto refIntelGTNotesSection = ArrayRef::fromAny(intelGtNotesSection.data(), intelGtNotesSection.size()); + auto decodeError = decodeIntelGTNoteSection(refIntelGTNotesSection, intelGTNotes, errors, warnings); + argHelper->printf(warnings.c_str()); + if (decodeError != NEO::DecodeError::Success) { + argHelper->printf(errors.c_str()); + } + return intelGTNotes; +} + +template +ErrorCode ZebinEncoder::loadSectionsInfo(std::vector §ionInfos) { + std::vector sectionsInfoLines; + argHelper->readFileToVectorOfStrings(getFilePath(sectionsInfoFilename.data()), sectionsInfoLines); + if (sectionsInfoLines.size() <= 2) { + return OclocErrorCode::INVALID_FILE; + } + + sectionInfos.resize(sectionsInfoLines.size() - 2); + for (size_t i = 2; i < sectionsInfoLines.size(); i++) { + auto elements = parseLine(sectionsInfoLines[i]); + UNRECOVERABLE_IF(elements.size() != 2); + auto §ionInfo = sectionInfos[i - 2]; + sectionInfo.name = elements[0]; + sectionInfo.type = static_cast(std::stoull(elements[1])); + } + return OclocErrorCode::SUCCESS; +} + +template +ErrorCode NEO::ZebinManipulator::ZebinEncoder::checkIfAllFilesExist(const std::vector §ionInfos) { + for (auto §ionInfo : sectionInfos) { + bool fileExists = argHelper->fileExists(getFilePath(sectionInfo.name)); + if (ConstStringRef(sectionInfo.name).startsWith(Elf::SectionsNamesZebin::textPrefix)) { + fileExists |= argHelper->fileExists(getFilePath(sectionInfo.name + ".asm")); + } + + if (false == fileExists) { + argHelper->printf("Error: Could not find the file \"%s\"\n", sectionInfo.name.c_str()); + return OclocErrorCode::INVALID_FILE; + } + } + return OclocErrorCode::SUCCESS; +} + +template +ErrorCode ZebinEncoder::appendSections(ElfEncoderT &encoder, const std::vector §ionInfos) { + SecNameToIdMapT secNameToId; + size_t symtabIdx = std::numeric_limits::max(); + for (size_t i = 0; i < sectionInfos.size(); i++) { + secNameToId[sectionInfos[i].name] = i + 1; + if (sectionInfos[i].name == Elf::SectionsNamesZebin::symtab) { + symtabIdx = i + 1; + } + } + + ErrorCode retVal = OclocErrorCode::SUCCESS; + for (const auto §ion : sectionInfos) { + if (section.type == Elf::SHT_SYMTAB) { + retVal |= appendSymtab(encoder, section, sectionInfos.size() + 1, secNameToId); + } else if (section.type == Elf::SHT_REL) { + retVal |= appendRel(encoder, section, secNameToId[section.name.substr(Elf::SpecialSectionNames::relPrefix.length())], symtabIdx); + } else if (section.type == Elf::SHT_RELA) { + retVal |= appendRela(encoder, section, secNameToId[section.name.substr(Elf::SpecialSectionNames::relaPrefix.length())], symtabIdx); + } else if (section.type == Elf::SHT_PROGBITS && ConstStringRef(section.name).startsWith(Elf::SectionsNamesZebin::textPrefix)) { + retVal |= appendKernel(encoder, section); + } else { + retVal |= appendOther(encoder, section); + } + } + return retVal; +} + +template +ErrorCode ZebinEncoder::appendRel(ElfEncoderT &encoder, const SectionInfo §ion, size_t targetSecId, size_t symtabSecId) { + std::vector relocationLines; + argHelper->readFileToVectorOfStrings(getFilePath(section.name), relocationLines); + if (relocationLines.empty()) { + argHelper->printf("Error: Empty relocations file: %s\n", section.name.c_str()); + return OclocErrorCode::INVALID_FILE; + } + auto relocs = parseRel(relocationLines); + auto &sec = encoder.appendSection(Elf::SHT_REL, section.name, ArrayRef::fromAny(relocs.data(), relocs.size())); + sec.info = static_cast(targetSecId); + sec.link = static_cast(symtabSecId); + return OclocErrorCode::SUCCESS; +} + +template +ErrorCode ZebinEncoder::appendRela(ElfEncoderT &encoder, const SectionInfo §ion, size_t targetSecId, size_t symtabSecId) { + std::vector relocationLines; + argHelper->readFileToVectorOfStrings(getFilePath(section.name), relocationLines); + if (relocationLines.empty()) { + argHelper->printf("Error: Empty relocations file: %s\n", section.name.c_str()); + return OclocErrorCode::INVALID_FILE; + } + auto relocs = parseRela(relocationLines); + auto &sec = encoder.appendSection(Elf::SHT_RELA, section.name, ArrayRef::fromAny(relocs.data(), relocs.size())); + sec.info = static_cast(targetSecId); + sec.link = static_cast(symtabSecId); + return OclocErrorCode::SUCCESS; +} + +template +std::string ZebinEncoder::getFilePath(const std::string &filename) { + return arguments.pathToDump + filename; +} + +template +std::string ZebinEncoder::parseKernelAssembly(ArrayRef kernelAssembly) { + std::string kernelAssemblyString(kernelAssembly.begin(), kernelAssembly.end()); + std::string outBinary; + if (iga->tryAssembleGenISA(kernelAssemblyString, outBinary)) { + return outBinary; + } + return {}; +} + +template +ErrorCode ZebinEncoder::appendKernel(ElfEncoderT &encoder, const SectionInfo §ion) { + if (argHelper->fileExists(getFilePath(section.name + ".asm"))) { + auto data = argHelper->readBinaryFile(getFilePath(section.name + ".asm")); + auto kernelBinary = parseKernelAssembly(ArrayRef::fromAny(data.data(), data.size())); + ArrayRef refKernelBinary = {reinterpret_cast(kernelBinary.data()), kernelBinary.size()}; + encoder.appendSection(section.type, section.name, refKernelBinary); + } else { + auto data = argHelper->readBinaryFile(getFilePath(section.name)); + encoder.appendSection(Elf::SHT_PROGBITS, section.name, ArrayRef::fromAny(data.data(), data.size())); + } + return OclocErrorCode::SUCCESS; +} + +template +ErrorCode ZebinEncoder::appendSymtab(ElfEncoderT &encoder, const SectionInfo §ion, size_t strtabSecId, SecNameToIdMapT secNameToId) { + std::vector symTabLines; + argHelper->readFileToVectorOfStrings(getFilePath(section.name), symTabLines); + if (symTabLines.empty()) { + argHelper->printf("Error: Empty symtab file: %s\n", section.name.c_str()); + return OclocErrorCode::INVALID_FILE; + } + + size_t numLocalSymbols = 0; + auto symbols = parseSymbols(symTabLines, encoder, numLocalSymbols, secNameToId); + + auto &symtabSection = encoder.appendSection(section.type, section.name, ArrayRef::fromAny(symbols.data(), symbols.size())); + symtabSection.info = static_cast(numLocalSymbols); + symtabSection.link = static_cast(strtabSecId); + return OclocErrorCode::SUCCESS; +} + +template +ErrorCode NEO::ZebinManipulator::ZebinEncoder::appendOther(ElfEncoderT &encoder, const SectionInfo §ion) { + auto sectionData = argHelper->readBinaryFile(getFilePath(section.name)); + encoder.appendSection(section.type, section.name, ArrayRef::fromAny(sectionData.data(), sectionData.size())); + return OclocErrorCode::SUCCESS; +} + +template +std::vector ZebinEncoder::parseLine(const std::string &line) { + std::vector out; + auto ss = std::stringstream(line); + while (ss.good()) { + auto &element = out.emplace_back(); + std::getline(ss, element, ','); + } + return out; +} + +template +std::vector::ElfRelT> ZebinEncoder::parseRel(const std::vector &relocationsFile) { + std::vector relocs; + relocs.resize(relocationsFile.size() - 1); + + for (size_t lineId = 1U; lineId < relocationsFile.size(); lineId++) { + auto elements = parseLine(relocationsFile[lineId]); + UNRECOVERABLE_IF(elements.size() != 3); + + auto &reloc = relocs[lineId - 1]; + reloc.offset = static_cast(std::stoull(elements[0])); + reloc.setRelocationType(static_cast(std::stoull(elements[1]))); + reloc.setSymbolTableIndex(static_cast(std::stoull(elements[2]))); + } + + return relocs; +} + +template +std::vector::ElfRelaT> ZebinEncoder::parseRela(const std::vector &relocationsFile) { + std::vector relocs; + relocs.resize(relocationsFile.size() - 1); + + for (size_t lineId = 1U; lineId < relocationsFile.size(); lineId++) { + auto elements = parseLine(relocationsFile[lineId]); + UNRECOVERABLE_IF(elements.size() != 4); + + auto &reloc = relocs[lineId - 1]; + reloc.offset = static_cast(std::stoull(elements[0])); + reloc.setRelocationType(static_cast(std::stoull(elements[1]))); + reloc.setSymbolTableIndex(static_cast(std::stoull(elements[2]))); + reloc.addend = static_cast(std::stoll(elements[3])); + } + + return relocs; +} + +template +std::vector::ElfSymT> ZebinEncoder::parseSymbols(const std::vector &symbolsFile, ElfEncoderT &encoder, size_t &outNumLocalSymbols, SecNameToIdMapT secNameToId) { + std::vector symbols; + symbols.resize(symbolsFile.size() - 1); + outNumLocalSymbols = 0U; + + for (size_t lineId = 1U; lineId < symbolsFile.size(); lineId++) { + auto &line = symbolsFile[lineId]; + auto elements = parseLine(line); + UNRECOVERABLE_IF(elements.size() != 7); + + auto symbolId = std::stoull(elements[0]); + auto symbolName = elements[1].substr(1); + auto sectionName = elements[2].substr(1); + auto symbolValue = std::stoull(elements[3]); + auto symbolType = std::stoi(elements[4]); + auto symbolVisibility = std::stoi(elements[5]); + auto symbolBinding = std::stoi(elements[6]); + + UNRECOVERABLE_IF(symbolId >= symbols.size()); + auto &symbol = symbols[static_cast(symbolId)]; + symbol.name = static_cast((symbolName == "UNDEF") ? 0 : encoder.appendSectionName(symbolName)); + symbol.shndx = static_cast((sectionName == "UNDEF") ? 0 : static_cast(secNameToId[sectionName])); + symbol.value = static_cast(symbolValue); + symbol.setType(static_cast(symbolType)); + symbol.setVisibility(static_cast(symbolVisibility)); + symbol.setBinding(static_cast(symbolBinding)); + + if (symbol.getBinding() == Elf::STB_LOCAL) { + outNumLocalSymbols = lineId; + } + } + + return symbols; +} + +} // namespace NEO::ZebinManipulator diff --git a/shared/offline_compiler/source/decoder/zebin_manipulator.h b/shared/offline_compiler/source/decoder/zebin_manipulator.h new file mode 100644 index 0000000000..27ed84028d --- /dev/null +++ b/shared/offline_compiler/source/decoder/zebin_manipulator.h @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/device_binary_format/elf/elf.h" +#include "shared/source/utilities/arrayref.h" +#include "shared/source/utilities/const_stringref.h" + +#include +#include +#include +#include +#include + +class OclocArgHelper; +struct IgaWrapper; +namespace NEO { + +namespace Elf { +struct IntelGTNote; + +template +struct Elf; + +template +struct ElfEncoder; +} // namespace Elf + +namespace ZebinManipulator { + +struct SectionInfo { + std::string name; + uint32_t type; +}; + +struct Arguments { + std::string pathToDump = ""; + std::string binaryFile = ""; + bool showHelp = false; + bool skipIGAdisassembly = false; +}; + +enum BinaryFormats { + PatchTokens, + Zebin32b, + Zebin64b +}; + +using ErrorCode = int; + +ErrorCode parseIntelGTNotesSectionForDevice(const std::vector &intelGTNotes, IgaWrapper *iga); +ErrorCode validateInput(const std::vector &args, IgaWrapper *iga, OclocArgHelper *argHelper, Arguments &outArguments); + +BinaryFormats getBinaryFormatForAssemble(OclocArgHelper *argHelper, const std::vector &args); +BinaryFormats getBinaryFormatForDisassemble(OclocArgHelper *argHelper, const std::vector &args); +bool is64BitZebin(OclocArgHelper *argHelper, const std::string §ionsInfoFilepath); + +constexpr ConstStringRef sectionsInfoFilename = "sections.txt"; + +template +class ZebinDecoder { + public: + using ElfT = Elf::Elf; + using ElfRelT = Elf::ElfRel; + using ElfRelaT = Elf::ElfRela; + using ElfSymT = Elf::ElfSymbolEntry; + + ZebinDecoder(OclocArgHelper *argHelper); + ~ZebinDecoder(); + ErrorCode decode(); + ErrorCode validateInput(const std::vector &args); + void printHelp(); + + protected: + MOCKABLE_VIRTUAL ErrorCode decodeZebin(ArrayRef zebin, ElfT &outElf); + MOCKABLE_VIRTUAL void dump(ConstStringRef name, ArrayRef data); + MOCKABLE_VIRTUAL void dumpKernelData(ConstStringRef name, ArrayRef data); + MOCKABLE_VIRTUAL void dumpRel(ConstStringRef name, ArrayRef data); + MOCKABLE_VIRTUAL void dumpRela(ConstStringRef name, ArrayRef data); + MOCKABLE_VIRTUAL void dumpSymtab(ElfT &elf, ArrayRef symtabData); + MOCKABLE_VIRTUAL std::vector dumpElfSections(ElfT &elf); + MOCKABLE_VIRTUAL void dumpSectionInfo(const std::vector §ionInfos); + MOCKABLE_VIRTUAL std::vector getIntelGTNotes(ElfT &elf); + + public: + bool &showHelp = arguments.showHelp; + + protected: + Arguments arguments; + OclocArgHelper *argHelper; + std::unique_ptr iga; +}; + +template +class ZebinEncoder { + public: + using ElfEncoderT = Elf::ElfEncoder; + using SecNameToIdMapT = std::unordered_map; + using ElfSecHdrT = Elf::ElfSectionHeader; + using ElfRelT = Elf::ElfRel; + using ElfRelaT = Elf::ElfRela; + using ElfSymT = Elf::ElfSymbolEntry; + + ZebinEncoder(OclocArgHelper *argHelper); + ~ZebinEncoder(); + ErrorCode encode(); + ErrorCode validateInput(const std::vector &args); + void printHelp(); + + protected: + MOCKABLE_VIRTUAL ErrorCode loadSectionsInfo(std::vector §ionInfos); + MOCKABLE_VIRTUAL ErrorCode checkIfAllFilesExist(const std::vector §ionInfos); + MOCKABLE_VIRTUAL std::vector getIntelGTNotesSection(const std::vector §ionInfos); + MOCKABLE_VIRTUAL std::vector getIntelGTNotes(const std::vector &intelGtNotesSection); + + MOCKABLE_VIRTUAL ErrorCode appendSections(ElfEncoderT &encoder, const std::vector §ionInfos); + MOCKABLE_VIRTUAL ErrorCode appendRel(ElfEncoderT &encoder, const SectionInfo §ion, size_t targetSecId, size_t symtabSecId); + MOCKABLE_VIRTUAL ErrorCode appendRela(ElfEncoderT &encoder, const SectionInfo §ion, size_t targetSecId, size_t symtabSecId); + MOCKABLE_VIRTUAL ErrorCode appendKernel(ElfEncoderT &encoder, const SectionInfo §ion); + MOCKABLE_VIRTUAL ErrorCode appendSymtab(ElfEncoderT &encoder, const SectionInfo §ion, size_t strtabId, SecNameToIdMapT secNameToId); + MOCKABLE_VIRTUAL ErrorCode appendOther(ElfEncoderT &encoder, const SectionInfo §ion); + + MOCKABLE_VIRTUAL std::string getFilePath(const std::string &filename); + MOCKABLE_VIRTUAL std::string parseKernelAssembly(ArrayRef kernelAssembly); + MOCKABLE_VIRTUAL std::vector parseLine(const std::string &line); + MOCKABLE_VIRTUAL std::vector parseRel(const std::vector &relocationsFile); + MOCKABLE_VIRTUAL std::vector parseRela(const std::vector &relocationsFile); + MOCKABLE_VIRTUAL std::vector parseSymbols(const std::vector &symbolsFile, ElfEncoderT &encoder, size_t &outNumLocalSymbols, SecNameToIdMapT secNameToId); + + public: + bool &showHelp = arguments.showHelp; + + protected: + Arguments arguments; + OclocArgHelper *argHelper; + std::unique_ptr iga; +}; + +}; // namespace ZebinManipulator +} // namespace NEO \ No newline at end of file diff --git a/shared/offline_compiler/source/ocloc_arg_helper.cpp b/shared/offline_compiler/source/ocloc_arg_helper.cpp index 9be80f7ef4..d1a09115ac 100644 --- a/shared/offline_compiler/source/ocloc_arg_helper.cpp +++ b/shared/offline_compiler/source/ocloc_arg_helper.cpp @@ -22,8 +22,9 @@ void Source::toVectorOfStrings(std::vector &lines, bool replaceTabs) { std::string line; const char *file = reinterpret_cast(data); + const char *end = file + length; - while (*file != '\0') { + while (file != end && *file != '\0') { if (replaceTabs && *file == '\t') { line += ' '; } else if (*file == '\n') { diff --git a/shared/offline_compiler/source/ocloc_interface.cpp b/shared/offline_compiler/source/ocloc_interface.cpp index 4e6dbccc4b..648b5aca0d 100644 --- a/shared/offline_compiler/source/ocloc_interface.cpp +++ b/shared/offline_compiler/source/ocloc_interface.cpp @@ -9,6 +9,7 @@ #include "shared/offline_compiler/source/decoder/binary_decoder.h" #include "shared/offline_compiler/source/decoder/binary_encoder.h" +#include "shared/offline_compiler/source/decoder/zebin_manipulator.h" #include "shared/offline_compiler/source/multi_command.h" #include "shared/offline_compiler/source/ocloc_concat.h" #include "shared/offline_compiler/source/ocloc_error_code.h" @@ -18,6 +19,9 @@ #include "shared/offline_compiler/source/offline_compiler.h" #include "shared/offline_compiler/source/offline_linker.h" #include "shared/offline_compiler/source/utilities/safety_caller.h" +#include "shared/source/device_binary_format/device_binary_formats.h" +#include "shared/source/device_binary_format/elf/elf_decoder.h" +#include "shared/source/device_binary_format/elf/zebin_elf.h" #include @@ -166,34 +170,48 @@ int link(OclocArgHelper *argHelper, const std::vector &args) { }; int disassemble(OclocArgHelper *argHelper, const std::vector &args) { - BinaryDecoder disasm(argHelper); - int retVal = disasm.validateInput(args); + const auto binaryFormat = ZebinManipulator::getBinaryFormatForDisassemble(argHelper, args); + auto decode = [&args](auto &decoder) -> int { + int retVal = decoder.validateInput(args); + if (decoder.showHelp) { + decoder.printHelp(); + return OclocErrorCode::SUCCESS; + } + return (retVal == OclocErrorCode::SUCCESS) ? decoder.decode() : retVal; + }; - if (disasm.showHelp) { - disasm.printHelp(); - return retVal; - } + if (binaryFormat == ZebinManipulator::BinaryFormats::PatchTokens) { + BinaryDecoder disasm(argHelper); + return decode(disasm); - if (retVal == 0) { - return disasm.decode(); + } else if (binaryFormat == ZebinManipulator::BinaryFormats::Zebin32b) { + ZebinManipulator::ZebinDecoder decoder(argHelper); + return decode(decoder); } else { - return retVal; + ZebinManipulator::ZebinDecoder decoder(argHelper); + return decode(decoder); } } int assemble(OclocArgHelper *argHelper, const std::vector &args) { - BinaryEncoder assembler(argHelper); - int retVal = assembler.validateInput(args); - - if (assembler.showHelp) { - assembler.printHelp(); - return retVal; - } - - if (retVal == 0) { - return assembler.encode(); + const auto binaryFormat = ZebinManipulator::getBinaryFormatForAssemble(argHelper, args); + auto encode = [&args](auto &encoder) -> int { + int retVal = encoder.validateInput(args); + if (encoder.showHelp) { + encoder.printHelp(); + return OclocErrorCode::SUCCESS; + } + return (retVal == OclocErrorCode::SUCCESS) ? encoder.encode() : retVal; + }; + if (binaryFormat == ZebinManipulator::BinaryFormats::PatchTokens) { + BinaryEncoder assembler(argHelper); + return encode(assembler); + } else if (binaryFormat == ZebinManipulator::BinaryFormats::Zebin32b) { + ZebinManipulator::ZebinEncoder encoder(argHelper); + return encode(encoder); } else { - return retVal; + ZebinManipulator::ZebinEncoder encoder(argHelper); + return encode(encoder); } } diff --git a/shared/source/device_binary_format/elf/zebin_elf.h b/shared/source/device_binary_format/elf/zebin_elf.h index 0200935a5f..cc336ea643 100644 --- a/shared/source/device_binary_format/elf/zebin_elf.h +++ b/shared/source/device_binary_format/elf/zebin_elf.h @@ -51,6 +51,7 @@ constexpr ConstStringRef dataGlobal = ".data.global"; constexpr ConstStringRef dataConstString = ".data.const.string"; constexpr ConstStringRef symtab = ".symtab"; constexpr ConstStringRef relTablePrefix = ".rel."; +constexpr ConstStringRef relaTablePrefix = ".rela."; constexpr ConstStringRef spv = ".spv"; constexpr ConstStringRef debugPrefix = ".debug_"; constexpr ConstStringRef debugInfo = ".debug_info"; diff --git a/shared/source/device_binary_format/zebin_decoder.cpp b/shared/source/device_binary_format/zebin_decoder.cpp index ec6a4781ae..2d30065768 100644 --- a/shared/source/device_binary_format/zebin_decoder.cpp +++ b/shared/source/device_binary_format/zebin_decoder.cpp @@ -108,52 +108,58 @@ DecodeError validateZeInfoVersion(const Elf::ZebinKernelMetadata::Types::Version return DecodeError::Success; } +template +DecodeError decodeIntelGTNoteSection(ArrayRef intelGTNotesSection, std::vector &intelGTNotes, std::string &outErrReason, std::string &outWarning) { + uint64_t currentPos = 0; + auto sectionSize = intelGTNotesSection.size(); + while (currentPos < sectionSize) { + auto intelGTNote = reinterpret_cast(intelGTNotesSection.begin() + currentPos); + auto nameSz = intelGTNote->nameSize; + auto descSz = intelGTNote->descSize; + + auto currOffset = sizeof(Elf::ElfNoteSection) + alignUp(nameSz, 4) + alignUp(descSz, 4); + if (currentPos + currOffset > sectionSize) { + intelGTNotes.clear(); + outErrReason.append("DeviceBinaryFormat::Zebin : Offseting will cause out-of-bound memory read! Section size: " + std::to_string(sectionSize) + + ", current section data offset: " + std::to_string(currentPos) + ", next offset : " + std::to_string(currOffset) + "\n"); + return DecodeError::InvalidBinary; + } + currentPos += currOffset; + + auto ownerName = reinterpret_cast(ptrOffset(intelGTNote, sizeof(Elf::ElfNoteSection))); + bool isValidGTNote = Elf::IntelGtNoteOwnerName.size() + 1 == nameSz; + isValidGTNote &= Elf::IntelGtNoteOwnerName == ConstStringRef(ownerName, nameSz - 1); + if (false == isValidGTNote) { + if (0u == nameSz) { + outWarning.append("DeviceBinaryFormat::Zebin : Empty owner name.\n"); + } else { + std::string invalidOwnerName{ownerName, nameSz}; + invalidOwnerName.erase(std::remove_if(invalidOwnerName.begin(), + invalidOwnerName.end(), + [](unsigned char c) { return '\0' == c; })); + outWarning.append("DeviceBinaryFormat::Zebin : Invalid owner name : " + invalidOwnerName + " for IntelGTNote - note will not be used.\n"); + } + continue; + } + auto notesData = ArrayRef(reinterpret_cast(ptrOffset(ownerName, nameSz)), descSz); + if (intelGTNote->type == Elf::IntelGTSectionType::ZebinVersion) { + isValidGTNote &= notesData[descSz - 1] == '\0'; + if (false == isValidGTNote) { + outWarning.append("DeviceBinaryFormat::Zebin : Versioning string is not null-terminated: " + ConstStringRef(reinterpret_cast(notesData.begin()), descSz).str() + " - note will not be used.\n"); + continue; + } + } + intelGTNotes.push_back(Elf::IntelGTNote{static_cast(intelGTNote->type), notesData}); + } + return DecodeError::Success; +} + template DecodeError getIntelGTNotes(const Elf::Elf &elf, std::vector &intelGTNotes, std::string &outErrReason, std::string &outWarning) { for (size_t i = 0; i < elf.sectionHeaders.size(); i++) { auto section = elf.sectionHeaders[i]; if (Elf::SHT_NOTE == section.header->type && Elf::SectionsNamesZebin::noteIntelGT == elf.getSectionName(static_cast(i))) { - uint64_t currentPos = 0; - auto sectionSize = section.header->size; - while (currentPos < sectionSize) { - auto intelGTNote = reinterpret_cast(section.data.begin() + currentPos); - auto nameSz = intelGTNote->nameSize; - auto descSz = intelGTNote->descSize; - - auto currOffset = sizeof(Elf::ElfNoteSection) + alignUp(nameSz, 4) + alignUp(descSz, 4); - if (currentPos + currOffset > sectionSize) { - intelGTNotes.clear(); - outErrReason.append("DeviceBinaryFormat::Zebin : Offseting will cause out-of-bound memory read! Section size: " + std::to_string(sectionSize) + - ", current section data offset: " + std::to_string(currentPos) + ", next offset : " + std::to_string(currOffset) + "\n"); - return DecodeError::InvalidBinary; - } - currentPos += currOffset; - - auto ownerName = reinterpret_cast(ptrOffset(intelGTNote, sizeof(Elf::ElfNoteSection))); - bool isValidGTNote = Elf::IntelGtNoteOwnerName.size() + 1 == nameSz; - isValidGTNote &= Elf::IntelGtNoteOwnerName == ConstStringRef(ownerName, nameSz - 1); - if (false == isValidGTNote) { - if (0u == nameSz) { - outWarning.append("DeviceBinaryFormat::Zebin : Empty owner name.\n"); - } else { - std::string invalidOwnerName{ownerName, nameSz}; - invalidOwnerName.erase(std::remove_if(invalidOwnerName.begin(), - invalidOwnerName.end(), - [](unsigned char c) { return '\0' == c; })); - outWarning.append("DeviceBinaryFormat::Zebin : Invalid owner name : " + invalidOwnerName + " for IntelGTNote - note will not be used.\n"); - } - continue; - } - auto notesData = ArrayRef(reinterpret_cast(ptrOffset(ownerName, nameSz)), descSz); - if (intelGTNote->type == Elf::IntelGTSectionType::ZebinVersion) { - isValidGTNote &= notesData[descSz - 1] == '\0'; - if (false == isValidGTNote) { - outWarning.append("DeviceBinaryFormat::Zebin : Versioning string is not null-terminated: " + ConstStringRef(reinterpret_cast(notesData.begin()), descSz).str() + " - note will not be used.\n"); - continue; - } - } - intelGTNotes.push_back(Elf::IntelGTNote{static_cast(intelGTNote->type), notesData}); - } + return decodeIntelGTNoteSection(section.data, intelGTNotes, outErrReason, outWarning); } } return DecodeError::Success; diff --git a/shared/source/device_binary_format/zebin_decoder.h b/shared/source/device_binary_format/zebin_decoder.h index 7982870376..6945ce1ef9 100644 --- a/shared/source/device_binary_format/zebin_decoder.h +++ b/shared/source/device_binary_format/zebin_decoder.h @@ -48,10 +48,14 @@ struct ZeInfoKernelSections { UniqueNode inlineSamplersNd; }; +DecodeError validateZeInfoVersion(const Elf::ZebinKernelMetadata::Types::Version &receivedZeInfoVersion, std::string &outErrReason, std::string &outWarning); + template bool validateTargetDevice(const Elf::Elf &elf, const TargetDevice &targetDevice, std::string &outErrReason, std::string &outWarning); -DecodeError validateZeInfoVersion(const Elf::ZebinKernelMetadata::Types::Version &receivedZeInfoVersion, std::string &outErrReason, std::string &outWarning); +template +DecodeError decodeIntelGTNoteSection(ArrayRef intelGTNotesSection, std::vector &intelGTNotes, std::string &outErrReason, std::string &outWarning); + template DecodeError getIntelGTNotes(const Elf::Elf &elf, std::vector &intelGTNotes, std::string &outErrReason, std::string &outWarning); diff --git a/shared/source/utilities/const_stringref.h b/shared/source/utilities/const_stringref.h index 67caf289de..77ce77c479 100644 --- a/shared/source/utilities/const_stringref.h +++ b/shared/source/utilities/const_stringref.h @@ -168,6 +168,23 @@ class ConstStringRef { return ('\0' == *rhs); } + constexpr bool startsWith(ConstStringRef subString) const noexcept { + if (subString.length() > len) { + return false; + } + const char *findEnd = ptr + subString.length(); + const char *lhs = ptr; + const char *rhs = subString.begin(); + while ((lhs < findEnd)) { + if (*lhs != *rhs) { + return false; + } + lhs++; + rhs++; + } + return true; + } + constexpr bool isEqualWithoutSeparator(const char separator, const char *subString) const noexcept { const char *end = ptr + len; const char *lhs = ptr; diff --git a/shared/test/common/mocks/mock_modules_zebin.h b/shared/test/common/mocks/mock_modules_zebin.h index b5ecc34d5c..ab69ca180c 100644 --- a/shared/test/common/mocks/mock_modules_zebin.h +++ b/shared/test/common/mocks/mock_modules_zebin.h @@ -9,6 +9,8 @@ #include "shared/source/device_binary_format/elf/zebin_elf.h" #include "shared/source/device_binary_format/zebin_decoder.h" +#include "shared/source/helpers/aligned_memory.h" +#include "shared/source/helpers/ptr_math.h" #include "shared/source/utilities/const_stringref.h" #include "shared/test/common/mocks/mock_elf.h" @@ -261,4 +263,42 @@ kernels: -)==="; }; +inline std::vector createIntelGTNoteSection(PRODUCT_FAMILY productFamily, GFXCORE_FAMILY coreFamily, NEO::Elf::ZebinTargetFlags flags, NEO::ConstStringRef version) { + std::array notes = {{{8, 4, NEO::Elf::IntelGTSectionType::ProductFamily}, + {8, 4, NEO::Elf::IntelGTSectionType::GfxCore}, + {8, 4, NEO::Elf::IntelGTSectionType::TargetMetadata}, + {8, static_cast(alignUp(version.length(), 4U)), NEO::Elf::IntelGTSectionType::ZebinVersion}}}; + + std::array, 4> descData; + descData[0].resize(4); + std::memcpy(descData[0].data(), &productFamily, 4); + + descData[1].resize(4); + std::memcpy(descData[1].data(), &coreFamily, 4); + + descData[2].resize(4); + std::memcpy(descData[2].data(), &flags.packed, 4); + + descData[3].resize(notes[3].descSize); + std::memcpy(descData[3].data(), version.data(), notes[3].descSize); + + size_t noteSectionSize = 0U; + for (auto ¬e : notes) { + noteSectionSize += sizeof(note) + note.descSize + note.nameSize; + } + + std::vector intelGTNotesSection(noteSectionSize); + auto intelGTNotes = intelGTNotesSection.data(); + for (size_t i = 0; i < 4U; i++) { + auto ¬e = notes[i]; + std::memcpy(intelGTNotes, ¬e, sizeof(NEO::Elf::ElfNoteSection)); + intelGTNotes = ptrOffset(intelGTNotes, sizeof(NEO::Elf::ElfNoteSection)); + std::memcpy(intelGTNotes, NEO::Elf::IntelGtNoteOwnerName.data(), note.nameSize); + intelGTNotes = ptrOffset(intelGTNotes, note.nameSize); + std::memcpy(intelGTNotes, descData[i].data(), note.descSize); + intelGTNotes = ptrOffset(intelGTNotes, note.descSize); + } + return intelGTNotesSection; +} + } // namespace ZebinTestData \ No newline at end of file diff --git a/shared/test/unit_test/utilities/const_stringref_tests.cpp b/shared/test/unit_test/utilities/const_stringref_tests.cpp index 9fb9444b8a..4eca78e247 100644 --- a/shared/test/unit_test/utilities/const_stringref_tests.cpp +++ b/shared/test/unit_test/utilities/const_stringref_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -238,3 +238,16 @@ TEST(ConstStringStartsWith, GivenInvalidPrefixThenReturnsFalse) { EXPECT_FALSE(str.startsWith("some text ")); EXPECT_FALSE(str.startsWith("substr some text")); } + +TEST(ConstStringStartsWithConstString, GivenRightPrefixThenReturnsTrue) { + ConstStringRef str = "some text"; + EXPECT_TRUE(str.startsWith(ConstStringRef("some"))); + EXPECT_TRUE(str.startsWith(ConstStringRef("some text"))); +} + +TEST(ConstStringStartsWithConstString, GivenInvalidPrefixThenReturnsFalse) { + ConstStringRef str = "some text"; + EXPECT_FALSE(str.startsWith(ConstStringRef("some text and more"))); + EXPECT_FALSE(str.startsWith(ConstStringRef("some diff"))); + EXPECT_FALSE(str.startsWith(ConstStringRef("ome text"))); +}