diff --git a/opencl/test/unit_test/offline_compiler/CMakeLists.txt b/opencl/test/unit_test/offline_compiler/CMakeLists.txt index 183d6a2845..4875b3ff23 100644 --- a/opencl/test/unit_test/offline_compiler/CMakeLists.txt +++ b/opencl/test/unit_test/offline_compiler/CMakeLists.txt @@ -11,6 +11,7 @@ set(IGDRCL_SRCS_cloc ${OCLOC_DIRECTORY}/source/decoder/binary_encoder.cpp ${OCLOC_DIRECTORY}/source/offline_compiler.cpp ${OCLOC_DIRECTORY}/source/offline_linker.cpp + ${OCLOC_DIRECTORY}/source/ocloc_concat.cpp ${OCLOC_DIRECTORY}/source/ocloc_fatbinary.cpp ) @@ -20,6 +21,7 @@ set(IGDRCL_SRCS_offline_compiler_mock ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_iga_wrapper.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_argument_helper.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_multi_command.h + ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_concat.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_fcl_facade.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_igc_facade.h ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_offline_compiler.h @@ -62,6 +64,7 @@ set(IGDRCL_SRCS_offline_compiler_tests ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_iga_dll.h ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_api_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_arg_helper_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_concat_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fatbinary_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fatbinary_tests.h ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fcl_facade_tests.cpp diff --git a/opencl/test/unit_test/offline_compiler/mock/mock_ocloc_concat.h b/opencl/test/unit_test/offline_compiler/mock/mock_ocloc_concat.h new file mode 100644 index 0000000000..19d6e952a2 --- /dev/null +++ b/opencl/test/unit_test/offline_compiler/mock/mock_ocloc_concat.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/offline_compiler/source/ocloc_concat.h" +#include "shared/source/device_binary_format/ar/ar_decoder.h" + +namespace NEO { +class MockOclocConcat : public OclocConcat { + public: + MockOclocConcat(OclocArgHelper *argHelper) : OclocConcat(argHelper){}; + + using OclocConcat::checkIfFatBinariesExist; + using OclocConcat::fatBinaryName; + using OclocConcat::fileNamesToConcat; + using OclocConcat::parseArguments; + + Ar::Ar decodeAr(const std::vector &arFile, std::string &outErrors, std::string &outWarnings) override { + outErrors.append(decodeArErrorMessage.str()); + return {}; + } + + bool shouldFailDecodingAr = false; + static constexpr ConstStringRef decodeArErrorMessage = "Error while decoding AR file\n"; +}; + +} // namespace NEO \ No newline at end of file diff --git a/opencl/test/unit_test/offline_compiler/ocloc_api_tests.cpp b/opencl/test/unit_test/offline_compiler/ocloc_api_tests.cpp index 62fe6f79e7..1e8e99bd13 100644 --- a/opencl/test/unit_test/offline_compiler/ocloc_api_tests.cpp +++ b/opencl/test/unit_test/offline_compiler/ocloc_api_tests.cpp @@ -7,9 +7,12 @@ #include "shared/offline_compiler/source/decoder/helper.h" #include "shared/offline_compiler/source/ocloc_api.h" +#include "shared/offline_compiler/source/ocloc_concat.h" #include "shared/offline_compiler/source/ocloc_error_code.h" #include "shared/offline_compiler/source/queries.h" #include "shared/offline_compiler/source/utilities/get_git_version_info.h" +#include "shared/source/device_binary_format/ar/ar_decoder.h" +#include "shared/source/device_binary_format/ar/ar_encoder.h" #include "shared/source/device_binary_format/elf/elf_decoder.h" #include "shared/source/device_binary_format/elf/ocl_elf.h" #include "shared/source/helpers/file_io.h" @@ -708,3 +711,101 @@ TEST(OclocApiTests, GivenInvalidParameterWhenLinkingThenErrorIsReturned) { const std::string expectedErrorMessage = expectedInitError + expectedExecuteError; EXPECT_EQ(expectedErrorMessage, output); } + +TEST(OclocApiTests, GivenInvalidCommandLineWhenConcatenatingThenErrorIsReturned) { + const char *argv[] = { + "ocloc", + "concat"}; + unsigned int argc = sizeof(argv) / sizeof(argv[0]); + testing::internal::CaptureStdout(); + int retVal = oclocInvoke(argc, argv, + 0, nullptr, nullptr, nullptr, + 0, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr); + std::string output = testing::internal::GetCapturedStdout(); + EXPECT_EQ(NEO::OclocErrorCode::INVALID_COMMAND_LINE, retVal); + const std::string emptyCommandLineError = "No files to concatenate were provided.\n"; + const std::string expectedErrorMessage = emptyCommandLineError + NEO::OclocConcat::helpMessage.str(); + EXPECT_EQ(expectedErrorMessage, output); +} + +TEST(OclocApiTests, GivenValidCommandLineAndFatBinariesWhenConcatenatingThenNewFatBinaryIsCreated) { + std::vector file1(32, 0x0); + std::vector file2(32, 0x10); + const std::string file1Name = "file1"; + const std::string file2Name = "file2"; + + std::vector fatBinary1; + { + NEO::Ar::ArEncoder arEncoder(true); + arEncoder.appendFileEntry(file1Name, ArrayRef::fromAny(file1.data(), file1.size())); + fatBinary1 = arEncoder.encode(); + } + std::vector fatBinary2; + { + NEO::Ar::ArEncoder arEncoder(true); + arEncoder.appendFileEntry(file2Name, ArrayRef::fromAny(file2.data(), file2.size())); + fatBinary2 = arEncoder.encode(); + } + + const uint8_t *sourcesData[2] = {fatBinary1.data(), + fatBinary2.data()}; + const uint64_t sourcesLen[2] = {fatBinary1.size(), + fatBinary2.size()}; + const char *sourcesName[2] = {"fatBinary1.ar", + "fatBinary2.ar"}; + + uint32_t numOutputs; + uint8_t **outputData; + uint64_t *outputLen; + char **outputName; + const char *argv[] = { + "ocloc", + "concat", + "fatBinary1.ar", + "fatBinary2.ar", + "-out", + "catFatBinary.ar"}; + unsigned int argc = sizeof(argv) / sizeof(argv[0]); + testing::internal::CaptureStdout(); + int retVal = oclocInvoke(argc, argv, + 2, sourcesData, sourcesLen, sourcesName, + 0, nullptr, nullptr, nullptr, + &numOutputs, &outputData, &outputLen, &outputName); + std::string output = testing::internal::GetCapturedStdout(); + EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal); + EXPECT_TRUE(output.empty()); + + uint32_t fatBinaryIdx = std::numeric_limits::max(); + for (uint32_t i = 0; i < numOutputs; i++) { + if (strcmp(argv[argc - 1], outputName[i]) == 0) { + fatBinaryIdx = i; + break; + } + } + ASSERT_NE(std::numeric_limits::max(), fatBinaryIdx); + + std::string errors, warnings; + auto ar = NEO::Ar::decodeAr(ArrayRef::fromAny(outputData[fatBinaryIdx], static_cast(outputLen[fatBinaryIdx])), + errors, warnings); + EXPECT_TRUE(errors.empty()); + EXPECT_TRUE(warnings.empty()); + + bool hasFatBinary1 = false; + bool hasFatBinary2 = false; + for (auto &file : ar.files) { + if (file.fileName == file1Name) { + hasFatBinary1 = true; + ASSERT_EQ(file1.size(), file.fileData.size()); + EXPECT_EQ(0, memcmp(file1.data(), file.fileData.begin(), file1.size())); + } else if (file.fileName == file2Name) { + hasFatBinary2 = true; + ASSERT_EQ(file2.size(), file.fileData.size()); + EXPECT_EQ(0, memcmp(file2.data(), file.fileData.begin(), file2.size())); + } + } + + EXPECT_TRUE(hasFatBinary1); + EXPECT_TRUE(hasFatBinary2); + oclocFreeOutput(&numOutputs, &outputData, &outputLen, &outputName); +} diff --git a/opencl/test/unit_test/offline_compiler/ocloc_concat_tests.cpp b/opencl/test/unit_test/offline_compiler/ocloc_concat_tests.cpp new file mode 100644 index 0000000000..af6a4373a4 --- /dev/null +++ b/opencl/test/unit_test/offline_compiler/ocloc_concat_tests.cpp @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/offline_compiler/source/ocloc_error_code.h" + +#include "gtest/gtest.h" +#include "mock/mock_argument_helper.h" +#include "mock/mock_ocloc_concat.h" + +namespace NEO { +TEST(OclocConcatTest, GivenNoArgumentsWhenInitializingThenErrorIsReturned) { + MockOclocArgHelper::FilesMap mockArgHelperFilesMap{}; + MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap}; + auto oclocConcat = MockOclocConcat(&mockArgHelper); + std::vector args = {"ocloc", "concat"}; + + ::testing::internal::CaptureStdout(); + auto error = oclocConcat.initialize(args); + const auto output = ::testing::internal::GetCapturedStdout(); + + EXPECT_EQ(static_cast(OclocErrorCode::INVALID_COMMAND_LINE), error); + const std::string expectedOutput = "No files to concatenate were provided.\n"; + EXPECT_EQ(expectedOutput, output); +} + +TEST(OclocConcatTest, GivenMissingFilesWhenInitializingThenErrorIsReturned) { + MockOclocArgHelper::FilesMap mockArgHelperFilesMap{}; + MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap}; + auto oclocConcat = MockOclocConcat(&mockArgHelper); + std::vector args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar"}; + + ::testing::internal::CaptureStdout(); + auto error = oclocConcat.initialize(args); + const auto output = ::testing::internal::GetCapturedStdout(); + + EXPECT_EQ(static_cast(OclocErrorCode::INVALID_COMMAND_LINE), error); + const std::string expectedOutput = "fatBinary1.ar doesn't exist!\nfatBinary2.ar doesn't exist!\n"; + EXPECT_EQ(expectedOutput, output); +} + +TEST(OclocConcatTest, GivenValidArgsWhenInitializingThenFileNamesToConcatAndOutputFileNameAreSetCorrectlyAndSuccessIsReturned) { + MockOclocArgHelper::FilesMap mockArgHelperFilesMap{ + {"fatBinary1.ar", "fatBinary1Data"}, + {"fatBinary2.ar", "fatBinary2Data"}}; + MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap}; + auto oclocConcat = MockOclocConcat(&mockArgHelper); + std::vector args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar", "-out", "fatBinary.ar"}; + + ::testing::internal::CaptureStdout(); + auto error = oclocConcat.initialize(args); + const auto output = ::testing::internal::GetCapturedStdout(); + + EXPECT_EQ(static_cast(OclocErrorCode::SUCCESS), error); + EXPECT_TRUE(output.empty()); + + EXPECT_EQ(args[2], oclocConcat.fileNamesToConcat[0]); + EXPECT_EQ(args[3], oclocConcat.fileNamesToConcat[1]); + EXPECT_EQ(args[5], oclocConcat.fatBinaryName); +} + +TEST(OclocConcatTest, GivenMissingOutFileNameAfterOutArgumentWhenInitalizingThenErrorIsReturned) { + MockOclocArgHelper::FilesMap mockArgHelperFilesMap{}; + MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap}; + auto oclocConcat = MockOclocConcat(&mockArgHelper); + std::vector args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar", "-out"}; + + ::testing::internal::CaptureStdout(); + auto error = oclocConcat.initialize(args); + const auto output = ::testing::internal::GetCapturedStdout(); + + EXPECT_EQ(static_cast(OclocErrorCode::INVALID_COMMAND_LINE), error); + const std::string expectedOutput = "Missing out file name after \"-out\" argument\n"; + EXPECT_EQ(expectedOutput, output); +} + +TEST(OclocConcatTest, GivenErrorDuringDecodingArWhenConcatenatingThenErrorIsReturned) { + MockOclocArgHelper::FilesMap mockArgHelperFilesMap{ + {"fatBinary1.ar", "fatBinary1Data"}, + {"fatBinary2.ar", "fatBinary2Data"}}; + MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap}; + auto oclocConcat = MockOclocConcat(&mockArgHelper); + oclocConcat.shouldFailDecodingAr = true; + oclocConcat.fileNamesToConcat = {"fatBinary1.ar", + "fatBinary2.ar"}; + + ::testing::internal::CaptureStdout(); + auto error = oclocConcat.concatenate(); + const auto output = ::testing::internal::GetCapturedStdout(); + + EXPECT_EQ(static_cast(OclocErrorCode::INVALID_FILE), error); + EXPECT_EQ(MockOclocConcat::decodeArErrorMessage.str(), output); +} + +} // namespace NEO \ No newline at end of file diff --git a/shared/offline_compiler/source/CMakeLists.txt b/shared/offline_compiler/source/CMakeLists.txt index aa91734b5d..36e1907cf8 100644 --- a/shared/offline_compiler/source/CMakeLists.txt +++ b/shared/offline_compiler/source/CMakeLists.txt @@ -60,6 +60,8 @@ set(CLOC_LIB_SRCS_LIB ${OCLOC_DIRECTORY}/source/ocloc_api.h ${OCLOC_DIRECTORY}/source/ocloc_arg_helper.cpp ${OCLOC_DIRECTORY}/source/ocloc_arg_helper.h + ${OCLOC_DIRECTORY}/source/ocloc_concat.cpp + ${OCLOC_DIRECTORY}/source/ocloc_concat.h ${OCLOC_DIRECTORY}/source/ocloc_dll_options.cpp ${OCLOC_DIRECTORY}/source/ocloc_dll_options.h ${OCLOC_DIRECTORY}/source/ocloc_error_code.h diff --git a/shared/offline_compiler/source/ocloc_api.cpp b/shared/offline_compiler/source/ocloc_api.cpp index 8afe8144b6..6258964d61 100644 --- a/shared/offline_compiler/source/ocloc_api.cpp +++ b/shared/offline_compiler/source/ocloc_api.cpp @@ -10,6 +10,7 @@ #include "shared/offline_compiler/source/decoder/binary_decoder.h" #include "shared/offline_compiler/source/decoder/binary_encoder.h" #include "shared/offline_compiler/source/multi_command.h" +#include "shared/offline_compiler/source/ocloc_concat.h" #include "shared/offline_compiler/source/ocloc_error_code.h" #include "shared/offline_compiler/source/ocloc_fatbinary.h" #include "shared/offline_compiler/source/ocloc_validator.h" @@ -65,6 +66,9 @@ Examples: Extract driver version ocloc query OCL_DRIVER_VERSION + + Concatenate fat binaries + ocloc concat ... [-out ] )==="; extern "C" { @@ -168,6 +172,16 @@ int oclocInvoke(unsigned int numArgs, const char *argv[], } return createResult | linkingResult; + } else if (numArgs > 1 && NEO::OclocConcat::commandStr == allArgs[1]) { + auto arConcat = NEO::OclocConcat(helper.get()); + auto error = arConcat.initialize(allArgs); + if (OclocErrorCode::SUCCESS != error) { + arConcat.printHelp(); + return error; + } + + error = arConcat.concatenate(); + return error; } else { int retVal = OclocErrorCode::SUCCESS; diff --git a/shared/offline_compiler/source/ocloc_concat.cpp b/shared/offline_compiler/source/ocloc_concat.cpp new file mode 100644 index 0000000000..9e956967e8 --- /dev/null +++ b/shared/offline_compiler/source/ocloc_concat.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/offline_compiler/source/ocloc_concat.h" + +#include "shared/offline_compiler/source/ocloc_arg_helper.h" +#include "shared/offline_compiler/source/ocloc_error_code.h" +#include "shared/source/device_binary_format/ar/ar_decoder.h" +#include "shared/source/device_binary_format/ar/ar_encoder.h" + +namespace NEO { +ErrorCode OclocConcat::initialize(const std::vector &args) { + auto error = parseArguments(args); + if (error) { + return error; + } + error = checkIfFatBinariesExist(); + return error; +} + +Ar::Ar OclocConcat::decodeAr(const std::vector &arFile, std::string &outErrors, std::string &outWarnings) { + return NEO::Ar::decodeAr({reinterpret_cast(arFile.data()), arFile.size()}, outErrors, outWarnings); +} + +ErrorCode OclocConcat::parseArguments(const std::vector &args) { + for (size_t i = 2; i < args.size(); i++) { + if (NEO::ConstStringRef("-out") == args[i]) { + if (i + 1 >= args.size()) { + argHelper->printf("Missing out file name after \"-out\" argument\n"); + return OclocErrorCode::INVALID_COMMAND_LINE; + } + fatBinaryName = args[++i]; + } else { + fileNamesToConcat.push_back(args[i]); + } + } + + if (fileNamesToConcat.empty()) { + argHelper->printf("No files to concatenate were provided.\n"); + return OclocErrorCode::INVALID_COMMAND_LINE; + } + + return OclocErrorCode::SUCCESS; +} + +ErrorCode OclocConcat::checkIfFatBinariesExist() { + bool filesExist = true; + for (auto &fileName : fileNamesToConcat) { + if (false == argHelper->fileExists(fileName)) { + filesExist = false; + auto errorMsg = fileName + " doesn't exist!\n"; + argHelper->printf(errorMsg.c_str()); + } + } + return filesExist ? OclocErrorCode::SUCCESS : OclocErrorCode::INVALID_COMMAND_LINE; +} + +ErrorCode OclocConcat::concatenate() { + NEO::Ar::ArEncoder arEncoder(true); + for (auto &fileName : fileNamesToConcat) { + auto arFile = argHelper->readBinaryFile(fileName); + + std::string warnings; + std::string errors; + auto ar = decodeAr(arFile, errors, warnings); + if (false == errors.empty()) { + argHelper->printf(errors.c_str()); + return OclocErrorCode::INVALID_FILE; + } + argHelper->printf(warnings.c_str()); + + for (auto &fileEntry : ar.files) { + if (NEO::ConstStringRef(fileEntry.fileName).startsWith("pad_")) { + continue; + } + + arEncoder.appendFileEntry(fileEntry.fileName, fileEntry.fileData); + } + } + + auto arFile = arEncoder.encode(); + argHelper->saveOutput(fatBinaryName, arFile.data(), arFile.size()); + return OclocErrorCode::SUCCESS; +} + +void OclocConcat::printHelp() { + argHelper->printf(helpMessage.data()); +} + +} // namespace NEO diff --git a/shared/offline_compiler/source/ocloc_concat.h b/shared/offline_compiler/source/ocloc_concat.h new file mode 100644 index 0000000000..06b41d41fa --- /dev/null +++ b/shared/offline_compiler/source/ocloc_concat.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/utilities/const_stringref.h" + +#include +#include + +class OclocArgHelper; +namespace NEO { +namespace Ar { +struct Ar; +} + +using ErrorCode = uint32_t; +class OclocConcat { + public: + OclocConcat() = delete; + OclocConcat(const OclocConcat &) = delete; + OclocConcat &operator=(const OclocConcat &) = delete; + + OclocConcat(OclocArgHelper *argHelper) : argHelper(argHelper){}; + ErrorCode initialize(const std::vector &args); + ErrorCode concatenate(); + void printHelp(); + + static constexpr ConstStringRef commandStr = "concat"; + static constexpr ConstStringRef helpMessage = R"===( +ocloc concat - concatenates fat binary files +Usage: ocloc concat ... [-out ] +)==="; + + protected: + MOCKABLE_VIRTUAL Ar::Ar decodeAr(const std::vector &arFile, std::string &outErrors, std::string &outWarnings); + ErrorCode parseArguments(const std::vector &args); + ErrorCode checkIfFatBinariesExist(); + + OclocArgHelper *argHelper; + std::vector fileNamesToConcat; + std::string fatBinaryName = "concat.ar"; +}; +} // namespace NEO \ No newline at end of file