feat(ocloc): concatenate multiple fat binaries

Signed-off-by: Krystian Chmielewski <krystian.chmielewski@intel.com>

opencl/test/unit_test/offline_compiler/CMakeLists.txt

@@ -11,6 +11,7 @@ set(IGDRCL_SRCS_cloc
     ${OCLOC_DIRECTORY}/source/decoder/binary_encoder.cpp
     ${OCLOC_DIRECTORY}/source/offline_compiler.cpp
     ${OCLOC_DIRECTORY}/source/offline_linker.cpp
+    ${OCLOC_DIRECTORY}/source/ocloc_concat.cpp
     ${OCLOC_DIRECTORY}/source/ocloc_fatbinary.cpp
 )
 
@@ -20,6 +21,7 @@ set(IGDRCL_SRCS_offline_compiler_mock
     ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_iga_wrapper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_argument_helper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_multi_command.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_concat.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_fcl_facade.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_igc_facade.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_offline_compiler.h
@@ -62,6 +64,7 @@ set(IGDRCL_SRCS_offline_compiler_tests
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_iga_dll.h
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_api_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_arg_helper_tests.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_concat_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fatbinary_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fatbinary_tests.h
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fcl_facade_tests.cpp

opencl/test/unit_test/offline_compiler/mock/mock_ocloc_concat.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "shared/offline_compiler/source/ocloc_concat.h"
+#include "shared/source/device_binary_format/ar/ar_decoder.h"
+
+namespace NEO {
+class MockOclocConcat : public OclocConcat {
+  public:
+    MockOclocConcat(OclocArgHelper *argHelper) : OclocConcat(argHelper){};
+
+    using OclocConcat::checkIfFatBinariesExist;
+    using OclocConcat::fatBinaryName;
+    using OclocConcat::fileNamesToConcat;
+    using OclocConcat::parseArguments;
+
+    Ar::Ar decodeAr(const std::vector<char> &arFile, std::string &outErrors, std::string &outWarnings) override {
+        outErrors.append(decodeArErrorMessage.str());
+        return {};
+    }
+
+    bool shouldFailDecodingAr = false;
+    static constexpr ConstStringRef decodeArErrorMessage = "Error while decoding AR file\n";
+};
+
+} // namespace NEO

opencl/test/unit_test/offline_compiler/ocloc_api_tests.cpp

@@ -7,9 +7,12 @@
 
 #include "shared/offline_compiler/source/decoder/helper.h"
 #include "shared/offline_compiler/source/ocloc_api.h"
+#include "shared/offline_compiler/source/ocloc_concat.h"
 #include "shared/offline_compiler/source/ocloc_error_code.h"
 #include "shared/offline_compiler/source/queries.h"
 #include "shared/offline_compiler/source/utilities/get_git_version_info.h"
+#include "shared/source/device_binary_format/ar/ar_decoder.h"
+#include "shared/source/device_binary_format/ar/ar_encoder.h"
 #include "shared/source/device_binary_format/elf/elf_decoder.h"
 #include "shared/source/device_binary_format/elf/ocl_elf.h"
 #include "shared/source/helpers/file_io.h"
@@ -708,3 +711,101 @@ TEST(OclocApiTests, GivenInvalidParameterWhenLinkingThenErrorIsReturned) {
     const std::string expectedErrorMessage = expectedInitError + expectedExecuteError;
     EXPECT_EQ(expectedErrorMessage, output);
 }
+
+TEST(OclocApiTests, GivenInvalidCommandLineWhenConcatenatingThenErrorIsReturned) {
+    const char *argv[] = {
+        "ocloc",
+        "concat"};
+    unsigned int argc = sizeof(argv) / sizeof(argv[0]);
+    testing::internal::CaptureStdout();
+    int retVal = oclocInvoke(argc, argv,
+                             0, nullptr, nullptr, nullptr,
+                             0, nullptr, nullptr, nullptr,
+                             nullptr, nullptr, nullptr, nullptr);
+    std::string output = testing::internal::GetCapturedStdout();
+    EXPECT_EQ(NEO::OclocErrorCode::INVALID_COMMAND_LINE, retVal);
+    const std::string emptyCommandLineError = "No files to concatenate were provided.\n";
+    const std::string expectedErrorMessage = emptyCommandLineError + NEO::OclocConcat::helpMessage.str();
+    EXPECT_EQ(expectedErrorMessage, output);
+}
+
+TEST(OclocApiTests, GivenValidCommandLineAndFatBinariesWhenConcatenatingThenNewFatBinaryIsCreated) {
+    std::vector<uint8_t> file1(32, 0x0);
+    std::vector<uint8_t> file2(32, 0x10);
+    const std::string file1Name = "file1";
+    const std::string file2Name = "file2";
+
+    std::vector<uint8_t> fatBinary1;
+    {
+        NEO::Ar::ArEncoder arEncoder(true);
+        arEncoder.appendFileEntry(file1Name, ArrayRef<const uint8_t>::fromAny(file1.data(), file1.size()));
+        fatBinary1 = arEncoder.encode();
+    }
+    std::vector<uint8_t> fatBinary2;
+    {
+        NEO::Ar::ArEncoder arEncoder(true);
+        arEncoder.appendFileEntry(file2Name, ArrayRef<const uint8_t>::fromAny(file2.data(), file2.size()));
+        fatBinary2 = arEncoder.encode();
+    }
+
+    const uint8_t *sourcesData[2] = {fatBinary1.data(),
+                                     fatBinary2.data()};
+    const uint64_t sourcesLen[2] = {fatBinary1.size(),
+                                    fatBinary2.size()};
+    const char *sourcesName[2] = {"fatBinary1.ar",
+                                  "fatBinary2.ar"};
+
+    uint32_t numOutputs;
+    uint8_t **outputData;
+    uint64_t *outputLen;
+    char **outputName;
+    const char *argv[] = {
+        "ocloc",
+        "concat",
+        "fatBinary1.ar",
+        "fatBinary2.ar",
+        "-out",
+        "catFatBinary.ar"};
+    unsigned int argc = sizeof(argv) / sizeof(argv[0]);
+    testing::internal::CaptureStdout();
+    int retVal = oclocInvoke(argc, argv,
+                             2, sourcesData, sourcesLen, sourcesName,
+                             0, nullptr, nullptr, nullptr,
+                             &numOutputs, &outputData, &outputLen, &outputName);
+    std::string output = testing::internal::GetCapturedStdout();
+    EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal);
+    EXPECT_TRUE(output.empty());
+
+    uint32_t fatBinaryIdx = std::numeric_limits<uint32_t>::max();
+    for (uint32_t i = 0; i < numOutputs; i++) {
+        if (strcmp(argv[argc - 1], outputName[i]) == 0) {
+            fatBinaryIdx = i;
+            break;
+        }
+    }
+    ASSERT_NE(std::numeric_limits<uint32_t>::max(), fatBinaryIdx);
+
+    std::string errors, warnings;
+    auto ar = NEO::Ar::decodeAr(ArrayRef<const uint8_t>::fromAny(outputData[fatBinaryIdx], static_cast<size_t>(outputLen[fatBinaryIdx])),
+                                errors, warnings);
+    EXPECT_TRUE(errors.empty());
+    EXPECT_TRUE(warnings.empty());
+
+    bool hasFatBinary1 = false;
+    bool hasFatBinary2 = false;
+    for (auto &file : ar.files) {
+        if (file.fileName == file1Name) {
+            hasFatBinary1 = true;
+            ASSERT_EQ(file1.size(), file.fileData.size());
+            EXPECT_EQ(0, memcmp(file1.data(), file.fileData.begin(), file1.size()));
+        } else if (file.fileName == file2Name) {
+            hasFatBinary2 = true;
+            ASSERT_EQ(file2.size(), file.fileData.size());
+            EXPECT_EQ(0, memcmp(file2.data(), file.fileData.begin(), file2.size()));
+        }
+    }
+
+    EXPECT_TRUE(hasFatBinary1);
+    EXPECT_TRUE(hasFatBinary2);
+    oclocFreeOutput(&numOutputs, &outputData, &outputLen, &outputName);
+}

opencl/test/unit_test/offline_compiler/ocloc_concat_tests.cpp

@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/offline_compiler/source/ocloc_error_code.h"
+
+#include "gtest/gtest.h"
+#include "mock/mock_argument_helper.h"
+#include "mock/mock_ocloc_concat.h"
+
+namespace NEO {
+TEST(OclocConcatTest, GivenNoArgumentsWhenInitializingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_COMMAND_LINE), error);
+    const std::string expectedOutput = "No files to concatenate were provided.\n";
+    EXPECT_EQ(expectedOutput, output);
+}
+
+TEST(OclocConcatTest, GivenMissingFilesWhenInitializingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_COMMAND_LINE), error);
+    const std::string expectedOutput = "fatBinary1.ar doesn't exist!\nfatBinary2.ar doesn't exist!\n";
+    EXPECT_EQ(expectedOutput, output);
+}
+
+TEST(OclocConcatTest, GivenValidArgsWhenInitializingThenFileNamesToConcatAndOutputFileNameAreSetCorrectlyAndSuccessIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{
+        {"fatBinary1.ar", "fatBinary1Data"},
+        {"fatBinary2.ar", "fatBinary2Data"}};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar", "-out", "fatBinary.ar"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::SUCCESS), error);
+    EXPECT_TRUE(output.empty());
+
+    EXPECT_EQ(args[2], oclocConcat.fileNamesToConcat[0]);
+    EXPECT_EQ(args[3], oclocConcat.fileNamesToConcat[1]);
+    EXPECT_EQ(args[5], oclocConcat.fatBinaryName);
+}
+
+TEST(OclocConcatTest, GivenMissingOutFileNameAfterOutArgumentWhenInitalizingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar", "-out"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_COMMAND_LINE), error);
+    const std::string expectedOutput = "Missing out file name after \"-out\" argument\n";
+    EXPECT_EQ(expectedOutput, output);
+}
+
+TEST(OclocConcatTest, GivenErrorDuringDecodingArWhenConcatenatingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{
+        {"fatBinary1.ar", "fatBinary1Data"},
+        {"fatBinary2.ar", "fatBinary2Data"}};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    oclocConcat.shouldFailDecodingAr = true;
+    oclocConcat.fileNamesToConcat = {"fatBinary1.ar",
+                                     "fatBinary2.ar"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.concatenate();
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_FILE), error);
+    EXPECT_EQ(MockOclocConcat::decodeArErrorMessage.str(), output);
+}
+
+} // namespace NEO

shared/offline_compiler/source/CMakeLists.txt

@@ -60,6 +60,8 @@ set(CLOC_LIB_SRCS_LIB
     ${OCLOC_DIRECTORY}/source/ocloc_api.h
     ${OCLOC_DIRECTORY}/source/ocloc_arg_helper.cpp
     ${OCLOC_DIRECTORY}/source/ocloc_arg_helper.h
+    ${OCLOC_DIRECTORY}/source/ocloc_concat.cpp
+    ${OCLOC_DIRECTORY}/source/ocloc_concat.h
     ${OCLOC_DIRECTORY}/source/ocloc_dll_options.cpp
     ${OCLOC_DIRECTORY}/source/ocloc_dll_options.h
     ${OCLOC_DIRECTORY}/source/ocloc_error_code.h

shared/offline_compiler/source/ocloc_api.cpp

@@ -10,6 +10,7 @@
 #include "shared/offline_compiler/source/decoder/binary_decoder.h"
 #include "shared/offline_compiler/source/decoder/binary_encoder.h"
 #include "shared/offline_compiler/source/multi_command.h"
+#include "shared/offline_compiler/source/ocloc_concat.h"
 #include "shared/offline_compiler/source/ocloc_error_code.h"
 #include "shared/offline_compiler/source/ocloc_fatbinary.h"
 #include "shared/offline_compiler/source/ocloc_validator.h"
@@ -65,6 +66,9 @@ Examples:
 
   Extract driver version
     ocloc query OCL_DRIVER_VERSION
+
+  Concatenate fat binaries
+    ocloc concat <fat binary> <fat binary> ... [-out <concatenated fat binary name>]
 )===";
 
 extern "C" {
@@ -168,6 +172,16 @@ int oclocInvoke(unsigned int numArgs, const char *argv[],
             }
 
             return createResult | linkingResult;
+        } else if (numArgs > 1 && NEO::OclocConcat::commandStr == allArgs[1]) {
+            auto arConcat = NEO::OclocConcat(helper.get());
+            auto error = arConcat.initialize(allArgs);
+            if (OclocErrorCode::SUCCESS != error) {
+                arConcat.printHelp();
+                return error;
+            }
+
+            error = arConcat.concatenate();
+            return error;
         } else {
             int retVal = OclocErrorCode::SUCCESS;
 

shared/offline_compiler/source/ocloc_concat.cpp

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/offline_compiler/source/ocloc_concat.h"
+
+#include "shared/offline_compiler/source/ocloc_arg_helper.h"
+#include "shared/offline_compiler/source/ocloc_error_code.h"
+#include "shared/source/device_binary_format/ar/ar_decoder.h"
+#include "shared/source/device_binary_format/ar/ar_encoder.h"
+
+namespace NEO {
+ErrorCode OclocConcat::initialize(const std::vector<std::string> &args) {
+    auto error = parseArguments(args);
+    if (error) {
+        return error;
+    }
+    error = checkIfFatBinariesExist();
+    return error;
+}
+
+Ar::Ar OclocConcat::decodeAr(const std::vector<char> &arFile, std::string &outErrors, std::string &outWarnings) {
+    return NEO::Ar::decodeAr({reinterpret_cast<const uint8_t *>(arFile.data()), arFile.size()}, outErrors, outWarnings);
+}
+
+ErrorCode OclocConcat::parseArguments(const std::vector<std::string> &args) {
+    for (size_t i = 2; i < args.size(); i++) {
+        if (NEO::ConstStringRef("-out") == args[i]) {
+            if (i + 1 >= args.size()) {
+                argHelper->printf("Missing out file name after \"-out\" argument\n");
+                return OclocErrorCode::INVALID_COMMAND_LINE;
+            }
+            fatBinaryName = args[++i];
+        } else {
+            fileNamesToConcat.push_back(args[i]);
+        }
+    }
+
+    if (fileNamesToConcat.empty()) {
+        argHelper->printf("No files to concatenate were provided.\n");
+        return OclocErrorCode::INVALID_COMMAND_LINE;
+    }
+
+    return OclocErrorCode::SUCCESS;
+}
+
+ErrorCode OclocConcat::checkIfFatBinariesExist() {
+    bool filesExist = true;
+    for (auto &fileName : fileNamesToConcat) {
+        if (false == argHelper->fileExists(fileName)) {
+            filesExist = false;
+            auto errorMsg = fileName + " doesn't exist!\n";
+            argHelper->printf(errorMsg.c_str());
+        }
+    }
+    return filesExist ? OclocErrorCode::SUCCESS : OclocErrorCode::INVALID_COMMAND_LINE;
+}
+
+ErrorCode OclocConcat::concatenate() {
+    NEO::Ar::ArEncoder arEncoder(true);
+    for (auto &fileName : fileNamesToConcat) {
+        auto arFile = argHelper->readBinaryFile(fileName);
+
+        std::string warnings;
+        std::string errors;
+        auto ar = decodeAr(arFile, errors, warnings);
+        if (false == errors.empty()) {
+            argHelper->printf(errors.c_str());
+            return OclocErrorCode::INVALID_FILE;
+        }
+        argHelper->printf(warnings.c_str());
+
+        for (auto &fileEntry : ar.files) {
+            if (NEO::ConstStringRef(fileEntry.fileName).startsWith("pad_")) {
+                continue;
+            }
+
+            arEncoder.appendFileEntry(fileEntry.fileName, fileEntry.fileData);
+        }
+    }
+
+    auto arFile = arEncoder.encode();
+    argHelper->saveOutput(fatBinaryName, arFile.data(), arFile.size());
+    return OclocErrorCode::SUCCESS;
+}
+
+void OclocConcat::printHelp() {
+    argHelper->printf(helpMessage.data());
+}
+
+} // namespace NEO

shared/offline_compiler/source/ocloc_concat.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "shared/source/utilities/const_stringref.h"
+
+#include <string>
+#include <vector>
+
+class OclocArgHelper;
+namespace NEO {
+namespace Ar {
+struct Ar;
+}
+
+using ErrorCode = uint32_t;
+class OclocConcat {
+  public:
+    OclocConcat() = delete;
+    OclocConcat(const OclocConcat &) = delete;
+    OclocConcat &operator=(const OclocConcat &) = delete;
+
+    OclocConcat(OclocArgHelper *argHelper) : argHelper(argHelper){};
+    ErrorCode initialize(const std::vector<std::string> &args);
+    ErrorCode concatenate();
+    void printHelp();
+
+    static constexpr ConstStringRef commandStr = "concat";
+    static constexpr ConstStringRef helpMessage = R"===(
+ocloc concat - concatenates fat binary files
+Usage: ocloc concat <fat binary> <fat binary> ... [-out <concatenated fat binary file name>]
+)===";
+
+  protected:
+    MOCKABLE_VIRTUAL Ar::Ar decodeAr(const std::vector<char> &arFile, std::string &outErrors, std::string &outWarnings);
+    ErrorCode parseArguments(const std::vector<std::string> &args);
+    ErrorCode checkIfFatBinariesExist();
+
+    OclocArgHelper *argHelper;
+    std::vector<std::string> fileNamesToConcat;
+    std::string fatBinaryName = "concat.ar";
+};
+} // namespace NEO