feat(ocloc): concatenate multiple fat binaries

Signed-off-by: Krystian Chmielewski <krystian.chmielewski@intel.com>

opencl/test/unit_test/offline_compiler/CMakeLists.txt

@@ -11,6 +11,7 @@ set(IGDRCL_SRCS_cloc
     ${OCLOC_DIRECTORY}/source/decoder/binary_encoder.cpp
     ${OCLOC_DIRECTORY}/source/offline_compiler.cpp
     ${OCLOC_DIRECTORY}/source/offline_linker.cpp
+    ${OCLOC_DIRECTORY}/source/ocloc_concat.cpp
     ${OCLOC_DIRECTORY}/source/ocloc_fatbinary.cpp
 )
 
@@ -20,6 +21,7 @@ set(IGDRCL_SRCS_offline_compiler_mock
     ${CMAKE_CURRENT_SOURCE_DIR}/decoder/mock/mock_iga_wrapper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_argument_helper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_multi_command.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_concat.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_fcl_facade.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_ocloc_igc_facade.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_offline_compiler.h
@@ -62,6 +64,7 @@ set(IGDRCL_SRCS_offline_compiler_tests
     ${CMAKE_CURRENT_SOURCE_DIR}/mock/mock_iga_dll.h
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_api_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_arg_helper_tests.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_concat_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fatbinary_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fatbinary_tests.h
     ${CMAKE_CURRENT_SOURCE_DIR}/ocloc_fcl_facade_tests.cpp

opencl/test/unit_test/offline_compiler/mock/mock_ocloc_concat.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "shared/offline_compiler/source/ocloc_concat.h"
+#include "shared/source/device_binary_format/ar/ar_decoder.h"
+
+namespace NEO {
+class MockOclocConcat : public OclocConcat {
+  public:
+    MockOclocConcat(OclocArgHelper *argHelper) : OclocConcat(argHelper){};
+
+    using OclocConcat::checkIfFatBinariesExist;
+    using OclocConcat::fatBinaryName;
+    using OclocConcat::fileNamesToConcat;
+    using OclocConcat::parseArguments;
+
+    Ar::Ar decodeAr(const std::vector<char> &arFile, std::string &outErrors, std::string &outWarnings) override {
+        outErrors.append(decodeArErrorMessage.str());
+        return {};
+    }
+
+    bool shouldFailDecodingAr = false;
+    static constexpr ConstStringRef decodeArErrorMessage = "Error while decoding AR file\n";
+};
+
+} // namespace NEO

opencl/test/unit_test/offline_compiler/ocloc_api_tests.cpp

@@ -7,9 +7,12 @@
 
 #include "shared/offline_compiler/source/decoder/helper.h"
 #include "shared/offline_compiler/source/ocloc_api.h"
+#include "shared/offline_compiler/source/ocloc_concat.h"
 #include "shared/offline_compiler/source/ocloc_error_code.h"
 #include "shared/offline_compiler/source/queries.h"
 #include "shared/offline_compiler/source/utilities/get_git_version_info.h"
+#include "shared/source/device_binary_format/ar/ar_decoder.h"
+#include "shared/source/device_binary_format/ar/ar_encoder.h"
 #include "shared/source/device_binary_format/elf/elf_decoder.h"
 #include "shared/source/device_binary_format/elf/ocl_elf.h"
 #include "shared/source/helpers/file_io.h"
@@ -708,3 +711,101 @@ TEST(OclocApiTests, GivenInvalidParameterWhenLinkingThenErrorIsReturned) {
     const std::string expectedErrorMessage = expectedInitError + expectedExecuteError;
     EXPECT_EQ(expectedErrorMessage, output);
 }
+
+TEST(OclocApiTests, GivenInvalidCommandLineWhenConcatenatingThenErrorIsReturned) {
+    const char *argv[] = {
+        "ocloc",
+        "concat"};
+    unsigned int argc = sizeof(argv) / sizeof(argv[0]);
+    testing::internal::CaptureStdout();
+    int retVal = oclocInvoke(argc, argv,
+                             0, nullptr, nullptr, nullptr,
+                             0, nullptr, nullptr, nullptr,
+                             nullptr, nullptr, nullptr, nullptr);
+    std::string output = testing::internal::GetCapturedStdout();
+    EXPECT_EQ(NEO::OclocErrorCode::INVALID_COMMAND_LINE, retVal);
+    const std::string emptyCommandLineError = "No files to concatenate were provided.\n";
+    const std::string expectedErrorMessage = emptyCommandLineError + NEO::OclocConcat::helpMessage.str();
+    EXPECT_EQ(expectedErrorMessage, output);
+}
+
+TEST(OclocApiTests, GivenValidCommandLineAndFatBinariesWhenConcatenatingThenNewFatBinaryIsCreated) {
+    std::vector<uint8_t> file1(32, 0x0);
+    std::vector<uint8_t> file2(32, 0x10);
+    const std::string file1Name = "file1";
+    const std::string file2Name = "file2";
+
+    std::vector<uint8_t> fatBinary1;
+    {
+        NEO::Ar::ArEncoder arEncoder(true);
+        arEncoder.appendFileEntry(file1Name, ArrayRef<const uint8_t>::fromAny(file1.data(), file1.size()));
+        fatBinary1 = arEncoder.encode();
+    }
+    std::vector<uint8_t> fatBinary2;
+    {
+        NEO::Ar::ArEncoder arEncoder(true);
+        arEncoder.appendFileEntry(file2Name, ArrayRef<const uint8_t>::fromAny(file2.data(), file2.size()));
+        fatBinary2 = arEncoder.encode();
+    }
+
+    const uint8_t *sourcesData[2] = {fatBinary1.data(),
+                                     fatBinary2.data()};
+    const uint64_t sourcesLen[2] = {fatBinary1.size(),
+                                    fatBinary2.size()};
+    const char *sourcesName[2] = {"fatBinary1.ar",
+                                  "fatBinary2.ar"};
+
+    uint32_t numOutputs;
+    uint8_t **outputData;
+    uint64_t *outputLen;
+    char **outputName;
+    const char *argv[] = {
+        "ocloc",
+        "concat",
+        "fatBinary1.ar",
+        "fatBinary2.ar",
+        "-out",
+        "catFatBinary.ar"};
+    unsigned int argc = sizeof(argv) / sizeof(argv[0]);
+    testing::internal::CaptureStdout();
+    int retVal = oclocInvoke(argc, argv,
+                             2, sourcesData, sourcesLen, sourcesName,
+                             0, nullptr, nullptr, nullptr,
+                             &numOutputs, &outputData, &outputLen, &outputName);
+    std::string output = testing::internal::GetCapturedStdout();
+    EXPECT_EQ(NEO::OclocErrorCode::SUCCESS, retVal);
+    EXPECT_TRUE(output.empty());
+
+    uint32_t fatBinaryIdx = std::numeric_limits<uint32_t>::max();
+    for (uint32_t i = 0; i < numOutputs; i++) {
+        if (strcmp(argv[argc - 1], outputName[i]) == 0) {
+            fatBinaryIdx = i;
+            break;
+        }
+    }
+    ASSERT_NE(std::numeric_limits<uint32_t>::max(), fatBinaryIdx);
+
+    std::string errors, warnings;
+    auto ar = NEO::Ar::decodeAr(ArrayRef<const uint8_t>::fromAny(outputData[fatBinaryIdx], static_cast<size_t>(outputLen[fatBinaryIdx])),
+                                errors, warnings);
+    EXPECT_TRUE(errors.empty());
+    EXPECT_TRUE(warnings.empty());
+
+    bool hasFatBinary1 = false;
+    bool hasFatBinary2 = false;
+    for (auto &file : ar.files) {
+        if (file.fileName == file1Name) {
+            hasFatBinary1 = true;
+            ASSERT_EQ(file1.size(), file.fileData.size());
+            EXPECT_EQ(0, memcmp(file1.data(), file.fileData.begin(), file1.size()));
+        } else if (file.fileName == file2Name) {
+            hasFatBinary2 = true;
+            ASSERT_EQ(file2.size(), file.fileData.size());
+            EXPECT_EQ(0, memcmp(file2.data(), file.fileData.begin(), file2.size()));
+        }
+    }
+
+    EXPECT_TRUE(hasFatBinary1);
+    EXPECT_TRUE(hasFatBinary2);
+    oclocFreeOutput(&numOutputs, &outputData, &outputLen, &outputName);
+}

opencl/test/unit_test/offline_compiler/ocloc_concat_tests.cpp

@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/offline_compiler/source/ocloc_error_code.h"
+
+#include "gtest/gtest.h"
+#include "mock/mock_argument_helper.h"
+#include "mock/mock_ocloc_concat.h"
+
+namespace NEO {
+TEST(OclocConcatTest, GivenNoArgumentsWhenInitializingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_COMMAND_LINE), error);
+    const std::string expectedOutput = "No files to concatenate were provided.\n";
+    EXPECT_EQ(expectedOutput, output);
+}
+
+TEST(OclocConcatTest, GivenMissingFilesWhenInitializingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_COMMAND_LINE), error);
+    const std::string expectedOutput = "fatBinary1.ar doesn't exist!\nfatBinary2.ar doesn't exist!\n";
+    EXPECT_EQ(expectedOutput, output);
+}
+
+TEST(OclocConcatTest, GivenValidArgsWhenInitializingThenFileNamesToConcatAndOutputFileNameAreSetCorrectlyAndSuccessIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{
+        {"fatBinary1.ar", "fatBinary1Data"},
+        {"fatBinary2.ar", "fatBinary2Data"}};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar", "-out", "fatBinary.ar"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::SUCCESS), error);
+    EXPECT_TRUE(output.empty());
+
+    EXPECT_EQ(args[2], oclocConcat.fileNamesToConcat[0]);
+    EXPECT_EQ(args[3], oclocConcat.fileNamesToConcat[1]);
+    EXPECT_EQ(args[5], oclocConcat.fatBinaryName);
+}
+
+TEST(OclocConcatTest, GivenMissingOutFileNameAfterOutArgumentWhenInitalizingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    std::vector<std::string> args = {"ocloc", "concat", "fatBinary1.ar", "fatBinary2.ar", "-out"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.initialize(args);
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_COMMAND_LINE), error);
+    const std::string expectedOutput = "Missing out file name after \"-out\" argument\n";
+    EXPECT_EQ(expectedOutput, output);
+}
+
+TEST(OclocConcatTest, GivenErrorDuringDecodingArWhenConcatenatingThenErrorIsReturned) {
+    MockOclocArgHelper::FilesMap mockArgHelperFilesMap{
+        {"fatBinary1.ar", "fatBinary1Data"},
+        {"fatBinary2.ar", "fatBinary2Data"}};
+    MockOclocArgHelper mockArgHelper{mockArgHelperFilesMap};
+    auto oclocConcat = MockOclocConcat(&mockArgHelper);
+    oclocConcat.shouldFailDecodingAr = true;
+    oclocConcat.fileNamesToConcat = {"fatBinary1.ar",
+                                     "fatBinary2.ar"};
+
+    ::testing::internal::CaptureStdout();
+    auto error = oclocConcat.concatenate();
+    const auto output = ::testing::internal::GetCapturedStdout();
+
+    EXPECT_EQ(static_cast<uint32_t>(OclocErrorCode::INVALID_FILE), error);
+    EXPECT_EQ(MockOclocConcat::decodeArErrorMessage.str(), output);
+}
+
+} // namespace NEO