intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2026-01-09 22:43:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adding support for AR input binary

Browse Source 
Related-To: NEO-3920
Change-Id: If11318fd01920016368f4c55aa582e5b09b77c5d
This commit is contained in:
Jaroslaw Chodor
2020-02-09 18:27:49 +01:00
parent c692502f04
commit 7c5af49e05
12 changed files with 420 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/device_binary_format/CMakeLists.txt
View File

@@ -11,6 +11,7 @@ set(NEO_DEVICE_BINARY_FORMAT
${CMAKE_CURRENT_SOURCE_DIR}/ar/ar_decoder.h
${CMAKE_CURRENT_SOURCE_DIR}/ar/ar_encoder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ar/ar_encoder.h
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_ar.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_ocl_elf.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_patchtokens.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_formats.cpp

20
core/device_binary_format/ar/ar_encoder.cpp
View File

@@ -22,14 +22,30 @@ ArFileEntryHeader *ArEncoder::appendFileEntry(const ConstStringRef fileName, con
if (fileName.empty()) {
return nullptr;
}
auto alignedFileSize = fileData.size() + (fileData.size() & 1U);
ArFileEntryHeader header = {};
if (padTo8Bytes && (0 != ((fileEntries.size() + sizeof(ArFileEntryHeader)) % 8))) {
ArFileEntryHeader paddingHeader = {};
auto paddingName = "pad_" + std::to_string(paddingEntry++);
UNRECOVERABLE_IF(paddingName.length() > sizeof(paddingHeader.identifier));
memcpy_s(paddingHeader.identifier, sizeof(paddingHeader.identifier), paddingName.c_str(), paddingName.size());
paddingHeader.identifier[paddingName.size()] = SpecialFileNames::fileNameTerminator;
size_t paddingSize = 8U - ((fileEntries.size() + 2 * sizeof(ArFileEntryHeader)) % 8);
auto padSizeString = std::to_string(paddingSize);
memcpy_s(paddingHeader.fileSizeInBytes, sizeof(paddingHeader.fileSizeInBytes), padSizeString.c_str(), padSizeString.size());
this->fileEntries.reserve(this->fileEntries.size() + sizeof(paddingHeader) + paddingSize + sizeof(header) + alignedFileSize);
this->fileEntries.insert(this->fileEntries.end(), reinterpret_cast<uint8_t *>(&paddingHeader), reinterpret_cast<uint8_t *>(&paddingHeader + 1));
this->fileEntries.resize(this->fileEntries.size() + paddingSize, ' ');
}
memcpy_s(header.identifier, sizeof(header.identifier), fileName.begin(), fileName.size());
header.identifier[fileName.size()] = SpecialFileNames::fileNameTerminator;
auto sizeString = std::to_string(fileData.size());
UNRECOVERABLE_IF(sizeString.length() > sizeof(header.fileSizeInBytes));
memcpy_s(header.fileSizeInBytes, sizeof(header.fileSizeInBytes), sizeString.c_str(), sizeString.size());
auto alignedFileSize = fileData.size() + (fileData.size() & 1U);
this->fileEntries.reserve(sizeof(header) + alignedFileSize);
this->fileEntries.reserve(this->fileEntries.size() + sizeof(header) + alignedFileSize);
auto newFileHeaderOffset = this->fileEntries.size();
this->fileEntries.insert(this->fileEntries.end(), reinterpret_cast<uint8_t *>(&header), reinterpret_cast<uint8_t *>(&header + 1));
this->fileEntries.insert(this->fileEntries.end(), fileData.begin(), fileData.end());

4
core/device_binary_format/ar/ar_encoder.h
View File

@@ -11,17 +11,21 @@
#include "core/utilities/arrayref.h"
#include "core/utilities/const_stringref.h"
#include <cstring>
#include <vector>
namespace NEO {
namespace Ar {
struct ArEncoder {
ArEncoder(bool padTo8Bytes = false) : padTo8Bytes(padTo8Bytes) {}
ArFileEntryHeader *appendFileEntry(const ConstStringRef fileName, const ArrayRef<const uint8_t> fileData);
std::vector<uint8_t> encode() const;
protected:
std::vector<uint8_t> fileEntries;
bool padTo8Bytes = false;
uint32_t paddingEntry = 0U;
};
} // namespace Ar

72
core/device_binary_format/device_binary_format_ar.cpp Normal file
View File

@@ -0,0 +1,72 @@
/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "core/device_binary_format/ar/ar.h"
#include "core/device_binary_format/ar/ar_decoder.h"
#include "core/device_binary_format/device_binary_formats.h"
namespace NEO {
template <>
bool isDeviceBinaryFormat<NEO::DeviceBinaryFormat::Archive>(const ArrayRef<const uint8_t> binary) {
return NEO::Ar::isAr(binary);
}
template <>
SingleDeviceBinary unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice &requestedTargetDevice,
std::string &outErrReason, std::string &outWarning) {
auto archiveData = NEO::Ar::decodeAr(archive, outErrReason, outWarning);
if (nullptr == archiveData.magic) {
return {};
}
std::string pointerSize = ((requestedTargetDevice.maxPointerSizeInBytes == 8) ? "64" : "32");
std::string filterPointerSizeAndPlatform = pointerSize + "." + requestedProductAbbreviation.str();
std::string filterPointerSizeAndPlatformAndStepping = filterPointerSizeAndPlatform + "." + std::to_string(requestedTargetDevice.stepping);
Ar::ArFileEntryHeaderAndData *matchedFiles[2] = {};
Ar::ArFileEntryHeaderAndData *&matchedPointerSizeAndPlatformAndStepping = matchedFiles[0]; // best match
Ar::ArFileEntryHeaderAndData *&matchedPointerSizeAndPlatform = matchedFiles[1];
for (auto &f : archiveData.files) {
if (ConstStringRef(f.fileName.begin(), filterPointerSizeAndPlatform.size()) != filterPointerSizeAndPlatform) {
continue;
}
if (ConstStringRef(f.fileName.begin(), filterPointerSizeAndPlatformAndStepping.size()) != filterPointerSizeAndPlatformAndStepping) {
matchedPointerSizeAndPlatform = &f;
continue;
}
matchedPointerSizeAndPlatformAndStepping = &f;
}
std::string unpackErrors;
std::string unpackWarnings;
for (auto matchedFile : matchedFiles) {
if (nullptr == matchedFile) {
continue;
}
auto unpacked = unpackSingleDeviceBinary(matchedFile->fileData, requestedProductAbbreviation, requestedTargetDevice, unpackErrors, unpackWarnings);
if (false == unpacked.deviceBinary.empty()) {
if (matchedFile != matchedPointerSizeAndPlatformAndStepping) {
outWarning = "Couldn't find perfectly matched binary (right stepping) in AR, using best usable";
}
return unpacked;
}
}
outErrReason = "Couldn't find matching binary in AR archive";
return {};
}
template <>
DecodeError decodeSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(ProgramInfo &dst, const SingleDeviceBinary &src, std::string &outErrReason, std::string &outWarning) {
// packed binary format
outErrReason = "Device binary format is packed";
return DecodeError::InvalidBinary;
}
} // namespace NEO

2
core/device_binary_format/device_binary_format_ocl_elf.cpp
View File

@@ -36,7 +36,7 @@ bool isDeviceBinaryFormat<NEO::DeviceBinaryFormat::OclElf>(const ArrayRef<const
}
template <>
SingleDeviceBinary unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::OclElf>(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice requestedTargetDevice,
SingleDeviceBinary unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::OclElf>(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice &requestedTargetDevice,
std::string &outErrReason, std::string &outWarning) {
auto elf = Elf::decodeElf<Elf::EI_CLASS_64>(archive, outErrReason, outWarning);
if (nullptr == elf.elfFileHeader) {

2
core/device_binary_format/device_binary_format_patchtokens.cpp
View File

@@ -21,7 +21,7 @@ bool isDeviceBinaryFormat<NEO::DeviceBinaryFormat::Patchtokens>(const ArrayRef<c
}
template <>
SingleDeviceBinary unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Patchtokens>(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice requestedTargetDevice,
SingleDeviceBinary unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Patchtokens>(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice &requestedTargetDevice,
std::string &outErrReason, std::string &outWarning) {
auto programHeader = NEO::PatchTokenBinary::decodeProgramHeader(archive);
if (nullptr == programHeader) {

28
core/device_binary_format/device_binary_formats.h
View File

@@ -24,7 +24,8 @@ enum class DeviceBinaryFormat : uint8_t {
OclElf,
OclLibrary,
OclCompiledObject,
Patchtokens
Patchtokens,
Archive
};
enum class DecodeError : uint8_t {
@@ -73,6 +74,8 @@ template <>
bool isDeviceBinaryFormat<DeviceBinaryFormat::OclElf>(const ArrayRef<const uint8_t>);
template <>
bool isDeviceBinaryFormat<DeviceBinaryFormat::Patchtokens>(const ArrayRef<const uint8_t>);
template <>
bool isDeviceBinaryFormat<DeviceBinaryFormat::Archive>(const ArrayRef<const uint8_t>);
inline bool isAnyDeviceBinaryFormat(const ArrayRef<const uint8_t> binary) {
if (isDeviceBinaryFormat<DeviceBinaryFormat::OclElf>(binary)) {
@@ -81,19 +84,24 @@ inline bool isAnyDeviceBinaryFormat(const ArrayRef<const uint8_t> binary) {
if (isDeviceBinaryFormat<DeviceBinaryFormat::Patchtokens>(binary)) {
return true;
}
if (isDeviceBinaryFormat<DeviceBinaryFormat::Archive>(binary)) {
return true;
}
return false;
}
template <DeviceBinaryFormat Format>
SingleDeviceBinary unpackSingleDeviceBinary(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice requestedTargetDevice,
SingleDeviceBinary unpackSingleDeviceBinary(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice &requestedTargetDevice,
std::string &outErrReason, std::string &outWarning);
template <>
SingleDeviceBinary unpackSingleDeviceBinary<DeviceBinaryFormat::OclElf>(const ArrayRef<const uint8_t>, const ConstStringRef, const TargetDevice, std::string &, std::string &);
SingleDeviceBinary unpackSingleDeviceBinary<DeviceBinaryFormat::OclElf>(const ArrayRef<const uint8_t>, const ConstStringRef, const TargetDevice &, std::string &, std::string &);
template <>
SingleDeviceBinary unpackSingleDeviceBinary<DeviceBinaryFormat::Patchtokens>(const ArrayRef<const uint8_t>, const ConstStringRef, const TargetDevice, std::string &, std::string &);
SingleDeviceBinary unpackSingleDeviceBinary<DeviceBinaryFormat::Patchtokens>(const ArrayRef<const uint8_t>, const ConstStringRef, const TargetDevice &, std::string &, std::string &);
template <>
SingleDeviceBinary unpackSingleDeviceBinary<DeviceBinaryFormat::Archive>(const ArrayRef<const uint8_t>, const ConstStringRef, const TargetDevice &, std::string &, std::string &);
inline SingleDeviceBinary unpackSingleDeviceBinary(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice requestedTargetDevice,
inline SingleDeviceBinary unpackSingleDeviceBinary(const ArrayRef<const uint8_t> archive, const ConstStringRef requestedProductAbbreviation, const TargetDevice &requestedTargetDevice,
std::string &outErrReason, std::string &outWarning) {
SingleDeviceBinary ret = {};
ret.format = DeviceBinaryFormat::Unknown;
@@ -101,6 +109,8 @@ inline SingleDeviceBinary unpackSingleDeviceBinary(const ArrayRef<const uint8_t>
return unpackSingleDeviceBinary<DeviceBinaryFormat::OclElf>(archive, requestedProductAbbreviation, requestedTargetDevice, outErrReason, outWarning);
} else if (isDeviceBinaryFormat<DeviceBinaryFormat::Patchtokens>(archive)) {
return unpackSingleDeviceBinary<DeviceBinaryFormat::Patchtokens>(archive, requestedProductAbbreviation, requestedTargetDevice, outErrReason, outWarning);
} else if (isDeviceBinaryFormat<DeviceBinaryFormat::Archive>(archive)) {
return unpackSingleDeviceBinary<DeviceBinaryFormat::Archive>(archive, requestedProductAbbreviation, requestedTargetDevice, outErrReason, outWarning);
} else {
outErrReason = "Unknown format";
}
@@ -119,6 +129,9 @@ inline bool isAnyPackedDeviceBinaryFormat(const ArrayRef<const uint8_t> binary)
if (isDeviceBinaryFormat<DeviceBinaryFormat::OclElf>(binary)) {
return true;
}
if (isDeviceBinaryFormat<DeviceBinaryFormat::Archive>(binary)) {
return true;
}
return false;
}
@@ -133,6 +146,8 @@ template <>
DecodeError decodeSingleDeviceBinary<DeviceBinaryFormat::OclElf>(ProgramInfo &, const SingleDeviceBinary &, std::string &, std::string &);
template <>
DecodeError decodeSingleDeviceBinary<DeviceBinaryFormat::Patchtokens>(ProgramInfo &, const SingleDeviceBinary &, std::string &, std::string &);
template <>
DecodeError decodeSingleDeviceBinary<DeviceBinaryFormat::Archive>(ProgramInfo &, const SingleDeviceBinary &, std::string &, std::string &);
inline std::pair<DecodeError, DeviceBinaryFormat> decodeSingleDeviceBinary(ProgramInfo &dst, const SingleDeviceBinary &src, std::string &outErrReason, std::string &outWarning) {
std::pair<DecodeError, DeviceBinaryFormat> ret;
@@ -144,6 +159,9 @@ inline std::pair<DecodeError, DeviceBinaryFormat> decodeSingleDeviceBinary(Progr
} else if (isDeviceBinaryFormat<DeviceBinaryFormat::Patchtokens>(src.deviceBinary)) {
ret.second = DeviceBinaryFormat::Patchtokens;
ret.first = decodeSingleDeviceBinary<DeviceBinaryFormat::Patchtokens>(dst, src, outErrReason, outWarning);
} else if (isDeviceBinaryFormat<DeviceBinaryFormat::Archive>(src.deviceBinary)) {
ret.second = DeviceBinaryFormat::Archive;
ret.first = decodeSingleDeviceBinary<DeviceBinaryFormat::Archive>(dst, src, outErrReason, outWarning);
} else {
outErrReason = "Unknown format";
}

1
core/unit_tests/device_binary_format/CMakeLists.txt
View File

@@ -10,6 +10,7 @@ set(NEO_DEVICE_BINARY_FORMAT_TESTS
${CMAKE_CURRENT_SOURCE_DIR}/ar/ar_encoder_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/elf/elf_decoder_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/elf/elf_encoder_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_ar_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_ocl_elf_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_format_patchtokens_tests.cpp
${CMAKE_CURRENT_SOURCE_DIR}/device_binary_formats_tests.cpp

44
core/unit_tests/device_binary_format/ar/ar_encoder_tests.cpp
View File

@@ -7,6 +7,7 @@
#include "core/compiler_interface/intermediate_representations.h"
#include "core/device_binary_format/ar/ar_encoder.h"
#include "core/helpers/ptr_math.h"
#include "core/helpers/string.h"
#include "test.h"
@@ -102,3 +103,46 @@ TEST(ArEncoder, GivenValidTwoFileEntriesWith2byteUnalignedDataThenPaddingIsImpli
EXPECT_EQ(0, memcmp(file1, &expectedSection1, sizeof(expectedSection1)));
EXPECT_EQ(0, memcmp(file1Data, data1, sizeof(data1)));
}
TEST(ArEncoder, GivenValidTwoFileEntriesWhen8BytePaddingIsRequestedThenPaddingFileEntriesAreAddedWhenNeeded) {
std::string fileName0 = "a";
std::string fileName1 = "b";
std::string fileName2 = "c";
const uint8_t data0[4] = "123"; // will require padding before
const uint8_t data1[8] = "9ABCDEF"; // won't require padding before
const uint8_t data2[16] = "9ABCDEF"; // will require padding before
ArEncoder encoder(true);
encoder.appendFileEntry(fileName0, data0);
encoder.appendFileEntry(fileName1, data1);
encoder.appendFileEntry(fileName2, data2);
auto arData = encoder.encode();
EXPECT_TRUE(NEO::hasSameMagic(arMagic, arData));
ASSERT_EQ(arData.size(), arMagic.size() + sizeof(data0) + sizeof(data1) + sizeof(data2) + 5 * sizeof(ArFileEntryHeader) + 16);
auto files = arData.data() + arMagic.size();
ArFileEntryHeader *pad0 = reinterpret_cast<ArFileEntryHeader *>(files);
auto pad0Data = reinterpret_cast<uint8_t *>(pad0) + sizeof(ArFileEntryHeader);
ArFileEntryHeader *file0 = reinterpret_cast<ArFileEntryHeader *>(pad0Data + 8);
auto file0Data = reinterpret_cast<uint8_t *>(file0) + sizeof(ArFileEntryHeader);
ArFileEntryHeader *file1 = reinterpret_cast<ArFileEntryHeader *>(file0Data + sizeof(data0));
auto file1Data = reinterpret_cast<uint8_t *>(file1) + sizeof(ArFileEntryHeader);
ArFileEntryHeader *pad1 = reinterpret_cast<ArFileEntryHeader *>(file1Data + sizeof(data1));
auto pad1Data = reinterpret_cast<uint8_t *>(pad1) + sizeof(ArFileEntryHeader);
ArFileEntryHeader *file2 = reinterpret_cast<ArFileEntryHeader *>(pad1Data + 8);
auto file2Data = reinterpret_cast<uint8_t *>(file2) + sizeof(ArFileEntryHeader);
EXPECT_EQ(0U, ptrDiff(file0Data, arData.data()) % 8);
EXPECT_EQ(0U, ptrDiff(file1Data, arData.data()) % 8);
EXPECT_EQ(0U, ptrDiff(file2Data, arData.data()) % 8);
EXPECT_EQ(0, memcmp(file0Data, data0, sizeof(data0)));
EXPECT_EQ(0, memcmp(file1Data, data1, sizeof(data1)));
EXPECT_EQ(0, memcmp(file2Data, data2, sizeof(data2)));
}

183
core/unit_tests/device_binary_format/device_binary_format_ar_tests.cpp Normal file
View File

@@ -0,0 +1,183 @@
/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "core/device_binary_format/ar/ar.h"
#include "core/device_binary_format/ar/ar_decoder.h"
#include "core/device_binary_format/ar/ar_encoder.h"
#include "core/device_binary_format/device_binary_formats.h"
#include "core/helpers/hw_info.h"
#include "core/unit_tests/device_binary_format/patchtokens_tests.h"
#include "test.h"
TEST(IsDeviceBinaryFormatAr, GivenValidBinaryReturnTrue) {
auto emptyArchive = ArrayRef<const uint8_t>::fromAny(NEO::Ar::arMagic.begin(), NEO::Ar::arMagic.size());
EXPECT_TRUE(NEO::isDeviceBinaryFormat<NEO::DeviceBinaryFormat::Archive>(emptyArchive));
}
TEST(IsDeviceBinaryFormatAr, GivenInvalidBinaryReturnTrue) {
const uint8_t binary[] = "not_ar";
EXPECT_FALSE(NEO::isDeviceBinaryFormat<NEO::DeviceBinaryFormat::Archive>(binary));
}
TEST(UnpackSingleDeviceBinaryAr, WhenFailedToDecodeArThenUnpackingFails) {
std::string unpackErrors;
std::string unpackWarnings;
auto unpackResult = NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>({}, "", {}, unpackErrors, unpackWarnings);
EXPECT_EQ(NEO::DeviceBinaryFormat::Unknown, unpackResult.format);
EXPECT_TRUE(unpackResult.deviceBinary.empty());
EXPECT_TRUE(unpackResult.debugData.empty());
EXPECT_TRUE(unpackResult.intermediateRepresentation.empty());
EXPECT_TRUE(unpackResult.buildOptions.empty());
EXPECT_TRUE(unpackWarnings.empty()) << unpackWarnings;
EXPECT_STREQ("Not an AR archive - mismatched file signature", unpackErrors.c_str());
}
TEST(UnpackSingleDeviceBinaryAr, WhenFailedToFindMatchingBinariesThenUnpackingFails) {
auto emptyArchive = ArrayRef<const uint8_t>::fromAny(NEO::Ar::arMagic.begin(), NEO::Ar::arMagic.size());
std::string unpackErrors;
std::string unpackWarnings;
auto unpackResult = NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(emptyArchive, "", {}, unpackErrors, unpackWarnings);
EXPECT_EQ(NEO::DeviceBinaryFormat::Unknown, unpackResult.format);
EXPECT_TRUE(unpackResult.deviceBinary.empty());
EXPECT_TRUE(unpackResult.debugData.empty());
EXPECT_TRUE(unpackResult.intermediateRepresentation.empty());
EXPECT_TRUE(unpackResult.buildOptions.empty());
EXPECT_TRUE(unpackWarnings.empty()) << unpackWarnings;
EXPECT_STREQ("Couldn't find matching binary in AR archive", unpackErrors.c_str());
}
TEST(UnpackSingleDeviceBinaryAr, WhenMultipleBinariesMatchedThenChooseBestMatch) {
PatchTokensTestData::ValidEmptyProgram programTokens;
NEO::Ar::ArEncoder encoder;
std::string requiredProduct = NEO::hardwarePrefix[productFamily];
std::string requiredStepping = std::to_string(programTokens.header->SteppingId);
std::string requiredPointerSize = (programTokens.header->GPUPointerSizeInBytes == 4) ? "32" : "64";
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "." + requiredProduct, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "." + requiredProduct + "." + requiredStepping, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "unk." + requiredStepping, programTokens.storage));
NEO::TargetDevice target;
target.coreFamily = static_cast<GFXCORE_FAMILY>(programTokens.header->Device);
target.stepping = programTokens.header->SteppingId;
target.maxPointerSizeInBytes = programTokens.header->GPUPointerSizeInBytes;
auto arData = encoder.encode();
std::string unpackErrors;
std::string unpackWarnings;
auto unpacked = NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(arData, requiredProduct, target, unpackErrors, unpackWarnings);
EXPECT_TRUE(unpackErrors.empty()) << unpackErrors;
EXPECT_TRUE(unpackWarnings.empty()) << unpackWarnings;
unpackErrors.clear();
unpackWarnings.clear();
auto decodedAr = NEO::Ar::decodeAr(arData, unpackErrors, unpackWarnings);
EXPECT_NE(nullptr, decodedAr.magic);
ASSERT_EQ(4U, decodedAr.files.size());
EXPECT_EQ(unpacked.deviceBinary.begin(), decodedAr.files[2].fileData.begin());
EXPECT_EQ(unpacked.deviceBinary.size(), decodedAr.files[2].fileData.size());
EXPECT_EQ(NEO::DeviceBinaryFormat::Patchtokens, unpacked.format);
}
TEST(UnpackSingleDeviceBinaryAr, WhenBestMatchIsntFullMatchThenChooseBestMatchButEmitWarnings) {
PatchTokensTestData::ValidEmptyProgram programTokens;
NEO::Ar::ArEncoder encoder;
std::string requiredProduct = NEO::hardwarePrefix[productFamily];
std::string requiredStepping = std::to_string(programTokens.header->SteppingId);
std::string requiredPointerSize = (programTokens.header->GPUPointerSizeInBytes == 4) ? "32" : "64";
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "." + requiredProduct, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "unk." + requiredStepping, programTokens.storage));
NEO::TargetDevice target;
target.coreFamily = static_cast<GFXCORE_FAMILY>(programTokens.header->Device);
target.stepping = programTokens.header->SteppingId;
target.maxPointerSizeInBytes = programTokens.header->GPUPointerSizeInBytes;
auto arData = encoder.encode();
std::string unpackErrors;
std::string unpackWarnings;
auto unpacked = NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(arData, requiredProduct, target, unpackErrors, unpackWarnings);
EXPECT_TRUE(unpackErrors.empty()) << unpackErrors;
EXPECT_FALSE(unpackWarnings.empty());
EXPECT_STREQ("Couldn't find perfectly matched binary (right stepping) in AR, using best usable", unpackWarnings.c_str());
unpackErrors.clear();
unpackWarnings.clear();
auto decodedAr = NEO::Ar::decodeAr(arData, unpackErrors, unpackWarnings);
EXPECT_NE(nullptr, decodedAr.magic);
ASSERT_EQ(3U, decodedAr.files.size());
EXPECT_EQ(unpacked.deviceBinary.begin(), decodedAr.files[1].fileData.begin());
EXPECT_EQ(unpacked.deviceBinary.size(), decodedAr.files[1].fileData.size());
EXPECT_EQ(NEO::DeviceBinaryFormat::Patchtokens, unpacked.format);
}
TEST(UnpackSingleDeviceBinaryAr, WhenFailedToUnpackBestMatchThenTryUnpackingAnyUsable) {
PatchTokensTestData::ValidEmptyProgram programTokens;
PatchTokensTestData::ValidEmptyProgram programTokensWrongTokenVersion;
programTokensWrongTokenVersion.headerMutable->Version -= 1;
NEO::Ar::ArEncoder encoder;
std::string requiredProduct = NEO::hardwarePrefix[productFamily];
std::string requiredStepping = std::to_string(programTokens.header->SteppingId);
std::string requiredPointerSize = (programTokens.header->GPUPointerSizeInBytes == 4) ? "32" : "64";
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "." + requiredProduct, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "." + requiredProduct + "." + requiredStepping, programTokensWrongTokenVersion.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "unk." + requiredStepping, programTokens.storage));
NEO::TargetDevice target;
target.coreFamily = static_cast<GFXCORE_FAMILY>(programTokens.header->Device);
target.stepping = programTokens.header->SteppingId;
target.maxPointerSizeInBytes = programTokens.header->GPUPointerSizeInBytes;
auto arData = encoder.encode();
std::string unpackErrors;
std::string unpackWarnings;
auto unpacked = NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(arData, requiredProduct, target, unpackErrors, unpackWarnings);
EXPECT_TRUE(unpackErrors.empty()) << unpackErrors;
EXPECT_FALSE(unpackWarnings.empty());
EXPECT_STREQ("Couldn't find perfectly matched binary (right stepping) in AR, using best usable", unpackWarnings.c_str());
unpackErrors.clear();
unpackWarnings.clear();
auto decodedAr = NEO::Ar::decodeAr(arData, unpackErrors, unpackWarnings);
EXPECT_NE(nullptr, decodedAr.magic);
ASSERT_EQ(4U, decodedAr.files.size());
EXPECT_EQ(unpacked.deviceBinary.begin(), decodedAr.files[1].fileData.begin());
EXPECT_EQ(unpacked.deviceBinary.size(), decodedAr.files[1].fileData.size());
EXPECT_EQ(NEO::DeviceBinaryFormat::Patchtokens, unpacked.format);
}
TEST(UnpackSingleDeviceBinaryAr, WhenCouldFindBinaryWithRightPointerSizeThenUnpackingFails) {
PatchTokensTestData::ValidEmptyProgram programTokens;
NEO::Ar::ArEncoder encoder;
std::string requiredProduct = NEO::hardwarePrefix[productFamily];
std::string requiredStepping = std::to_string(programTokens.header->SteppingId);
std::string requiredPointerSize = (programTokens.header->GPUPointerSizeInBytes == 4) ? "32" : "64";
std::string wrongPointerSize = (programTokens.header->GPUPointerSizeInBytes == 8) ? "32" : "64";
ASSERT_TRUE(encoder.appendFileEntry(wrongPointerSize, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(wrongPointerSize + "." + requiredProduct, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(wrongPointerSize + "." + requiredProduct + "." + requiredStepping, programTokens.storage));
ASSERT_TRUE(encoder.appendFileEntry(requiredPointerSize + "unk." + requiredStepping, programTokens.storage));
NEO::TargetDevice target;
target.coreFamily = static_cast<GFXCORE_FAMILY>(programTokens.header->Device);
target.stepping = programTokens.header->SteppingId;
target.maxPointerSizeInBytes = programTokens.header->GPUPointerSizeInBytes;
auto arData = encoder.encode();
std::string unpackErrors;
std::string unpackWarnings;
auto unpacked = NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::Archive>(arData, requiredProduct, target, unpackErrors, unpackWarnings);
EXPECT_EQ(NEO::DeviceBinaryFormat::Unknown, unpacked.format);
EXPECT_TRUE(unpacked.deviceBinary.empty());
EXPECT_TRUE(unpacked.debugData.empty());
EXPECT_TRUE(unpacked.intermediateRepresentation.empty());
EXPECT_TRUE(unpacked.buildOptions.empty());
EXPECT_TRUE(unpackWarnings.empty()) << unpackWarnings;
EXPECT_STREQ("Couldn't find matching binary in AR archive", unpackErrors.c_str());
}

4
core/unit_tests/device_binary_format/device_binary_format_patchtokens_tests.cpp
View File

@@ -10,12 +10,12 @@
#include "core/unit_tests/device_binary_format/patchtokens_tests.h"
#include "test.h"
TEST(IsDeviceBinaryFormatPatchtokens, GiveValidBinaryReturnTrue) {
TEST(IsDeviceBinaryFormatPatchtokens, GivenValidBinaryReturnTrue) {
PatchTokensTestData::ValidProgramWithKernel programTokens;
EXPECT_TRUE(NEO::isDeviceBinaryFormat<NEO::DeviceBinaryFormat::Patchtokens>(programTokens.storage));
}
TEST(IsDeviceBinaryFormatPatchtokens, GiveInvalidBinaryReturnTrue) {
TEST(IsDeviceBinaryFormatPatchtokens, GivenInvalidBinaryReturnTrue) {
const uint8_t binary[] = "not_patchtokens";
EXPECT_FALSE(NEO::isDeviceBinaryFormat<NEO::DeviceBinaryFormat::Patchtokens>(binary));
}

72
core/unit_tests/device_binary_format/device_binary_formats_tests.cpp
View File

@@ -5,6 +5,7 @@
*
*/
#include "core/device_binary_format/ar/ar_encoder.h"
#include "core/device_binary_format/device_binary_formats.h"
#include "core/device_binary_format/elf/elf_encoder.h"
#include "core/device_binary_format/elf/ocl_elf.h"
@@ -35,6 +36,10 @@ TEST(IsAnyDeviceBinaryFormat, GivenOclElfFormatThenReturnsTrue) {
EXPECT_TRUE(NEO::isAnyDeviceBinaryFormat(elfEnc.encode()));
}
TEST(IsAnyDeviceBinaryFormat, GivenArFormatThenReturnsTrue) {
EXPECT_TRUE(NEO::isAnyDeviceBinaryFormat(ArrayRef<const uint8_t>::fromAny(NEO::Ar::arMagic.begin(), NEO::Ar::arMagic.size())));
}
TEST(UnpackSingleDeviceBinary, GivenUnknownBinaryThenReturnError) {
const uint8_t data[] = "none of known formats";
ConstStringRef requestedProductAbbreviation = "unk";
@@ -59,7 +64,7 @@ TEST(UnpackSingleDeviceBinary, GivenPatchtoknsBinaryThenReturnSelf) {
ConstStringRef requestedProductAbbreviation = "unk";
NEO::TargetDevice requestedTargetDevice;
requestedTargetDevice.coreFamily = static_cast<GFXCORE_FAMILY>(patchtokensProgram.header->Device);
requestedTargetDevice.stepping = patchtokensProgram.header->Device;
requestedTargetDevice.stepping = patchtokensProgram.header->SteppingId;
requestedTargetDevice.maxPointerSizeInBytes = patchtokensProgram.header->GPUPointerSizeInBytes;
std::string outErrors;
std::string outWarnings;
@@ -88,7 +93,7 @@ TEST(UnpackSingleDeviceBinary, GivenOclElfBinaryThenReturnPatchtokensBinary) {
ConstStringRef requestedProductAbbreviation = "unk";
NEO::TargetDevice requestedTargetDevice;
requestedTargetDevice.coreFamily = static_cast<GFXCORE_FAMILY>(patchtokensProgram.header->Device);
requestedTargetDevice.stepping = patchtokensProgram.header->Device;
requestedTargetDevice.stepping = patchtokensProgram.header->SteppingId;
requestedTargetDevice.maxPointerSizeInBytes = patchtokensProgram.header->GPUPointerSizeInBytes;
std::string outErrors;
std::string outWarnings;
@@ -109,6 +114,44 @@ TEST(UnpackSingleDeviceBinary, GivenOclElfBinaryThenReturnPatchtokensBinary) {
EXPECT_EQ(0, memcmp(patchtokensProgram.storage.data(), unpacked.deviceBinary.begin(), unpacked.deviceBinary.size()));
}
TEST(UnpackSingleDeviceBinary, GivenArBinaryWithOclElfThenReturnPatchtokensBinary) {
PatchTokensTestData::ValidEmptyProgram patchtokensProgram;
NEO::Elf::ElfEncoder<NEO::Elf::EI_CLASS_64> elfEnc;
elfEnc.getElfFileHeader().type = NEO::Elf::ET_OPENCL_EXECUTABLE;
elfEnc.appendSection(NEO::Elf::SHT_OPENCL_DEV_BINARY, NEO::Elf::SectionNamesOpenCl::deviceBinary, patchtokensProgram.storage);
NEO::TargetDevice requestedTargetDevice;
requestedTargetDevice.coreFamily = static_cast<GFXCORE_FAMILY>(patchtokensProgram.header->Device);
requestedTargetDevice.stepping = patchtokensProgram.header->SteppingId;
requestedTargetDevice.maxPointerSizeInBytes = patchtokensProgram.header->GPUPointerSizeInBytes;
std::string outErrors;
std::string outWarnings;
auto elfData = elfEnc.encode();
std::string requiredProduct = NEO::hardwarePrefix[productFamily];
std::string requiredStepping = std::to_string(patchtokensProgram.header->SteppingId);
std::string requiredPointerSize = (patchtokensProgram.header->GPUPointerSizeInBytes == 4) ? "32" : "64";
NEO::Ar::ArEncoder arEnc(true);
ASSERT_TRUE(arEnc.appendFileEntry(requiredPointerSize + "." + requiredProduct + "." + requiredStepping, elfData));
auto arData = arEnc.encode();
auto unpacked = NEO::unpackSingleDeviceBinary(arData, requiredProduct, requestedTargetDevice, outErrors, outWarnings);
EXPECT_TRUE(unpacked.buildOptions.empty());
EXPECT_TRUE(unpacked.debugData.empty());
EXPECT_TRUE(unpacked.intermediateRepresentation.empty());
EXPECT_EQ(NEO::DeviceBinaryFormat::Patchtokens, unpacked.format);
EXPECT_EQ(requestedTargetDevice.coreFamily, unpacked.targetDevice.coreFamily);
EXPECT_EQ(requestedTargetDevice.stepping, unpacked.targetDevice.stepping);
EXPECT_EQ(patchtokensProgram.header->GPUPointerSizeInBytes, unpacked.targetDevice.maxPointerSizeInBytes);
EXPECT_TRUE(outWarnings.empty()) << outWarnings;
EXPECT_TRUE(outErrors.empty()) << outErrors;
EXPECT_FALSE(unpacked.deviceBinary.empty());
ASSERT_EQ(patchtokensProgram.storage.size(), unpacked.deviceBinary.size());
EXPECT_EQ(0, memcmp(patchtokensProgram.storage.data(), unpacked.deviceBinary.begin(), unpacked.deviceBinary.size()));
}
TEST(IsAnyPackedDeviceBinaryFormat, GivenUnknownFormatThenReturnFalse) {
const uint8_t data[] = "none of known formats";
EXPECT_FALSE(NEO::isAnyPackedDeviceBinaryFormat(data));
@@ -125,6 +168,10 @@ TEST(IsAnyPackedDeviceBinaryFormat, GivenOclElfFormatThenReturnsTrue) {
EXPECT_TRUE(NEO::isAnyPackedDeviceBinaryFormat(elfEnc.encode()));
}
TEST(IsAnyPackedDeviceBinaryFormat, GivenArFormatThenReturnsTrue) {
EXPECT_TRUE(NEO::isAnyPackedDeviceBinaryFormat(ArrayRef<const uint8_t>::fromAny(NEO::Ar::arMagic.begin(), NEO::Ar::arMagic.size())));
}
TEST(IsAnySingleDeviceBinaryFormat, GivenUnknownFormatThenReturnFalse) {
const uint8_t data[] = "none of known formats";
EXPECT_FALSE(NEO::isAnySingleDeviceBinaryFormat(data));
@@ -141,6 +188,10 @@ TEST(IsAnySingleDeviceBinaryFormat, GivenOclElfFormatThenReturnsFalse) {
EXPECT_FALSE(NEO::isAnySingleDeviceBinaryFormat(elfEnc.encode()));
}
TEST(IsAnySingleDeviceBinaryFormat, GivenArFormatThenReturnsFalse) {
EXPECT_FALSE(NEO::isAnySingleDeviceBinaryFormat(ArrayRef<const uint8_t>::fromAny(NEO::Ar::arMagic.begin(), NEO::Ar::arMagic.size())));
}
TEST(DecodeSingleDeviceBinary, GivenUnknownFormatThenReturnFalse) {
const uint8_t data[] = "none of known formats";
NEO::ProgramInfo programInfo;
@@ -195,6 +246,23 @@ TEST(DecodeSingleDeviceBinary, GivenOclElfFormatThenDecodingFails) {
EXPECT_STREQ("Device binary format is packed", decodeErrors.c_str());
}
TEST(DecodeSingleDeviceBinary, GivenArFormatThenDecodingFails) {
NEO::Ar::ArEncoder arEnc;
auto arData = arEnc.encode();
NEO::ProgramInfo programInfo;
std::string decodeErrors;
std::string decodeWarnings;
NEO::SingleDeviceBinary bin;
bin.deviceBinary = arData;
NEO::DecodeError status;
NEO::DeviceBinaryFormat format;
std::tie(status, format) = NEO::decodeSingleDeviceBinary(programInfo, bin, decodeErrors, decodeWarnings);
EXPECT_EQ(NEO::DecodeError::InvalidBinary, status);
EXPECT_EQ(NEO::DeviceBinaryFormat::Archive, format);
EXPECT_TRUE(decodeWarnings.empty());
EXPECT_STREQ("Device binary format is packed", decodeErrors.c_str());
}
TEST(PackDeviceBinary, GivenRequestToPackThenUsesOclElfFormat) {
NEO::SingleDeviceBinary deviceBinary;