Media/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-09-15 13:01:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

ocloc - checksum recalculation during reassembly

Browse Source 
Resolves: NEO-2696

Change-Id: I2c049ac511e437679df9b58d00e4fb8d995bbe3e
This commit is contained in:
Chodor, Jaroslaw
2019-06-19 00:27:59 +02:00
committed by sys_ocldev
parent 0fa5ceea37
commit 2e5ef30009
5 changed files with 245 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
offline_compiler/decoder/binary_decoder.cpp
View File

@ -257,9 +257,6 @@ int BinaryDecoder::processBinary(void *&ptr, std::ostream &ptmFile) {
}
dumpField(ptr, v, ptmFile);
}
if (patchListSize == 0) {
messagePrinter.printf("Warning! Program's patch list size is 0.\n");
}
if (numberOfKernels == 0) {
messagePrinter.printf("Warning! Number of Kernels is 0.\n");
}

143
offline_compiler/decoder/binary_encoder.cpp
View File

@ -8,7 +8,9 @@
#include "binary_encoder.h"
#include "elf/writer.h"
#include "runtime/helpers/aligned_memory.h"
#include "runtime/helpers/file_io.h"
#include "runtime/helpers/hash.h"
#include "CL/cl.h"
#include "helper.h"
@ -16,6 +18,7 @@
#include <algorithm>
#include <cstring>
#include <fstream>
#include <sstream>
void BinaryEncoder::setMessagePrinter(const MessagePrinter &messagePrinter) {
this->messagePrinter = messagePrinter;
@ -49,11 +52,11 @@ void BinaryEncoder::calculatePatchListSizes(std::vector<std::string> &ptmFile) {
}
}
int BinaryEncoder::copyBinaryToBinary(const std::string &srcFileName, std::ostream &outBinary) {
bool BinaryEncoder::copyBinaryToBinary(const std::string &srcFileName, std::ostream &outBinary, uint32_t *binaryLength) {
std::ifstream ifs(srcFileName, std::ios::binary);
if (!ifs.good()) {
messagePrinter.printf("Cannot open %s.\n", srcFileName.c_str());
return -1;
return false;
}
ifs.seekg(0, ifs.end);
auto length = static_cast<size_t>(ifs.tellg());
@ -61,7 +64,12 @@ int BinaryEncoder::copyBinaryToBinary(const std::string &srcFileName, std::ostre
std::vector<char> binary(length);
ifs.read(binary.data(), length);
outBinary.write(binary.data(), length);
return 0;
if (binaryLength) {
*binaryLength = static_cast<uint32_t>(length);
}
return true;
}
int BinaryEncoder::createElf() {
@ -169,56 +177,103 @@ int BinaryEncoder::processBinary(const std::vector<std::string> &ptmFile, std::o
return 0;
}
int BinaryEncoder::processKernel(size_t &i, const std::vector<std::string> &ptmFile, std::ostream &deviceBinary) {
uint32_t kernelNameSize = 0;
while (i < ptmFile.size()) {
if (ptmFile[i].find("KernelName ") != std::string::npos) {
break;
} else if (ptmFile[i].find("KernelNameSize") != std::string::npos) {
std::stringstream ss(ptmFile[i]);
ss.ignore(32, ' ');
ss.ignore(32, ' ');
ss >> kernelNameSize;
void BinaryEncoder::addPadding(std::ostream &out, size_t numBytes) {
for (size_t i = 0; i < numBytes; ++i) {
const char nullByte = 0;
out.write(&nullByte, 1U);
}
if (writeDeviceBinary(ptmFile[i++], deviceBinary)) {
}
int BinaryEncoder::processKernel(size_t &line, const std::vector<std::string> &ptmFileLines, std::ostream &deviceBinary) {
auto kernelInfoBeginMarker = line;
auto kernelInfoEndMarker = ptmFileLines.size();
auto kernelNameMarker = ptmFileLines.size();
auto kernelPatchtokensMarker = ptmFileLines.size();
std::stringstream kernelBlob;
// Normally these are added by the compiler, need to take or of them when reassembling
constexpr size_t isaPaddingSizeInBytes = 128;
constexpr uint32_t kernelHeapAlignmentInBytes = 64;
uint32_t kernelNameSizeInBinary = 0;
std::string kernelName;
// Scan PTM lines for kernel info
while (line < ptmFileLines.size()) {
if (ptmFileLines[line].find("KernelName ") != std::string::npos) {
kernelName = std::string(ptmFileLines[line], ptmFileLines[line].find(' ') + 1);
kernelNameMarker = line;
kernelPatchtokensMarker = kernelNameMarker + 1; // patchtokens come after name
} else if (ptmFileLines[line].find("KernelNameSize") != std::string::npos) {
std::stringstream ss(ptmFileLines[line]);
ss.ignore(32, ' ');
ss.ignore(32, ' ');
ss >> kernelNameSizeInBinary;
} else if (ptmFileLines[line].find("Kernel #") != std::string::npos) {
kernelInfoEndMarker = line;
break;
}
++line;
}
// Write KernelName and padding
kernelBlob.write(kernelName.c_str(), kernelName.size());
addPadding(kernelBlob, kernelNameSizeInBinary - kernelName.size());
// Write KernelHeap and padding
uint32_t kernelSizeUnpadded = 0U;
bool heapsCopiedSuccesfully = copyBinaryToBinary(pathToDump + kernelName + "_KernelHeap.bin", kernelBlob, &kernelSizeUnpadded);
// Adding padding and alignment
addPadding(kernelBlob, isaPaddingSizeInBytes);
const uint32_t kernelHeapPaddedSize = kernelSizeUnpadded + isaPaddingSizeInBytes;
const uint32_t kernelHeapAlignedSize = alignUp(kernelHeapPaddedSize, kernelHeapAlignmentInBytes);
addPadding(kernelBlob, kernelHeapAlignedSize - kernelHeapPaddedSize);
// Write GeneralStateHeap, DynamicStateHeap, SurfaceStateHeap
if (fileExists(pathToDump + kernelName + "_GeneralStateHeap.bin")) {
heapsCopiedSuccesfully = heapsCopiedSuccesfully && copyBinaryToBinary(pathToDump + kernelName + "_GeneralStateHeap.bin", kernelBlob);
}
heapsCopiedSuccesfully = heapsCopiedSuccesfully && copyBinaryToBinary(pathToDump + kernelName + "_DynamicStateHeap.bin", kernelBlob);
heapsCopiedSuccesfully = heapsCopiedSuccesfully && copyBinaryToBinary(pathToDump + kernelName + "_SurfaceStateHeap.bin", kernelBlob);
if (false == heapsCopiedSuccesfully) {
return -1;
}
// Write kernel patchtokens
for (size_t i = kernelPatchtokensMarker; i < kernelInfoEndMarker; ++i) {
if (writeDeviceBinary(ptmFileLines[i], kernelBlob)) {
messagePrinter.printf("Error while writing to binary.\n");
return -1;
}
}
//KernelName
if (i == ptmFile.size()) {
messagePrinter.printf("Couldn't find KernelName line.\n");
auto kernelBlobData = kernelBlob.str();
uint64_t hashValue = NEO::Hash::hash(reinterpret_cast<const char *>(kernelBlobData.data()), kernelBlobData.size());
uint32_t calcCheckSum = hashValue & 0xFFFFFFFF;
// Add kernel header
for (size_t i = kernelInfoBeginMarker; i < kernelNameMarker; ++i) {
if (ptmFileLines[i].find("CheckSum") != std::string::npos) {
static_assert(std::is_same<decltype(calcCheckSum), uint32_t>::value, "");
deviceBinary.write(reinterpret_cast<char *>(&calcCheckSum), sizeof(uint32_t));
} else if (ptmFileLines[i].find("KernelHeapSize") != std::string::npos) {
static_assert(sizeof(kernelHeapAlignedSize) == sizeof(uint32_t), "");
deviceBinary.write(reinterpret_cast<const char *>(&kernelHeapAlignedSize), sizeof(uint32_t));
} else if (ptmFileLines[i].find("KernelUnpaddedSize") != std::string::npos) {
static_assert(sizeof(kernelSizeUnpadded) == sizeof(uint32_t), "");
deviceBinary.write(reinterpret_cast<char *>(&kernelSizeUnpadded), sizeof(uint32_t));
} else {
if (writeDeviceBinary(ptmFileLines[i], deviceBinary)) {
messagePrinter.printf("Error while writing to binary.\n");
return -1;
}
std::string kernelName(ptmFile[i], ptmFile[i].find(' ') + 1);
i++;
deviceBinary.write(kernelName.c_str(), kernelName.size());
for (auto j = kernelName.size(); j < kernelNameSize; ++j) {
uint8_t nullByte = 0;
deviceBinary.write(reinterpret_cast<const char *>(&nullByte), sizeof(uint8_t));
}
}
// Writing KernelHeap, DynamicStateHeap, SurfaceStateHeap
if (fileExists(pathToDump + kernelName + "_GeneralStateHeap.bin")) {
messagePrinter.printf("Warning! Adding GeneralStateHeap.\n");
if (copyBinaryToBinary(pathToDump + kernelName + "_GeneralStateHeap.bin", deviceBinary)) {
messagePrinter.printf("Couldn't copy %s_GeneralStateHeap.bin\n", kernelName.c_str());
return -1;
}
}
if (copyBinaryToBinary(pathToDump + kernelName + "_KernelHeap.bin", deviceBinary)) {
messagePrinter.printf("Couldn't copy %s_KernelHeap.bin\n", kernelName.c_str());
return -1;
}
if (copyBinaryToBinary(pathToDump + kernelName + "_DynamicStateHeap.bin", deviceBinary)) {
messagePrinter.printf("Couldn't copy %s_DynamicStateHeap.bin\n", kernelName.c_str());
return -1;
}
if (copyBinaryToBinary(pathToDump + kernelName + "_SurfaceStateHeap.bin", deviceBinary)) {
messagePrinter.printf("Couldn't copy %s_SurfaceStateHeap.bin\n", kernelName.c_str());
return -1;
}
// Add kernel blob after the header
deviceBinary.write(kernelBlobData.c_str(), kernelBlobData.size());
return 0;
}

8
offline_compiler/decoder/binary_encoder.h
View File

@ -27,12 +27,16 @@ class BinaryEncoder {
MessagePrinter messagePrinter;
void calculatePatchListSizes(std::vector<std::string> &ptmFile);
int copyBinaryToBinary(const std::string &srcFileName, std::ostream &outBinary);
MOCKABLE_VIRTUAL bool copyBinaryToBinary(const std::string &srcFileName, std::ostream &outBinary, uint32_t *binaryLength);
bool copyBinaryToBinary(const std::string &srcFileName, std::ostream &outBinary) {
return copyBinaryToBinary(srcFileName, outBinary, nullptr);
}
int createElf();
void printHelp();
int processBinary(const std::vector<std::string> &ptmFile, std::ostream &deviceBinary);
int processKernel(size_t &i, const std::vector<std::string> &ptmFile, std::ostream &deviceBinary);
int processKernel(size_t &i, const std::vector<std::string> &ptmFileLines, std::ostream &deviceBinary);
template <typename T>
void write(std::stringstream &in, std::ostream &deviceBinary);
int writeDeviceBinary(const std::string &line, std::ostream &deviceBinary);
void addPadding(std::ostream &out, size_t numBytes);
};

150
unit_tests/offline_compiler/decoder/encoder_tests.cpp
View File

@ -56,8 +56,8 @@ TEST(EncoderTests, GivenWrongParametersWhenParsingParametersThenErrorCodeIsRetur
TEST(EncoderTests, WhenTryingToCopyNonExistingFileThenErrorCodeIsReturned) {
MockEncoder encoder;
std::stringstream ss;
int retVal = encoder.copyBinaryToBinary("test_files/non_existing.bin", ss);
EXPECT_NE(0, retVal);
auto retVal = encoder.copyBinaryToBinary("test_files/non_existing.bin", ss);
EXPECT_FALSE(retVal);
}
TEST(EncoderTests, WhenWritingValuesToBinaryThenValuesAreWrittenCorrectly) {
@ -156,11 +156,10 @@ TEST(EncoderTests, GivenIncorrectPatchListSizeWhileCalculatingPatchListSizeThenP
EXPECT_EQ("\t4 PatchListSize 29", ptmFile[2]);
}
TEST(EncoderTests, GivenCorrectPTMFileWhileProcessingThenCorrectBinaryIsExpected) {
TEST(EncoderTests, GivenCorrectPTMFileWhileProcessingThenCorrectProgramHeaderExpected) {
std::stringstream expectedBinary;
uint8_t byte;
uint32_t byte4;
uint64_t byte8;
byte4 = 1229870147;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
@ -188,28 +187,7 @@ TEST(EncoderTests, GivenCorrectPTMFileWhileProcessingThenCorrectBinaryIsExpected
expectedBinary.write(reinterpret_cast<char *>(&byte), sizeof(uint8_t));
byte = 0x65;
expectedBinary.write(reinterpret_cast<char *>(&byte), sizeof(uint8_t));
byte4 = 2316678223;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte8 = 4988534869940066475;
expectedBinary.write(reinterpret_cast<char *>(&byte8), sizeof(uint64_t));
byte4 = 12;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 520;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
std::string kernelName = "kernel";
expectedBinary.write(kernelName.c_str(), kernelName.length());
byte = 0;
for (size_t i = kernelName.length(); i < 12; ++i) {
expectedBinary.write(reinterpret_cast<char *>(&byte), sizeof(uint8_t));
}
std::vector<std::string> ptmFile;
ptmFile.push_back("ProgramBinaryHeader:");
ptmFile.push_back("\t4 Magic 1229870147");
@ -238,9 +216,123 @@ TEST(EncoderTests, GivenCorrectPTMFileWhileProcessingThenCorrectBinaryIsExpected
ptmFile.push_back("\tKernelName kernel");
std::stringstream binary;
MockEncoder encoder;
int retVal = encoder.processBinary(ptmFile, binary);
EXPECT_EQ(-1, retVal);
MockEncoder().processBinary(ptmFile, binary);
EXPECT_EQ(expectedBinary.str(), binary.str());
}
TEST(EncoderTests, WhenAddPaddingIsCalledThenProperNumberOfZerosIsAdded) {
std::stringstream stream;
stream << "aa";
MockEncoder().addPadding(stream, 8);
std::string asString = stream.str();
ASSERT_EQ(10U, asString.size());
char expected[] = {'a', 'a', 0, 0, 0, 0, 0, 0, 0, 0};
EXPECT_EQ(0, memcmp(asString.c_str(), expected, 10U));
}
TEST(EncoderTests, WhenProcessingDeviceBinaryThenProperChecksumIsCalculated) {
std::stringstream expectedBinary;
uint8_t byte;
uint32_t byte4;
uint64_t byte8;
MockEncoder encoder;
std::string kernelName = "kernel";
encoder.filesMap["kernel_DynamicStateHeap.bin"] = std::string(16, 2);
encoder.filesMap["kernel_KernelHeap.bin"] = std::string(16, 4);
encoder.filesMap["kernel_SurfaceStateHeap.bin"] = std::string(16, 8);
std::stringstream kernelBlob;
kernelBlob << kernelName;
kernelBlob.write(encoder.filesMap["kernel_KernelHeap.bin"].data(), encoder.filesMap["kernel_KernelHeap.bin"].size());
encoder.addPadding(kernelBlob, 128); // isa prefetch padding
encoder.addPadding(kernelBlob, 64 - (encoder.filesMap["kernel_KernelHeap.bin"].size() + 128) % 64); // isa alignment
size_t kernelHeapSize = encoder.filesMap["kernel_KernelHeap.bin"].size();
kernelHeapSize = alignUp(kernelHeapSize + 128, 64);
kernelBlob.write(encoder.filesMap["kernel_DynamicStateHeap.bin"].data(), encoder.filesMap["kernel_DynamicStateHeap.bin"].size());
kernelBlob.write(encoder.filesMap["kernel_SurfaceStateHeap.bin"].data(), encoder.filesMap["kernel_SurfaceStateHeap.bin"].size());
auto kernelBlobData = kernelBlob.str();
uint64_t hashValue = NEO::Hash::hash(reinterpret_cast<const char *>(kernelBlobData.data()), kernelBlobData.size());
uint32_t checksum = hashValue & 0xFFFFFFFF;
byte4 = 1229870147;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 1042;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 12;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 4;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 1;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 2;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 18;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 42;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 16;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 2;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte = 0x48;
expectedBinary.write(reinterpret_cast<char *>(&byte), sizeof(uint8_t));
byte = 0x65;
expectedBinary.write(reinterpret_cast<char *>(&byte), sizeof(uint8_t));
byte4 = checksum;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte8 = 4988534869940066475;
expectedBinary.write(reinterpret_cast<char *>(&byte8), sizeof(uint64_t));
byte4 = static_cast<uint32_t>(kernelName.size());
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = static_cast<uint32_t>(kernelHeapSize);
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 0;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = 16;
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
byte4 = static_cast<uint32_t>(encoder.filesMap["kernel_KernelHeap.bin"].size());
expectedBinary.write(reinterpret_cast<char *>(&byte4), sizeof(uint32_t));
expectedBinary.write(kernelName.c_str(), kernelName.length());
std::vector<std::string> ptmFile;
ptmFile.push_back("ProgramBinaryHeader:");
ptmFile.push_back("\t4 Magic 1229870147");
ptmFile.push_back("\t4 Version 1042");
ptmFile.push_back("\t4 Device 12");
ptmFile.push_back("\t4 GPUPointerSizeInBytes 4");
ptmFile.push_back("\t4 NumberOfKernels 1");
ptmFile.push_back("\t4 SteppingId 2");
ptmFile.push_back("\t4 PatchListSize 18");
ptmFile.push_back("PATCH_TOKEN_ALLOCATE_CONSTANT_MEMORY_SURFACE_PROGRAM_BINARY_INFO:");
ptmFile.push_back("\t4 Token 42");
ptmFile.push_back("\t4 Size 16");
ptmFile.push_back("\t4 ConstantBufferIndex 0");
ptmFile.push_back("\t4 InlineDataSize 2");
ptmFile.push_back("\tHex 48 65");
ptmFile.push_back("Kernel #0");
ptmFile.push_back("KernelBinaryHeader:");
ptmFile.push_back("\t4 CheckSum " + std::to_string(checksum));
ptmFile.push_back("\t8 ShaderHashCode 4988534869940066475");
ptmFile.push_back("\t4 KernelNameSize " + std::to_string(kernelName.size()));
ptmFile.push_back("\t4 PatchListSize 0");
ptmFile.push_back("\t4 KernelHeapSize 16");
ptmFile.push_back("\t4 GeneralStateHeapSize 0");
ptmFile.push_back("\t4 DynamicStateHeapSize 16");
ptmFile.push_back("\t4 KernelUnpaddedSize 16");
ptmFile.push_back("\tKernelName " + kernelName);
std::stringstream result;
auto ret = encoder.processBinary(ptmFile, result);
auto resultAsString = result.str();
EXPECT_EQ(0, ret);
auto expectedBinaryAsString = expectedBinary.str();
resultAsString.resize(expectedBinaryAsString.size()); // don't test beyond kernel header
EXPECT_EQ(expectedBinaryAsString, resultAsString);
}
} // namespace NEO

20
unit_tests/offline_compiler/decoder/mock/mock_encoder.h
View File

@ -7,6 +7,10 @@
#pragma once
#include "offline_compiler/decoder/binary_encoder.h"
#include "runtime/helpers/hash.h"
#include <map>
#include <string>
struct MockEncoder : public BinaryEncoder {
MockEncoder() : MockEncoder("", ""){};
@ -14,6 +18,20 @@ struct MockEncoder : public BinaryEncoder {
: BinaryEncoder(dump, elf) {
setMessagePrinter(MessagePrinter{true});
};
bool copyBinaryToBinary(const std::string &srcFileName, std::ostream &outBinary, uint32_t *binaryLength) override {
auto it = filesMap.find(srcFileName);
if (it == filesMap.end()) {
return false;
}
outBinary.write(it->second.c_str(), it->second.size());
if (binaryLength != nullptr) {
*binaryLength = static_cast<uint32_t>(it->second.size());
}
return true;
}
using BinaryEncoder::addPadding;
using BinaryEncoder::calculatePatchListSizes;
using BinaryEncoder::copyBinaryToBinary;
using BinaryEncoder::createElf;
@ -24,4 +42,6 @@ struct MockEncoder : public BinaryEncoder {
using BinaryEncoder::processKernel;
using BinaryEncoder::write;
using BinaryEncoder::writeDeviceBinary;
std::map<std::string, std::string> filesMap;
};