Media/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-09-20 13:11:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: use virtualFileSystem in ULTs

Browse Source 
reducing the number of tests that have interactions with filesystem.
writeDataToFile() saves filename and content in std::map.
fileExistsHasSize() checks if file was previously written to virtualFileSystem
loadDataFromVirtualFile() fetches data from std::map based on filename

Related-To: NEO-7006
Signed-off-by: Marcel Skierkowski <marcel.skierkowski@intel.com>
This commit is contained in:
Marcel Skierkowski
2024-12-12 15:55:13 +00:00
committed by Compute-Runtime-Automation
parent 9f23867d30
commit e27a6dc280
18 changed files with 160 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
shared/offline_compiler/source/decoder/helper.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2023 Intel Corporation
* Copyright (C) 2018-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -51,23 +51,27 @@ std::vector<char> readBinaryFile(const std::string &fileName) {
}
}
void readFileToVectorOfStrings(std::vector<std::string> &lines, const std::string &fileName, bool replaceTabs) {
std::ifstream file(fileName);
if (file.good()) {
void istreamToVectorOfStrings(std::istream &input, std::vector<std::string> &lines, bool replaceTabs) {
if (input.good()) {
if (replaceTabs) {
for (std::string line; std::getline(file, line);) {
for (std::string line; std::getline(input, line);) {
std::replace_if(
line.begin(), line.end(), [](auto c) { return c == '\t'; }, ' ');
lines.push_back(std::move(line));
}
} else {
for (std::string line; std::getline(file, line);) {
for (std::string line; std::getline(input, line);) {
lines.push_back(std::move(line));
}
}
}
}
void readFileToVectorOfStrings(std::vector<std::string> &lines, const std::string &fileName, bool replaceTabs) {
std::ifstream file(fileName);
istreamToVectorOfStrings(file, lines, replaceTabs);
}
size_t findPos(const std::vector<std::string> &lines, const std::string &whatToFind) {
for (size_t i = 0; i < lines.size(); ++i) {
auto it = lines[i].find(whatToFind);

3
shared/offline_compiler/source/decoder/helper.h
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2018-2023 Intel Corporation
* Copyright (C) 2018-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
@ -26,6 +26,7 @@ void addSlash(std::string &path);
std::vector<char> readBinaryFile(const std::string &fileName);
void istreamToVectorOfStrings(std::istream &input, std::vector<std::string> &lines, bool replaceTabs = false);
void readFileToVectorOfStrings(std::vector<std::string> &lines, const std::string &fileName, bool replaceTabs = false);
void setProductFamilyForIga(const std::string &device, IgaWrapper *iga, OclocArgHelper *argHelper);

7
shared/source/utilities/logger.cpp
View File

@ -9,6 +9,7 @@
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/helpers/file_io.h"
#include "shared/source/helpers/timestamp_packet.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/utilities/io_functions.h"
@ -44,11 +45,7 @@ FileLogger<debugLevel>::~FileLogger() = default;
template <DebugFunctionalityLevel debugLevel>
void FileLogger<debugLevel>::writeToFile(std::string filename, const char *str, size_t length, std::ios_base::openmode mode) {
std::lock_guard theLock(mutex);
std::ofstream outFile(filename, mode);
if (outFile.is_open()) {
outFile.write(str, length);
outFile.close();
}
writeDataToFile(filename.c_str(), str, length);
}
template <DebugFunctionalityLevel debugLevel>

41
shared/test/common/helpers/file_io.cpp
View File

@ -11,11 +11,12 @@
#include "shared/source/helpers/stdio.h"
#include <cstring>
#include <map>
#include <new>
#include <set>
#include <sstream>
namespace NEO {
extern std::set<std::string> virtualFileList;
extern std::map<std::string, std::stringstream> virtualFileList;
}
size_t writeDataToFile(
@ -26,7 +27,7 @@ size_t writeDataToFile(
DEBUG_BREAK_IF(nullptr == pData);
DEBUG_BREAK_IF(nullptr == filename);
NEO::virtualFileList.insert(filename);
NEO::virtualFileList[filename] << std::string(static_cast<const char *>(pData), dataSize);
return dataSize;
}
@ -55,6 +56,13 @@ bool fileExistsHasSize(const std::string &fileName) {
DEBUG_BREAK_IF(fileName.empty());
DEBUG_BREAK_IF(fileName == "");
auto it = NEO::virtualFileList.find(fileName);
if (it != NEO::virtualFileList.end()) {
std::stringstream &ss = it->second;
ss.seekg(0, std::ios::end);
return ss.tellg() > 0;
}
fopen_s(&pFile, fileName.c_str(), "rb");
if (pFile) {
fseek(pFile, 0, SEEK_END);
@ -74,3 +82,30 @@ bool virtualFileExists(const std::string &fileName) {
}
return false;
}
std::unique_ptr<char[]> loadDataFromVirtualFile(
const char *filename,
size_t &retSize) {
if (!virtualFileExists(filename)) {
retSize = 0;
return nullptr;
}
std::unique_ptr<char[]> ret;
auto it = NEO::virtualFileList.find(filename);
std::stringstream &fileStream = it->second;
fileStream.seekg(0, std::ios::end);
retSize = static_cast<size_t>(fileStream.tellg());
fileStream.seekg(0, std::ios::beg);
ret.reset(new (std::nothrow) char[retSize + 1]);
if (ret) {
memset(ret.get(), 0x00, retSize + 1);
fileStream.read(ret.get(), retSize);
} else {
retSize = 0;
}
return ret;
}

3
shared/test/common/helpers/mock_file_io.h
View File

@ -9,3 +9,6 @@
bool virtualFileExists(const std::string &fileName);
void removeVirtualFile(const std::string &fileName);
std::unique_ptr<char[]> loadDataFromVirtualFile(
const char *filename,
size_t &retSize);

2
shared/test/common/helpers/virtual_file_system_listener.cpp
View File

@ -10,7 +10,7 @@
#include "shared/test/common/helpers/mock_file_io.h"
namespace NEO {
std::set<std::string> virtualFileList;
std::map<std::string, std::stringstream> virtualFileList;
void VirtualFileSystemListener::OnTestStart(const ::testing::TestInfo &testInfo) {
}

35
shared/test/common/utilities/logger_tests.h
View File

@ -7,24 +7,32 @@
#pragma once
#include "shared/source/helpers/file_io.h"
#include "shared/source/helpers/string_helpers.h"
#include "shared/source/utilities/directory.h"
#include "shared/source/utilities/logger.h"
#include "shared/test/common/helpers/mock_file_io.h"
#include <map>
namespace NEO {
extern std::map<std::string, std::stringstream> virtualFileList;
} // namespace NEO
template <DebugFunctionalityLevel debugLevel>
class TestFileLogger : public NEO::FileLogger<debugLevel> {
public:
using NEO::FileLogger<debugLevel>::FileLogger;
~TestFileLogger() override {
std::remove(NEO::FileLogger<debugLevel>::logFileName.c_str());
TestFileLogger(std::string filename, const NEO::DebugVariables &flags) : NEO::FileLogger<debugLevel>(filename, flags) {
if (NEO::FileLogger<debugLevel>::enabled() && virtualFileExists(this->getLogFileName())) {
removeVirtualFile(this->getLogFileName());
}
}
void useRealFiles(bool value) {
mockFileSystem = !value;
~TestFileLogger() override {
if (virtualFileExists(this->getLogFileName())) {
removeVirtualFile(this->getLogFileName());
}
}
void writeToFile(std::string filename,
@ -32,27 +40,20 @@ class TestFileLogger : public NEO::FileLogger<debugLevel> {
size_t length,
std::ios_base::openmode mode) override {
savedFiles[filename] << std::string(str, str + length);
if (mockFileSystem == false) {
NEO::FileLogger<debugLevel>::writeToFile(filename, str, length, mode);
}
};
NEO::FileLogger<debugLevel>::writeToFile(filename, str, length, mode);
}
int32_t createdFilesCount() {
return static_cast<int32_t>(savedFiles.size());
return static_cast<int32_t>(NEO::virtualFileList.size());
}
bool wasFileCreated(std::string filename) {
return savedFiles.find(filename) != savedFiles.end();
return virtualFileExists(filename);
}
std::string getFileString(std::string filename) {
return savedFiles[filename].str();
return NEO::virtualFileList[filename].str();
}
protected:
bool mockFileSystem = true;
std::map<std::string, std::stringstream> savedFiles;
};
using FullyEnabledFileLogger = TestFileLogger<DebugFunctionalityLevel::full>;

4
shared/test/unit_test/built_ins/sip_tests.cpp
View File

@ -30,11 +30,11 @@
#include "common/StateSaveAreaHeader.h"
#include <set>
#include <map>
using namespace NEO;
namespace NEO {
extern std::set<std::string> virtualFileList;
extern std::map<std::string, std::stringstream> virtualFileList;
}
struct RawBinarySipFixture : public DeviceWithoutSipFixture {

13
shared/test/unit_test/debug_settings/debug_settings_manager_tests.cpp
View File

@ -13,6 +13,7 @@
#include "shared/test/common/debug_settings/debug_settings_manager_fixture.h"
#include "shared/test/common/helpers/debug_manager_state_restore.h"
#include "shared/test/common/helpers/gtest_helpers.h"
#include "shared/test/common/helpers/mock_file_io.h"
#include "shared/test/common/helpers/variable_backup.h"
#include "shared/test/common/mocks/mock_io_functions.h"
#include "shared/test/common/mocks/mock_product_helper.h"
@ -402,21 +403,25 @@ TEST(DebugSettingsManager, GivenLogsEnabledAndDumpToFileWhenPrintDebuggerLogCall
auto logFile = NEO::fileLoggerInstance().getLogFileName();
std::remove(logFile);
if (virtualFileExists(logFile)) {
removeVirtualFile(logFile);
}
::testing::internal::CaptureStdout();
PRINT_DEBUGGER_LOG(stdout, "test %s", "log");
auto output = ::testing::internal::GetCapturedStdout();
EXPECT_EQ(0u, output.size());
auto logFileExists = fileExists(logFile);
auto logFileExists = virtualFileExists(logFile);
EXPECT_TRUE(logFileExists);
size_t retSize;
auto data = loadDataFromFile(logFile, retSize);
auto data = loadDataFromVirtualFile(logFile, retSize);
EXPECT_STREQ("test log", data.get());
std::remove(logFile);
if (virtualFileExists(logFile)) {
removeVirtualFile(logFile);
}
}
TEST(DebugSettingsManager, GivenLogsDisabledAndDumpToFileWhenPrintDebuggerLogCalledThenStringIsNotPrintedToFile) {

5
shared/test/unit_test/device_binary_format/device_binary_format_zebin_tests.cpp
View File

@ -8,7 +8,6 @@
#include "shared/source/device_binary_format/device_binary_formats.h"
#include "shared/source/device_binary_format/elf/elf.h"
#include "shared/source/device_binary_format/zebin/zebin_elf.h"
#include "shared/source/helpers/file_io.h"
#include "shared/source/helpers/hw_info.h"
#include "shared/source/helpers/ptr_math.h"
#include "shared/source/helpers/string.h"
@ -135,8 +134,8 @@ TEST(UnpackSingleDeviceBinaryZebin, givenDumpZEBinFlagSetWhenUnpackingZebinBinar
std::string unpackWarnings;
std::string fileName = "dumped_zebin_module.elf";
std::string fileNameInc = "dumped_zebin_module_0.elf";
EXPECT_FALSE(fileExists(fileName));
EXPECT_FALSE(fileExists(fileNameInc));
EXPECT_FALSE(virtualFileExists(fileName));
EXPECT_FALSE(virtualFileExists(fileNameInc));
NEO::unpackSingleDeviceBinary<NEO::DeviceBinaryFormat::zebin>(ArrayRef<const uint8_t>::fromAny(&zebin, 1U), "", targetDevice, unpackErrors, unpackWarnings);
EXPECT_TRUE(virtualFileExists(fileName));

35
shared/test/unit_test/helpers/file_io_tests.cpp
View File

@ -1,12 +1,12 @@
/*
* Copyright (C) 2018-2022 Intel Corporation
* Copyright (C) 2018-2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/helpers/file_io.h"
#include "shared/source/helpers/stdio.h"
#include "shared/test/common/helpers/mock_file_io.h"
#include "gtest/gtest.h"
@ -14,29 +14,30 @@
TEST(FileIO, GivenNonEmptyFileWhenCheckingIfHasSizeThenReturnTrue) {
std::string fileName("fileIO.bin");
std::remove(fileName.c_str());
ASSERT_FALSE(fileExists(fileName.c_str()));
if (virtualFileExists(fileName)) {
removeVirtualFile(fileName);
}
FILE *fp = nullptr;
fopen_s(&fp, fileName.c_str(), "wb");
ASSERT_NE(nullptr, fp);
fprintf(fp, "TEST");
fclose(fp);
ASSERT_FALSE(virtualFileExists(fileName.c_str()));
EXPECT_TRUE(fileExists(fileName.c_str()));
writeDataToFile(fileName.c_str(), "TEST", 4);
EXPECT_TRUE(virtualFileExists(fileName.c_str()));
EXPECT_TRUE(fileExistsHasSize(fileName.c_str()));
removeVirtualFile(fileName);
}
TEST(FileIO, GivenEmptyFileWhenCheckingIfHasSizeThenReturnFalse) {
std::string fileName("fileIO.bin");
std::remove(fileName.c_str());
ASSERT_FALSE(fileExists(fileName.c_str()));
if (virtualFileExists(fileName)) {
removeVirtualFile(fileName);
}
FILE *fp = nullptr;
fopen_s(&fp, fileName.c_str(), "wb");
ASSERT_NE(nullptr, fp);
fclose(fp);
ASSERT_FALSE(virtualFileExists(fileName.c_str()));
EXPECT_TRUE(fileExists(fileName.c_str()));
writeDataToFile(fileName.c_str(), "", 0);
EXPECT_TRUE(virtualFileExists(fileName.c_str()));
EXPECT_FALSE(fileExistsHasSize(fileName.c_str()));
removeVirtualFile(fileName);
}

4
shared/test/unit_test/os_interface/windows/file_logger_win_tests.cpp
View File

@ -25,7 +25,7 @@ TEST_F(FileLoggerTests, GivenLogAllocationMemoryPoolFlagThenLogsCorrectInfo) {
FullyEnabledFileLogger fileLogger(testFile, flags);
// Log file not created
bool logFileCreated = fileExists(fileLogger.getLogFileName());
bool logFileCreated = virtualFileExists(fileLogger.getLogFileName());
EXPECT_FALSE(logFileCreated);
auto executionEnvironment = std::unique_ptr<ExecutionEnvironment>(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u));
executionEnvironment->rootDeviceEnvironments[0]->initGmm();
@ -82,7 +82,7 @@ TEST_F(FileLoggerTests, GivenLogAllocationMemoryPoolFlagSetFalseThenAllocationIs
FullyEnabledFileLogger fileLogger(testFile, flags);
// Log file not created
bool logFileCreated = fileExists(fileLogger.getLogFileName());
bool logFileCreated = virtualFileExists(fileLogger.getLogFileName());
EXPECT_FALSE(logFileCreated);
auto executionEnvironment = std::unique_ptr<ExecutionEnvironment>(MockDevice::prepareExecutionEnvironment(defaultHwInfo.get(), 0u));

29
shared/test/unit_test/utilities/logger_tests.cpp
View File

@ -93,7 +93,7 @@ TEST(FileLogger, GivenDisabledDebugFunctinalityWhenLoggingApiCallsThenFileIsNotC
FullyDisabledFileLogger fileLogger(std::string(" "), flags);
// Log file not created
bool logFileCreated = fileExists(fileLogger.getLogFileName());
bool logFileCreated = virtualFileExists(fileLogger.getLogFileName());
EXPECT_FALSE(logFileCreated);
fileLogger.logApiCall("searchString", true, 0);
@ -109,22 +109,21 @@ TEST(FileLogger, GivenIncorrectFilenameFileWhenLoggingApiCallsThenFileIsNotCreat
DebugVariables flags;
flags.LogApiCalls.set(true);
FullyEnabledFileLogger fileLogger(std::string("test.log"), flags);
fileLogger.useRealFiles(true);
fileLogger.writeToFile("", "", 0, std::ios_base::in | std::ios_base::out);
EXPECT_FALSE(fileLogger.wasFileCreated(fileLogger.getLogFileName()));
EXPECT_FALSE(virtualFileExists(fileLogger.getLogFileName()));
}
TEST(FileLogger, GivenCorrectFilenameFileWhenLoggingApiCallsThenFileIsCreated) {
std::string testFile = "testfile";
DebugVariables flags;
FullyEnabledFileLogger fileLogger(testFile, flags);
fileLogger.useRealFiles(true);
fileLogger.writeToFile(testFile, "test", 4, std::fstream::out);
EXPECT_TRUE(fileExists(testFile));
if (fileExists(testFile)) {
std::remove(testFile.c_str());
EXPECT_TRUE(virtualFileExists(fileLogger.getLogFileName()));
if (virtualFileExists(fileLogger.getLogFileName())) {
removeVirtualFile(fileLogger.getLogFileName());
}
}
@ -133,12 +132,11 @@ TEST(FileLogger, GivenSameFileNameWhenCreatingNewInstanceThenOldFileIsRemoved) {
DebugVariables flags;
flags.LogApiCalls.set(true);
FullyEnabledFileLogger fileLogger(testFile, flags);
fileLogger.useRealFiles(true);
fileLogger.writeToFile(fileLogger.getLogFileName(), "test", 4, std::fstream::out);
EXPECT_TRUE(fileExists(fileLogger.getLogFileName()));
EXPECT_TRUE(virtualFileExists(fileLogger.getLogFileName()));
FullyEnabledFileLogger fileLogger2(testFile, flags);
EXPECT_FALSE(fileExists(fileLogger.getLogFileName()));
EXPECT_FALSE(virtualFileExists(fileLogger.getLogFileName()));
}
TEST(FileLogger, GivenSameFileNameWhenCreatingNewFullyDisabledLoggerThenOldFileIsNotRemoved) {
@ -146,13 +144,14 @@ TEST(FileLogger, GivenSameFileNameWhenCreatingNewFullyDisabledLoggerThenOldFileI
DebugVariables flags;
flags.LogApiCalls.set(true);
FullyEnabledFileLogger fileLogger(testFile, flags);
fileLogger.useRealFiles(true);
fileLogger.writeToFile(fileLogger.getLogFileName(), "test", 4, std::fstream::out);
EXPECT_TRUE(fileExists(fileLogger.getLogFileName()));
EXPECT_TRUE(virtualFileExists(fileLogger.getLogFileName()));
FullyDisabledFileLogger fileLogger2(testFile, flags);
EXPECT_TRUE(fileExists(fileLogger.getLogFileName()));
std::remove(fileLogger.getLogFileName());
EXPECT_TRUE(virtualFileExists(fileLogger.getLogFileName()));
if (virtualFileExists(fileLogger.getLogFileName())) {
removeVirtualFile(fileLogger.getLogFileName());
}
}
TEST(FileLogger, GivenFlagIsFalseWhenLoggingThenOnlyCustomLogsAreDumped) {
@ -163,7 +162,7 @@ TEST(FileLogger, GivenFlagIsFalseWhenLoggingThenOnlyCustomLogsAreDumped) {
FullyEnabledFileLogger fileLogger(testFile, flags);
// Log file not created
bool logFileCreated = fileExists(fileLogger.getLogFileName());
bool logFileCreated = virtualFileExists(fileLogger.getLogFileName());
EXPECT_FALSE(logFileCreated);
fileLogger.logApiCall("searchString", true, 0);
@ -554,7 +553,7 @@ TEST(AllocationTypeLoggingSingle, givenLogAllocationTypeWhenLoggingAllocationThe
GraphicsAllocation graphicsAllocation(0, 1u /*num gmms*/, AllocationType::commandBuffer, nullptr, 0, 0, MemoryPool::memoryNull, MemoryManager::maxOsContextCount, 0llu);
// Log file not created
bool logFileCreated = fileExists(fileLogger.getLogFileName());
bool logFileCreated = virtualFileExists(fileLogger.getLogFileName());
EXPECT_FALSE(logFileCreated);
testing::internal::CaptureStdout();