From 913a926fd4e7cbdf383b926d3ebefbe29830cdfb Mon Sep 17 00:00:00 2001 From: Compute-Runtime-Validation Date: Tue, 19 Sep 2023 13:53:41 +0200 Subject: [PATCH] Revert "feature: Optimize intra-module kernel ISA allocations" This reverts commit c3488314702e27a4521093bc4ed808425b040f85. Signed-off-by: Compute-Runtime-Validation --- level_zero/core/source/kernel/kernel.h | 18 +- level_zero/core/source/kernel/kernel_imp.cpp | 55 ++- level_zero/core/source/kernel/kernel_imp.h | 1 - level_zero/core/source/module/module_imp.cpp | 145 ++------ level_zero/core/source/module/module_imp.h | 9 +- .../unit_tests/fixtures/module_fixture.cpp | 27 +- .../test/unit_tests/fixtures/module_fixture.h | 17 +- .../core/test/unit_tests/mocks/CMakeLists.txt | 1 - .../test/unit_tests/mocks/mock_module.cpp | 26 -- .../core/test/unit_tests/mocks/mock_module.h | 18 - .../sources/cmdlist/test_cmdlist_5.cpp | 11 +- .../test_cmdlist_append_launch_kernel_2.cpp | 6 +- .../sources/debugger/l0_debugger_fixture.h | 7 - .../debugger/test_module_with_debug.cpp | 7 - .../unit_tests/sources/kernel/test_kernel.cpp | 111 ++++-- .../unit_tests/sources/module/test_module.cpp | 315 +----------------- .../command_encoder_bdw_and_later.inl | 2 +- .../command_encoder_xehp_and_later.inl | 2 +- shared/source/helpers/gfx_core_helper.h | 5 - .../dispatch_kernel_encoder_interface.h | 1 - .../encoders/test_encode_dispatch_kernel.cpp | 39 --- .../mock_dispatch_kernel_encoder_interface.h | 1 - 22 files changed, 176 insertions(+), 648 deletions(-) delete mode 100644 level_zero/core/test/unit_tests/mocks/mock_module.cpp diff --git a/level_zero/core/source/kernel/kernel.h b/level_zero/core/source/kernel/kernel.h index 19f82413e9..8f5059b8f2 100644 --- a/level_zero/core/source/kernel/kernel.h +++ b/level_zero/core/source/kernel/kernel.h @@ -35,9 +35,9 @@ struct KernelImmutableData { KernelImmutableData(L0::Device *l0device = nullptr); virtual ~KernelImmutableData(); - MOCKABLE_VIRTUAL ze_result_t initialize(NEO::KernelInfo *kernelInfo, Device *device, uint32_t computeUnitsUsedForSratch, - NEO::GraphicsAllocation *globalConstBuffer, NEO::GraphicsAllocation *globalVarBuffer, - bool internalKernel); + ze_result_t initialize(NEO::KernelInfo *kernelInfo, Device *device, uint32_t computeUnitsUsedForSratch, + NEO::GraphicsAllocation *globalConstBuffer, NEO::GraphicsAllocation *globalVarBuffer, + bool internalKernel); const std::vector &getResidencyContainer() const { return residencyContainer; @@ -48,14 +48,7 @@ struct KernelImmutableData { } uint32_t getIsaSize() const; - NEO::GraphicsAllocation *getIsaGraphicsAllocation() const; - void setIsaPerKernelAllocation(NEO::GraphicsAllocation *allocation); - inline NEO::GraphicsAllocation *getIsaParentAllocation() const { return isaParentAllocation; } - inline void setIsaParentAllocation(NEO::GraphicsAllocation *allocation) { isaParentAllocation = allocation; }; - inline size_t getIsaOffsetInParentAllocation() const { return isaSubAllocationOffset; } - inline void setIsaSubAllocationOffset(size_t offset) { isaSubAllocationOffset = offset; } - inline void setIsaSubAllocationSize(size_t size) { isaSubAllocationSize = size; } - inline size_t getIsaSubAllocationSize() const { return isaSubAllocationSize; } + NEO::GraphicsAllocation *getIsaGraphicsAllocation() const { return isaGraphicsAllocation.get(); } const uint8_t *getCrossThreadDataTemplate() const { return crossThreadDataTemplate.get(); } @@ -87,9 +80,6 @@ struct KernelImmutableData { NEO::KernelInfo *kernelInfo = nullptr; NEO::KernelDescriptor *kernelDescriptor = nullptr; std::unique_ptr isaGraphicsAllocation = nullptr; - NEO::GraphicsAllocation *isaParentAllocation = nullptr; - size_t isaSubAllocationOffset = 0lu; - size_t isaSubAllocationSize = 0lu; uint32_t crossThreadDataSize = 0; std::unique_ptr crossThreadDataTemplate = nullptr; diff --git a/level_zero/core/source/kernel/kernel_imp.cpp b/level_zero/core/source/kernel/kernel_imp.cpp index 83a3ad0db9..8375344b94 100644 --- a/level_zero/core/source/kernel/kernel_imp.cpp +++ b/level_zero/core/source/kernel/kernel_imp.cpp @@ -75,6 +75,20 @@ ze_result_t KernelImmutableData::initialize(NEO::KernelInfo *kernelInfo, Device DeviceImp *deviceImp = static_cast(device); auto neoDevice = deviceImp->getActiveDevice(); + auto memoryManager = neoDevice->getMemoryManager(); + + auto kernelIsaSize = kernelInfo->heapInfo.kernelHeapSize; + UNRECOVERABLE_IF(kernelIsaSize == 0); + UNRECOVERABLE_IF(!kernelInfo->heapInfo.pKernelHeap); + const auto allocType = internalKernel ? NEO::AllocationType::KERNEL_ISA_INTERNAL : NEO::AllocationType::KERNEL_ISA; + + auto allocation = memoryManager->allocateGraphicsMemoryWithProperties( + {neoDevice->getRootDeviceIndex(), kernelIsaSize, allocType, neoDevice->getDeviceBitfield()}); + if (!allocation) { + return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; + } + + isaGraphicsAllocation.reset(allocation); if (neoDevice->getDebugger() && kernelInfo->kernelDescriptor.external.debugData.get()) { createRelocatedDebugData(globalConstBuffer, globalVarBuffer); @@ -189,41 +203,18 @@ void KernelImmutableData::createRelocatedDebugData(NEO::GraphicsAllocation *glob } } -NEO::GraphicsAllocation *KernelImmutableData::getIsaGraphicsAllocation() const { - if (auto allocation = this->getIsaParentAllocation(); allocation != nullptr) { - DEBUG_BREAK_IF(this->device->getL0Debugger() != nullptr); - DEBUG_BREAK_IF(this->isaGraphicsAllocation != nullptr); - return allocation; - } else { - DEBUG_BREAK_IF(this->isaGraphicsAllocation.get() == nullptr); - return this->isaGraphicsAllocation.get(); - } -} - -uint32_t KernelImmutableData::getIsaSize() const { - if (this->getIsaParentAllocation()) { - DEBUG_BREAK_IF(this->device->getL0Debugger() != nullptr); - DEBUG_BREAK_IF(this->isaGraphicsAllocation != nullptr); - return static_cast(this->isaSubAllocationSize); - } else { - return static_cast(this->isaGraphicsAllocation->getUnderlyingBufferSize()); - } -} - -void KernelImmutableData::setIsaPerKernelAllocation(NEO::GraphicsAllocation *allocation) { - DEBUG_BREAK_IF(this->isaParentAllocation != nullptr); - this->isaGraphicsAllocation.reset(allocation); -} - ze_result_t KernelImp::getBaseAddress(uint64_t *baseAddress) { if (baseAddress) { auto gmmHelper = module->getDevice()->getNEODevice()->getGmmHelper(); - *baseAddress = gmmHelper->decanonize(this->kernelImmData->getIsaGraphicsAllocation()->getGpuAddress() + - this->kernelImmData->getIsaOffsetInParentAllocation()); + *baseAddress = gmmHelper->decanonize(this->kernelImmData->getIsaGraphicsAllocation()->getGpuAddress()); } return ZE_RESULT_SUCCESS; } +uint32_t KernelImmutableData::getIsaSize() const { + return static_cast(isaGraphicsAllocation->getUnderlyingBufferSize()); +} + KernelImp::KernelImp(Module *module) : module(module) {} KernelImp::~KernelImp() { @@ -957,12 +948,10 @@ ze_result_t KernelImp::initialize(const ze_kernel_desc_t *desc) { UNRECOVERABLE_IF(!this->kernelImmData->getKernelInfo()->heapInfo.pKernelHeap); if (isaAllocation->getAllocationType() == NEO::AllocationType::KERNEL_ISA_INTERNAL) { - isaAllocation->setTbxWritable(true, std::numeric_limits::max()); - isaAllocation->setAubWritable(true, std::numeric_limits::max()); NEO::MemoryTransferHelper::transferMemoryToAllocation(productHelper.isBlitCopyRequiredForLocalMemory(neoDevice->getRootDeviceEnvironment(), *isaAllocation), *neoDevice, isaAllocation, - this->kernelImmData->getIsaOffsetInParentAllocation(), + 0, this->kernelImmData->getKernelInfo()->heapInfo.pKernelHeap, static_cast(this->kernelImmData->getKernelInfo()->heapInfo.kernelHeapSize)); } @@ -1220,10 +1209,6 @@ NEO::GraphicsAllocation *KernelImp::getIsaAllocation() const { return getImmutableData()->getIsaGraphicsAllocation(); } -uint64_t KernelImp::getIsaOffsetInParentAllocation() const { - return static_cast(getImmutableData()->getIsaOffsetInParentAllocation()); -} - ze_result_t KernelImp::setSchedulingHintExp(ze_scheduling_hint_exp_desc_t *pHint) { auto &threadArbitrationPolicy = const_cast(getKernelDescriptor().kernelAttributes.threadArbitrationPolicy); if (pHint->flags == ZE_SCHEDULING_HINT_EXP_FLAG_OLDEST_FIRST) { diff --git a/level_zero/core/source/kernel/kernel_imp.h b/level_zero/core/source/kernel/kernel_imp.h index 4a90e1fbcc..e6ec3b0ab2 100644 --- a/level_zero/core/source/kernel/kernel_imp.h +++ b/level_zero/core/source/kernel/kernel_imp.h @@ -141,7 +141,6 @@ struct KernelImp : Kernel { } NEO::GraphicsAllocation *getIsaAllocation() const override; - uint64_t getIsaOffsetInParentAllocation() const override; uint32_t getRequiredWorkgroupOrder() const override { return requiredWorkgroupOrder; } bool requiresGenerationOfLocalIdsByRuntime() const override { return kernelRequiresGenerationOfLocalIdsByRuntime; } diff --git a/level_zero/core/source/module/module_imp.cpp b/level_zero/core/source/module/module_imp.cpp index bf137d8fd4..92e37d4778 100644 --- a/level_zero/core/source/module/module_imp.cpp +++ b/level_zero/core/source/module/module_imp.cpp @@ -32,7 +32,6 @@ #include "shared/source/helpers/kernel_helpers.h" #include "shared/source/helpers/string.h" #include "shared/source/kernel/kernel_descriptor.h" -#include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/memory_manager.h" #include "shared/source/memory_manager/memory_operations_handler.h" #include "shared/source/memory_manager/unified_memory_manager.h" @@ -495,19 +494,12 @@ void ModuleTranslationUnit::processDebugData() { ModuleImp::ModuleImp(Device *device, ModuleBuildLog *moduleBuildLog, ModuleType type) : device(device), translationUnit(std::make_unique(device)), - moduleBuildLog(moduleBuildLog), kernelsIsaParentRegion(nullptr), type(type) { - auto &gfxCoreHelper = device->getGfxCoreHelper(); - auto &hwInfo = device->getHwInfo(); - this->isaAllocationPageSize = gfxCoreHelper.useSystemMemoryPlacementForISA(hwInfo) ? MemoryConstants::pageSize : MemoryConstants::pageSize64k; - this->productFamily = hwInfo.platform.eProductFamily; + moduleBuildLog(moduleBuildLog), type(type) { + productFamily = device->getHwInfo().platform.eProductFamily; } ModuleImp::~ModuleImp() { - this->kernelImmDatas.clear(); - if (this->kernelsIsaParentRegion) { - DEBUG_BREAK_IF(this->device->getNEODevice()->getMemoryManager() == nullptr); - this->device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(this->kernelsIsaParentRegion.release()); - } + kernelImmDatas.clear(); } NEO::Zebin::Debug::Segments ModuleImp::getZebinSegments() { @@ -562,8 +554,7 @@ ze_result_t ModuleImp::initialize(const ze_module_desc_t *desc, NEO::Device *neo for (auto &ki : kernelImmDatas) { if (!ki->isIsaCopiedToAllocation()) { - ki->getIsaGraphicsAllocation()->setTbxWritable(true, std::numeric_limits::max()); - ki->getIsaGraphicsAllocation()->setAubWritable(true, std::numeric_limits::max()); + NEO::MemoryTransferHelper::transferMemoryToAllocation(productHelper.isBlitCopyRequiredForLocalMemory(rootDeviceEnvironment, *ki->getIsaGraphicsAllocation()), *neoDevice, ki->getIsaGraphicsAllocation(), 0, ki->getKernelInfo()->heapInfo.pKernelHeap, static_cast(ki->getKernelInfo()->heapInfo.kernelHeapSize)); @@ -696,98 +687,23 @@ inline ze_result_t ModuleImp::checkIfBuildShouldBeFailed(NEO::Device *neoDevice) return ZE_RESULT_SUCCESS; } -ze_result_t ModuleImp::initializeKernelImmutableDatas() { - if (size_t kernelsCount = this->translationUnit->programInfo.kernelInfos.size(); kernelsCount > 0lu) { - ze_result_t result; - if (result = this->allocateKernelImmutableDatas(kernelsCount); result != ZE_RESULT_SUCCESS) { +inline ze_result_t ModuleImp::initializeKernelImmutableDatas() { + ze_result_t result = ZE_RESULT_ERROR_MODULE_BUILD_FAILURE; + + kernelImmDatas.reserve(this->translationUnit->programInfo.kernelInfos.size()); + for (auto &ki : this->translationUnit->programInfo.kernelInfos) { + std::unique_ptr kernelImmData{new KernelImmutableData(this->device)}; + result = kernelImmData->initialize(ki, device, device->getNEODevice()->getDeviceInfo().computeUnitsUsedForScratch, + this->translationUnit->globalConstBuffer, this->translationUnit->globalVarBuffer, + this->type == ModuleType::Builtin); + if (result != ZE_RESULT_SUCCESS) { return result; } - for (size_t i = 0lu; i < kernelsCount; i++) { - result = kernelImmDatas[i]->initialize(this->translationUnit->programInfo.kernelInfos[i], - device, - device->getNEODevice()->getDeviceInfo().computeUnitsUsedForScratch, - this->translationUnit->globalConstBuffer, - this->translationUnit->globalVarBuffer, - this->type == ModuleType::Builtin); - if (result != ZE_RESULT_SUCCESS) { - kernelImmDatas[i].reset(); - return result; - } - } + kernelImmDatas.push_back(std::move(kernelImmData)); } return ZE_RESULT_SUCCESS; } -ze_result_t ModuleImp::allocateKernelImmutableDatas(size_t kernelsCount) { - if (this->kernelImmDatas.size() == kernelsCount) { - return ZE_RESULT_SUCCESS; - } - - this->kernelImmDatas.reserve(kernelsCount); - for (size_t i = 0lu; i < kernelsCount; i++) { - this->kernelImmDatas.emplace_back(new KernelImmutableData(this->device)); - } - return this->setIsaGraphicsAllocations(); -} - -ze_result_t ModuleImp::setIsaGraphicsAllocations() { - size_t kernelsCount = this->kernelImmDatas.size(); - - auto kernelsChunks = std::vector>(kernelsCount); - size_t kernelsIsaTotalSize = 0lu; - for (auto i = 0lu; i < kernelsCount; i++) { - auto kernelInfo = this->translationUnit->programInfo.kernelInfos[i]; - DEBUG_BREAK_IF(kernelInfo->heapInfo.kernelHeapSize == 0lu); - DEBUG_BREAK_IF(!kernelInfo->heapInfo.pKernelHeap); - auto chunkOffset = kernelsIsaTotalSize; - auto chunkSize = this->computeKernelIsaAllocationAlignedSizeWithPadding(kernelInfo->heapInfo.kernelHeapSize); - kernelsIsaTotalSize += chunkSize; - kernelsChunks[i] = {chunkOffset, chunkSize}; - } - - bool debuggerDisabled = (this->device->getL0Debugger() == nullptr); - if (debuggerDisabled && kernelsIsaTotalSize <= isaAllocationPageSize) { - if (auto allocation = this->allocateKernelsIsaMemory(kernelsIsaTotalSize); allocation == nullptr) { - return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; - } else { - this->kernelsIsaParentRegion.reset(allocation); - } - - for (auto i = 0lu; i < kernelsCount; i++) { - auto [isaOffset, isaSize] = kernelsChunks[i]; - this->kernelImmDatas[i]->setIsaParentAllocation(this->kernelsIsaParentRegion.get()); - this->kernelImmDatas[i]->setIsaSubAllocationOffset(isaOffset); - this->kernelImmDatas[i]->setIsaSubAllocationSize(isaSize); - } - } else { - for (auto i = 0lu; i < kernelsCount; i++) { - auto kernelInfo = this->translationUnit->programInfo.kernelInfos[i]; - if (auto allocation = this->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize); allocation == nullptr) { - return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY; - } else { - this->kernelImmDatas[i]->setIsaPerKernelAllocation(allocation); - } - } - } - return ZE_RESULT_SUCCESS; -} - -size_t ModuleImp::computeKernelIsaAllocationAlignedSizeWithPadding(size_t isaSize) { - auto isaPadding = this->device->getGfxCoreHelper().getPaddingForISAAllocation(); - auto kernelStartPointerAlignment = this->device->getGfxCoreHelper().getKernelIsaPointerAlignment(); - auto isaAllocationSize = alignUp(isaPadding + isaSize, kernelStartPointerAlignment); - return isaAllocationSize; -} - -NEO::GraphicsAllocation *ModuleImp::allocateKernelsIsaMemory(size_t size) { - auto allocType = (this->type == ModuleType::Builtin ? NEO::AllocationType::KERNEL_ISA_INTERNAL : NEO::AllocationType::KERNEL_ISA); - auto neoDevice = this->device->getNEODevice(); - return neoDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties({neoDevice->getRootDeviceIndex(), - size, - allocType, - neoDevice->getDeviceBitfield()}); -} - void ModuleImp::createDebugZebin() { auto refBin = ArrayRef::fromAny(translationUnit->unpackedDeviceBinary.get(), translationUnit->unpackedDeviceBinarySize); auto segments = getZebinSegments(); @@ -964,10 +880,7 @@ void ModuleImp::copyPatchedSegments(const NEO::Linker::PatchableSegments &isaSeg auto segmentId = &kernelImmData - &this->kernelImmDatas[0]; NEO::MemoryTransferHelper::transferMemoryToAllocation(productHelper.isBlitCopyRequiredForLocalMemory(rootDeviceEnvironment, *kernelImmData->getIsaGraphicsAllocation()), - *device->getNEODevice(), - kernelImmData->getIsaGraphicsAllocation(), - kernelImmData->getIsaOffsetInParentAllocation(), - isaSegmentsForPatching[segmentId].hostPointer, + *device->getNEODevice(), kernelImmData->getIsaGraphicsAllocation(), 0, isaSegmentsForPatching[segmentId].hostPointer, isaSegmentsForPatching[segmentId].segmentSize); kernelImmData->setIsaCopiedToAllocation(); @@ -1012,9 +925,8 @@ bool ModuleImp::linkBinary() { if (linkerInput->getExportedFunctionsSegmentId() >= 0) { auto exportedFunctionHeapId = linkerInput->getExportedFunctionsSegmentId(); this->exportedFunctionsSurface = this->kernelImmDatas[exportedFunctionHeapId]->getIsaGraphicsAllocation(); - auto offsetInParentAllocation = this->kernelImmDatas[exportedFunctionHeapId]->getIsaOffsetInParentAllocation(); - exportedFunctions.gpuAddress = static_cast(exportedFunctionsSurface->getGpuAddressToPatch() + offsetInParentAllocation); - exportedFunctions.segmentSize = this->kernelImmDatas[exportedFunctionHeapId]->getIsaSize(); + exportedFunctions.gpuAddress = static_cast(exportedFunctionsSurface->getGpuAddressToPatch()); + exportedFunctions.segmentSize = exportedFunctionsSurface->getUnderlyingBufferSize(); } Linker::KernelDescriptorsT kernelDescriptors; @@ -1026,9 +938,7 @@ bool ModuleImp::linkBinary() { auto &kernHeapInfo = kernelInfo->heapInfo; const char *originalIsa = reinterpret_cast(kernHeapInfo.pKernelHeap); patchedIsaTempStorage.push_back(std::vector(originalIsa, originalIsa + kernHeapInfo.kernelHeapSize)); - auto isaAddressToPatch = static_cast(kernelImmDatas.at(i)->getIsaGraphicsAllocation()->getGpuAddressToPatch() + - kernelImmDatas.at(i)->getIsaOffsetInParentAllocation()); - isaSegmentsForPatching.push_back(Linker::PatchableSegment{patchedIsaTempStorage.rbegin()->data(), isaAddressToPatch, kernHeapInfo.kernelHeapSize}); + isaSegmentsForPatching.push_back(Linker::PatchableSegment{patchedIsaTempStorage.rbegin()->data(), static_cast(kernelImmDatas.at(i)->getIsaGraphicsAllocation()->getGpuAddressToPatch()), kernHeapInfo.kernelHeapSize}); kernelDescriptors.push_back(&kernelInfo->kernelDescriptor); } } @@ -1091,10 +1001,10 @@ ze_result_t ModuleImp::getFunctionPointer(const char *pFunctionName, void **pfnF auto kernelImmData = this->getKernelImmutableData(pFunctionName); if (kernelImmData != nullptr) { auto isaAllocation = kernelImmData->getIsaGraphicsAllocation(); - *pfnFunction = reinterpret_cast(isaAllocation->getGpuAddress() + kernelImmData->getIsaOffsetInParentAllocation()); + *pfnFunction = reinterpret_cast(isaAllocation->getGpuAddress()); // Ensure that any kernel in this module which uses this kernel module function pointer has access to the memory. for (auto &data : this->getKernelImmutableDataVector()) { - if (data.get() != kernelImmData && data.get()->getIsaOffsetInParentAllocation() == 0lu) { + if (data.get() != kernelImmData) { data.get()->getResidencyContainer().insert(data.get()->getResidencyContainer().end(), isaAllocation); } } @@ -1345,9 +1255,7 @@ ze_result_t ModuleImp::performDynamicLink(uint32_t numModules, auto &kernHeapInfo = kernelInfo->heapInfo; const char *originalIsa = reinterpret_cast(kernHeapInfo.pKernelHeap); patchedIsaTempStorage.push_back(std::vector(originalIsa, originalIsa + kernHeapInfo.kernelHeapSize)); - auto isaAddressToPatch = static_cast(kernelImmDatas.at(i)->getIsaGraphicsAllocation()->getGpuAddressToPatch() + - kernelImmDatas.at(i)->getIsaOffsetInParentAllocation()); - isaSegmentsForPatching.push_back(NEO::Linker::PatchableSegment{patchedIsaTempStorage.rbegin()->data(), isaAddressToPatch, kernHeapInfo.kernelHeapSize}); + isaSegmentsForPatching.push_back(NEO::Linker::PatchableSegment{patchedIsaTempStorage.rbegin()->data(), static_cast(kernelImmDatas.at(i)->getIsaGraphicsAllocation()->getGpuAddressToPatch()), kernHeapInfo.kernelHeapSize}); } } for (const auto &unresolvedExternal : moduleId->unresolvedExternalsInfo) { @@ -1567,13 +1475,8 @@ void ModuleImp::notifyModuleDestroy() { StackVec ModuleImp::getModuleAllocations() { StackVec allocs; - if (auto isaParentAllocation = this->getKernelsIsaParentAllocation(); isaParentAllocation != nullptr) { - allocs.push_back(isaParentAllocation); - } else { - // ISA allocations not optimized - for (auto &kernImmData : kernelImmDatas) { - allocs.push_back(kernImmData->getIsaGraphicsAllocation()); - } + for (auto &kernImmData : kernelImmDatas) { + allocs.push_back(kernImmData->getIsaGraphicsAllocation()); } if (translationUnit) { diff --git a/level_zero/core/source/module/module_imp.h b/level_zero/core/source/module/module_imp.h index 13c025f686..d9a6909bd0 100644 --- a/level_zero/core/source/module/module_imp.h +++ b/level_zero/core/source/module/module_imp.h @@ -124,7 +124,6 @@ struct ModuleImp : public Module { const KernelImmutableData *getKernelImmutableData(const char *kernelName) const override; const std::vector> &getKernelImmutableDataVector() const override { return kernelImmDatas; } - NEO::GraphicsAllocation *getKernelsIsaParentAllocation() const { return kernelsIsaParentRegion.get(); } uint32_t getMaxGroupSize(const NEO::KernelDescriptor &kernelDescriptor) const override; @@ -156,9 +155,8 @@ struct ModuleImp : public Module { } protected: - MOCKABLE_VIRTUAL ze_result_t initializeTranslationUnit(const ze_module_desc_t *desc, NEO::Device *neoDevice); + ze_result_t initializeTranslationUnit(const ze_module_desc_t *desc, NEO::Device *neoDevice); ze_result_t checkIfBuildShouldBeFailed(NEO::Device *neoDevice); - ze_result_t allocateKernelImmutableDatas(size_t kernelsCount); ze_result_t initializeKernelImmutableDatas(); void copyPatchedSegments(const NEO::Linker::PatchableSegments &isaSegmentsForPatching); void verifyDebugCapabilities(); @@ -169,9 +167,6 @@ struct ModuleImp : public Module { void notifyModuleCreate(); void notifyModuleDestroy(); bool populateHostGlobalSymbolsMap(std::unordered_map &devToHostNameMapping); - ze_result_t setIsaGraphicsAllocations(); - MOCKABLE_VIRTUAL size_t computeKernelIsaAllocationAlignedSizeWithPadding(size_t isaSize); - MOCKABLE_VIRTUAL NEO::GraphicsAllocation *allocateKernelsIsaMemory(size_t size); StackVec getModuleAllocations(); Device *device = nullptr; @@ -179,7 +174,6 @@ struct ModuleImp : public Module { std::unique_ptr translationUnit; ModuleBuildLog *moduleBuildLog = nullptr; NEO::GraphicsAllocation *exportedFunctionsSurface = nullptr; - std::unique_ptr kernelsIsaParentRegion; std::vector> kernelImmDatas; NEO::Linker::RelocatedSymbolsMap symbols; @@ -204,7 +198,6 @@ struct ModuleImp : public Module { uint32_t debugElfHandle = 0; uint32_t profileFlags = 0; uint64_t moduleLoadAddress = std::numeric_limits::max(); - size_t isaAllocationPageSize = 0; NEO::Linker::PatchableSegments isaSegmentsForPatching; std::vector> patchedIsaTempStorage; diff --git a/level_zero/core/test/unit_tests/fixtures/module_fixture.cpp b/level_zero/core/test/unit_tests/fixtures/module_fixture.cpp index aaa997174f..a062219ba2 100644 --- a/level_zero/core/test/unit_tests/fixtures/module_fixture.cpp +++ b/level_zero/core/test/unit_tests/fixtures/module_fixture.cpp @@ -124,13 +124,24 @@ void ModuleImmutableDataFixture::tearDown() { DeviceFixture::tearDown(); } -void ModuleFixture::setUp(bool skipCreatingModules) { +ModuleFixture::ProxyModuleImp *ModuleFixture::ProxyModuleImp::create(L0::Device *device, const ze_module_desc_t *desc, + ModuleBuildLog *moduleBuildLog, ModuleType type, ze_result_t *result) { + auto module = new ProxyModuleImp(device, moduleBuildLog, type); + + *result = module->initialize(desc, device->getNEODevice()); + if (*result != ZE_RESULT_SUCCESS) { + module->destroy(); + return nullptr; + } + + return module; +} + +void ModuleFixture::setUp() { DebugManager.flags.FailBuildProgramWithStatefulAccess.set(0); DeviceFixture::setUp(); - if (skipCreatingModules == false) { - createModuleFromMockBinary(); - } + createModuleFromMockBinary(); } void ModuleFixture::createModuleFromMockBinary(ModuleType type) { @@ -144,13 +155,7 @@ void ModuleFixture::createModuleFromMockBinary(ModuleType type) { ModuleBuildLog *moduleBuildLog = nullptr; ze_result_t result = ZE_RESULT_SUCCESS; - if (!module) { - module.reset(new WhiteBox<::L0::Module>{device, moduleBuildLog, type}); - } - result = module->initialize(&moduleDesc, device->getNEODevice()); - if (result != ZE_RESULT_SUCCESS) { - module->destroy(); - } + module.reset(ProxyModuleImp::create(device, &moduleDesc, moduleBuildLog, type, &result)); } void ModuleFixture::createKernel() { diff --git a/level_zero/core/test/unit_tests/fixtures/module_fixture.h b/level_zero/core/test/unit_tests/fixtures/module_fixture.h index 700762ca90..7b7252152b 100644 --- a/level_zero/core/test/unit_tests/fixtures/module_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/module_fixture.h @@ -123,8 +123,19 @@ struct ModuleImmutableDataFixture : public DeviceFixture { struct ModuleFixture : public DeviceFixture { - void setUp(bool skipCreatingModules); - void setUp() { this->setUp(false); } + struct ProxyModuleImp : public WhiteBox<::L0::Module> { + using BaseClass = WhiteBox<::L0::Module>; + using BaseClass::BaseClass; + + std::vector> &getKernelImmDatas() { + return kernelImmDatas; + } + + static ModuleFixture::ProxyModuleImp *create(L0::Device *device, const ze_module_desc_t *desc, + ModuleBuildLog *moduleBuildLog, ModuleType type, ze_result_t *result); + }; + + void setUp(); void createModuleFromMockBinary(ModuleType type = ModuleType::User); @@ -136,7 +147,7 @@ struct ModuleFixture : public DeviceFixture { const std::string kernelName = "test"; const uint32_t numKernelArguments = 6; - std::unique_ptr> module; + std::unique_ptr module; std::unique_ptr> kernel; std::unique_ptr zebinData; DebugManagerStateRestore restore; diff --git a/level_zero/core/test/unit_tests/mocks/CMakeLists.txt b/level_zero/core/test/unit_tests/mocks/CMakeLists.txt index 4e41a79419..862a2bff47 100644 --- a/level_zero/core/test/unit_tests/mocks/CMakeLists.txt +++ b/level_zero/core/test/unit_tests/mocks/CMakeLists.txt @@ -34,7 +34,6 @@ set(L0_MOCKS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/mock_kernel.h ${CMAKE_CURRENT_SOURCE_DIR}/mock_memory_manager.h ${CMAKE_CURRENT_SOURCE_DIR}/mock_module.h - ${CMAKE_CURRENT_SOURCE_DIR}/mock_module.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mock_sampler.h ) diff --git a/level_zero/core/test/unit_tests/mocks/mock_module.cpp b/level_zero/core/test/unit_tests/mocks/mock_module.cpp deleted file mode 100644 index 6895e3140a..0000000000 --- a/level_zero/core/test/unit_tests/mocks/mock_module.cpp +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2023 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "level_zero/core/test/unit_tests/mocks/mock_module.h" - -#include "level_zero/core/source/device/device.h" - -namespace L0 { -namespace ult { - -ze_result_t WhiteBox<::L0::Module>::initializeTranslationUnit(const ze_module_desc_t *desc, NEO::Device *neoDevice) { - auto result = this->BaseClass::initializeTranslationUnit(desc, neoDevice); - if (this->mockGlobalConstBuffer) { - this->translationUnit->globalConstBuffer = this->mockGlobalConstBuffer; - } - if (this->mockGlobalVarBuffer) { - this->translationUnit->globalVarBuffer = this->mockGlobalVarBuffer; - } - return result; -} -} // namespace ult -} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/mocks/mock_module.h b/level_zero/core/test/unit_tests/mocks/mock_module.h index 158af07a62..de70f6e007 100644 --- a/level_zero/core/test/unit_tests/mocks/mock_module.h +++ b/level_zero/core/test/unit_tests/mocks/mock_module.h @@ -13,7 +13,6 @@ #include "shared/test/common/test_macros/mock_method_macros.h" #include "level_zero/core/source/module/module_imp.h" -#include "level_zero/core/test/unit_tests/mock.h" #include "level_zero/core/test/unit_tests/white_box.h" #include "gtest/gtest.h" @@ -49,7 +48,6 @@ constexpr inline MockModuleTranslationUnit *toMockPtr(L0::ModuleTranslationUnit template <> struct WhiteBox<::L0::Module> : public ::L0::ModuleImp { using BaseClass = ::L0::ModuleImp; - using BaseClass::allocateKernelsIsaMemory; using BaseClass::allocatePrivateMemoryPerDispatch; using BaseClass::BaseClass; using BaseClass::builtFromSPIRv; @@ -63,7 +61,6 @@ struct WhiteBox<::L0::Module> : public ::L0::ModuleImp { using BaseClass::isFunctionSymbolExportEnabled; using BaseClass::isGlobalSymbolExportEnabled; using BaseClass::kernelImmDatas; - using BaseClass::setIsaGraphicsAllocations; using BaseClass::symbols; using BaseClass::translationUnit; using BaseClass::type; @@ -73,11 +70,6 @@ struct WhiteBox<::L0::Module> : public ::L0::ModuleImp { : ::L0::ModuleImp{device, moduleBuildLog, type} { this->translationUnit.reset(new MockModuleTranslationUnit{device}); } - - ze_result_t initializeTranslationUnit(const ze_module_desc_t *desc, NEO::Device *neoDevice) override; - - NEO::GraphicsAllocation *mockGlobalVarBuffer = nullptr; - NEO::GraphicsAllocation *mockGlobalConstBuffer = nullptr; }; using Module = WhiteBox<::L0::Module>; @@ -99,23 +91,15 @@ struct Mock : public Module { (uint32_t numModules, ze_module_handle_t *phModules, ze_module_build_log_handle_t *phLinkLog)); ADDMETHOD_NOBASE(getProperties, ze_result_t, ZE_RESULT_SUCCESS, (ze_module_properties_t * pModuleProperties)); ADDMETHOD_NOBASE(getGlobalPointer, ze_result_t, ZE_RESULT_SUCCESS, (const char *pGlobalName, size_t *pSize, void **pPtr)); - ADDMETHOD(allocateKernelsIsaMemory, NEO::GraphicsAllocation *, true, nullptr, (size_t isaSize), (isaSize)); - ADDMETHOD(computeKernelIsaAllocationAlignedSizeWithPadding, size_t, true, 0ul, (size_t isaSize), (isaSize)); }; struct MockModule : public L0::ModuleImp { - using ModuleImp::allocateKernelImmutableDatas; - using ModuleImp::allocateKernelsIsaMemory; - using ModuleImp::computeKernelIsaAllocationAlignedSizeWithPadding; using ModuleImp::debugModuleHandle; using ModuleImp::getModuleAllocations; - using ModuleImp::initializeKernelImmutableDatas; - using ModuleImp::isaAllocationPageSize; using ModuleImp::isFunctionSymbolExportEnabled; using ModuleImp::isGlobalSymbolExportEnabled; using ModuleImp::kernelImmDatas; using ModuleImp::populateHostGlobalSymbolsMap; - using ModuleImp::setIsaGraphicsAllocations; using ModuleImp::symbols; using ModuleImp::translationUnit; @@ -131,8 +115,6 @@ struct MockModule : public L0::ModuleImp { return kernelImmData; } - std::vector> &getKernelImmutableDataVectorRef() { return kernelImmDatas; } - KernelImmutableData *kernelImmData = nullptr; }; diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp index 625f7f4146..109e3401ee 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_5.cpp @@ -457,17 +457,11 @@ HWTEST2_F(AppendQueryKernelTimestamps, givenCommandListWhenAppendQueryKernelTime HWTEST2_F(AppendQueryKernelTimestamps, givenEventWhenAppendQueryIsCalledThenSetAllEventData, IsAtLeastSkl) { class MockQueryKernelTimestampsKernel : public L0::KernelImp { public: - MockQueryKernelTimestampsKernel(MockModule *module) : KernelImp{module}, module{module} { + MockQueryKernelTimestampsKernel(L0::Module *module) : KernelImp(module) { mockKernelImmutableData.kernelDescriptor = &mockKernelDescriptor; - size_t stubKernelHeapSize = 42; - mockKernelImmutableData.setIsaPerKernelAllocation(module->allocateKernelsIsaMemory(stubKernelHeapSize)); this->kernelImmData = &mockKernelImmutableData; } - ~MockQueryKernelTimestampsKernel() override { - this->module->getDevice()->getNEODevice()->getMemoryManager()->freeGraphicsMemory(mockKernelImmutableData.isaGraphicsAllocation.release()); - } - ze_result_t setArgBufferWithAlloc(uint32_t argIndex, uintptr_t argVal, NEO::GraphicsAllocation *allocation, NEO::SvmAllocationData *peerAllocData) override { if (argIndex == 0) { index0Allocation = allocation; @@ -486,13 +480,12 @@ HWTEST2_F(AppendQueryKernelTimestamps, givenEventWhenAppendQueryIsCalledThenSetA NEO::GraphicsAllocation *index0Allocation = nullptr; KernelDescriptor mockKernelDescriptor = {}; WhiteBox<::L0::KernelImmutableData> mockKernelImmutableData = {}; - MockModule *module = nullptr; }; struct MockBuiltinFunctionsForQueryKernelTimestamps : BuiltinFunctionsLibImpl { MockBuiltinFunctionsForQueryKernelTimestamps(L0::Device *device, NEO::BuiltIns *builtInsLib) : BuiltinFunctionsLibImpl(device, builtInsLib) { tmpModule = std::make_unique(device, nullptr, ModuleType::Builtin); - tmpMockKernel = std::make_unique(tmpModule.get()); + tmpMockKernel = std::make_unique(static_cast(tmpModule.get())); } MockQueryKernelTimestampsKernel *getFunction(Builtin func) override { return tmpMockKernel.get(); diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_2.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_2.cpp index f350d3d602..06a7f3121f 100644 --- a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_2.cpp +++ b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_2.cpp @@ -1382,7 +1382,8 @@ HWTEST2_F(CommandListAppendLaunchKernel, givenTwoKernelPrivateAllocsWhichTogethe auto overAllocMinSize = static_cast(devInfo.globalMemSize / kernelsNb / devInfo.computeUnitsUsedForScratch) + margin1KB; auto kernelNames = std::array{"test1", "test2"}; - auto &kernelImmDatas = this->module->kernelImmDatas; + auto proxyModuleImpl = static_cast(this->module.get()); + auto &kernelImmDatas = proxyModuleImpl->getKernelImmDatas(); for (size_t i = 0; i < kernelsNb; i++) { auto &kernelDesc = const_cast(kernelImmDatas[i]->getDescriptor()); kernelDesc.kernelAttributes.perHwThreadPrivateMemorySize = overAllocMinSize + static_cast(i * MemoryConstants::cacheLineSize); @@ -1418,7 +1419,8 @@ HWTEST2_F(CommandListAppendLaunchKernel, givenTwoKernelPrivateAllocsWhichDontExc auto underAllocSize = static_cast(devInfo.globalMemSize / kernelsNb / devInfo.computeUnitsUsedForScratch) - margin128KB; auto kernelNames = std::array{"test1", "test2"}; - auto &kernelImmDatas = this->module->kernelImmDatas; + auto proxyModuleImpl = static_cast(this->module.get()); + auto &kernelImmDatas = proxyModuleImpl->getKernelImmDatas(); for (size_t i = 0; i < kernelsNb; i++) { auto &kernelDesc = const_cast(kernelImmDatas[i]->getDescriptor()); kernelDesc.kernelAttributes.perHwThreadPrivateMemorySize = underAllocSize; diff --git a/level_zero/core/test/unit_tests/sources/debugger/l0_debugger_fixture.h b/level_zero/core/test/unit_tests/sources/debugger/l0_debugger_fixture.h index 7a871f2698..be6c0f6f96 100644 --- a/level_zero/core/test/unit_tests/sources/debugger/l0_debugger_fixture.h +++ b/level_zero/core/test/unit_tests/sources/debugger/l0_debugger_fixture.h @@ -8,7 +8,6 @@ #pragma once #include "shared/source/helpers/gfx_core_helper.h" -#include "shared/source/memory_manager/allocation_properties.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_builtins.h" #include "shared/test/common/mocks/mock_compilers.h" @@ -85,12 +84,6 @@ struct L0DebuggerHwFixture : public L0DebuggerFixture { MockDebuggerL0Hw *getMockDebuggerL0Hw() { return static_cast *>(debuggerHw); } - - NEO::GraphicsAllocation *allocateIsaMemory(size_t size, bool internal) { - auto allocType = (internal ? NEO::AllocationType::KERNEL_ISA_INTERNAL : NEO::AllocationType::KERNEL_ISA); - return neoDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties({neoDevice->getRootDeviceIndex(), size, allocType, neoDevice->getDeviceBitfield()}); - } - DebuggerL0 *debuggerHw = nullptr; }; diff --git a/level_zero/core/test/unit_tests/sources/debugger/test_module_with_debug.cpp b/level_zero/core/test/unit_tests/sources/debugger/test_module_with_debug.cpp index 66cb7a0469..9c9fb7fa86 100644 --- a/level_zero/core/test/unit_tests/sources/debugger/test_module_with_debug.cpp +++ b/level_zero/core/test/unit_tests/sources/debugger/test_module_with_debug.cpp @@ -60,7 +60,6 @@ TEST_F(KernelInitializeTest, givenDebuggingEnabledWhenKernelsAreInitializedThenA kernelInfo.heapInfo.pKernelHeap = &kernelHeap; KernelImmutableData kernelImmutableData(device); - kernelImmutableData.setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo.heapInfo.kernelHeapSize, false)); memoryOperationsHandler->makeResidentCalledCount = 0; kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, false); @@ -328,14 +327,12 @@ HWTEST_F(ModuleWithZebinAndL0DebuggerTest, GivenZebinDebugDataWhenInitializingMo kernelInfo->kernelDescriptor.kernelMetadata.kernelName = ZebinTestData::ValidEmptyProgram<>::kernelName; auto kernelImmutableData = ::std::make_unique(device); - kernelImmutableData->setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo->heapInfo.kernelHeapSize, false)); kernelImmutableData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); std::unique_ptr moduleMock = std::make_unique(device, nullptr, ModuleType::User); moduleMock->translationUnit = std::make_unique(device); moduleMock->kernelImmDatas.push_back(std::move(kernelImmutableData)); kernelImmutableData = ::std::make_unique(device); - kernelImmutableData->setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo->heapInfo.kernelHeapSize, false)); kernelImmutableData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); moduleMock->kernelImmDatas.push_back(std::move(kernelImmutableData)); @@ -374,12 +371,10 @@ HWTEST_F(ModuleWithZebinAndL0DebuggerTest, GivenDumpElfFlagAndZebinWhenInitializ kernelInfo->kernelDescriptor.kernelMetadata.kernelName = ZebinTestData::ValidEmptyProgram<>::kernelName; auto kernelImmutableData = ::std::make_unique(device); - kernelImmutableData->setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo->heapInfo.kernelHeapSize, false)); kernelImmutableData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); moduleMock->kernelImmDatas.push_back(std::move(kernelImmutableData)); kernelImmutableData = ::std::make_unique(device); - kernelImmutableData->setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo->heapInfo.kernelHeapSize, false)); kernelImmutableData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); moduleMock->kernelImmDatas.push_back(std::move(kernelImmutableData)); @@ -438,7 +433,6 @@ HWTEST_F(ModuleWithZebinAndL0DebuggerTest, GivenZebinWhenModuleIsInitializedAndD kernelInfo->kernelDescriptor.kernelMetadata.kernelName = ZebinTestData::ValidEmptyProgram<>::kernelName; auto kernelImmutableData = ::std::make_unique(device); - kernelImmutableData->setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo->heapInfo.kernelHeapSize, false)); kernelImmutableData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); std::unique_ptr moduleMock = std::make_unique(device, nullptr, ModuleType::User); moduleMock->translationUnit = std::make_unique(device); @@ -573,7 +567,6 @@ HWTEST_F(ModuleWithZebinAndL0DebuggerTest, GivenModuleDebugHandleZeroWhenInitial kernelInfo->kernelDescriptor.kernelMetadata.kernelName = ZebinTestData::ValidEmptyProgram<>::kernelName; auto kernelImmutableData = ::std::make_unique(device); - kernelImmutableData->setIsaPerKernelAllocation(this->allocateIsaMemory(kernelInfo->heapInfo.kernelHeapSize, false)); kernelImmutableData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); std::unique_ptr moduleMock = std::make_unique(device, nullptr, ModuleType::User); moduleMock->kernelImmDatas.push_back(std::move(kernelImmutableData)); diff --git a/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp b/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp index 3d8b7dff7e..e78ce84640 100644 --- a/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp +++ b/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp @@ -1865,9 +1865,9 @@ TEST_F(KernelLocalIdsTest, WhenKernelIsCreatedThenDefaultLocalIdGenerationbyRunt EXPECT_TRUE(kernel->requiresGenerationOfLocalIdsByRuntime()); } -struct KernelIsaFixture : ModuleFixture { - void setUp() { - ModuleFixture::setUp(true); +struct KernelIsaTests : Test { + void SetUp() override { + Test::SetUp(); auto &capabilityTable = device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->capabilityTable; bool createBcsEngine = !capabilityTable.blitterOperationsSupported; @@ -1883,98 +1883,137 @@ struct KernelIsaFixture : ModuleFixture { } std::unique_ptr bcsOsContext; - uint32_t testKernelHeap = 0; }; -using KernelIsaTests = Test; - TEST_F(KernelIsaTests, givenKernelAllocationInLocalMemoryWhenCreatingWithoutAllowedCpuAccessThenUseBcsForTransfer) { DebugManagerStateRestore restore; DebugManager.flags.ForceLocalMemoryAccessMode.set(static_cast(LocalMemoryAccessMode::CpuAccessDisallowed)); DebugManager.flags.ForceNonSystemMemoryPlacement.set(1 << (static_cast(NEO::AllocationType::KERNEL_ISA) - 1)); - this->createModuleFromMockBinary(ModuleType::User); + + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; + + KernelImmutableData kernelImmutableData(device); auto bcsCsr = device->getNEODevice()->getEngine(aub_stream::EngineType::ENGINE_BCS, EngineUsage::Regular).commandStreamReceiver; auto initialTaskCount = bcsCsr->peekTaskCount(); - auto &kernelImmutableData = this->module->kernelImmDatas.back(); - if (kernelImmutableData->getIsaGraphicsAllocation()->isAllocatedInLocalMemoryPool()) { + kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, false); + + if (kernelImmutableData.getIsaGraphicsAllocation()->isAllocatedInLocalMemoryPool()) { EXPECT_EQ(initialTaskCount + 1, bcsCsr->peekTaskCount()); } else { EXPECT_EQ(initialTaskCount, bcsCsr->peekTaskCount()); } + + device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(kernelInfo.kernelAllocation); } TEST_F(KernelIsaTests, givenKernelAllocationInLocalMemoryWhenCreatingWithAllowedCpuAccessThenDontUseBcsForTransfer) { DebugManagerStateRestore restore; DebugManager.flags.ForceLocalMemoryAccessMode.set(static_cast(LocalMemoryAccessMode::CpuAccessAllowed)); DebugManager.flags.ForceNonSystemMemoryPlacement.set(1 << (static_cast(NEO::AllocationType::KERNEL_ISA) - 1)); - this->createModuleFromMockBinary(ModuleType::User); + + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; + + KernelImmutableData kernelImmutableData(device); auto bcsCsr = device->getNEODevice()->getEngine(aub_stream::EngineType::ENGINE_BCS, EngineUsage::Regular).commandStreamReceiver; auto initialTaskCount = bcsCsr->peekTaskCount(); + kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, false); + EXPECT_EQ(initialTaskCount, bcsCsr->peekTaskCount()); + + device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(kernelInfo.kernelAllocation); } TEST_F(KernelIsaTests, givenKernelAllocationInLocalMemoryWhenCreatingWithDisallowedCpuAccessAndDisabledBlitterThenFallbackToCpuCopy) { DebugManagerStateRestore restore; DebugManager.flags.ForceLocalMemoryAccessMode.set(static_cast(LocalMemoryAccessMode::CpuAccessDisallowed)); DebugManager.flags.ForceNonSystemMemoryPlacement.set(1 << (static_cast(NEO::AllocationType::KERNEL_ISA) - 1)); - this->createModuleFromMockBinary(ModuleType::User); device->getNEODevice()->getExecutionEnvironment()->rootDeviceEnvironments[0]->getMutableHardwareInfo()->capabilityTable.blitterOperationsSupported = false; + + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; + + KernelImmutableData kernelImmutableData(device); + auto bcsCsr = device->getNEODevice()->getEngine(aub_stream::EngineType::ENGINE_BCS, EngineUsage::Regular).commandStreamReceiver; auto initialTaskCount = bcsCsr->peekTaskCount(); + kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, false); + EXPECT_EQ(initialTaskCount, bcsCsr->peekTaskCount()); + + device->getNEODevice()->getMemoryManager()->freeGraphicsMemory(kernelInfo.kernelAllocation); } TEST_F(KernelIsaTests, givenKernelInfoWhenInitializingImmutableDataWithInternalIsaThenCorrectAllocationTypeIsUsed) { - this->createModuleFromMockBinary(ModuleType::Builtin); + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; - auto &kernelImmutableData = this->module->kernelImmDatas.back(); - EXPECT_EQ(NEO::AllocationType::KERNEL_ISA_INTERNAL, kernelImmutableData->getIsaGraphicsAllocation()->getAllocationType()); + KernelImmutableData kernelImmutableData(device); + + kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, true); + EXPECT_EQ(NEO::AllocationType::KERNEL_ISA_INTERNAL, kernelImmutableData.getIsaGraphicsAllocation()->getAllocationType()); } TEST_F(KernelIsaTests, givenKernelInfoWhenInitializingImmutableDataWithNonInternalIsaThenCorrectAllocationTypeIsUsed) { - this->createModuleFromMockBinary(ModuleType::User); + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; - auto &kernelImmutableData = this->module->kernelImmDatas.back(); - EXPECT_EQ(NEO::AllocationType::KERNEL_ISA, kernelImmutableData->getIsaGraphicsAllocation()->getAllocationType()); + KernelImmutableData kernelImmutableData(device); + + kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, false); + EXPECT_EQ(NEO::AllocationType::KERNEL_ISA, kernelImmutableData.getIsaGraphicsAllocation()->getAllocationType()); } TEST_F(KernelIsaTests, givenKernelInfoWhenInitializingImmutableDataWithIsaThenPaddingIsAdded) { - this->createModuleFromMockBinary(ModuleType::User); + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; - auto &kernelImmutableData = this->module->kernelImmDatas.back(); - auto kernelHeapSize = kernelImmutableData->getKernelInfo()->heapInfo.kernelHeapSize; - auto &helper = device->getNEODevice()->getGfxCoreHelper(); + KernelImmutableData kernelImmutableData(device); + kernelImmutableData.initialize(&kernelInfo, device, 0, nullptr, nullptr, false); + auto graphicsAllocation = kernelImmutableData.getIsaGraphicsAllocation(); + auto &helper = device->getNEODevice()->getRootDeviceEnvironment().getHelper(); size_t isaPadding = helper.getPaddingForISAAllocation(); - EXPECT_EQ(kernelImmutableData->getIsaSize(), kernelHeapSize + isaPadding); + EXPECT_EQ(graphicsAllocation->getUnderlyingBufferSize(), kernelInfo.heapInfo.kernelHeapSize + isaPadding); } TEST_F(KernelIsaTests, givenGlobalBuffersWhenCreatingKernelImmutableDataThenBuffersAreAddedToResidencyContainer) { + uint32_t kernelHeap = 0; + KernelInfo kernelInfo; + kernelInfo.heapInfo.kernelHeapSize = 1; + kernelInfo.heapInfo.pKernelHeap = &kernelHeap; + + KernelImmutableData kernelImmutableData(device); + uint64_t gpuAddress = 0x1200; void *buffer = reinterpret_cast(gpuAddress); size_t size = 0x1100; NEO::MockGraphicsAllocation globalVarBuffer(buffer, gpuAddress, size); NEO::MockGraphicsAllocation globalConstBuffer(buffer, gpuAddress, size); - ModuleBuildLog *moduleBuildLog = nullptr; - this->module.reset(new WhiteBox<::L0::Module>{this->device, moduleBuildLog, ModuleType::User}); - this->module->mockGlobalVarBuffer = &globalVarBuffer; - this->module->mockGlobalConstBuffer = &globalConstBuffer; - - this->createModuleFromMockBinary(ModuleType::User); - - for (auto &kernelImmData : this->module->kernelImmDatas) { - auto &resCont = kernelImmData->getResidencyContainer(); - EXPECT_EQ(1, std::count(resCont.begin(), resCont.end(), &globalVarBuffer)); - EXPECT_EQ(1, std::count(resCont.begin(), resCont.end(), &globalConstBuffer)); - } - this->module->translationUnit->globalConstBuffer = nullptr; - this->module->translationUnit->globalVarBuffer = nullptr; + kernelImmutableData.initialize(&kernelInfo, device, 0, + &globalConstBuffer, &globalVarBuffer, false); + auto &resCont = kernelImmutableData.getResidencyContainer(); + EXPECT_EQ(1, std::count(resCont.begin(), resCont.end(), &globalVarBuffer)); + EXPECT_EQ(1, std::count(resCont.begin(), resCont.end(), &globalConstBuffer)); } using KernelImpPatchBindlessTest = Test; diff --git a/level_zero/core/test/unit_tests/sources/module/test_module.cpp b/level_zero/core/test/unit_tests/sources/module/test_module.cpp index 119a4f85f9..13fb1e4c52 100644 --- a/level_zero/core/test/unit_tests/sources/module/test_module.cpp +++ b/level_zero/core/test/unit_tests/sources/module/test_module.cpp @@ -26,7 +26,6 @@ #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_elf.h" #include "shared/test/common/mocks/mock_graphics_allocation.h" -#include "shared/test/common/mocks/mock_l0_debugger.h" #include "shared/test/common/mocks/mock_memory_operations_handler.h" #include "shared/test/common/mocks/mock_modules_zebin.h" #include "shared/test/common/test_macros/hw_test.h" @@ -105,71 +104,7 @@ HWTEST_F(ModuleTest, givenUserModuleWhenCreatedThenCorrectAllocationTypeIsUsedFo EXPECT_EQ(NEO::AllocationType::KERNEL_ISA, kernel->getIsaAllocation()->getAllocationType()); } -template -struct ModuleKernelIsaAllocationsFixture : public ModuleFixture { - static constexpr size_t isaAllocationPageSize = (localMemEnabled ? MemoryConstants::pageSize64k : MemoryConstants::pageSize); - using Module = WhiteBox<::L0::Module>; - - void setUp() { - this->dbgRestorer = std::make_unique(); - DebugManager.flags.EnableLocalMemory.set(localMemEnabled); - ModuleFixture::setUp(); - - ModuleBuildLog *moduleBuildLog = nullptr; - auto type = ModuleType::User; - this->module.reset(new Mock{device, moduleBuildLog, type}); - - zebinData = std::make_unique(device->getHwInfo()); - const auto &src = zebinData->storage; - this->moduleDesc.format = ZE_MODULE_FORMAT_NATIVE; - this->moduleDesc.pInputModule = reinterpret_cast(src.data()); - this->moduleDesc.inputSize = src.size(); - - this->mockModule = static_cast *>(this->module.get()); - } - - void givenIsaMemoryRegionSharedBetweenKernelsWhenGraphicsAllocationFailsThenProperErrorReturned() { - mockModule->allocateKernelsIsaMemoryCallBase = false; - auto result = module->initialize(&this->moduleDesc, device->getNEODevice()); - EXPECT_EQ(result, ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY); - } - - void givenSeparateIsaMemoryRegionPerKernelWhenGraphicsAllocationFailsThenProperErrorReturned() { - mockModule->allocateKernelsIsaMemoryCallBase = false; - mockModule->computeKernelIsaAllocationAlignedSizeWithPaddingCallBase = false; - mockModule->computeKernelIsaAllocationAlignedSizeWithPaddingResult = isaAllocationPageSize; - - auto result = module->initialize(&this->moduleDesc, device->getNEODevice()); - EXPECT_EQ(result, ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY); - } - - Mock *mockModule = nullptr; - ze_module_desc_t moduleDesc = {}; - std::unique_ptr dbgRestorer = nullptr; -}; - -using ModuleKernelIsaAllocationsInLocalMemoryTests = Test>; - -HWTEST_F(ModuleKernelIsaAllocationsInLocalMemoryTests, givenIsaMemoryRegionSharedBetweenKernelsWhenGraphicsAllocationFailsThenProperErrorReturned) { - this->givenIsaMemoryRegionSharedBetweenKernelsWhenGraphicsAllocationFailsThenProperErrorReturned(); -} - -HWTEST_F(ModuleKernelIsaAllocationsInLocalMemoryTests, givenSeparateIsaMemoryRegionPerKernelWhenGraphicsAllocationFailsThenProperErrorReturned) { - this->givenSeparateIsaMemoryRegionPerKernelWhenGraphicsAllocationFailsThenProperErrorReturned(); -} - -using ModuleKernelIsaAllocationsInSharedMemoryTests = Test>; - -HWTEST_F(ModuleKernelIsaAllocationsInSharedMemoryTests, givenIsaMemoryRegionSharedBetweenKernelsWhenGraphicsAllocationFailsThenProperErrorReturned) { - this->givenIsaMemoryRegionSharedBetweenKernelsWhenGraphicsAllocationFailsThenProperErrorReturned(); -} - -HWTEST_F(ModuleKernelIsaAllocationsInSharedMemoryTests, givenSeparateIsaMemoryRegionPerKernelWhenGraphicsAllocationFailsThenProperErrorReturned) { - this->givenSeparateIsaMemoryRegionPerKernelWhenGraphicsAllocationFailsThenProperErrorReturned(); -} - HWTEST_F(ModuleTest, givenBuiltinModuleWhenCreatedThenCorrectAllocationTypeIsUsedForIsa) { - this->module.reset(); createModuleFromMockBinary(ModuleType::Builtin); createKernel(); EXPECT_EQ(NEO::AllocationType::KERNEL_ISA_INTERNAL, kernel->getIsaAllocation()->getAllocationType()); @@ -2327,8 +2262,16 @@ TEST_F(ModuleFunctionPointerTests, givenModuleWithExportedSymbolThenGetFunctionP EXPECT_EQ(reinterpret_cast(functionPointer), module0->kernelImmDatas[0]->getIsaGraphicsAllocation()->getGpuAddress()); } -class DeviceModuleSetArgBufferFixture : public ModuleFixture { +class DeviceModuleSetArgBufferTest : public ModuleFixture, public ::testing::Test { public: + void SetUp() override { + ModuleFixture::setUp(); + } + + void TearDown() override { + ModuleFixture::tearDown(); + } + void createKernelAndAllocMemory(uint32_t rootDeviceIndex, void **ptr, ze_kernel_handle_t *kernelHandle) { ze_kernel_desc_t kernelDesc = {}; kernelDesc.pKernelName = kernelName.c_str(); @@ -2340,11 +2283,11 @@ class DeviceModuleSetArgBufferFixture : public ModuleFixture { EXPECT_EQ(ZE_RESULT_SUCCESS, res); } }; -using DeviceModuleSetArgBufferTest = Test; HWTEST_F(DeviceModuleSetArgBufferTest, givenValidMemoryUsedinFirstCallToSetArgBufferThenNullptrSetOnTheSecondCallThenArgBufferisUpdatedInEachCallAndSuccessIsReturned) { uint32_t rootDeviceIndex = 0; + createModuleFromMockBinary(); ze_kernel_handle_t kernelHandle; void *validBufferPtr = nullptr; @@ -3531,227 +3474,6 @@ TEST_F(ModuleTest, whenContainsStatefulAccessIsCalledThenResultIsCorrect) { } } -template -struct ModuleIsaAllocationsFixture : public DeviceFixture { - static constexpr size_t isaAllocationPageSize = (localMemEnabled ? MemoryConstants::pageSize64k : MemoryConstants::pageSize); - static constexpr NEO::MemoryPool isaAllocationMemoryPool = (localMemEnabled ? NEO::MemoryPool::LocalMemory : NEO::MemoryPool::System4KBPagesWith32BitGpuAddressing); - - void setUp() { - this->dbgRestorer = std::make_unique(); - DebugManager.flags.EnableLocalMemory.set(localMemEnabled); - - DeviceFixture::setUp(); - - this->neoDevice = this->device->getNEODevice(); - this->isaPadding = this->neoDevice->getGfxCoreHelper().getPaddingForISAAllocation(); - this->kernelStartPointerAlignment = this->neoDevice->getGfxCoreHelper().getKernelIsaPointerAlignment(); - this->mockMemoryManager = static_cast(this->neoDevice->getMemoryManager()); - this->mockMemoryManager->localMemorySupported[this->neoDevice->getRootDeviceIndex()] = true; - this->mockModule.reset(new MockModule{this->device, nullptr, ModuleType::User}); - this->mockModule->translationUnit.reset(new MockModuleTranslationUnit{this->device}); - } - - void tearDown() { - this->mockModule->translationUnit.reset(); - this->mockModule.reset(); - DeviceFixture::tearDown(); - } - - void prepareKernelInfoAndAddToTranslationUnit(size_t isaSize) { - auto kernelInfo = new KernelInfo{}; - kernelInfo->heapInfo.pKernelHeap = reinterpret_cast(0xdeadbeef0000); - kernelInfo->heapInfo.kernelHeapSize = static_cast(isaSize); - this->mockModule->translationUnit->programInfo.kernelInfos.push_back(kernelInfo); - } - - size_t computeKernelIsaAllocationSizeWithPadding(size_t isaSize) { - auto isaPadding = this->neoDevice->getGfxCoreHelper().getPaddingForISAAllocation(); - return isaPadding + isaSize; - } - - template - void givenMultipleKernelIsasWhichFitInSinglePageAndDebuggerEnabledWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations() { - auto requestedSize = 0x40; - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize); - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize); - - auto debugger = MockDebuggerL0Hw::allocate(neoDevice); - this->neoDevice->getRootDeviceEnvironmentRef().debugger.reset(debugger); - - this->mockModule->initializeKernelImmutableDatas(); - auto &kernelImmDatas = this->mockModule->getKernelImmutableDataVector(); - EXPECT_EQ(nullptr, kernelImmDatas[0]->getIsaParentAllocation()); - EXPECT_NE(nullptr, kernelImmDatas[0]->getIsaGraphicsAllocation()); - EXPECT_EQ(nullptr, kernelImmDatas[1]->getIsaParentAllocation()); - EXPECT_NE(nullptr, kernelImmDatas[1]->getIsaGraphicsAllocation()); - } - - void givenMultipleKernelIsasWhichExceedSinglePageWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations() { - auto maxAllocationSizeInPage = alignDown(isaAllocationPageSize - this->isaPadding, this->kernelStartPointerAlignment); - this->prepareKernelInfoAndAddToTranslationUnit(maxAllocationSizeInPage); - - auto tinyAllocationSize = 0x8; - this->prepareKernelInfoAndAddToTranslationUnit(tinyAllocationSize); - - this->mockModule->initializeKernelImmutableDatas(); - auto &kernelImmDatas = this->mockModule->getKernelImmutableDataVector(); - EXPECT_EQ(nullptr, kernelImmDatas[0]->getIsaParentAllocation()); - EXPECT_NE(nullptr, kernelImmDatas[0]->getIsaGraphicsAllocation()); - EXPECT_EQ(kernelImmDatas[0]->getIsaOffsetInParentAllocation(), 0lu); - EXPECT_EQ(kernelImmDatas[0]->getIsaSubAllocationSize(), 0lu); - EXPECT_EQ(nullptr, kernelImmDatas[1]->getIsaParentAllocation()); - EXPECT_NE(nullptr, kernelImmDatas[1]->getIsaGraphicsAllocation()); - EXPECT_EQ(kernelImmDatas[1]->getIsaOffsetInParentAllocation(), 0lu); - EXPECT_EQ(kernelImmDatas[1]->getIsaSubAllocationSize(), 0lu); - if constexpr (localMemEnabled) { - EXPECT_EQ(isaAllocationPageSize, kernelImmDatas[0]->getIsaSize()); - EXPECT_EQ(isaAllocationPageSize, kernelImmDatas[1]->getIsaSize()); - } else { - EXPECT_EQ(this->computeKernelIsaAllocationSizeWithPadding(maxAllocationSizeInPage), kernelImmDatas[0]->getIsaSize()); - EXPECT_EQ(this->computeKernelIsaAllocationSizeWithPadding(tinyAllocationSize), kernelImmDatas[1]->getIsaSize()); - } - } - - struct ProxyKernelImmutableData : public KernelImmutableData { - using BaseClass = KernelImmutableData; - using BaseClass::BaseClass; - - ~ProxyKernelImmutableData() override { this->KernelImmutableData::~KernelImmutableData(); } - - ADDMETHOD(initialize, ze_result_t, true, ZE_RESULT_ERROR_UNKNOWN, - (NEO::KernelInfo * kernelInfo, L0::Device *device, uint32_t computeUnitsUsedForScratch, NEO::GraphicsAllocation *globalConstBuffer, NEO::GraphicsAllocation *globalVarBuffer, bool internalKernel), - (kernelInfo, device, computeUnitsUsedForScratch, globalConstBuffer, globalVarBuffer, internalKernel)); - }; - - void givenMultipleKernelIsasWhenKernelInitializationFailsThenItIsProperlyCleanedAndPreviouslyInitializedKernelsLeftUntouched() { - auto requestedSize = 0x40; - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize); - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize); - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize); - - auto &kernelImmDatas = this->mockModule->getKernelImmutableDataVectorRef(); - { - auto kernelsCount = 3ul; - kernelImmDatas.reserve(kernelsCount); - for (size_t i = 0lu; i < kernelsCount; i++) { - kernelImmDatas.emplace_back(new ProxyKernelImmutableData(this->device)); - } - auto result = this->mockModule->setIsaGraphicsAllocations(); - EXPECT_EQ(result, ZE_RESULT_SUCCESS); - } - - static_cast(kernelImmDatas[2].get())->initializeCallBase = false; - auto result = this->mockModule->initializeKernelImmutableDatas(); - EXPECT_EQ(result, ZE_RESULT_ERROR_UNKNOWN); - ASSERT_NE(kernelImmDatas[0].get(), nullptr); - ASSERT_NE(kernelImmDatas[1].get(), nullptr); - EXPECT_EQ(kernelImmDatas[2].get(), nullptr); - EXPECT_NE(kernelImmDatas[0]->getIsaGraphicsAllocation(), nullptr); - EXPECT_NE(kernelImmDatas[1]->getIsaGraphicsAllocation(), nullptr); - } - - size_t isaPadding; - size_t kernelStartPointerAlignment; - NEO::Device *neoDevice = nullptr; - MockMemoryManager *mockMemoryManager = nullptr; - std::unique_ptr mockModule = nullptr; - std::unique_ptr dbgRestorer = nullptr; -}; -using ModuleIsaAllocationsInLocalMemoryTest = Test>; - -TEST_F(ModuleIsaAllocationsInLocalMemoryTest, givenMultipleKernelIsasWhichFitInSinglePage64KWhenKernelImmutableDatasInitializedThenKernelIsasShareParentAllocation) { - EXPECT_EQ(this->mockModule->isaAllocationPageSize, isaAllocationPageSize); - - auto requestedSize1 = 0x40; - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize1); - auto isaAllocationSize1 = this->mockModule->computeKernelIsaAllocationAlignedSizeWithPadding(requestedSize1); - - auto requestedSize2 = isaAllocationPageSize - isaAllocationSize1 - this->isaPadding; - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize2); - auto isaAllocationSize2 = this->mockModule->computeKernelIsaAllocationAlignedSizeWithPadding(requestedSize2); - - this->mockModule->initializeKernelImmutableDatas(); - auto &kernelImmDatas = this->mockModule->getKernelImmutableDataVector(); - EXPECT_EQ(kernelImmDatas[0]->getIsaGraphicsAllocation(), kernelImmDatas[0]->getIsaParentAllocation()); - EXPECT_EQ(kernelImmDatas[0]->getIsaOffsetInParentAllocation(), 0lu); - EXPECT_EQ(kernelImmDatas[0]->getIsaSubAllocationSize(), isaAllocationSize1); - EXPECT_EQ(kernelImmDatas[1]->getIsaGraphicsAllocation(), kernelImmDatas[1]->getIsaParentAllocation()); - EXPECT_EQ(kernelImmDatas[1]->getIsaOffsetInParentAllocation(), isaAllocationSize1); - EXPECT_EQ(kernelImmDatas[1]->getIsaSubAllocationSize(), isaAllocationSize2); - - EXPECT_EQ(kernelImmDatas[0]->getIsaSize(), isaAllocationSize1); - EXPECT_EQ(kernelImmDatas[0]->getIsaGraphicsAllocation()->getMemoryPool(), isaAllocationMemoryPool); - EXPECT_EQ(kernelImmDatas[1]->getIsaSize(), isaAllocationSize2); - EXPECT_EQ(kernelImmDatas[1]->getIsaGraphicsAllocation()->getMemoryPool(), isaAllocationMemoryPool); -} - -HWTEST_F(ModuleIsaAllocationsInLocalMemoryTest, givenMultipleKernelIsasWhichFitInSinglePage64KAndDebuggerEnabledWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations) { - this->givenMultipleKernelIsasWhichFitInSinglePageAndDebuggerEnabledWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations(); -} - -TEST_F(ModuleIsaAllocationsInLocalMemoryTest, givenMultipleKernelIsasWhichExceedSinglePage64KWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations) { - this->givenMultipleKernelIsasWhichExceedSinglePageWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations(); -} - -TEST_F(ModuleIsaAllocationsInLocalMemoryTest, givenMultipleKernelIsasWhenKernelInitializationFailsThenItIsProperlyCleanedAndPreviouslyInitializedKernelsLeftUntouched) { - this->givenMultipleKernelIsasWhenKernelInitializationFailsThenItIsProperlyCleanedAndPreviouslyInitializedKernelsLeftUntouched(); -} - -using ModuleIsaAllocationsInSystemMemoryTest = Test>; - -TEST_F(ModuleIsaAllocationsInSystemMemoryTest, givenKernelIsaWhichCouldFitInPages4KBWhenKernelImmutableDatasInitializedThenKernelIsasCanGetSeparateAllocationsDependingOnPaddingSize) { - EXPECT_EQ(this->mockModule->isaAllocationPageSize, isaAllocationPageSize); - - const auto requestedSize1 = 0x8; - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize1); - auto isaAllocationAlignedSize1 = this->mockModule->computeKernelIsaAllocationAlignedSizeWithPadding(requestedSize1); - - const auto requestedSize2 = 0x4; - this->prepareKernelInfoAndAddToTranslationUnit(requestedSize2); - auto isaAllocationAlignedSize2 = this->mockModule->computeKernelIsaAllocationAlignedSizeWithPadding(requestedSize2); - - // for 4kB pages, 2x isaPaddings alone could exceed isaAllocationPageSize, which precludes page sharing - const bool isasShouldShareSamePage = (isaAllocationAlignedSize1 + isaAllocationAlignedSize2 <= isaAllocationPageSize); - - this->mockModule->initializeKernelImmutableDatas(); - auto &kernelImmDatas = this->mockModule->getKernelImmutableDataVector(); - if (isasShouldShareSamePage) { - EXPECT_EQ(kernelImmDatas[0]->getIsaGraphicsAllocation(), kernelImmDatas[0]->getIsaParentAllocation()); - EXPECT_EQ(kernelImmDatas[0]->getIsaOffsetInParentAllocation(), 0lu); - EXPECT_EQ(kernelImmDatas[0]->getIsaSize(), isaAllocationAlignedSize1); - EXPECT_EQ(kernelImmDatas[1]->getIsaGraphicsAllocation(), kernelImmDatas[1]->getIsaParentAllocation()); - EXPECT_EQ(kernelImmDatas[1]->getIsaOffsetInParentAllocation(), isaAllocationAlignedSize1); - EXPECT_EQ(kernelImmDatas[1]->getIsaSubAllocationSize(), isaAllocationAlignedSize2); - EXPECT_EQ(kernelImmDatas[0]->getIsaSize(), isaAllocationAlignedSize2); - } else { - EXPECT_EQ(nullptr, kernelImmDatas[0]->getIsaParentAllocation()); - EXPECT_NE(nullptr, kernelImmDatas[0]->getIsaGraphicsAllocation()); - EXPECT_EQ(kernelImmDatas[0]->getIsaOffsetInParentAllocation(), 0lu); - EXPECT_EQ(kernelImmDatas[0]->getIsaSubAllocationSize(), 0lu); - EXPECT_EQ(kernelImmDatas[0]->getIsaSize(), computeKernelIsaAllocationSizeWithPadding(requestedSize1)); - EXPECT_EQ(nullptr, kernelImmDatas[1]->getIsaParentAllocation()); - EXPECT_NE(nullptr, kernelImmDatas[1]->getIsaGraphicsAllocation()); - EXPECT_EQ(kernelImmDatas[1]->getIsaOffsetInParentAllocation(), 0lu); - EXPECT_EQ(kernelImmDatas[1]->getIsaSubAllocationSize(), 0lu); - EXPECT_EQ(kernelImmDatas[1]->getIsaSize(), computeKernelIsaAllocationSizeWithPadding(requestedSize2)); - } - - EXPECT_EQ(kernelImmDatas[0]->getIsaGraphicsAllocation()->getMemoryPool(), isaAllocationMemoryPool); - EXPECT_EQ(kernelImmDatas[1]->getIsaGraphicsAllocation()->getMemoryPool(), isaAllocationMemoryPool); -} - -HWTEST_F(ModuleIsaAllocationsInSystemMemoryTest, givenMultipleKernelIsasWhichFitInSinglePageAndDebuggerEnabledWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations) { - this->givenMultipleKernelIsasWhichFitInSinglePageAndDebuggerEnabledWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations(); -} - -TEST_F(ModuleIsaAllocationsInSystemMemoryTest, givenMultipleKernelIsasWhichExceedSinglePageWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations) { - this->givenMultipleKernelIsasWhichExceedSinglePageWhenKernelImmutableDatasAreInitializedThenKernelIsasGetSeparateAllocations(); -} - -TEST_F(ModuleIsaAllocationsInSystemMemoryTest, givenMultipleKernelIsasWhenKernelInitializationFailsThenItIsProperlyCleanedAndPreviouslyInitializedKernelsLeftUntouched) { - this->givenMultipleKernelIsasWhenKernelInitializationFailsThenItIsProperlyCleanedAndPreviouslyInitializedKernelsLeftUntouched(); -} - using ModuleInitializeTest = Test; TEST_F(ModuleInitializeTest, whenModuleInitializeIsCalledThenCorrectResultIsReturned) { @@ -3844,7 +3566,9 @@ TEST_F(ModuleDebugDataTest, GivenDebugDataWithRelocationsWhenCreatingRelocatedDe moduleDesc.inputSize = 10; ModuleBuildLog *moduleBuildLog = nullptr; - auto module = std::make_unique(device, moduleBuildLog, ModuleType::User); + std::unique_ptr module = std::make_unique(device, + moduleBuildLog, + ModuleType::User); module->translationUnit = std::make_unique(device); module->translationUnit->globalVarBuffer = neoDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties( @@ -3868,8 +3592,7 @@ TEST_F(ModuleDebugDataTest, GivenDebugDataWithRelocationsWhenCreatingRelocatedDe // pass kernelInfo ownership to programInfo module->translationUnit->programInfo.kernelInfos.push_back(kernelInfo); - auto kernelImmData = std::make_unique>(this->device); - kernelImmData->setIsaPerKernelAllocation(module->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); + std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; kernelImmData->initialize(kernelInfo, device, 0, module->translationUnit->globalConstBuffer, module->translationUnit->globalVarBuffer, false); kernelImmData->createRelocatedDebugData(module->translationUnit->globalConstBuffer, module->translationUnit->globalVarBuffer); @@ -4041,7 +3764,6 @@ TEST_F(ModuleTests, whenCopyingPatchedSegmentsThenAllocationsAreSetWritableForTb kernelInfo->heapInfo.pKernelHeap = data; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, false); pModule->kernelImmDatas.push_back(std::move(kernelImmData)); @@ -4071,7 +3793,6 @@ TEST_F(ModuleTests, givenConstDataStringSectionWhenLinkingModuleThenSegmentIsPat kernelInfo->heapInfo.pKernelHeap = data; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo, device, 0, nullptr, nullptr, false); auto patchAddr = reinterpret_cast(ptrOffset(kernelImmData->isaGraphicsAllocation->getUnderlyingBuffer(), 0x8)); pModule->kernelImmDatas.push_back(std::move(kernelImmData)); @@ -4102,7 +3823,6 @@ TEST_F(ModuleTests, givenImplicitArgsRelocationAndStackCallsWhenLinkingBuiltinMo kernelInfo->heapInfo.pKernelHeap = data; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo, device, 0, nullptr, nullptr, true); kernelImmData->kernelDescriptor->kernelAttributes.flags.useStackCalls = true; @@ -4148,7 +3868,6 @@ TEST_F(ModuleTests, givenFullyLinkedModuleAndSlmSizeExceedingLocalMemorySizeWhen auto slmInlineSizeCopy = kernelInfo->kernelDescriptor.kernelAttributes.slmInlineSize; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, true); pModule->kernelImmDatas.push_back(std::move(kernelImmData)); @@ -4191,7 +3910,6 @@ TEST_F(ModuleTests, givenFullyLinkedModuleWhenCreatingKernelThenDebugMsgOnPrivat kernelInfo->heapInfo.pKernelHeap = data; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo.get(), device, 0, nullptr, nullptr, true); pModule->kernelImmDatas.push_back(std::move(kernelImmData)); @@ -4235,7 +3953,6 @@ TEST_F(ModuleTests, givenImplicitArgsRelocationAndStackCallsWhenLinkingModuleThe kernelInfo->heapInfo.pKernelHeap = data; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo, device, 0, nullptr, nullptr, false); kernelImmData->kernelDescriptor->kernelAttributes.flags.useStackCalls = true; @@ -4266,7 +3983,6 @@ TEST_F(ModuleTests, givenImplicitArgsRelocationAndNoDebuggerOrStackCallsWhenLink kernelInfo->heapInfo.pKernelHeap = data; std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - kernelImmData->setIsaPerKernelAllocation(pModule->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize)); kernelImmData->initialize(kernelInfo, device, 0, nullptr, nullptr, false); kernelImmData->kernelDescriptor->kernelAttributes.flags.useStackCalls = false; @@ -4307,9 +4023,6 @@ TEST_F(ModuleTests, givenModuleWithGlobalAndConstAllocationsWhenGettingModuleAll module->translationUnit->programInfo.kernelInfos.push_back(kernelInfo); std::unique_ptr> kernelImmData{new WhiteBox<::L0::KernelImmutableData>(this->device)}; - auto isaAlloc = module->allocateKernelsIsaMemory(kernelInfo->heapInfo.kernelHeapSize); - ASSERT_NE(isaAlloc, nullptr); - kernelImmData->setIsaPerKernelAllocation(isaAlloc); kernelImmData->initialize(kernelInfo, device, 0, module->translationUnit->globalConstBuffer, module->translationUnit->globalVarBuffer, false); module->kernelImmDatas.push_back(std::move(kernelImmData)); diff --git a/shared/source/command_container/command_encoder_bdw_and_later.inl b/shared/source/command_container/command_encoder_bdw_and_later.inl index de8350bd8c..b77301b3c7 100644 --- a/shared/source/command_container/command_encoder_bdw_and_later.inl +++ b/shared/source/command_container/command_encoder_bdw_and_later.inl @@ -77,7 +77,7 @@ void EncodeDispatchKernel::encode(CommandContainer &container, EncodeDis { auto alloc = args.dispatchInterface->getIsaAllocation(); UNRECOVERABLE_IF(nullptr == alloc); - auto offset = alloc->getGpuAddressToPatch() + args.dispatchInterface->getIsaOffsetInParentAllocation(); + auto offset = alloc->getGpuAddressToPatch(); idd.setKernelStartPointer(offset); idd.setKernelStartPointerHigh(0u); } diff --git a/shared/source/command_container/command_encoder_xehp_and_later.inl b/shared/source/command_container/command_encoder_xehp_and_later.inl index 19355787a6..80acc56d36 100644 --- a/shared/source/command_container/command_encoder_xehp_and_later.inl +++ b/shared/source/command_container/command_encoder_xehp_and_later.inl @@ -87,7 +87,7 @@ void EncodeDispatchKernel::encode(CommandContainer &container, EncodeDis { auto alloc = args.dispatchInterface->getIsaAllocation(); UNRECOVERABLE_IF(nullptr == alloc); - auto offset = alloc->getGpuAddressToPatch() + args.dispatchInterface->getIsaOffsetInParentAllocation(); + auto offset = alloc->getGpuAddressToPatch(); if (!localIdsGenerationByRuntime) { offset += kernelDescriptor.entryPoints.skipPerThreadDataLoad; } diff --git a/shared/source/helpers/gfx_core_helper.h b/shared/source/helpers/gfx_core_helper.h index 1c61dda07c..c0bca817f4 100644 --- a/shared/source/helpers/gfx_core_helper.h +++ b/shared/source/helpers/gfx_core_helper.h @@ -53,7 +53,6 @@ class GfxCoreHelper { static std::unique_ptr create(const GFXCORE_FAMILY gfxCoreFamily); virtual size_t getMaxBarrierRegisterPerSlice() const = 0; virtual size_t getPaddingForISAAllocation() const = 0; - virtual size_t getKernelIsaPointerAlignment() const = 0; virtual uint32_t getComputeUnitsUsedForScratch(const RootDeviceEnvironment &rootDeviceEnvironment) const = 0; virtual uint32_t getPitchAlignmentForImage(const RootDeviceEnvironment &rootDeviceEnvironment) const = 0; virtual void adjustDefaultEngineType(HardwareInfo *pHwInfo, const ProductHelper &productHelper) = 0; @@ -216,10 +215,6 @@ class GfxCoreHelperHw : public GfxCoreHelper { size_t getPaddingForISAAllocation() const override; - size_t getKernelIsaPointerAlignment() const override { - return static_cast(GfxFamily::cmdInitInterfaceDescriptorData.KERNELSTARTPOINTER_ALIGN_SIZE); - } - uint32_t getComputeUnitsUsedForScratch(const RootDeviceEnvironment &rootDeviceEnvironment) const override; uint32_t getPitchAlignmentForImage(const RootDeviceEnvironment &rootDeviceEnvironment) const override; diff --git a/shared/source/kernel/dispatch_kernel_encoder_interface.h b/shared/source/kernel/dispatch_kernel_encoder_interface.h index da974cdaa8..cdc419f386 100644 --- a/shared/source/kernel/dispatch_kernel_encoder_interface.h +++ b/shared/source/kernel/dispatch_kernel_encoder_interface.h @@ -40,7 +40,6 @@ struct DispatchKernelEncoderI { virtual uint32_t getSurfaceStateHeapDataSize() const = 0; virtual GraphicsAllocation *getIsaAllocation() const = 0; - virtual uint64_t getIsaOffsetInParentAllocation() const = 0; virtual const uint8_t *getDynamicStateHeapData() const = 0; virtual uint32_t getRequiredWorkgroupOrder() const = 0; diff --git a/shared/test/unit_test/encoders/test_encode_dispatch_kernel.cpp b/shared/test/unit_test/encoders/test_encode_dispatch_kernel.cpp index b1d33b5456..5ee17f4450 100644 --- a/shared/test/unit_test/encoders/test_encode_dispatch_kernel.cpp +++ b/shared/test/unit_test/encoders/test_encode_dispatch_kernel.cpp @@ -997,45 +997,6 @@ HWTEST2_F(EncodeDispatchKernelTest, givenBindlessKernelWhenDispatchingKernelThen EXPECT_NE(usedAfter, usedBefore); } -HWTEST2_F(EncodeDispatchKernelTest, givenKernelsSharingISAParentAllocationsWhenProgrammingWalkerThenKernelStartPointerHasProperOffset, IsBeforeXeHpCore) { - using INTERFACE_DESCRIPTOR_DATA = typename FamilyType::INTERFACE_DESCRIPTOR_DATA; - - auto dispatchInterface = std::make_unique(); - dispatchInterface->getIsaOffsetInParentAllocationResult = 8 << INTERFACE_DESCRIPTOR_DATA::KERNELSTARTPOINTER_BIT_SHIFT; - uint32_t dims[] = {2, 1, 1}; - bool requiresUncachedMocs = false; - EncodeDispatchKernelArgs dispatchArgs = createDefaultDispatchKernelArgs(pDevice, dispatchInterface.get(), dims, requiresUncachedMocs); - EncodeDispatchKernel::encode(*cmdContainer.get(), dispatchArgs); - - auto idd = static_cast(cmdContainer->getIddBlock()); - EXPECT_EQ(idd->getKernelStartPointer(), dispatchInterface->getIsaAllocation()->getGpuAddressToPatch() + dispatchInterface->getIsaOffsetInParentAllocation()); -} - -HWTEST_F(EncodeDispatchKernelTest, givenKernelStartPointerAlignmentInInterfaceDescriptorWhenHelperGetterUsedThenCorrectValueReturned) { - using INTERFACE_DESCRIPTOR_DATA = typename FamilyType::INTERFACE_DESCRIPTOR_DATA; - EXPECT_EQ(INTERFACE_DESCRIPTOR_DATA::KERNELSTARTPOINTER_ALIGN_SIZE, pDevice->getGfxCoreHelper().getKernelIsaPointerAlignment()); -} - -HWTEST2_F(EncodeDispatchKernelTest, givenKernelsSharingISAParentAllocationsWhenProgrammingWalkerThenKernelStartPointerHasProperOffset, IsAtLeastXeHpCore) { - using WALKER_TYPE = typename FamilyType::WALKER_TYPE; - using INTERFACE_DESCRIPTOR_DATA = typename FamilyType::INTERFACE_DESCRIPTOR_DATA; - - auto dispatchInterface = std::make_unique(); - dispatchInterface->getIsaOffsetInParentAllocationResult = 8 << INTERFACE_DESCRIPTOR_DATA::KERNELSTARTPOINTER_BIT_SHIFT; - uint32_t dims[] = {2, 1, 1}; - bool requiresUncachedMocs = false; - EncodeDispatchKernelArgs dispatchArgs = createDefaultDispatchKernelArgs(pDevice, dispatchInterface.get(), dims, requiresUncachedMocs); - EncodeDispatchKernel::encode(*cmdContainer.get(), dispatchArgs); - - GenCmdList commands; - CmdParse::parseCommandBuffer(commands, ptrOffset(cmdContainer->getCommandStream()->getCpuBase(), 0), cmdContainer->getCommandStream()->getUsed()); - auto itor = find(commands.begin(), commands.end()); - ASSERT_NE(itor, commands.end()); - - auto walkerCmd = genCmdCast(*itor); - EXPECT_EQ(walkerCmd->getInterfaceDescriptor().getKernelStartPointer(), dispatchInterface->getIsaAllocation()->getGpuAddressToPatch() + dispatchInterface->getIsaOffsetInParentAllocation()); -} - HWTEST_F(EncodeDispatchKernelTest, givenNonBindlessOrStatelessArgWhenDispatchingKernelThenSurfaceStateOffsetInCrossThreadDataIsNotPatched) { using BINDING_TABLE_STATE = typename FamilyType::BINDING_TABLE_STATE; using DataPortBindlessSurfaceExtendedMessageDescriptor = typename FamilyType::DataPortBindlessSurfaceExtendedMessageDescriptor; diff --git a/shared/test/unit_test/mocks/mock_dispatch_kernel_encoder_interface.h b/shared/test/unit_test/mocks/mock_dispatch_kernel_encoder_interface.h index 56763a99a0..69ac2c59c2 100644 --- a/shared/test/unit_test/mocks/mock_dispatch_kernel_encoder_interface.h +++ b/shared/test/unit_test/mocks/mock_dispatch_kernel_encoder_interface.h @@ -55,6 +55,5 @@ struct MockDispatchKernelEncoder : public DispatchKernelEncoderI { ADDMETHOD_CONST_NOBASE(getDynamicStateHeapData, const uint8_t *, nullptr, ()); ADDMETHOD_CONST_NOBASE(requiresGenerationOfLocalIdsByRuntime, bool, true, ()); ADDMETHOD_CONST_NOBASE(getSlmPolicy, SlmPolicy, SlmPolicy::SlmPolicyNone, ()); - ADDMETHOD_CONST_NOBASE(getIsaOffsetInParentAllocation, uint64_t, 0lu, ()); }; } // namespace NEO