diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config index 27d069a065..b7fd883b58 100644 --- a/opencl/test/unit_test/test_files/igdrcl.config +++ b/opencl/test/unit_test/test_files/igdrcl.config @@ -28,6 +28,7 @@ AUBDumpAllocsOnEnqueueSVMMemcpyOnly = 0 AUBDumpForceAllToLocalMemory = 0 ForceDeviceId = unk ForceL1Caching = -1 +UseKmdMigration = -1 SchedulerSimulationReturnInstance = 0 SchedulerGWS = 0 EnableExperimentalCommandBuffer = 0 diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index fe4eb6afc6..02bc25b5bc 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -207,6 +207,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, OverrideThreadArbitrationPolicy, -1, "-1 (dont o DECLARE_DEBUG_VARIABLE(int32_t, OverrideAubDeviceId, -1, "-1 dont override, any other: use this value for AUB generation device id") DECLARE_DEBUG_VARIABLE(int32_t, EnableTimestampPacket, -1, "-1: default, 0: disable, 1:enable. Write Timestamp Packet for each set of gpu walkers") DECLARE_DEBUG_VARIABLE(int32_t, AllocateSharedAllocationsWithCpuAndGpuStorage, -1, "When enabled driver creates cpu & gpu storage for shared unified memory allocations. (-1 - devices default mode, 0 - disable, 1 - enable)") +DECLARE_DEBUG_VARIABLE(int32_t, UseKmdMigration, -1, "-1: devices default mode (currently - disabled), 0: disable - pagefault handling by UMD using handler for SIGSEGV, 1: enable - pagefault handling by KMD, GEM objects migrated by KMD upon access)") DECLARE_DEBUG_VARIABLE(int32_t, ForceSemaphoreDelayBetweenWaits, -1, "Specifies the minimum number of microseconds allowed for command streamer to wait before re-fetching the data. 0 - poll interval will be equal to the memory latency of the read completion") DECLARE_DEBUG_VARIABLE(int32_t, ForceLocalMemoryAccessMode, -1, "-1: don't override, 0: default rules apply, 1: CPU can access local memory, 3: CPU never accesses local memory") DECLARE_DEBUG_VARIABLE(int32_t, ForceUserptrAlignment, -1, "-1: no force (4kb), >0: n kb alignment") diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index 924a9e4894..900e3888ee 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -94,7 +94,8 @@ class GraphicsAllocation : public IDNode { SEMAPHORE_BUFFER, DEBUG_CONTEXT_SAVE_AREA, DEBUG_SBA_TRACKING_BUFFER, - DEBUG_MODULE_AREA + DEBUG_MODULE_AREA, + UNIFIED_SHARED_MEMORY }; ~GraphicsAllocation() override; diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index 290b17e218..839596a084 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -215,23 +215,72 @@ void *SVMAllocsManager::createSharedUnifiedMemoryAllocation(uint32_t rootDeviceI } if (supportDualStorageSharedMemory) { - auto unifiedMemoryPointer = createUnifiedAllocationWithDeviceStorage(rootDeviceIndex, size, {}, memoryProperties); - if (!unifiedMemoryPointer) { - return nullptr; + bool useKmdMigration = false; + + if (DebugManager.flags.UseKmdMigration.get() != -1) { + useKmdMigration = DebugManager.flags.UseKmdMigration.get(); } + + void *unifiedMemoryPointer = nullptr; + if (useKmdMigration) { + unifiedMemoryPointer = createUnifiedKmdMigratedAllocation(rootDeviceIndex, size, {}, memoryProperties); + if (!unifiedMemoryPointer) { + return nullptr; + } + } else { + unifiedMemoryPointer = createUnifiedAllocationWithDeviceStorage(rootDeviceIndex, size, {}, memoryProperties); + if (!unifiedMemoryPointer) { + return nullptr; + } + + UNRECOVERABLE_IF(cmdQ == nullptr); + auto pageFaultManager = this->memoryManager->getPageFaultManager(); + pageFaultManager->insertAllocation(unifiedMemoryPointer, size, this, cmdQ, memoryProperties.allocationFlags); + } + auto unifiedMemoryAllocation = this->getSVMAlloc(unifiedMemoryPointer); unifiedMemoryAllocation->memoryType = memoryProperties.memoryType; unifiedMemoryAllocation->allocationFlagsProperty = memoryProperties.allocationFlags; - UNRECOVERABLE_IF(cmdQ == nullptr); - auto pageFaultManager = this->memoryManager->getPageFaultManager(); - pageFaultManager->insertAllocation(unifiedMemoryPointer, size, this, cmdQ, memoryProperties.allocationFlags); - return unifiedMemoryPointer; } return createUnifiedMemoryAllocation(rootDeviceIndex, size, memoryProperties); } +void *SVMAllocsManager::createUnifiedKmdMigratedAllocation(uint32_t rootDeviceIndex, size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties) { + size_t alignedSize = alignUp(size, 2 * MemoryConstants::megaByte); + AllocationProperties gpuProperties{rootDeviceIndex, + true, + alignedSize, + GraphicsAllocation::AllocationType::UNIFIED_SHARED_MEMORY, + unifiedMemoryProperties.subdeviceBitfield.count() > 1, + false, + unifiedMemoryProperties.subdeviceBitfield}; + + gpuProperties.alignment = 2 * MemoryConstants::megaByte; + MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false); + GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties); + if (!allocationGpu) { + return nullptr; + } + setUnifiedAllocationProperties(allocationGpu, svmProperties); + + SvmAllocationData allocData(rootDeviceIndex); + allocData.gpuAllocations.addAllocation(allocationGpu); + allocData.cpuAllocation = nullptr; + allocData.device = unifiedMemoryProperties.device; + allocData.size = size; + + std::unique_lock lock(mtx); + this->SVMAllocs.insert(allocData); + return allocationGpu->getUnderlyingBuffer(); +} + +void SVMAllocsManager::setUnifiedAllocationProperties(GraphicsAllocation *allocation, const SvmAllocationProperties &svmProperties) { + allocation->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly); + allocation->setCoherent(svmProperties.coherent); +} + SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) { std::unique_lock lock(mtx); return SVMAllocs.get(ptr); @@ -309,8 +358,7 @@ void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(uint32_t rootDe if (!allocationCpu) { return nullptr; } - allocationCpu->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly); - allocationCpu->setCoherent(svmProperties.coherent); + setUnifiedAllocationProperties(allocationCpu, svmProperties); void *svmPtr = allocationCpu->getUnderlyingBuffer(); AllocationProperties gpuProperties{rootDeviceIndex, @@ -328,8 +376,7 @@ void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(uint32_t rootDe memoryManager->freeGraphicsMemory(allocationCpu); return nullptr; } - allocationGpu->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly); - allocationGpu->setCoherent(svmProperties.coherent); + setUnifiedAllocationProperties(allocationGpu, svmProperties); SvmAllocationData allocData(rootDeviceIndex); allocData.gpuAllocations.addAllocation(allocationGpu); diff --git a/shared/source/memory_manager/unified_memory_manager.h b/shared/source/memory_manager/unified_memory_manager.h index 5cc30654be..ce9974b52d 100644 --- a/shared/source/memory_manager/unified_memory_manager.h +++ b/shared/source/memory_manager/unified_memory_manager.h @@ -114,6 +114,11 @@ class SVMAllocsManager { size_t size, const UnifiedMemoryProperties &svmProperties, void *cmdQ); + void *createUnifiedKmdMigratedAllocation(uint32_t rootDeviceIndex, + size_t size, + const SvmAllocationProperties &svmProperties, + const UnifiedMemoryProperties &unifiedMemoryProperties); + void setUnifiedAllocationProperties(GraphicsAllocation *allocation, const SvmAllocationProperties &svmProperties); SvmAllocationData *getSVMAlloc(const void *ptr); bool freeSVMAlloc(void *ptr, bool blocking); bool freeSVMAlloc(void *ptr) { return freeSVMAlloc(ptr, false); } diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index c89032ff42..a31cd569fe 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -94,6 +94,7 @@ class DrmMemoryManager : public MemoryManager { GraphicsAllocation *allocateShareableMemory(const AllocationData &allocationData) override; GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) override; GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override; + GraphicsAllocation *createSharedUnifiedMemoryAllocation(const AllocationData &allocationData); void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override; void *lockResourceInLocalMemoryImpl(GraphicsAllocation &graphicsAllocation); diff --git a/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool.cpp b/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool.cpp index 43f8beebd9..dfa916f190 100644 --- a/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool.cpp @@ -21,6 +21,10 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData & return createAllocWithAlignmentFromUserptr(allocationData, size, alignment, alignedSize, gpuAddress); } +GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) { + return nullptr; +} + GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) { status = AllocationStatus::RetryInNonDevicePool; return nullptr; diff --git a/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool_dg1.cpp b/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool_dg1.cpp index e13da00d34..34cd372526 100644 --- a/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool_dg1.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager_allocate_in_device_pool_dg1.cpp @@ -64,6 +64,10 @@ BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(Drm *drm, return bo; } +GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const AllocationData &allocationData) { + return nullptr; +} + DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress) { bool useBooMmap = this->getDrm(allocationData.rootDeviceIndex).getMemoryInfo() && allocationData.useMmapObject;