From ae56d50b4f8171d4888915831623992600c552f8 Mon Sep 17 00:00:00 2001 From: Krzysztof Gibala Date: Mon, 30 May 2022 14:18:50 +0000 Subject: [PATCH] Pass canonized gpuAddress to GraphicsAllocation Related-To: NEO-6523 Signed-off-by: Krzysztof Gibala --- .../windows/mock_wddm_allocation.h | 2 +- .../os_interface/windows/wddm20_tests.cpp | 41 +++++++++++++------ .../windows/wddm_memory_manager_tests.cpp | 18 +++++--- .../wddm_residency_controller_tests.cpp | 25 +++++++---- .../memory_manager/graphics_allocation.cpp | 4 +- .../memory_manager/graphics_allocation.h | 2 +- .../os_agnostic_memory_manager.cpp | 23 +++++++---- .../os_agnostic_memory_manager.h | 12 +++--- .../os_interface/linux/drm_allocation.h | 16 ++++---- .../os_interface/linux/drm_memory_manager.cpp | 29 +++++++++---- ...y_manager_create_multi_host_allocation.cpp | 4 +- .../os_interface/windows/wddm_allocation.h | 8 ++-- .../windows/wddm_memory_manager.cpp | 39 +++++++++++++----- 13 files changed, 145 insertions(+), 78 deletions(-) diff --git a/opencl/test/unit_test/os_interface/windows/mock_wddm_allocation.h b/opencl/test/unit_test/os_interface/windows/mock_wddm_allocation.h index a32c4dc268..1caed31e9a 100644 --- a/opencl/test/unit_test/os_interface/windows/mock_wddm_allocation.h +++ b/opencl/test/unit_test/os_interface/windows/mock_wddm_allocation.h @@ -16,7 +16,7 @@ class MockWddmAllocation : public WddmAllocation { public: MockWddmAllocation(GmmHelper *gmmHelper) : MockWddmAllocation(gmmHelper, EngineLimits::maxHandleCount) {} MockWddmAllocation(GmmHelper *gmmHelper, uint32_t numGmms) : WddmAllocation(0, numGmms, AllocationType::UNKNOWN, - nullptr, 0, nullptr, MemoryPool::MemoryNull, 0u, 3u), + nullptr, 0, 0, nullptr, MemoryPool::MemoryNull, 0u, 3u), gpuPtr(gpuAddress), handle(handles[0]) { for (uint32_t i = 0; i < numGmms; i++) { setGmm(new MockGmm(gmmHelper), i); diff --git a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp index 701ff15522..8dbcb84776 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm20_tests.cpp @@ -95,7 +95,9 @@ TEST_F(Wddm20Tests, givenNullPageTableManagerAndCompressedResourceWhenMappingGpu mockGmmRes->setUnifiedAuxTranslationCapable(); void *fakePtr = reinterpret_cast(0x100); - WddmAllocation allocation(0, AllocationType::UNKNOWN, fakePtr, 0x2100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(fakePtr))); + WddmAllocation allocation(0, AllocationType::UNKNOWN, fakePtr, canonizedAddress, 0x2100, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation.setDefaultGmm(gmm.get()); allocation.getHandleToModify(0u) = ALLOCATION_HANDLE; @@ -248,7 +250,10 @@ TEST_F(Wddm20Tests, whenInitPrivateDataThenDefaultValuesAreSet) { TEST_F(Wddm20Tests, WhenCreatingAllocationAndDestroyingAllocationThenCorrectResultReturned) { OsAgnosticMemoryManager mm(*executionEnvironment); - WddmAllocation allocation(0, AllocationType::UNKNOWN, mm.allocateSystemMemory(100, 0), 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = getGmmHelper(); + auto ptr = mm.allocateSystemMemory(100, 0); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); Gmm *gmm = GmmHelperFunctions::getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize(), getGmmHelper()); allocation.setDefaultGmm(gmm); @@ -272,7 +277,9 @@ TEST_F(Wddm20WithMockGdiDllTests, givenAllocationSmallerUnderlyingThanAlignedSiz size_t underlyingPages = underlyingSize / MemoryConstants::pageSize; size_t alignedPages = alignedSize / MemoryConstants::pageSize; - WddmAllocation allocation(0, AllocationType::UNKNOWN, ptr, 0x2100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 0x2100, nullptr, MemoryPool::MemoryNull, 0u, 1u); Gmm *gmm = GmmHelperFunctions::getGmm(allocation.getAlignedCpuPtr(), allocation.getAlignedSize(), getGmmHelper()); allocation.setDefaultGmm(gmm); @@ -314,7 +321,9 @@ TEST_F(Wddm20WithMockGdiDllTests, givenReserveCallWhenItIsCalledWithProperParamt TEST_F(Wddm20WithMockGdiDllTests, givenWddmAllocationWhenMappingGpuVaThenUseGmmSize) { void *fakePtr = reinterpret_cast(0x123); - WddmAllocation allocation(0, AllocationType::UNKNOWN, fakePtr, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(fakePtr))); + WddmAllocation allocation(0, AllocationType::UNKNOWN, fakePtr, canonizedAddress, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); std::unique_ptr gmm(GmmHelperFunctions::getGmm(allocation.getAlignedCpuPtr(), allocation.getAlignedSize(), getGmmHelper())); allocation.setDefaultGmm(gmm.get()); @@ -401,7 +410,10 @@ TEST_F(Wddm20WithMockGdiDllTests, GivenThreeOsHandlesWhenAskedForDestroyAllocati TEST_F(Wddm20Tests, WhenMappingAndFreeingGpuVaThenReturnIsCorrect) { OsAgnosticMemoryManager mm(*executionEnvironment); - WddmAllocation allocation(0, AllocationType::UNKNOWN, mm.allocateSystemMemory(100, 0), 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = getGmmHelper(); + auto ptr = mm.allocateSystemMemory(100, 0); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); Gmm *gmm = GmmHelperFunctions::getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize(), getGmmHelper()); allocation.setDefaultGmm(gmm); @@ -427,7 +439,7 @@ TEST_F(Wddm20Tests, WhenMappingAndFreeingGpuVaThenReturnIsCorrect) { TEST_F(Wddm20Tests, givenNullAllocationWhenCreateThenAllocateAndMap) { OsAgnosticMemoryManager mm(*executionEnvironment); - WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, 0, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); auto gmm = std::unique_ptr(GmmHelperFunctions::getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize(), getGmmHelper())); allocation.setDefaultGmm(gmm.get()); @@ -447,7 +459,7 @@ TEST_F(Wddm20Tests, givenNullAllocationWhenCreateThenAllocateAndMap) { TEST_F(WddmTestWithMockGdiDll, givenShareableAllocationWhenCreateThenCreateResourceFlagIsEnabled) { init(); - WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, true, 1u); + WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, 0, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, true, 1u); auto gmm = std::unique_ptr(GmmHelperFunctions::getGmm(nullptr, MemoryConstants::pageSize, getGmmHelper())); allocation.setDefaultGmm(gmm.get()); auto status = wddm->createAllocation(&allocation); @@ -464,7 +476,7 @@ TEST_F(WddmTestWithMockGdiDll, givenShareableAllocationWhenCreateThenSharedHandl using WddmMemoryManager::WddmMemoryManager; }; MemoryManagerCreate memoryManager(false, false, *executionEnvironment); - WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, true, 1u); + WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, 0, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, true, 1u); auto gmm = std::unique_ptr(GmmHelperFunctions::getGmm(nullptr, MemoryConstants::pageSize, getGmmHelper())); allocation.setDefaultGmm(gmm.get()); auto status = memoryManager.createGpuAllocationsWithRetry(&allocation); @@ -473,7 +485,7 @@ TEST_F(WddmTestWithMockGdiDll, givenShareableAllocationWhenCreateThenSharedHandl } TEST(WddmAllocationTest, whenAllocationIsShareableThenSharedHandleToModifyIsSharedHandleOfAllocation) { - WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, true, 1u); + WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, 0, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, true, 1u); auto sharedHandleToModify = allocation.getSharedHandleToModify(); EXPECT_NE(nullptr, sharedHandleToModify); *sharedHandleToModify = 1234u; @@ -481,14 +493,17 @@ TEST(WddmAllocationTest, whenAllocationIsShareableThenSharedHandleToModifyIsShar } TEST(WddmAllocationTest, whenAllocationIsNotShareableThenItDoesntReturnSharedHandleToModify) { - WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, false, 1u); + WddmAllocation allocation(0, AllocationType::UNKNOWN, nullptr, 0, MemoryConstants::pageSize, nullptr, MemoryPool::MemoryNull, false, 1u); auto sharedHandleToModify = allocation.getSharedHandleToModify(); EXPECT_EQ(nullptr, sharedHandleToModify); } TEST_F(Wddm20Tests, WhenMakingResidentAndEvictingThenReturnIsCorrect) { OsAgnosticMemoryManager mm(*executionEnvironment); - WddmAllocation allocation(0, AllocationType::UNKNOWN, mm.allocateSystemMemory(100, 0), 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = getGmmHelper(); + auto ptr = mm.allocateSystemMemory(100, 0); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); Gmm *gmm = GmmHelperFunctions::getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize(), getGmmHelper()); allocation.setDefaultGmm(gmm); @@ -925,8 +940,10 @@ TEST_F(Wddm20Tests, whenCreateAllocation64kFailsThenReturnFalse) { gdi->createAllocation2 = FailingCreateAllocation::mockCreateAllocation2; void *fakePtr = reinterpret_cast(0x123); + auto gmmHelper = getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(fakePtr))); auto gmm = std::make_unique(rootDeviceEnvironment->getGmmHelper(), fakePtr, 100, 0, GMM_RESOURCE_USAGE_OCL_BUFFER, false, StorageInfo{}, true); - WddmAllocation allocation(0, AllocationType::UNKNOWN, fakePtr, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation(0, AllocationType::UNKNOWN, fakePtr, canonizedAddress, 100, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation.setDefaultGmm(gmm.get()); EXPECT_FALSE(wddm->createAllocation64k(&allocation)); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index 86c68deeaa..c5a959bb5b 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -121,15 +121,15 @@ TEST(WddmAllocationTest, givenRequestedContextIdTooLargeWhenGettingTrimCandidate } TEST(WddmAllocationTest, givenAllocationTypeWhenPassedToWddmAllocationConstructorThenAllocationTypeIsStored) { - WddmAllocation allocation{0, AllocationType::COMMAND_BUFFER, nullptr, 0, nullptr, MemoryPool::MemoryNull, 0u, 1u}; + WddmAllocation allocation{0, AllocationType::COMMAND_BUFFER, nullptr, 0, 0, nullptr, MemoryPool::MemoryNull, 0u, 1u}; EXPECT_EQ(AllocationType::COMMAND_BUFFER, allocation.getAllocationType()); } TEST(WddmAllocationTest, givenMemoryPoolWhenPassedToWddmAllocationConstructorThenMemoryPoolIsStored) { - WddmAllocation allocation{0, AllocationType::COMMAND_BUFFER, nullptr, 0, nullptr, MemoryPool::System64KBPages, 0u, 1u}; + WddmAllocation allocation{0, AllocationType::COMMAND_BUFFER, nullptr, 0, 0, nullptr, MemoryPool::System64KBPages, 0u, 1u}; EXPECT_EQ(MemoryPool::System64KBPages, allocation.getMemoryPool()); - WddmAllocation allocation2{0, AllocationType::COMMAND_BUFFER, nullptr, 0, 0u, MemoryPool::SystemCpuInaccessible, 0u, 1u}; + WddmAllocation allocation2{0, AllocationType::COMMAND_BUFFER, nullptr, 0, 0, 0u, MemoryPool::SystemCpuInaccessible, 0u, 1u}; EXPECT_EQ(MemoryPool::SystemCpuInaccessible, allocation2.getMemoryPool()); } @@ -1325,7 +1325,9 @@ TEST_F(WddmMemoryManagerTest, givenManagerWithDisabledDeferredDeleterWhenMapGpuV memoryManager->setDeferredDeleter(nullptr); setMapGpuVaFailConfigFcn(0, 1); - WddmAllocation allocation(0, AllocationType::BUFFER, ptr, size, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = rootDeviceEnvironment->getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::BUFFER, ptr, canonizedAddress, size, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation.setDefaultGmm(gmm.get()); bool ret = memoryManager->createWddmAllocation(&allocation, allocation.getAlignedCpuPtr()); EXPECT_FALSE(ret); @@ -1341,7 +1343,9 @@ TEST_F(WddmMemoryManagerTest, givenManagerWithEnabledDeferredDeleterWhenFirstMap setMapGpuVaFailConfigFcn(0, 1); - WddmAllocation allocation(0, AllocationType::BUFFER, ptr, size, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = rootDeviceEnvironment->getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::BUFFER, ptr, canonizedAddress, size, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation.setDefaultGmm(gmm.get()); bool ret = memoryManager->createWddmAllocation(&allocation, allocation.getAlignedCpuPtr()); EXPECT_TRUE(ret); @@ -1357,7 +1361,9 @@ TEST_F(WddmMemoryManagerTest, givenManagerWithEnabledDeferredDeleterWhenFirstAnd setMapGpuVaFailConfigFcn(0, 2); - WddmAllocation allocation(0, AllocationType::BUFFER, ptr, size, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = rootDeviceEnvironment->getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation(0, AllocationType::BUFFER, ptr, canonizedAddress, size, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation.setDefaultGmm(gmm.get()); bool ret = memoryManager->createWddmAllocation(&allocation, allocation.getAlignedCpuPtr()); EXPECT_FALSE(ret); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp index f9d2b83021..3694335240 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_controller_tests.cpp @@ -714,9 +714,12 @@ TEST_F(WddmResidencyControllerWithGdiTest, GivenNumBytesToTrimIsNotZeroWhenTrimm } TEST_F(WddmResidencyControllerWithGdiTest, GivenNumBytesToTrimIsZeroWhenTrimmingToBudgetThenEvictingStops) { - WddmAllocation allocation1(0, AllocationType::UNKNOWN, reinterpret_cast(0x1000), 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); - WddmAllocation allocation2(0, AllocationType::UNKNOWN, reinterpret_cast(0x1000), 0x3000, nullptr, MemoryPool::MemoryNull, 0u, 1u); - WddmAllocation allocation3(0, AllocationType::UNKNOWN, reinterpret_cast(0x1000), 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto ptr = reinterpret_cast(0x1000); + auto gmmHelper = rootDeviceEnvironment->getGmmHelper(); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation1(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation2(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 0x3000, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation3(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation1.getResidencyData().resident[osContextId] = true; allocation1.getResidencyData().updateCompletionData(0, osContextId); @@ -814,8 +817,10 @@ TEST_F(WddmResidencyControllerWithGdiAndMemoryManagerTest, WhenTrimmingToBudgetT } gdi->setNonZeroNumBytesToTrimInEvict(); void *ptr = reinterpret_cast(wddm->virtualAllocAddress + 0x1000); - WddmAllocation allocation1(0, AllocationType::UNKNOWN, ptr, 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); - WddmAllocation allocation2(0, AllocationType::UNKNOWN, ptr, 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = memoryManager->getGmmHelper(0); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptr))); + WddmAllocation allocation1(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation2(0, AllocationType::UNKNOWN, ptr, canonizedAddress, 0x1000, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation1.getResidencyData().resident[osContextId] = true; allocation1.getResidencyData().updateCompletionData(0, osContextId); @@ -873,9 +878,10 @@ TEST_F(WddmResidencyControllerWithGdiTest, givenThreeAllocationsAlignedSizeBigge void *ptr2 = reinterpret_cast(wddm->virtualAllocAddress + 0x3000); void *ptr3 = reinterpret_cast(wddm->virtualAllocAddress + 0x5000); - WddmAllocation allocation1(0, AllocationType::UNKNOWN, ptr1, underlyingSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); - WddmAllocation allocation2(0, AllocationType::UNKNOWN, ptr2, underlyingSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); - WddmAllocation allocation3(0, AllocationType::UNKNOWN, ptr3, underlyingSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto gmmHelper = rootDeviceEnvironment->getGmmHelper(); + WddmAllocation allocation1(0, AllocationType::UNKNOWN, ptr1, gmmHelper->canonize(castToUint64(const_cast(ptr1))), underlyingSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation2(0, AllocationType::UNKNOWN, ptr2, gmmHelper->canonize(castToUint64(const_cast(ptr2))), underlyingSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); + WddmAllocation allocation3(0, AllocationType::UNKNOWN, ptr3, gmmHelper->canonize(castToUint64(const_cast(ptr3))), underlyingSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocation1.getResidencyData().resident[osContextId] = true; allocation1.getResidencyData().updateCompletionData(0, osContextId); @@ -1086,7 +1092,8 @@ TEST_F(WddmResidencyControllerWithMockWddmTest, givenMakeResidentFailsAndTrimToB MockWddmAllocation allocation1(gmmHelper); void *cpuPtr = reinterpret_cast(wddm->getWddmMinAddress() + 0x1000); size_t allocationSize = 0x1000; - WddmAllocation allocationToTrim(0, AllocationType::UNKNOWN, cpuPtr, allocationSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(cpuPtr))); + WddmAllocation allocationToTrim(0, AllocationType::UNKNOWN, cpuPtr, canonizedAddress, allocationSize, nullptr, MemoryPool::MemoryNull, 0u, 1u); allocationToTrim.getResidencyData().updateCompletionData(residencyController->getMonitoredFence().lastSubmittedFence, osContext->getContextId()); diff --git a/shared/source/memory_manager/graphics_allocation.cpp b/shared/source/memory_manager/graphics_allocation.cpp index d48be31852..f18919bc84 100644 --- a/shared/source/memory_manager/graphics_allocation.cpp +++ b/shared/source/memory_manager/graphics_allocation.cpp @@ -20,11 +20,11 @@ void GraphicsAllocation::setAllocationType(AllocationType allocationType) { fileLoggerInstance().logAllocation(this); } -GraphicsAllocation::GraphicsAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t gpuAddress, +GraphicsAllocation::GraphicsAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t canonizedGpuAddress, uint64_t baseAddress, size_t sizeIn, MemoryPool::Type pool, size_t maxOsContextCount) : rootDeviceIndex(rootDeviceIndex), gpuBaseAddress(baseAddress), - gpuAddress(GmmHelper::canonize(gpuAddress)), + gpuAddress(canonizedGpuAddress), size(sizeIn), cpuPtr(cpuPtrIn), memoryPool(pool), diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index b77fb5f10f..20723ba619 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -75,7 +75,7 @@ class GraphicsAllocation : public IDNode { : GraphicsAllocation(rootDeviceIndex, 1, allocationType, cpuPtrIn, sizeIn, sharedHandleIn, pool, maxOsContextCount) {} GraphicsAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, - uint64_t gpuAddress, uint64_t baseAddress, size_t sizeIn, MemoryPool::Type pool, size_t maxOsContextCount); + uint64_t canonizedGpuAddress, uint64_t baseAddress, size_t sizeIn, MemoryPool::Type pool, size_t maxOsContextCount); GraphicsAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandleIn, MemoryPool::Type pool, size_t maxOsContextCount); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.cpp b/shared/source/memory_manager/os_agnostic_memory_manager.cpp index 459beb4397..573b247a41 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.cpp +++ b/shared/source/memory_manager/os_agnostic_memory_manager.cpp @@ -186,9 +186,11 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(con return nullptr; } uint64_t offset = static_cast(reinterpret_cast(allocationData.hostPtr) & MemoryConstants::pageMask); + + auto canonizedGpuAddress = gmmHelper->canonize(gpuVirtualAddress + offset); MemoryAllocation *memAlloc = new MemoryAllocation( allocationData.rootDeviceIndex, allocationData.type, nullptr, const_cast(allocationData.hostPtr), - gmmHelper->canonize(gpuVirtualAddress + offset), allocationData.size, + canonizedGpuAddress, allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false, false, maxOsContextCount); memAlloc->set32BitAllocation(true); @@ -213,8 +215,9 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(con MemoryAllocation *memoryAllocation = nullptr; if (ptrAlloc != nullptr) { + auto canonizedGpuAddress = gmmHelper->canonize(gpuAddress); memoryAllocation = new MemoryAllocation(allocationData.rootDeviceIndex, allocationData.type, ptrAlloc, ptrAlloc, - gmmHelper->canonize(gpuAddress), + canonizedGpuAddress, allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false, allocationData.flags.flushL3, maxOsContextCount); @@ -419,8 +422,10 @@ MemoryAllocation *OsAgnosticMemoryManager::createMemoryAllocation(AllocationType void *pMem, uint64_t gpuAddress, size_t memSize, uint64_t count, MemoryPool::Type pool, uint32_t rootDeviceIndex, bool uncacheable, bool flushL3Required, bool requireSpecificBitness) { + auto gmmHelper = getGmmHelper(rootDeviceIndex); if (!isLimitedRange(rootDeviceIndex)) { - return new MemoryAllocation(rootDeviceIndex, allocationType, driverAllocatedCpuPointer, pMem, gpuAddress, memSize, + auto canonizedGpuAddress = gmmHelper->canonize(gpuAddress); + return new MemoryAllocation(rootDeviceIndex, allocationType, driverAllocatedCpuPointer, pMem, canonizedGpuAddress, memSize, count, pool, uncacheable, flushL3Required, maxOsContextCount); } @@ -430,8 +435,8 @@ MemoryAllocation *OsAgnosticMemoryManager::createMemoryAllocation(AllocationType auto gfxPartition = getGfxPartition(rootDeviceIndex); uint64_t limitedGpuAddress = gfxPartition->heapAllocate(heap, alignedSize); - auto gmmHelper = getGmmHelper(rootDeviceIndex); - auto memoryAllocation = new MemoryAllocation(rootDeviceIndex, allocationType, driverAllocatedCpuPointer, pMem, limitedGpuAddress, memSize, + auto canonizedGpuAddress = gmmHelper->canonize(limitedGpuAddress); + auto memoryAllocation = new MemoryAllocation(rootDeviceIndex, allocationType, driverAllocatedCpuPointer, pMem, canonizedGpuAddress, memSize, count, pool, uncacheable, flushL3Required, maxOsContextCount); if (heap == HeapIndex::HEAP_EXTERNAL) { @@ -461,6 +466,7 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool( MemoryAllocation *allocation = nullptr; status = AllocationStatus::RetryInNonDevicePool; auto numHandles = allocationData.storageInfo.getNumBanks(); + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); if (!this->localMemorySupported[allocationData.rootDeviceIndex]) { return nullptr; @@ -477,7 +483,8 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool( allocation = static_cast(allocate32BitGraphicsMemoryImpl(adjustedAllocationData, true)); } else if (allocationData.type == AllocationType::SVM_GPU) { auto storage = allocateSystemMemory(allocationData.size, MemoryConstants::pageSize2Mb); - allocation = new MemoryAllocation(allocationData.rootDeviceIndex, numHandles, allocationData.type, storage, storage, reinterpret_cast(allocationData.hostPtr), + auto canonizedGpuAddress = gmmHelper->canonize(reinterpret_cast(allocationData.hostPtr)); + allocation = new MemoryAllocation(allocationData.rootDeviceIndex, numHandles, allocationData.type, storage, storage, canonizedGpuAddress, allocationData.size, counter, MemoryPool::LocalMemory, false, allocationData.flags.flushL3, maxOsContextCount); counter++; } else { @@ -518,9 +525,9 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool( } auto sizeOfHeapChunk = sizeAligned64k; auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); - auto gpuAddress = gmmHelper->canonize(gfxPartition->heapAllocate(heapIndex, sizeOfHeapChunk)); + auto canonizedGpuAddress = gmmHelper->canonize(gfxPartition->heapAllocate(heapIndex, sizeOfHeapChunk)); allocation = new MemoryAllocation(allocationData.rootDeviceIndex, numHandles, allocationData.type, systemMemory, systemMemory, - gpuAddress, sizeAligned64k, counter, + canonizedGpuAddress, sizeAligned64k, counter, MemoryPool::LocalMemory, false, allocationData.flags.flushL3, maxOsContextCount); counter++; allocation->setDefaultGmm(gmm.release()); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.h b/shared/source/memory_manager/os_agnostic_memory_manager.h index c8e850028b..d4db0cc856 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.h +++ b/shared/source/memory_manager/os_agnostic_memory_manager.h @@ -22,9 +22,9 @@ class MemoryAllocation : public GraphicsAllocation { MemoryPool::Type pool, size_t maxOsContextCount) : MemoryAllocation(rootDeviceIndex, 1, allocationType, cpuPtrIn, gpuAddress, baseAddress, sizeIn, pool, maxOsContextCount) {} - MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t gpuAddress, uint64_t baseAddress, size_t sizeIn, + MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t canonizedGpuAddress, uint64_t baseAddress, size_t sizeIn, MemoryPool::Type pool, size_t maxOsContextCount) - : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, gpuAddress, baseAddress, sizeIn, pool, maxOsContextCount), + : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, canonizedGpuAddress, baseAddress, sizeIn, pool, maxOsContextCount), id(0), uncacheable(false) {} MemoryAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandleIn, MemoryPool::Type pool, size_t maxOsContextCount) @@ -34,14 +34,14 @@ class MemoryAllocation : public GraphicsAllocation { : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, sizeIn, sharedHandleIn, pool, maxOsContextCount), id(0), uncacheable(false) {} - MemoryAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize, + MemoryAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t canonizedGpuAddress, size_t memSize, uint64_t count, MemoryPool::Type pool, bool uncacheable, bool flushL3Required, size_t maxOsContextCount) - : MemoryAllocation(rootDeviceIndex, 1, allocationType, driverAllocatedCpuPointer, pMem, gpuAddress, memSize, + : MemoryAllocation(rootDeviceIndex, 1, allocationType, driverAllocatedCpuPointer, pMem, canonizedGpuAddress, memSize, count, pool, uncacheable, flushL3Required, maxOsContextCount) {} - MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize, + MemoryAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t canonizedGpuAddress, size_t memSize, uint64_t count, MemoryPool::Type pool, bool uncacheable, bool flushL3Required, size_t maxOsContextCount) - : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, pMem, gpuAddress, 0u, memSize, pool, maxOsContextCount), + : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, pMem, canonizedGpuAddress, 0u, memSize, pool, maxOsContextCount), id(count), uncacheable(uncacheable) { this->driverAllocatedCpuPointer = driverAllocatedCpuPointer; diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 650363c151..a78d4c5845 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -41,19 +41,19 @@ class DrmAllocation : public GraphicsAllocation { bufferObjects[0] = bo; } - DrmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, BufferObject *bo, void *ptrIn, uint64_t gpuAddress, size_t sizeIn, MemoryPool::Type pool) - : DrmAllocation(rootDeviceIndex, 1, allocationType, bo, ptrIn, gpuAddress, sizeIn, pool) {} + DrmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, BufferObject *bo, void *ptrIn, uint64_t canonizedGpuAddress, size_t sizeIn, MemoryPool::Type pool) + : DrmAllocation(rootDeviceIndex, 1, allocationType, bo, ptrIn, canonizedGpuAddress, sizeIn, pool) {} - DrmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, BufferObject *bo, void *ptrIn, uint64_t gpuAddress, size_t sizeIn, MemoryPool::Type pool) - : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, ptrIn, gpuAddress, 0, sizeIn, pool, MemoryManager::maxOsContextCount), bufferObjects(EngineLimits::maxHandleCount) { + DrmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, BufferObject *bo, void *ptrIn, uint64_t canonizedGpuAddress, size_t sizeIn, MemoryPool::Type pool) + : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, ptrIn, canonizedGpuAddress, 0, sizeIn, pool, MemoryManager::maxOsContextCount), bufferObjects(EngineLimits::maxHandleCount) { bufferObjects[0] = bo; } - DrmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, BufferObjects &bos, void *ptrIn, uint64_t gpuAddress, size_t sizeIn, MemoryPool::Type pool) - : DrmAllocation(rootDeviceIndex, 1, allocationType, bos, ptrIn, gpuAddress, sizeIn, pool) {} + DrmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, BufferObjects &bos, void *ptrIn, uint64_t canonizedGpuAddress, size_t sizeIn, MemoryPool::Type pool) + : DrmAllocation(rootDeviceIndex, 1, allocationType, bos, ptrIn, canonizedGpuAddress, sizeIn, pool) {} - DrmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, BufferObjects &bos, void *ptrIn, uint64_t gpuAddress, size_t sizeIn, MemoryPool::Type pool) - : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, ptrIn, gpuAddress, 0, sizeIn, pool, MemoryManager::maxOsContextCount), + DrmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, BufferObjects &bos, void *ptrIn, uint64_t canonizedGpuAddress, size_t sizeIn, MemoryPool::Type pool) + : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, ptrIn, canonizedGpuAddress, 0, sizeIn, pool, MemoryManager::maxOsContextCount), bufferObjects(bos) { } diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 0450e12915..50f3b3161a 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -274,7 +274,9 @@ void DrmMemoryManager::emitPinningRequest(BufferObject *bo, const AllocationData DrmAllocation *DrmMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) { auto hostPtr = const_cast(allocationData.hostPtr); - auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, nullptr, hostPtr, castToUint64(hostPtr), allocationData.size, MemoryPool::System4KBPages); + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(castToUint64(hostPtr)); + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, nullptr, hostPtr, canonizedGpuAddress, allocationData.size, MemoryPool::System4KBPages); allocation->fragmentsStorage = handleStorage; if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { return nullptr; @@ -347,7 +349,9 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignmentFromUserptr(const Alloc obtainGpuAddress(allocationData, bo.get(), gpuAddress); emitPinningRequest(bo.get(), allocationData); - auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), res, bo->peekAddress(), size, MemoryPool::System4KBPages); + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(bo->peekAddress()); + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), res, canonizedGpuAddress, size, MemoryPool::System4KBPages); allocation->setDriverAllocatedCpuPtr(res); allocation->setReservedAddressRange(reinterpret_cast(gpuAddress), alignedSVMSize); if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { @@ -597,8 +601,9 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemoryImpl(const Allocatio bo->setAddress(gpuVirtualAddress); auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(ptrOffset(gpuVirtualAddress, inputPointerOffset)); auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), const_cast(allocationData.hostPtr), - gmmHelper->canonize(ptrOffset(gpuVirtualAddress, inputPointerOffset)), + canonizedGpuAddress, allocationSize, MemoryPool::System4KBPagesWith32BitGpuAddressing); allocation->set32BitAllocation(true); allocation->setGpuBaseAddress(gmmHelper->canonize(gfxPartition->getHeapBase(allocatorToUse))); @@ -635,8 +640,9 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemoryImpl(const Allocatio auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); // softpin to the GPU address, res if it uses limitedRange Allocation + auto canonizedGpuAddress = gmmHelper->canonize(gpuVA); auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), ptrAlloc, - gmmHelper->canonize(gpuVA), alignedAllocationSize, + canonizedGpuAddress, alignedAllocationSize, MemoryPool::System4KBPagesWith32BitGpuAddressing); allocation->set32BitAllocation(true); @@ -875,8 +881,9 @@ GraphicsAllocation *DrmMemoryManager::createPaddedAllocation(GraphicsAllocation } bo->setAddress(gpuRange); auto gmmHelper = getGmmHelper(rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(ptrOffset(gpuRange, offset)); auto allocation = new DrmAllocation(rootDeviceIndex, inputGraphicsAllocation->getAllocationType(), bo.get(), srcPtr, - gmmHelper->canonize(ptrOffset(gpuRange, offset)), sizeWithPadding, + canonizedGpuAddress, sizeWithPadding, inputGraphicsAllocation->getMemoryPool()); allocation->setReservedAddressRange(reinterpret_cast(gpuRange), sizeWithPadding); @@ -1471,8 +1478,8 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const A auto gpuAddress = getGpuAddress(this->alignmentSelector, this->heapAssigner, *hwInfo, allocationData.type, gfxPartition, sizeAllocated, allocationData.hostPtr, allocationData.flags.resource48Bit, allocationData.flags.use32BitFrontWindow, *gmmHelper); - - auto allocation = std::make_unique(allocationData.rootDeviceIndex, numHandles, allocationData.type, nullptr, nullptr, gpuAddress, sizeAligned, MemoryPool::LocalMemory); + auto canonizedGpuAddress = gmmHelper->canonize(gpuAddress); + auto allocation = std::make_unique(allocationData.rootDeviceIndex, numHandles, allocationData.type, nullptr, nullptr, canonizedGpuAddress, sizeAligned, MemoryPool::LocalMemory); DrmAllocation *drmAllocation = static_cast(allocation.get()); GraphicsAllocation *graphicsAllocation = static_cast(allocation.get()); @@ -1728,7 +1735,9 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData & obtainGpuAddress(allocationData, bo.get(), gpuAddress); emitPinningRequest(bo.get(), allocationData); - auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), alignedSize, MemoryPool::System4KBPages); + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(bo->peekAddress()); + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, canonizedGpuAddress, alignedSize, MemoryPool::System4KBPages); allocation->setMmapPtr(cpuPointer); allocation->setMmapSize(alignedSize); if (pointerDiff != 0) { @@ -1864,7 +1873,9 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const bo->setAddress(reinterpret_cast(cpuPointer)); - auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, bo->peekAddress(), size, MemoryPool::System4KBPages); + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(bo->peekAddress()); + auto allocation = std::make_unique(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, canonizedGpuAddress, size, MemoryPool::System4KBPages); allocation->setMmapPtr(cpuBasePointer); allocation->setMmapSize(totalSizeToAlloc); if (!allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion))) { diff --git a/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp b/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp index c2af2402c1..af2fa607bc 100644 --- a/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager_create_multi_host_allocation.cpp @@ -36,8 +36,10 @@ DrmAllocation *DrmMemoryManager::createMultiHostAllocation(const AllocationData gpuAddress = allocationData.gpuAddress; } + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedGpuAddress = gmmHelper->canonize(gpuAddress); auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, numTiles, allocationData.type, - nullptr /*bo*/, cpuBasePointer, gpuAddress, sizePerTile, MemoryPool::System4KBPages); + nullptr /*bo*/, cpuBasePointer, canonizedGpuAddress, sizePerTile, MemoryPool::System4KBPages); allocation->storageInfo = allocationData.storageInfo; allocation->setFlushL3Required(true); diff --git a/shared/source/os_interface/windows/wddm_allocation.h b/shared/source/os_interface/windows/wddm_allocation.h index 6631ba17ec..71e8377dca 100644 --- a/shared/source/os_interface/windows/wddm_allocation.h +++ b/shared/source/os_interface/windows/wddm_allocation.h @@ -24,12 +24,12 @@ constexpr size_t trimListUnusedPosition = std::numeric_limits::max(); class WddmAllocation : public GraphicsAllocation { public: - WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool, uint32_t shareable, size_t maxOsContextCount) - : WddmAllocation(rootDeviceIndex, 1, allocationType, cpuPtrIn, sizeIn, reservedAddr, pool, shareable, maxOsContextCount) {} + WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, uint64_t canonizedAddress, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool, uint32_t shareable, size_t maxOsContextCount) + : WddmAllocation(rootDeviceIndex, 1, allocationType, cpuPtrIn, canonizedAddress, sizeIn, reservedAddr, pool, shareable, maxOsContextCount) {} - WddmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, + WddmAllocation(uint32_t rootDeviceIndex, size_t numGmms, AllocationType allocationType, void *cpuPtrIn, uint64_t canonizedAddress, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool, uint32_t shareable, size_t maxOsContextCount) - : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool, maxOsContextCount), + : GraphicsAllocation(rootDeviceIndex, numGmms, allocationType, cpuPtrIn, canonizedAddress, 0llu, sizeIn, pool, maxOsContextCount), shareable(shareable), trimCandidateListPositions(maxOsContextCount, trimListUnusedPosition) { reservedAddressRangeInfo.addressPtr = reservedAddr; reservedAddressRangeInfo.rangeSize = sizeIn; diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index 28461f0c9e..577ab1d32c 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -78,7 +78,7 @@ GraphicsAllocation *WddmMemoryManager::allocateMemoryByKMD(const AllocationData CacheSettingsHelper::getGmmUsageType(allocationData.type, !!allocationData.flags.uncacheable, *hwInfo), false, systemMemoryStorageInfo, true); auto allocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, nullptr, allocationData.size, nullptr, + allocationData.type, nullptr, 0, allocationData.size, nullptr, MemoryPool::SystemCpuInaccessible, allocationData.flags.shareable, maxOsContextCount); allocation->setDefaultGmm(gmm.get()); if (!createWddmAllocation(allocation.get(), nullptr)) { @@ -93,7 +93,7 @@ GraphicsAllocation *WddmMemoryManager::allocateMemoryByKMD(const AllocationData GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr gmm) { auto allocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, nullptr, allocationData.imgInfo->size, + allocationData.type, nullptr, 0, allocationData.imgInfo->size, nullptr, MemoryPool::SystemCpuInaccessible, 0u, // shareable maxOsContextCount); @@ -119,7 +119,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryUsingKmdAndMapItToC auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, nullptr, + allocationData.type, nullptr, 0, sizeAligned, nullptr, allowLargePages ? MemoryPool::System64KBPages : MemoryPool::System4KBPages, 0u, // shareable maxOsContextCount); @@ -177,8 +177,10 @@ GraphicsAllocation *WddmMemoryManager::allocateHugeGraphicsMemory(const Allocati auto chunkSize = getHugeGfxMemoryChunkSize(GfxMemoryAllocationMethod::UseUmdSystemPtr); auto numGmms = (alignedSize + chunkSize - 1) / chunkSize; + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(hostPtr))); auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, numGmms, - allocationData.type, hostPtr, allocationData.size, + allocationData.type, hostPtr, canonizedAddress, allocationData.size, nullptr, memoryPool, 0u, // shareable maxOsContextCount); @@ -241,9 +243,11 @@ GraphicsAllocation *WddmMemoryManager::allocateSystemMemoryAndCreateGraphicsAllo return nullptr; } + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedAddress = gmmHelper->canonize(castToUint64(pSysMem)); auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, pSysMem, sizeAligned, + allocationData.type, pSysMem, canonizedAddress, sizeAligned, nullptr, MemoryPool::System4KBPages, 0u, // shareable maxOsContextCount); @@ -286,9 +290,11 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(co return allocateHugeGraphicsMemory(allocationData, false); } + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(allocationData.hostPtr))); auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, const_cast(allocationData.hostPtr), + allocationData.type, const_cast(allocationData.hostPtr), canonizedAddress, allocationData.size, nullptr, MemoryPool::System4KBPages, 0u, // shareable maxOsContextCount); @@ -328,9 +334,14 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithHostPtr(const A return nullptr; } + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(inputPtr))); auto allocation = new WddmAllocation(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, const_cast(inputPtr), allocationData.size, + allocationData.type, + const_cast(inputPtr), + canonizedAddress, + allocationData.size, reserve, MemoryPool::System4KBPages, 0u, // shareable maxOsContextCount); @@ -372,9 +383,12 @@ GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const All sizeAligned = alignUp(sizeAligned, MemoryConstants::allocationAlignment); } + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(ptrAligned))); auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, const_cast(ptrAligned), sizeAligned, nullptr, + allocationData.type, const_cast(ptrAligned), canonizedAddress, + sizeAligned, nullptr, MemoryPool::System4KBPagesWith32BitGpuAddressing, 0u, // shareable maxOsContextCount); @@ -395,7 +409,6 @@ GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const All return nullptr; } auto baseAddress = getGfxPartition(allocationData.rootDeviceIndex)->getHeapBase(heapAssigner.get32BitHeapIndex(allocationData.type, useLocalMemory, *hwInfo, allocationData.flags.use32BitFrontWindow)); - auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); wddmAllocation->setGpuBaseAddress(gmmHelper->canonize(baseAddress)); if (preferredAllocationMethod != GfxMemoryAllocationMethod::UseUmdSystemPtr) { @@ -696,9 +709,13 @@ void WddmMemoryManager::obtainGpuAddressFromFragments(WddmAllocation *allocation } GraphicsAllocation *WddmMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) { + auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); + auto canonizedAddress = gmmHelper->canonize(castToUint64(const_cast(allocationData.hostPtr))); auto allocation = new WddmAllocation(allocationData.rootDeviceIndex, 1u, // numGmms - allocationData.type, const_cast(allocationData.hostPtr), + allocationData.type, + const_cast(allocationData.hostPtr), + canonizedAddress, allocationData.size, nullptr, MemoryPool::System4KBPages, 0u, // shareable maxOsContextCount); @@ -1035,7 +1052,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const const size_t numGmms = (static_cast(sizeAligned) + chunkSize - 1) / chunkSize; auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, singleBankAllocation ? numGmms : numBanks, - allocationData.type, nullptr, sizeAligned, nullptr, MemoryPool::LocalMemory, allocationData.flags.shareable, maxOsContextCount); + allocationData.type, nullptr, 0, sizeAligned, nullptr, MemoryPool::LocalMemory, allocationData.flags.shareable, maxOsContextCount); if (singleBankAllocation) { if (numGmms > 1) { splitGmmsInAllocation(gmmHelper, wddmAllocation.get(), alignment, chunkSize, const_cast(allocationData.storageInfo));