diff --git a/shared/source/memory_manager/allocation_properties.h b/shared/source/memory_manager/allocation_properties.h index d507db9e12..baa7a45e7c 100644 --- a/shared/source/memory_manager/allocation_properties.h +++ b/shared/source/memory_manager/allocation_properties.h @@ -47,6 +47,8 @@ struct AllocationProperties { uint64_t gpuAddress = 0; OsContext *osContext = nullptr; bool useMmapObject = true; + bool forceKMDAllocation = false; + bool makeGPUVaDifferentThanCPUPtr = false; uint32_t cacheRegion = 0; bool makeDeviceBufferLockable = false; @@ -126,6 +128,8 @@ struct AllocationData { uint32_t rootDeviceIndex = 0; OsContext *osContext = nullptr; bool useMmapObject = true; + bool forceKMDAllocation = false; + bool makeGPUVaDifferentThanCPUPtr = false; uint32_t cacheRegion = 0; }; } // namespace NEO diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 7bde03ffb8..030400927f 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -426,6 +426,8 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo allocationData.flags.preferCompressed |= CompressionSelector::preferCompressedAllocation(properties, hwInfo); allocationData.flags.multiOsContextCapable = properties.flags.multiOsContextCapable; allocationData.usmInitialPlacement = properties.usmInitialPlacement; + allocationData.forceKMDAllocation = properties.forceKMDAllocation; + allocationData.makeGPUVaDifferentThanCPUPtr = properties.makeGPUVaDifferentThanCPUPtr; if (GraphicsAllocation::isDebugSurfaceAllocationType(properties.allocationType)) { allocationData.flags.zeroMemory = 1; diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index 57a476cbc5..e281fced25 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -493,6 +493,8 @@ void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(size_t size, co subDevices}; cpuProperties.alignment = MemoryConstants::pageSize2Mb; cpuProperties.flags.isUSMHostAllocation = useExternalHostPtrForCpu; + cpuProperties.forceKMDAllocation = true; + cpuProperties.makeGPUVaDifferentThanCPUPtr = true; auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags); MemoryPropertiesHelper::fillCachePolicyInProperties(cpuProperties, false, svmProperties.readOnly, false, cacheRegion); GraphicsAllocation *allocationCpu = memoryManager->allocateGraphicsMemoryWithProperties(cpuProperties, externalPtr); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index af69de99bd..4dcabae5f1 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -118,6 +118,10 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryUsingKmdAndMapItToC return allocateHugeGraphicsMemory(allocationData, false); } + if (preferredAllocationMethod == GfxMemoryAllocationMethod::AllocateByKmd && allocationData.makeGPUVaDifferentThanCPUPtr) { + sizeAligned = sizeAligned + allocationData.alignment; + } + auto wddmAllocation = std::make_unique(allocationData.rootDeviceIndex, 1u, // numGmms allocationData.type, nullptr, 0, @@ -146,10 +150,16 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryUsingKmdAndMapItToC [[maybe_unused]] auto status = true; - if (executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo()->capabilityTable.gpuAddressSpace >= MemoryConstants::max64BitAppAddress || is32bit) { + if ((!(preferredAllocationMethod == GfxMemoryAllocationMethod::AllocateByKmd && allocationData.makeGPUVaDifferentThanCPUPtr) && executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHardwareInfo()->capabilityTable.gpuAddressSpace >= MemoryConstants::max64BitAppAddress) || is32bit) { status = mapGpuVirtualAddress(wddmAllocation.get(), cpuPtr); } else { status = mapGpuVirtualAddress(wddmAllocation.get(), nullptr); + + if (preferredAllocationMethod == GfxMemoryAllocationMethod::AllocateByKmd) { + void *tempCPUPtr = cpuPtr; + cpuPtr = alignUp(cpuPtr, MemoryConstants::pageSize64k); + wddmAllocation->setGpuAddress(wddmAllocation->getGpuAddress() + ptrDiff(cpuPtr, tempCPUPtr)); + } } DEBUG_BREAK_IF(!status); wddmAllocation->setCpuAddress(cpuPtr); @@ -224,7 +234,7 @@ GraphicsAllocation *WddmMemoryManager::allocateUSMHostGraphicsMemory(const Alloc GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) { auto pageSize = NEO::OSInterface::osEnabled64kbPages ? MemoryConstants::pageSize64k : MemoryConstants::pageSize; bool requiresNonStandardAlignment = allocationData.alignment > pageSize; - if ((preferredAllocationMethod == GfxMemoryAllocationMethod::UseUmdSystemPtr) || requiresNonStandardAlignment) { + if ((preferredAllocationMethod == GfxMemoryAllocationMethod::UseUmdSystemPtr) || (requiresNonStandardAlignment && allocationData.forceKMDAllocation == false)) { return allocateSystemMemoryAndCreateGraphicsAllocationFromIt(allocationData); } else { return allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocationData, NEO::OSInterface::osEnabled64kbPages); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.h b/shared/source/os_interface/windows/wddm_memory_manager.h index aec55d759f..531d836033 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.h +++ b/shared/source/os_interface/windows/wddm_memory_manager.h @@ -107,7 +107,7 @@ class WddmMemoryManager : public MemoryManager { GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle, AllocationType allocationType, uint32_t rootDeviceIndex); static bool validateAllocation(WddmAllocation *alloc); MOCKABLE_VIRTUAL bool createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr); - bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr); + MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr); bool mapGpuVaForOneHandleAllocation(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr); bool mapMultiHandleAllocationWithRetry(WddmAllocation *allocation, const void *requiredGpuPtr); bool createGpuAllocationsWithRetry(WddmAllocation *graphicsAllocation); diff --git a/shared/test/unit_test/os_interface/windows/CMakeLists.txt b/shared/test/unit_test/os_interface/windows/CMakeLists.txt index c0a2fa8133..2f52361285 100644 --- a/shared/test/unit_test/os_interface/windows/CMakeLists.txt +++ b/shared/test/unit_test/os_interface/windows/CMakeLists.txt @@ -24,6 +24,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/wddm_address_space_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_command_stream_l0_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_mapper_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_preemption_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_shared_allocations_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_special_heap_test.cpp diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_test.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_test.cpp new file mode 100644 index 0000000000..70adb308de --- /dev/null +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_test.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/test/common/helpers/execution_environment_helper.h" +#include "shared/test/common/mocks/mock_device.h" +#include "shared/test/common/mocks/mock_memory_manager.h" +#include "shared/test/common/os_interface/windows/mock_wddm_memory_manager.h" +#include "shared/test/common/os_interface/windows/wddm_fixture.h" + +using namespace NEO; +using namespace ::testing; + +class MockAllocateGraphicsMemoryWithAlignmentWddm : public MemoryManagerCreate { + public: + using WddmMemoryManager::allocateGraphicsMemoryWithAlignment; + MockAllocateGraphicsMemoryWithAlignmentWddm(ExecutionEnvironment &executionEnvironment) : MemoryManagerCreate(false, false, executionEnvironment) {} + bool allocateSystemMemoryAndCreateGraphicsAllocationFromItCalled = false; + bool allocateGraphicsMemoryUsingKmdAndMapItToCpuVACalled = false; + bool mapGpuVirtualAddressWithCpuPtr = false; + + GraphicsAllocation *allocateSystemMemoryAndCreateGraphicsAllocationFromIt(const AllocationData &allocationData) override { + allocateSystemMemoryAndCreateGraphicsAllocationFromItCalled = true; + + return nullptr; + } + GraphicsAllocation *allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(const AllocationData &allocationData, bool allowLargePages) override { + allocateGraphicsMemoryUsingKmdAndMapItToCpuVACalled = true; + + return nullptr; + } + bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr) override { + if (requiredGpuPtr != nullptr) { + mapGpuVirtualAddressWithCpuPtr = true; + } else { + mapGpuVirtualAddressWithCpuPtr = false; + } + + return true; + } +}; + +class WddmMemoryManagerTests : public ::testing::Test { + public: + MockAllocateGraphicsMemoryWithAlignmentWddm *memoryManager = nullptr; + WddmMock *wddm = nullptr; + ExecutionEnvironment *executionEnvironment = nullptr; + + void SetUp() override { + HardwareInfo *hwInfo = nullptr; + executionEnvironment = getExecutionEnvironmentImpl(hwInfo, 1); + + memoryManager = new MockAllocateGraphicsMemoryWithAlignmentWddm(*executionEnvironment); + executionEnvironment->memoryManager.reset(memoryManager); + wddm = static_cast(executionEnvironment->rootDeviceEnvironments[0]->osInterface->getDriverModel()->as()); + } + + void TearDown() override { + delete executionEnvironment; + } +}; + +TEST_F(WddmMemoryManagerTests, GivenAllocDataWithSVMCPUSetWhenAllocateGraphicsMemoryWithAlignmentThenProperFunctionIsUsed) { + NEO::AllocationData allocData = {}; + allocData.type = NEO::AllocationType::SVM_CPU; + allocData.forceKMDAllocation = true; + allocData.makeGPUVaDifferentThanCPUPtr = true; + memoryManager->allocateGraphicsMemoryWithAlignment(allocData); + + if (preferredAllocationMethod == GfxMemoryAllocationMethod::AllocateByKmd) { + EXPECT_TRUE(memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVACalled); + } else { + EXPECT_TRUE(memoryManager->allocateSystemMemoryAndCreateGraphicsAllocationFromItCalled); + } +} + +class MockAllocateGraphicsMemoryUsingKmdAndMapItToCpuVAWddm : public MemoryManagerCreate { + public: + using WddmMemoryManager::allocateGraphicsMemoryUsingKmdAndMapItToCpuVA; + using WddmMemoryManager::mapGpuVirtualAddress; + MockAllocateGraphicsMemoryUsingKmdAndMapItToCpuVAWddm(ExecutionEnvironment &executionEnvironment) : MemoryManagerCreate(false, false, executionEnvironment) {} + + bool mapGpuVirtualAddressWithCpuPtr = false; + + bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr) override { + if (requiredGpuPtr != nullptr) { + mapGpuVirtualAddressWithCpuPtr = true; + } else { + mapGpuVirtualAddressWithCpuPtr = false; + } + + return true; + } +}; + +class WddmMemoryManagerAllocPathTests : public ::testing::Test { + public: + MockAllocateGraphicsMemoryUsingKmdAndMapItToCpuVAWddm *memoryManager = nullptr; + WddmMock *wddm = nullptr; + ExecutionEnvironment *executionEnvironment = nullptr; + + void SetUp() override { + HardwareInfo *hwInfo = nullptr; + executionEnvironment = getExecutionEnvironmentImpl(hwInfo, 1); + + memoryManager = new MockAllocateGraphicsMemoryUsingKmdAndMapItToCpuVAWddm(*executionEnvironment); + executionEnvironment->memoryManager.reset(memoryManager); + wddm = static_cast(executionEnvironment->rootDeviceEnvironments[0]->osInterface->getDriverModel()->as()); + } + + void TearDown() override { + delete executionEnvironment; + } +}; + +TEST_F(WddmMemoryManagerAllocPathTests, givenAllocateGraphicsMemoryUsingKmdAndMapItToCpuVAWhenPreferedAllocationMethodThenProperArgumentsAreSet) { + { + NEO::AllocationData allocData = {}; + allocData.type = NEO::AllocationType::SVM_CPU; + allocData.forceKMDAllocation = true; + allocData.makeGPUVaDifferentThanCPUPtr = true; + auto graphicsAllocation = memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocData, false); + + if (preferredAllocationMethod == GfxMemoryAllocationMethod::AllocateByKmd) { + EXPECT_FALSE(memoryManager->mapGpuVirtualAddressWithCpuPtr); + } else { + EXPECT_TRUE(memoryManager->mapGpuVirtualAddressWithCpuPtr); + } + + memoryManager->freeGraphicsMemory(graphicsAllocation); + } + { + NEO::AllocationData allocData = {}; + allocData.type = NEO::AllocationType::EXTERNAL_HOST_PTR; + auto graphicsAllocation = memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocData, false); + + EXPECT_TRUE(memoryManager->mapGpuVirtualAddressWithCpuPtr); + + memoryManager->freeGraphicsMemory(graphicsAllocation); + } +}