fix: Use Gmm to map virtual gpu addresses

Related-To: NEO-5842, NEO-7946

Signed-off-by: Maciej Plewka <maciej.plewka@intel.com>

shared/source/gmm_helper/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2019-2022 Intel Corporation
+# Copyright (C) 2019-2023 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -8,7 +8,9 @@ set(NEO_CORE_GMM_HELPER
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
     ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}resource_info_${DRIVER_MODEL}.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/client_context/gmm_client_context.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_context/gmm_client_context_${DRIVER_MODEL}.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/client_context/gmm_client_context.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/client_context/map_gpu_va_gmm.h
     ${CMAKE_CURRENT_SOURCE_DIR}/cache_settings_helper.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/cache_settings_helper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/gmm.cpp

shared/source/gmm_helper/client_context/gmm_client_context.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2022 Intel Corporation
+ * Copyright (C) 2018-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -14,6 +14,8 @@ namespace NEO {
 class GmmClientContext;
 struct RootDeviceEnvironment;
 class GmmHandleAllocator;
+class MapGpuVirtualAddressGmm;
+class FreeGpuVirtualAddressGmm;
 
 class GmmClientContext {
   public:
@@ -26,6 +28,8 @@ class GmmClientContext {
     MOCKABLE_VIRTUAL GMM_RESOURCE_INFO *createResInfoObject(GMM_RESCREATE_PARAMS *pCreateParams);
     MOCKABLE_VIRTUAL GMM_RESOURCE_INFO *copyResInfoObject(GMM_RESOURCE_INFO *pSrcRes);
     MOCKABLE_VIRTUAL void destroyResInfoObject(GMM_RESOURCE_INFO *pResInfo);
+    MOCKABLE_VIRTUAL uint64_t mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa);
+    MOCKABLE_VIRTUAL uint64_t freeGpuVirtualAddress(FreeGpuVirtualAddressGmm *pFreeGpuVa);
     GMM_CLIENT_CONTEXT *getHandle() const;
     template <typename T>
     static std::unique_ptr<GmmClientContext> create(const RootDeviceEnvironment &rootDeviceEnvironment) {

shared/source/gmm_helper/client_context/gmm_client_context_drm.cpp

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
+
+namespace NEO {
+uint64_t GmmClientContext::mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) {
+    return 0;
+}
+uint64_t GmmClientContext::freeGpuVirtualAddress(FreeGpuVirtualAddressGmm *pFreeGpuVa) {
+    return 0;
+}
+
+} // namespace NEO

shared/source/gmm_helper/client_context/gmm_client_context_drm_or_wddm.cpp

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
+#include "shared/source/gmm_helper/client_context/map_gpu_va_gmm.h"
+#include "shared/source/os_interface/windows/gdi_interface.h"
+
+namespace NEO {
+uint64_t GmmClientContext::mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) {
+    return pMapGpuVa->gdi->mapGpuVirtualAddress(pMapGpuVa->mapGpuVirtualAddressParams);
+}
+uint64_t GmmClientContext::freeGpuVirtualAddress(FreeGpuVirtualAddressGmm *pFreeGpuVa) {
+    return 0;
+}
+
+} // namespace NEO

shared/source/gmm_helper/client_context/gmm_client_context_wddm.cpp

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/gmm_helper/client_context/gmm_client_context.h"
+#include "shared/source/gmm_helper/client_context/map_gpu_va_gmm.h"
+
+namespace NEO {
+uint64_t GmmClientContext::mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) {
+    GMM_MAPGPUVIRTUALADDRESS gmmMapAddress = {pMapGpuVa->mapGpuVirtualAddressParams, 1, pMapGpuVa->resourceInfoHandle, pMapGpuVa->outVirtualAddress};
+    return clientContext->MapGpuVirtualAddress(&gmmMapAddress);
+}
+uint64_t GmmClientContext::freeGpuVirtualAddress(FreeGpuVirtualAddressGmm *pFreeGpuVa) {
+    GMM_FREEGPUVIRTUALADDRESS gmmFreeAddress = {pFreeGpuVa->hAdapter, pFreeGpuVa->baseAddress, pFreeGpuVa->size, 1, pFreeGpuVa->resourceInfoHandle};
+    return clientContext->FreeGpuVirtualAddress(&gmmFreeAddress);
+}
+
+} // namespace NEO

shared/source/gmm_helper/client_context/map_gpu_va_gmm.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/gmm_helper/gmm_lib.h"
+
+namespace NEO {
+class Gdi;
+
+class MapGpuVirtualAddressGmm {
+  public:
+    MapGpuVirtualAddressGmm(D3DDDI_MAPGPUVIRTUALADDRESS *mapGpuVirtualAddressParams, GMM_RESOURCE_INFO **resourceInfoHandle, D3DGPU_VIRTUAL_ADDRESS *outVirtualAddress, Gdi *gdi) : mapGpuVirtualAddressParams(mapGpuVirtualAddressParams), resourceInfoHandle(resourceInfoHandle), outVirtualAddress(outVirtualAddress), gdi(gdi) {}
+    D3DDDI_MAPGPUVIRTUALADDRESS *mapGpuVirtualAddressParams;
+    GMM_RESOURCE_INFO **resourceInfoHandle;
+    D3DGPU_VIRTUAL_ADDRESS *outVirtualAddress;
+    Gdi *gdi;
+};
+
+class FreeGpuVirtualAddressGmm {
+  public:
+    FreeGpuVirtualAddressGmm(D3DKMT_HANDLE hAdapter, D3DGPU_VIRTUAL_ADDRESS baseAddress, D3DGPU_SIZE_T size, GMM_RESOURCE_INFO **resourceInfoHandle, Gdi *gdi) : hAdapter(hAdapter), baseAddress(baseAddress), size(size), resourceInfoHandle(resourceInfoHandle), gdi(gdi) {}
+    D3DKMT_HANDLE hAdapter;
+    D3DGPU_VIRTUAL_ADDRESS baseAddress;
+    D3DGPU_SIZE_T size;
+    GMM_RESOURCE_INFO **resourceInfoHandle;
+    Gdi *gdi;
+};
+
+} // namespace NEO

shared/source/os_interface/windows/wddm/wddm.cpp

@@ -13,6 +13,7 @@
 #include "shared/source/execution_environment/root_device_environment.h"
 #include "shared/source/gmm_helper/client_context/gmm_client_context.h"
 #include "shared/source/gmm_helper/client_context/gmm_handle_allocator.h"
+#include "shared/source/gmm_helper/client_context/map_gpu_va_gmm.h"
 #include "shared/source/gmm_helper/gmm.h"
 #include "shared/source/gmm_helper/gmm_helper.h"
 #include "shared/source/gmm_helper/page_table_mngr.h"
@@ -536,8 +537,9 @@ bool Wddm::mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_A
     mapGPUVA.MaximumAddress = maximumAddress;
 
     applyAdditionalMapGPUVAFields(mapGPUVA, gmm);
-
-    NTSTATUS status = getGdi()->mapGpuVirtualAddress(&mapGPUVA);
+    auto resourceInfo = gmm->gmmResourceInfo->peekGmmResourceInfo();
+    MapGpuVirtualAddressGmm gmmMapGpuVa = {&mapGPUVA, &resourceInfo, &gpuPtr, getGdi()};
+    auto status = gmm->getGmmHelper()->getClientContext()->mapGpuVirtualAddress(&gmmMapGpuVa);
 
     auto gmmHelper = gmm->getGmmHelper();
     gpuPtr = gmmHelper->canonize(mapGPUVA.VirtualAddress);
@@ -588,6 +590,14 @@ NTSTATUS Wddm::reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS baseAddress,
     return status;
 }
 
+uint64_t Wddm::freeGmmGpuVirtualAddress(Gmm *gmm, D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size) {
+    uint64_t status = STATUS_SUCCESS;
+    auto resourceInfo = gmm->gmmResourceInfo->peekGmmResourceInfo();
+    FreeGpuVirtualAddressGmm freeGpuva = {getAdapter(), rootDeviceEnvironment.getGmmHelper()->decanonize(gpuPtr), size, &resourceInfo, getGdi()};
+    status = gmm->getGmmHelper()->getClientContext()->freeGpuVirtualAddress(&freeGpuva);
+    return status;
+}
+
 bool Wddm::freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size) {
     NTSTATUS status = STATUS_SUCCESS;
     D3DKMT_FREEGPUVIRTUALADDRESS freeGpuva = {};

shared/source/os_interface/windows/wddm/wddm.h

@@ -70,6 +70,7 @@ class Wddm : public DriverModel {
     MOCKABLE_VIRTUAL bool createContext(OsContextWin &osContext);
     MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext);
     MOCKABLE_VIRTUAL void applyAdditionalMapGPUVAFields(D3DDDI_MAPGPUVIRTUALADDRESS &mapGPUVA, Gmm *gmm);
+    MOCKABLE_VIRTUAL uint64_t freeGmmGpuVirtualAddress(Gmm *gmm, D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
     MOCKABLE_VIRTUAL bool freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
     MOCKABLE_VIRTUAL NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, uint64_t *outSharedHandle);
     MOCKABLE_VIRTUAL bool createAllocation(const Gmm *gmm, D3DKMT_HANDLE &outHandle);

shared/source/os_interface/windows/wddm_memory_manager.cpp

@@ -639,7 +639,12 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation
         }
     }
     for (auto handleId = 0u; handleId < gfxAllocation->getNumGmms(); handleId++) {
-        delete gfxAllocation->getGmm(handleId);
+        auto gmm = gfxAllocation->getGmm(handleId);
+        if (gmm) {
+            auto gpuAddress = input->getGpuAddress();
+            getWddm(gfxAllocation->getRootDeviceIndex()).freeGmmGpuVirtualAddress(gfxAllocation->getGmm(handleId), gpuAddress, input->getAlignedSize());
+        }
+        delete gmm;
     }
     uint64_t handle = 0;
     int ret = input->peekInternalHandle(nullptr, handle);

shared/test/common/mocks/CMakeLists.txt

@@ -57,6 +57,7 @@ set(NEO_CORE_tests_mocks
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_client_context.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_client_context.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_client_context_base.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_client_context_base_${DRIVER_MODEL}.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_client_context_base.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_page_table_mngr.h
     ${CMAKE_CURRENT_SOURCE_DIR}/mock_gmm_resource_info.h

shared/test/common/mocks/mock_gmm_client_context_base.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021 Intel Corporation
+ * Copyright (C) 2018-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,7 +10,6 @@
 #include "gtest/gtest.h"
 
 namespace NEO {
-
 GMM_RESOURCE_INFO *MockGmmClientContextBase::createResInfoObject(GMM_RESCREATE_PARAMS *pCreateParams) {
     return reinterpret_cast<GMM_RESOURCE_INFO *>(new char[1]);
 }
@@ -41,4 +40,8 @@ void MockGmmClientContextBase::setGmmDeviceInfo(GMM_DEVICE_INFO *deviceInfo) {
     GMM_DEVICE_CALLBACKS_INT emptyStruct{};
     EXPECT_EQ(0, memcmp(deviceInfo->pDeviceCb, &emptyStruct, sizeof(GMM_DEVICE_CALLBACKS_INT)));
 }
+uint64_t MockGmmClientContextBase::freeGpuVirtualAddress(FreeGpuVirtualAddressGmm *pFreeGpuVa) {
+    freeGpuVirtualAddressCalled++;
+    return 0;
+}
 } // namespace NEO

shared/test/common/mocks/mock_gmm_client_context_base.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2022 Intel Corporation
+ * Copyright (C) 2018-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -25,11 +25,15 @@ class MockGmmClientContextBase : public GmmClientContext {
     uint8_t getSurfaceStateCompressionFormat(GMM_RESOURCE_FORMAT format) override;
     uint8_t getMediaSurfaceStateCompressionFormat(GMM_RESOURCE_FORMAT format) override;
     void setGmmDeviceInfo(GMM_DEVICE_INFO *deviceInfo) override;
+    uint64_t mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) override;
+    uint64_t freeGpuVirtualAddress(FreeGpuVirtualAddressGmm *pMapGpuVa) override;
 
     GMM_RESOURCE_FORMAT capturedFormat = GMM_FORMAT_INVALID;
     uint8_t compressionFormatToReturn = 1;
     uint32_t getSurfaceStateCompressionFormatCalled = 0u;
     uint32_t getMediaSurfaceStateCompressionFormatCalled = 0u;
+    uint32_t mapGpuVirtualAddressCalled = 0u;
+    uint32_t freeGpuVirtualAddressCalled = 0u;
     bool returnErrorOnPatIndexQuery = false;
 
     bool passedCompressedSettingForGetPatIndexQuery = false;

shared/test/common/mocks/mock_gmm_client_context_base_drm.cpp

@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/common/mocks/mock_gmm_client_context_base.h"
+
+namespace NEO {
+uint64_t MockGmmClientContextBase::mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) {
+    mapGpuVirtualAddressCalled++;
+    return 0;
+}
+} // namespace NEO

shared/test/common/mocks/mock_gmm_client_context_base_drm_or_wddm.cpp

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/gmm_helper/client_context/map_gpu_va_gmm.h"
+#include "shared/source/os_interface/windows/gdi_interface.h"
+#include "shared/test/common/mocks/mock_gmm_client_context_base.h"
+
+namespace NEO {
+uint64_t MockGmmClientContextBase::mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) {
+    mapGpuVirtualAddressCalled++;
+    return pMapGpuVa->gdi->mapGpuVirtualAddress(pMapGpuVa->mapGpuVirtualAddressParams);
+}
+} // namespace NEO

shared/test/common/mocks/mock_gmm_client_context_base_wddm.cpp

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2018-2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/gmm_helper/client_context/map_gpu_va_gmm.h"
+#include "shared/source/os_interface/windows/gdi_interface.h"
+#include "shared/test/common/mocks/mock_gmm_client_context_base.h"
+
+namespace NEO {
+uint64_t MockGmmClientContextBase::mapGpuVirtualAddress(MapGpuVirtualAddressGmm *pMapGpuVa) {
+    mapGpuVirtualAddressCalled++;
+    return pMapGpuVa->gdi->mapGpuVirtualAddress(pMapGpuVa->mapGpuVirtualAddressParams);
+}
+} // namespace NEO

shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp

@@ -5,9 +5,11 @@
  *
  */
 
+#include "shared/source/gmm_helper/gmm_helper.h"
 #include "shared/test/common/helpers/execution_environment_helper.h"
 #include "shared/test/common/mocks/mock_device.h"
 #include "shared/test/common/mocks/mock_gmm.h"
+#include "shared/test/common/mocks/mock_gmm_client_context_base.h"
 #include "shared/test/common/mocks/mock_memory_manager.h"
 #include "shared/test/common/os_interface/windows/mock_wddm_memory_manager.h"
 #include "shared/test/common/os_interface/windows/wddm_fixture.h"
@@ -261,4 +263,18 @@ TEST_F(WddmMemoryManagerAllocPathTests, GivenValidAllocationWithFailingCreateInt
     EXPECT_EQ(1, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle));
 
     memoryManager->freeGraphicsMemory(graphicsAllocation);
+}
+TEST_F(WddmMemoryManagerTests, GivenAllocationWhenAllocationIsFreeThenFreeToGmmClientContextCalled) {
+    NEO::AllocationData allocData = {};
+    allocData.type = NEO::AllocationType::BUFFER;
+
+    allocData.forceKMDAllocation = true;
+    allocData.makeGPUVaDifferentThanCPUPtr = true;
+
+    memoryManager->callBaseAllocateGraphicsMemoryUsingKmdAndMapItToCpuVA = true;
+    auto graphicsAllocation = memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocData, false);
+    auto gmmHelper = executionEnvironment->rootDeviceEnvironments[0]->getGmmHelper();
+
+    memoryManager->freeGraphicsMemory(graphicsAllocation);
+    EXPECT_GT(reinterpret_cast<MockGmmClientContextBase *>(gmmHelper->getClientContext())->freeGpuVirtualAddressCalled, 0u);
 }

shared/test/unit_test/os_interface/windows/wddm_tests.cpp

@@ -6,11 +6,13 @@
  */
 
 #include "shared/source/gmm_helper/gmm.h"
+#include "shared/source/gmm_helper/gmm_helper.h"
 #include "shared/source/helpers/string.h"
 #include "shared/source/os_interface/driver_info.h"
 #include "shared/source/os_interface/windows/wddm/um_km_data_translator.h"
 #include "shared/source/os_interface/windows/wddm_allocation.h"
 #include "shared/test/common/helpers/debug_manager_state_restore.h"
+#include "shared/test/common/mocks/mock_gmm_client_context_base.h"
 #include "shared/test/common/mocks/mock_io_functions.h"
 #include "shared/test/common/os_interface/windows/wddm_fixture.h"
 #include "shared/test/common/test_macros/hw_test.h"
@@ -395,6 +397,18 @@ TEST_F(WddmTests, GivenPlatformNotSupportEvictIfNecessaryWhenAdjustingEvictNeede
     EXPECT_TRUE(value);
 }
 
+TEST_F(WddmTests, GivenWddmWhenMapGpuVaCalledThenGmmClientCallsMapGpuVa) {
+    NEO::AllocationData allocData = {};
+    allocData.type = NEO::AllocationType::BUFFER;
+    wddm->callBaseDestroyAllocations = false;
+    wddm->pagingQueue = PAGINGQUEUE_HANDLE;
+    auto memoryManager = std::make_unique<MockWddmMemoryManager>(*executionEnvironment);
+    auto allocation = static_cast<WddmAllocation *>(memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{0, MemoryConstants::pageSize}));
+    wddm->mapGpuVirtualAddress(allocation);
+    EXPECT_GT(reinterpret_cast<MockGmmClientContextBase *>(allocation->getDefaultGmm()->getGmmHelper()->getClientContext())->mapGpuVirtualAddressCalled, 0u);
+    memoryManager->freeGraphicsMemory(allocation);
+}
+
 uint64_t waitForSynchronizationObjectFromCpuCounter = 0u;
 
 NTSTATUS __stdcall waitForSynchronizationObjectFromCpuNoOpMock(const D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *waitStruct) {