Add NTHandle sharing

Signed-off-by: Kamil Diedrich <kamil.diedrich@intel.com>

shared/source/memory_manager/graphics_allocation.h

@@ -29,6 +29,7 @@ namespace NEO {
 
 using osHandle = unsigned int;
 inline osHandle toOsHandle(const void *handle) {
+
     return static_cast<osHandle>(castToUint64(handle));
 }
 

shared/source/os_interface/windows/CMakeLists.txt

@@ -86,6 +86,7 @@ set(NEO_CORE_OS_INTERFACE_WDDM
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm/configure_device_address_space_${DRIVER_MODEL}.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm/set_gmm_input_args_${DRIVER_MODEL}.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm/max_mem_alloc_size_${DRIVER_MODEL}.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/wddm/helper_${DRIVER_MODEL}.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_defs.h
@@ -108,12 +109,14 @@ set(NEO_CORE_OS_INTERFACE_WDDM
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.h
     ${CMAKE_CURRENT_SOURCE_DIR}/windows_defs.h
     ${CMAKE_CURRENT_SOURCE_DIR}/windows_wrapper.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/sys_calls_wrapper.h
 )
 
 if(NOT WIN32 AND NOT DISABLE_WDDM_LINUX)
   list(APPEND NEO_CORE_OS_INTERFACE_WDDM
        ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory_create_dxcore.cpp
        ${CMAKE_CURRENT_SOURCE_DIR}/trim_callback_stub.cpp
+       ${CMAKE_CURRENT_SOURCE_DIR}/sys_calls_wrapper_drm_or_wddm.cpp
   )
 endif()
 

shared/source/os_interface/windows/gdi_interface.cpp

@@ -40,6 +40,7 @@ bool Gdi::setupHwQueueProcAddresses() {
 bool Gdi::getAllProcAddresses() {
     openAdapterFromLuid = gdiDll->getProcAddress("D3DKMTOpenAdapterFromLuid");
     createAllocation_ = gdiDll->getProcAddress("D3DKMTCreateAllocation");
+    shareObjects = reinterpret_cast<decltype(shareObjects)>(gdiDll->getProcAddress("D3DKMTShareObjects"));
     createAllocation2 = gdiDll->getProcAddress("D3DKMTCreateAllocation2");
     destroyAllocation2 = gdiDll->getProcAddress("D3DKMTDestroyAllocation2");
     queryAdapterInfo = gdiDll->getProcAddress("D3DKMTQueryAdapterInfo");
@@ -82,7 +83,7 @@ bool Gdi::getAllProcAddresses() {
 
     // clang-format off
     if (openAdapterFromLuid && createAllocation2 
-        && destroyAllocation2 && queryAdapterInfo && closeAdapter && createDevice 
+        && destroyAllocation2 && shareObjects && queryAdapterInfo && closeAdapter && createDevice
         && destroyDevice && escape && createContext && destroyContext 
         && openResource && queryResourceInfo
         && createSynchronizationObject && createSynchronizationObject2 

shared/source/os_interface/windows/gdi_interface.h

@@ -24,6 +24,8 @@ class Gdi {
     ThkWrapper<IN OUT D3DKMT_OPENADAPTERFROMLUID *> openAdapterFromLuid{};
     ThkWrapper<IN OUT D3DKMT_CREATEALLOCATION *> createAllocation_{};
     ThkWrapper<IN OUT D3DKMT_CREATEALLOCATION *> createAllocation2{};
+    NTSTATUS(APIENTRY *shareObjects)
+    (UINT cObjects, const D3DKMT_HANDLE *hObjects, POBJECT_ATTRIBUTES pObjectAttributes, DWORD dwDesiredAccess, HANDLE *phSharedNtHandle) = {};
     ThkWrapper<IN CONST D3DKMT_DESTROYALLOCATION *> destroyAllocation{};
     ThkWrapper<IN CONST D3DKMT_DESTROYALLOCATION2 *> destroyAllocation2{};
     ThkWrapper<IN CONST D3DKMT_QUERYADAPTERINFO *> queryAdapterInfo{};

shared/source/os_interface/windows/sys_calls_wrapper.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+#include "shared/source/os_interface/windows/d3dkmthk_wrapper.h"
+
+namespace NEO {
+namespace SysCalls {
+BOOL closeHandle(HANDLE hObject);
+}
+} // namespace NEO

shared/source/os_interface/windows/sys_calls_wrapper_drm_or_wddm.cpp

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/windows/sys_calls_wrapper.h"
+
+namespace NEO {
+namespace SysCalls {
+BOOL closeHandle(HANDLE hObject) {
+    return TRUE;
+}
+} // namespace SysCalls
+} // namespace NEO

shared/source/os_interface/windows/wddm/helper_drm_or_wddm.cpp

@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/windows/gdi_interface.h"
+#include "shared/source/os_interface/windows/wddm/wddm.h"
+
+namespace NEO {
+NTSTATUS Wddm::createNTHandle(const D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle) {
+    return getGdi()->shareObjects(1, resourceHandle, nullptr, SHARED_ALLOCATION_WRITE, ntHandle);
+}
+} // namespace NEO

shared/source/os_interface/windows/wddm/helper_wddm.cpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/windows/gdi_interface.h"
+#include "shared/source/os_interface/windows/wddm/wddm.h"
+
+namespace NEO {
+
+NTSTATUS Wddm::createNTHandle(const D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle) {
+    OBJECT_ATTRIBUTES objAttr = {};
+    objAttr.Length = sizeof(OBJECT_ATTRIBUTES);
+
+    return getGdi()->shareObjects(1, resourceHandle, &objAttr, SHARED_ALLOCATION_WRITE, ntHandle);
+}
+} // namespace NEO

shared/source/os_interface/windows/wddm/wddm.cpp

@@ -495,7 +495,7 @@ bool Wddm::freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size)
     return status == STATUS_SUCCESS;
 }
 
-NTSTATUS Wddm::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, D3DKMT_HANDLE *outSharedHandle) {
+NTSTATUS Wddm::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, uint64_t *outSharedHandle) {
     NTSTATUS status = STATUS_UNSUCCESSFUL;
     D3DDDI_ALLOCATIONINFO2 AllocationInfo = {};
     D3DKMT_CREATEALLOCATION CreateAllocation = {};
@@ -522,7 +522,17 @@ NTSTATUS Wddm::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKM
     outHandle = AllocationInfo.hAllocation;
     outResourceHandle = CreateAllocation.hResource;
     if (outSharedHandle) {
-        *outSharedHandle = CreateAllocation.hGlobalShare;
+        HANDLE ntSharedHandle = NULL;
+        status = this->createNTHandle(&outResourceHandle, &ntSharedHandle);
+        if (status != STATUS_SUCCESS) {
+            DEBUG_BREAK_IF(true);
+            [[maybe_unused]] auto destroyStatus = this->destroyAllocations(&outHandle, 1, outResourceHandle);
+            outHandle = NULL_HANDLE;
+            outResourceHandle = NULL_HANDLE;
+            DEBUG_BREAK_IF(destroyStatus != STATUS_SUCCESS);
+            return status;
+        }
+        *outSharedHandle = castToUint64(ntSharedHandle);
     }
     kmDafListener->notifyWriteTarget(featureTable->ftrKmdDaf, getAdapter(), device, outHandle, getGdi()->escape);
 
@@ -531,7 +541,7 @@ NTSTATUS Wddm::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKM
 
 bool Wddm::createAllocation(const Gmm *gmm, D3DKMT_HANDLE &outHandle) {
     D3DKMT_HANDLE outResourceHandle = NULL_HANDLE;
-    D3DKMT_HANDLE *outSharedHandle = nullptr;
+    uint64_t *outSharedHandle = nullptr;
     auto result = this->createAllocation(nullptr, gmm, outHandle, outResourceHandle, outSharedHandle);
     return STATUS_SUCCESS == result;
 }

shared/source/os_interface/windows/wddm/wddm.h

@@ -75,7 +75,7 @@ class Wddm : public DriverModel {
     MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext, const HardwareInfo &hwInfo);
     MOCKABLE_VIRTUAL void applyAdditionalMapGPUVAFields(D3DDDI_MAPGPUVIRTUALADDRESS &MapGPUVA, Gmm *gmm);
     MOCKABLE_VIRTUAL bool freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
-    MOCKABLE_VIRTUAL NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, D3DKMT_HANDLE *outSharedHandle);
+    MOCKABLE_VIRTUAL NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, uint64_t *outSharedHandle);
     MOCKABLE_VIRTUAL bool createAllocation(const Gmm *gmm, D3DKMT_HANDLE &outHandle);
     MOCKABLE_VIRTUAL NTSTATUS createAllocationsAndMapGpuVa(OsHandleStorage &osHandles);
     MOCKABLE_VIRTUAL bool destroyAllocations(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
@@ -92,6 +92,7 @@ class Wddm : public DriverModel {
 
     MOCKABLE_VIRTUAL bool destroyContext(D3DKMT_HANDLE context);
     MOCKABLE_VIRTUAL bool queryAdapterInfo();
+    MOCKABLE_VIRTUAL NTSTATUS createNTHandle(const D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle);
 
     MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments);
     MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence);

shared/source/os_interface/windows/wddm_allocation.h

@@ -69,9 +69,13 @@ class WddmAllocation : public GraphicsAllocation {
         handles[handleIndex] = handle;
     }
 
-    D3DKMT_HANDLE *getSharedHandleToModify() {
+    uint64_t peekInternalHandle(MemoryManager *memoryManager) override {
+        return ntSecureHandle;
+    }
+
+    uint64_t *getSharedHandleToModify() {
         if (shareable) {
-            return &sharingInfo.sharedHandle;
+            return &ntSecureHandle;
         }
         return nullptr;
     }
@@ -102,6 +106,7 @@ class WddmAllocation : public GraphicsAllocation {
     bool allocInFrontWindowPool = false;
 
   protected:
+    uint64_t ntSecureHandle = 0u;
     std::string getHandleInfoString() const {
         std::stringstream ss;
         for (auto &handle : handles) {

shared/source/os_interface/windows/wddm_memory_manager.cpp

@@ -26,6 +26,7 @@
 #include "shared/source/os_interface/hw_info_config.h"
 #include "shared/source/os_interface/os_interface.h"
 #include "shared/source/os_interface/windows/os_context_win.h"
+#include "shared/source/os_interface/windows/sys_calls_wrapper.h"
 #include "shared/source/os_interface/windows/wddm/wddm.h"
 #include "shared/source/os_interface/windows/wddm_allocation.h"
 #include "shared/source/os_interface/windows/wddm_residency_allocations_container.h"
@@ -499,13 +500,17 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation
     for (auto handleId = 0u; handleId < gfxAllocation->getNumGmms(); handleId++) {
         delete gfxAllocation->getGmm(handleId);
     }
+    if (input->peekInternalHandle(nullptr) != 0u) {
+        [[maybe_unused]] auto status = SysCalls::closeHandle(reinterpret_cast<void *>(reinterpret_cast<uintptr_t *>(input->peekInternalHandle(nullptr))));
+        DEBUG_BREAK_IF(!status);
+    }
 
     if (input->peekSharedHandle() == false &&
         input->getDriverAllocatedCpuPtr() == nullptr &&
         input->fragmentsStorage.fragmentCount > 0) {
         cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
     } else {
-        if (input->peekSharedHandle()) {
+        if (input->peekSharedHandle() || input->peekInternalHandle(nullptr) != 0) {
             [[maybe_unused]] auto status = tryDeferDeletions(nullptr, 0, input->resourceHandle, gfxAllocation->getRootDeviceIndex());
             DEBUG_BREAK_IF(!status);
         } else {

shared/source/os_interface/windows/windows_wrapper.h

@@ -8,6 +8,9 @@
 #pragma once
 #if _WIN32
 #include <windows.h>
+
+#include <winternl.h>
+
 #pragma warning(push)
 #pragma warning(disable : 4005)
 #include <ntstatus.h>
@@ -24,7 +27,9 @@
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wunused-value"
 #endif
+
 #include <winadapter.h>
+
 #if __clang__
 #pragma clang diagnostic pop
 #endif

shared/test/common/mock_gdi/gdi32_mock.def

@@ -17,6 +17,7 @@ D3DKMTCloseAdapter
 D3DKMTCreateDevice
 D3DKMTDestroyDevice
 D3DKMTEscape
+D3DKMTShareObjects
 D3DKMTCreateContextVirtual
 D3DKMTDestroyContext
 D3DKMTOpenResource

shared/test/common/mock_gdi/mock_gdi.cpp

@@ -168,6 +168,11 @@ NTSTATUS __stdcall D3DKMTCreateAllocation2(IN OUT D3DKMT_CREATEALLOCATION *alloc
     return STATUS_SUCCESS;
 }
 
+NTSTATUS __stdcall D3DKMTShareObjects(UINT cObjects, const D3DKMT_HANDLE *hObjects, POBJECT_ATTRIBUTES pObjectAttributes, DWORD dwDesiredAccess, HANDLE *phSharedNtHandle) {
+    *phSharedNtHandle = reinterpret_cast<HANDLE>(reinterpret_cast<void *>(reinterpret_cast<uintptr_t *>(0x123)));
+    return STATUS_SUCCESS;
+}
+
 static unsigned int DestroyAllocationWithResourceHandleCalled = 0u;
 static D3DKMT_DESTROYALLOCATION2 destroyalloc2 = {0};
 static D3DKMT_HANDLE LastDestroyedResourceHandle = 0;

shared/test/common/mocks/mock_wddm.cpp

@@ -89,7 +89,7 @@ NTSTATUS WddmMock::createAllocation(WddmAllocation *wddmAllocation) {
     }
     return false;
 }
-NTSTATUS WddmMock::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, D3DKMT_HANDLE *outSharedHandle) {
+NTSTATUS WddmMock::createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResourceHandle, uint64_t *outSharedHandle) {
     createAllocationResult.called++;
     if (callBaseDestroyAllocations) {
         createAllocationStatus = Wddm::createAllocation(alignedCpuPtr, gmm, outHandle, outResourceHandle, outSharedHandle);

shared/test/common/mocks/mock_wddm.h

@@ -57,7 +57,7 @@ class WddmMock : public Wddm {
     bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr) override;
     bool mapGpuVirtualAddress(WddmAllocation *allocation);
     bool freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size) override;
-    NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResource, D3DKMT_HANDLE *outSharedHandle) override;
+    NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, D3DKMT_HANDLE &outResource, uint64_t *outSharedHandle) override;
     bool createAllocation(const Gmm *gmm, D3DKMT_HANDLE &outHandle) override;
     bool destroyAllocations(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle) override;
 

shared/test/unit_test/os_interface/windows/CMakeLists.txt

@@ -16,6 +16,7 @@ set(NEO_CORE_OS_INTERFACE_TESTS_WINDOWS
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm_preemption_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm_special_heap_test.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/wddm_tests.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/wddm_shared_allocations_test.cpp
 )
 
 set_property(GLOBAL PROPERTY NEO_CORE_OS_INTERFACE_TESTS_WINDOWS ${NEO_CORE_OS_INTERFACE_TESTS_WINDOWS})

shared/test/unit_test/os_interface/windows/wddm_shared_allocations_test.cpp

@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/gmm_helper/gmm.h"
+#include "shared/test/common/fixtures/mock_execution_environment_gmm_fixture.h"
+#include "shared/test/common/mocks/mock_wddm.h"
+#include "shared/test/common/os_interface/windows/wddm_fixture.h"
+
+#include "test.h"
+
+using namespace NEO;
+
+class WddmSharedAllocationsMock : public WddmMock {
+  public:
+    WddmSharedAllocationsMock(RootDeviceEnvironment &rootDeviceEnvironment) : WddmMock(rootDeviceEnvironment) {}
+    WddmSharedAllocationsMock(std::unique_ptr<HwDeviceIdWddm> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment) : WddmMock(std::move(hwDeviceId), rootDeviceEnvironment) {}
+    NTSTATUS createNTHandle(const D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle) override {
+        return static_cast<NTSTATUS>(0xfff);
+    }
+};
+
+class WddmSharedTestsFixture : public GdiDllFixture, public MockExecutionEnvironmentGmmFixture {
+  public:
+    void SetUp() override {
+        MockExecutionEnvironmentGmmFixture::SetUp();
+        GdiDllFixture::SetUp();
+        rootDeviceEnvironment = executionEnvironment->rootDeviceEnvironments[0].get();
+        wddm = new WddmSharedAllocationsMock(*rootDeviceEnvironment);
+        wddmMockInterface = new WddmMockInterface20(*wddm);
+        wddm->wddmInterface.reset(wddmMockInterface);
+        rootDeviceEnvironment->osInterface = std::make_unique<OSInterface>();
+        rootDeviceEnvironment->osInterface->setDriverModel(std::unique_ptr<DriverModel>(wddm));
+        rootDeviceEnvironment->memoryOperationsInterface = std::make_unique<WddmMemoryOperationsHandler>(wddm);
+        osInterface = rootDeviceEnvironment->osInterface.get();
+    }
+
+    void init() {
+        auto preemptionMode = PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo);
+        wddmMockInterface = static_cast<WddmMockInterface20 *>(wddm->wddmInterface.release());
+        wddm->init();
+        wddm->wddmInterface.reset(wddmMockInterface);
+
+        auto hwInfo = rootDeviceEnvironment->getHardwareInfo();
+        auto engine = HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*hwInfo)[0];
+        osContext = std::make_unique<OsContextWin>(*osInterface->getDriverModel()->as<Wddm>(), 0u, EngineDescriptorHelper::getDefaultDescriptor(engine, preemptionMode));
+        osContext->ensureContextInitialized();
+    }
+
+    void TearDown() override {
+        GdiDllFixture::TearDown();
+    }
+
+    WddmSharedAllocationsMock *wddm = nullptr;
+    OSInterface *osInterface;
+    std::unique_ptr<OsContextWin> osContext;
+    WddmMockInterface20 *wddmMockInterface = nullptr;
+    RootDeviceEnvironment *rootDeviceEnvironment = nullptr;
+};
+
+using WdmmSharedTests = Test<WddmSharedTestsFixture>;
+
+TEST_F(WdmmSharedTests, WhenCreatingSharedAllocationAndGetNTHandleFailedThenAllocationIsDeletedAndHandlesAreSetToZero) {
+    init();
+
+    D3DKMT_HANDLE handle = 32u;
+    D3DKMT_HANDLE resourceHandle = 32u;
+    uint64_t ntHandle = 0u;
+    Gmm gmm(executionEnvironment->rootDeviceEnvironments[0]->getGmmClientContext(), nullptr, 20, 0, false, true, true, {});
+
+    EXPECT_NE(STATUS_SUCCESS, wddm->createAllocation(nullptr, &gmm, handle, resourceHandle, &ntHandle));
+    EXPECT_EQ(wddm->destroyAllocationResult.called++, 1u);
+    EXPECT_EQ(handle, 0u);
+    EXPECT_EQ(resourceHandle, 0u);
+}