Wddm interface [7/n]: Add 2.3 interface with HW queue support

Change-Id: Ia0e829b8616b7060e39170aea0f1d2f123d73399

runtime/dll/CMakeLists.txt

@@ -56,7 +56,7 @@ set(RUNTIME_SRCS_DLL_WINDOWS
   ${IGDRCL_SOURCE_DIR}/runtime/gmm_helper/page_table_mngr.cpp
   ${IGDRCL_SOURCE_DIR}/runtime/os_interface/windows/sys_calls.cpp
   ${IGDRCL_SOURCE_DIR}/runtime/os_interface/windows/wddm/wddm_calls.cpp
-  ${IGDRCL_SOURCE_DIR}/runtime/os_interface/windows/wddm${BRANCH_DIR_SUFFIX}/wddm_create.cpp
+  ${IGDRCL_SOURCE_DIR}/runtime/os_interface/windows/wddm/wddm_create.cpp
 )
 
 target_sources(${NEO_DYNAMIC_LIB_NAME} PRIVATE ${RUNTIME_SRCS_DLL_BASE})

runtime/os_interface/DebugVariables.inl

@@ -87,7 +87,6 @@ DECLARE_DEBUG_VARIABLE(int32_t, CsrDispatchMode, 0, "Chooses DispatchMode for Cs
 /*DRIVER TOGGLES*/
 DECLARE_DEBUG_VARIABLE(int32_t, ForceOCLVersion, 0, "Force specific OpenCL API version")
 DECLARE_DEBUG_VARIABLE(int32_t, ForcePreemptionMode, -1, "Keep this variable in sync with PreemptionMode enum. -1 - devices default mode, 1 - disable, 2 - midBatch, 3 - threadGroup, 4 - midThread")
-DECLARE_DEBUG_VARIABLE(int32_t, ForceWddmInterfaceVersion, 0, "Windows only. Force internal interface version. 0 is default value. Example: set 20 to force 2.0")
 DECLARE_DEBUG_VARIABLE(int32_t, NodeOrdinal, -1, "-1: default do not override, 0: ENGINE_RCS")
 DECLARE_DEBUG_VARIABLE(int32_t, OverrideThreadArbitrationPolicy, -1, "-1 (dont override) or any valid config (0: Age Based, 1: Round Robin)")
 DECLARE_DEBUG_VARIABLE(bool, HwQueueSupported, false, "Windows only. Pass flag to KMD during Wddm Context creation")

runtime/os_interface/windows/gdi_interface.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -31,6 +31,17 @@ Gdi::Gdi() : gdiDll(Os::gdiDllName),
     }
 }
 
+bool Gdi::setupHwQueueProcAddresses() {
+    createHwQueue = reinterpret_cast<PFND3DKMT_CREATEHWQUEUE>(gdiDll.getProcAddress("D3DKMTCreateHwQueue"));
+    destroyHwQueue = reinterpret_cast<PFND3DKMT_DESTROYHWQUEUE>(gdiDll.getProcAddress("D3DKMTDestroyHwQueue"));
+    submitCommandToHwQueue = reinterpret_cast<PFND3DKMT_SUBMITCOMMANDTOHWQUEUE>(gdiDll.getProcAddress("D3DKMTSubmitCommandToHwQueue"));
+
+    if (!createHwQueue || !destroyHwQueue || !submitCommandToHwQueue) {
+        return false;
+    }
+    return true;
+}
+
 bool Gdi::getAllProcAddresses() {
     openAdapterFromHdc = reinterpret_cast<PFND3DKMT_OPENADAPTERFROMHDC>(gdiDll.getProcAddress("D3DKMTOpenAdapterFromHdc"));
     openAdapterFromLuid = reinterpret_cast<PFND3DKMT_OPENADAPTERFROMLUID>(gdiDll.getProcAddress("D3DKMTOpenAdapterFromLuid"));

runtime/os_interface/windows/gdi_interface.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Intel Corporation
+ * Copyright (c) 2017 - 2018, Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -79,6 +79,11 @@ class Gdi {
     ThkWrapper<OCL_RUNTIME_PROFILING, IN D3DKMT_REGISTERTRIMNOTIFICATION *> registerTrimNotification;
     ThkWrapper<OCL_RUNTIME_PROFILING, IN D3DKMT_UNREGISTERTRIMNOTIFICATION *> unregisterTrimNotification;
 
+    // HW queue
+    ThkWrapper<OCL_RUNTIME_PROFILING, IN OUT D3DKMT_CREATEHWQUEUE *> createHwQueue;
+    ThkWrapper<OCL_RUNTIME_PROFILING, IN CONST D3DKMT_DESTROYHWQUEUE *> destroyHwQueue;
+    ThkWrapper<OCL_RUNTIME_PROFILING, IN CONST D3DKMT_SUBMITCOMMANDTOHWQUEUE *> submitCommandToHwQueue;
+
     // For debug purposes
     ThkWrapper<OCL_RUNTIME_PROFILING, IN OUT D3DKMT_GETDEVICESTATE *> getDeviceState;
 
@@ -86,8 +91,10 @@ class Gdi {
         return initialized;
     }
 
+    MOCKABLE_VIRTUAL bool setupHwQueueProcAddresses();
+
   protected:
-    virtual bool getAllProcAddresses();
+    MOCKABLE_VIRTUAL bool getAllProcAddresses();
     bool initialized;
 
   private:

runtime/os_interface/windows/thk_wrapper.h

@@ -72,6 +72,9 @@ enum SystemCallsIds {
     SYSTIMER_ID_UNREGISTERTRIMNOTIFICATION = 44,
     SYSTIMER_ID_QUERYRESOURCEINFOFROMNTHANDLE = 45,
     SYSTIMER_ID_OPENRESOURCEFROMNTHANDLE = 46,
+    SYSTIMER_ID_CREATEHWQUEUE = 47,
+    SYSTIMER_ID_DESTROYHWQUEUE = 48,
+    SYSTIMER_ID_SUBMITCOMMANDTOHWQUEUE = 49,
 
     SYSTIMER_ID_SLEEP_0 = 100,
     SYSTIMER_ID_WAIT_FOR_KMD = 200,
@@ -172,5 +175,8 @@ class ThkWrapper {
     GET_ID(D3DKMT_UNREGISTERTRIMNOTIFICATION *, SYSTIMER_ID_UNREGISTERTRIMNOTIFICATION)
     GET_ID(D3DKMT_OPENRESOURCEFROMNTHANDLE *, SYSTIMER_ID_OPENRESOURCEFROMNTHANDLE)
     GET_ID(D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *, SYSTIMER_ID_QUERYRESOURCEINFOFROMNTHANDLE)
+    GET_ID(D3DKMT_CREATEHWQUEUE *, SYSTIMER_ID_CREATEHWQUEUE)
+    GET_ID(CONST D3DKMT_DESTROYHWQUEUE *, SYSTIMER_ID_DESTROYHWQUEUE)
+    GET_ID(CONST D3DKMT_SUBMITCOMMANDTOHWQUEUE *, SYSTIMER_ID_SUBMITCOMMANDTOHWQUEUE)
 };
 } // namespace OCLRT

runtime/os_interface/windows/wddm/wddm.cpp

@@ -62,8 +62,7 @@ Wddm::Wddm() : initialized(false),
                pagingFenceAddress(nullptr),
                currentPagingFenceValue(0),
                hwContextId(0),
-               trimCallbackHandle(nullptr),
-               wddmInterfaceVersion(WddmInterfaceVersion::Wddm20) {
+               trimCallbackHandle(nullptr) {
     featureTable.reset(new FeatureTable());
     waTable.reset(new WorkaroundTable());
     gtSystemInfo.reset(new GT_SYSTEM_INFO);
@@ -235,12 +234,9 @@ bool Wddm::createMonitoredFence() {
 
     DEBUG_BREAK_IF(STATUS_SUCCESS != Status);
 
-    monitoredFence.currentFenceValue = 1;
-    monitoredFence.fenceHandle = CreateSynchronizationObject.hSyncObject;
-    monitoredFence.cpuAddress = reinterpret_cast<UINT64 *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress);
-    monitoredFence.lastSubmittedFence = 0;
-
-    monitoredFence.gpuAddress = CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress;
+    resetMonitoredFenceParams(CreateSynchronizationObject.hSyncObject,
+                              reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress),
+                              CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress);
 
     return Status == STATUS_SUCCESS;
 }
@@ -728,7 +724,7 @@ bool Wddm::createContext() {
 
     CreateContext.EngineAffinity = 0;
     CreateContext.Flags.NullRendering = static_cast<UINT>(DebugManager.flags.EnableNullHardware.get());
-    CreateContext.Flags.HwQueueSupported = static_cast<UINT>(DebugManager.flags.HwQueueSupported.get());
+    CreateContext.Flags.HwQueueSupported = hwQueuesSupported();
 
     if (preemptionMode >= PreemptionMode::MidBatch) {
         CreateContext.Flags.DisableGpuTimeout = readEnablePreemptionRegKey();
@@ -951,8 +947,17 @@ bool Wddm::reserveValidAddressRange(size_t size, void *&reservedMem) {
 void *Wddm::virtualAlloc(void *inPtr, size_t size, unsigned long flags, unsigned long type) {
     return virtualAllocFnc(inPtr, size, flags, type);
 }
+
 int Wddm::virtualFree(void *ptr, size_t size, unsigned long flags) {
     return virtualFreeFnc(ptr, size, flags);
 }
 
+void Wddm::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress) {
+    monitoredFence.lastSubmittedFence = 0;
+    monitoredFence.currentFenceValue = 1;
+    monitoredFence.fenceHandle = handle;
+    monitoredFence.cpuAddress = cpuAddress;
+    monitoredFence.gpuAddress = gpuAddress;
+}
+
 } // namespace OCLRT

runtime/os_interface/windows/wddm/wddm.h

@@ -51,6 +51,7 @@ struct KmDafListener;
 
 namespace WddmInterfaceVersion {
 constexpr uint32_t Wddm20 = 20;
+constexpr uint32_t Wddm23 = 23;
 } // namespace WddmInterfaceVersion
 
 class Wddm {
@@ -59,7 +60,6 @@ class Wddm {
     typedef void(WINAPI *GetSystemInfoFcn)(SYSTEM_INFO *pSystemInfo);
     typedef BOOL(WINAPI *VirtualFreeFcn)(LPVOID ptr, SIZE_T size, DWORD flags);
     typedef LPVOID(WINAPI *VirtualAllocFcn)(LPVOID inPtr, SIZE_T size, DWORD flags, DWORD type);
-    const uint32_t wddmInterfaceVersion;
 
     virtual ~Wddm();
 
@@ -72,6 +72,7 @@ class Wddm {
     bool mapGpuVirtualAddress(WddmAllocation *allocation, void *cpuPtr, uint64_t size, bool allocation32bit, bool use64kbPages, bool useHeap1);
     bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData, bool allocation32bit, bool use64kbPages);
     MOCKABLE_VIRTUAL bool createContext();
+    virtual bool createHwQueue() { return false; }
     MOCKABLE_VIRTUAL bool freeGpuVirtualAddres(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
     MOCKABLE_VIRTUAL NTSTATUS createAllocation(WddmAllocation *alloc);
     MOCKABLE_VIRTUAL bool createAllocation64k(WddmAllocation *alloc);
@@ -87,7 +88,7 @@ class Wddm {
     MOCKABLE_VIRTUAL bool destroyContext(D3DKMT_HANDLE context);
     MOCKABLE_VIRTUAL bool queryAdapterInfo();
 
-    MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader);
+    virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader);
     MOCKABLE_VIRTUAL bool waitOnGPU();
     MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue);
 
@@ -207,12 +208,14 @@ class Wddm {
     bool destroyPagingQueue();
     bool destroyDevice();
     bool closeAdapter();
-    bool createMonitoredFence();
+    virtual bool createMonitoredFence();
     void getDeviceState();
     void handleCompletion();
     unsigned int readEnablePreemptionRegKey();
     bool initGmmContext();
     void destroyGmmContext();
+    void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress);
+    virtual const bool hwQueuesSupported() const { return false; }
 
     static CreateDXGIFactoryFcn createDxgiFactory;
     static GetSystemInfoFcn getSystemInfo;

runtime/os_interface/windows/wddm/wddm.inl

@@ -61,6 +61,9 @@ bool Wddm::init() {
         if (!createContext()) {
             return false;
         }
+        if (hwQueuesSupported() && !createHwQueue()) {
+            return false;
+        }
         if (!createMonitoredFence()) {
             return false;
         }

runtime/os_interface/windows/wddm/wddm23.cpp

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "runtime/os_interface/windows/gdi_interface.h"
+#include "runtime/os_interface/windows/wddm/wddm23.h"
+
+namespace OCLRT {
+Wddm23::Wddm23() : Wddm20() {}
+
+Wddm23::~Wddm23() {
+    destroyHwQueue();
+}
+
+bool Wddm23::createHwQueue() {
+    D3DKMT_CREATEHWQUEUE createHwQueue = {};
+
+    if (!gdi->setupHwQueueProcAddresses()) {
+        return false;
+    }
+
+    createHwQueue.hHwContext = context;
+    if (preemptionMode >= PreemptionMode::MidBatch) {
+        createHwQueue.Flags.DisableGpuTimeout = readEnablePreemptionRegKey();
+    }
+
+    auto status = gdi->createHwQueue(&createHwQueue);
+    UNRECOVERABLE_IF(status != STATUS_SUCCESS);
+    hwQueueHandle = createHwQueue.hHwQueue;
+
+    resetMonitoredFenceParams(createHwQueue.hHwQueueProgressFence,
+                              reinterpret_cast<uint64_t *>(createHwQueue.HwQueueProgressFenceCPUVirtualAddress),
+                              createHwQueue.HwQueueProgressFenceGPUVirtualAddress);
+
+    return status == STATUS_SUCCESS;
+}
+
+void Wddm23::destroyHwQueue() {
+    if (hwQueueHandle) {
+        D3DKMT_DESTROYHWQUEUE destroyHwQueue = {};
+        destroyHwQueue.hHwQueue = hwQueueHandle;
+
+        auto status = gdi->destroyHwQueue(&destroyHwQueue);
+        DEBUG_BREAK_IF(status != STATUS_SUCCESS);
+    }
+}
+
+bool Wddm23::submit(uint64_t commandBuffer, size_t size, void *commandHeader) {
+    D3DKMT_SUBMITCOMMANDTOHWQUEUE submitCommand = {};
+    submitCommand.hHwQueue = hwQueueHandle;
+    submitCommand.HwQueueProgressFenceId = monitoredFence.fenceHandle;
+    submitCommand.CommandBuffer = commandBuffer;
+    submitCommand.CommandLength = static_cast<UINT>(size);
+
+    COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandHeader);
+    pHeader->MonitorFenceVA = monitoredFence.gpuAddress;
+    pHeader->MonitorFenceValue = monitoredFence.currentFenceValue;
+
+    submitCommand.pPrivateDriverData = commandHeader;
+    submitCommand.PrivateDriverDataSize = sizeof(COMMAND_BUFFER_HEADER);
+
+    if (currentPagingFenceValue > *pagingFenceAddress && !waitOnGPU()) {
+        return false;
+    }
+
+    DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", monitoredFence.currentFenceValue);
+
+    auto status = gdi->submitCommandToHwQueue(&submitCommand);
+    UNRECOVERABLE_IF(status != STATUS_SUCCESS);
+
+    if (STATUS_SUCCESS == status) {
+        monitoredFence.lastSubmittedFence = monitoredFence.currentFenceValue;
+        monitoredFence.currentFenceValue++;
+    }
+
+    getDeviceState();
+
+    return status == STATUS_SUCCESS;
+}
+} // namespace OCLRT

runtime/os_interface/windows/wddm/wddm23.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+#include "runtime/os_interface/windows/wddm/wddm.h"
+
+namespace OCLRT {
+class Wddm23 : public Wddm20 {
+  protected:
+    friend Wddm20;
+    Wddm23();
+    ~Wddm23();
+
+    bool createHwQueue() override;
+    void destroyHwQueue();
+    bool createMonitoredFence() override { return true; }
+    const bool hwQueuesSupported() const override { return true; }
+    bool submit(uint64_t commandBuffer, size_t size, void *commandHeader) override;
+
+    D3DKMT_HANDLE hwQueueHandle = 0;
+};
+} // namespace OCLRT

runtime/os_interface/windows/wddm/wddm_create.cpp

@@ -20,10 +20,24 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include "runtime/os_interface/debug_settings_manager.h"
 #include "runtime/os_interface/windows/wddm/wddm.h"
+#include "runtime/os_interface/windows/wddm/wddm23.h"
 
 namespace OCLRT {
 Wddm *Wddm::createWddm(uint32_t interfaceVersion) {
-    return new Wddm20();
+    if (DebugManager.flags.HwQueueSupported.get()) {
+        interfaceVersion = WddmInterfaceVersion::Wddm23;
+    }
+
+    switch (interfaceVersion) {
+    case WddmInterfaceVersion::Wddm20:
+        return new Wddm20();
+    case WddmInterfaceVersion::Wddm23:
+        return new Wddm23();
+    default:
+        UNRECOVERABLE_IF(true);
+        return nullptr;
+    }
 }
 } // namespace OCLRT

runtime/os_interface/windows/windows_defs.h

@@ -30,11 +30,11 @@ namespace OCLRT {
 constexpr uintptr_t windowsMinAddress = 0x200000;
 
 struct MonitoredFence {
-    D3DKMT_HANDLE fenceHandle;
-    D3DGPU_VIRTUAL_ADDRESS gpuAddress;
-    volatile uint64_t *cpuAddress;
-    volatile uint64_t currentFenceValue;
-    uint64_t lastSubmittedFence;
+    D3DKMT_HANDLE fenceHandle = 0;
+    D3DGPU_VIRTUAL_ADDRESS gpuAddress = 0;
+    volatile uint64_t *cpuAddress = nullptr;
+    volatile uint64_t currentFenceValue = 0;
+    uint64_t lastSubmittedFence = 0;
 };
 
 } // namespace OCLRT