diff --git a/opencl/source/dll/windows/options_windows.cpp b/opencl/source/dll/windows/options_windows.cpp index 776e998064..a9dc7be82f 100644 --- a/opencl/source/dll/windows/options_windows.cpp +++ b/opencl/source/dll/windows/options_windows.cpp @@ -13,6 +13,7 @@ namespace Os { const char *frontEndDllName = FCL_LIBRARY_NAME; const char *igcDllName = IGC_LIBRARY_NAME; const char *gdiDllName = "gdi32.dll"; +const char *dxcoreDllName = "dxcore.dll"; // Os specific Metrics Library name #if _WIN64 diff --git a/opencl/test/unit_test/os_interface/windows/options.cpp b/opencl/test/unit_test/os_interface/windows/options.cpp index ab31aef730..664f933598 100644 --- a/opencl/test/unit_test/os_interface/windows/options.cpp +++ b/opencl/test/unit_test/os_interface/windows/options.cpp @@ -18,6 +18,7 @@ namespace Os { const char *frontEndDllName = ""; const char *igcDllName = ""; const char *gdiDllName = "gdi32_mock.dll"; +const char *dxcoreDllName = ""; const char *testDllName = "test_dynamic_lib.dll"; const char *metricsLibraryDllName = ""; } // namespace Os diff --git a/shared/source/os_interface/linux/driver_info_linux.cpp b/shared/source/os_interface/linux/driver_info_linux.cpp index 10ba0a0e7c..cd29fae4ca 100644 --- a/shared/source/os_interface/linux/driver_info_linux.cpp +++ b/shared/source/os_interface/linux/driver_info_linux.cpp @@ -17,10 +17,7 @@ namespace NEO { DriverInfo *DriverInfo::create(const HardwareInfo *hwInfo, const OSInterface *osInterface) { PhysicalDevicePciBusInfo pciBusInfo(PhysicalDevicePciBusInfo::InvalidValue, PhysicalDevicePciBusInfo::InvalidValue, PhysicalDevicePciBusInfo::InvalidValue, PhysicalDevicePciBusInfo::InvalidValue); if (osInterface) { - auto drm = osInterface->getDriverModel()->as(); - UNRECOVERABLE_IF(drm == nullptr); - - pciBusInfo = drm->getPciBusInfo(); + pciBusInfo = osInterface->getDriverModel()->getPciBusInfo(); } if (hwInfo) { auto imageSupport = hwInfo->capabilityTable.supportsImages; diff --git a/shared/source/os_interface/linux/drm_neo.h b/shared/source/os_interface/linux/drm_neo.h index c0726b9484..c4cd701818 100644 --- a/shared/source/os_interface/linux/drm_neo.h +++ b/shared/source/os_interface/linux/drm_neo.h @@ -135,7 +135,7 @@ class Drm : public DriverModel { void setupSystemInfo(HardwareInfo *hwInfo, SystemInfo &sysInfo); void setupCacheInfo(const HardwareInfo &hwInfo); - PhysicalDevicePciBusInfo getPciBusInfo() const; + PhysicalDevicePciBusInfo getPciBusInfo() const override; bool areNonPersistentContextsSupported() const { return nonPersistentContextsSupported; } void checkNonPersistentContextsSupport(); diff --git a/shared/source/os_interface/os_interface.h b/shared/source/os_interface/os_interface.h index b5328edd7c..c640533b3d 100644 --- a/shared/source/os_interface/os_interface.h +++ b/shared/source/os_interface/os_interface.h @@ -8,6 +8,7 @@ #pragma once #include "shared/source/helpers/debug_helpers.h" #include "shared/source/helpers/non_copyable_or_moveable.h" +#include "shared/source/os_interface/driver_info.h" #include #include @@ -75,6 +76,8 @@ class DriverModel : public NonCopyableClass { return driverModelType; } + virtual PhysicalDevicePciBusInfo getPciBusInfo() const = 0; + protected: DriverModelType driverModelType; }; diff --git a/shared/source/os_interface/windows/CMakeLists.txt b/shared/source/os_interface/windows/CMakeLists.txt index fbb91b7efb..bf0bfb4135 100644 --- a/shared/source/os_interface/windows/CMakeLists.txt +++ b/shared/source/os_interface/windows/CMakeLists.txt @@ -4,7 +4,7 @@ # SPDX-License-Identifier: MIT # -if(KMDAF_HEADERS_DIR) +if(KMDAF_HEADERS_DIR AND WIN32) set(KMDAF_FILE_SUFFIX "") else() set(KMDAF_FILE_SUFFIX "_stub") @@ -15,6 +15,7 @@ set(NEO_CORE_OS_INTERFACE_WINDOWS ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_additional_context_flags.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_allocation.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_engine_mapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/d3dkmthk_wrapper.h ${CMAKE_CURRENT_SOURCE_DIR}/debug_registry_reader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/debug_registry_reader.h ${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.cpp @@ -53,8 +54,16 @@ set(NEO_CORE_OS_INTERFACE_WINDOWS ${CMAKE_CURRENT_SOURCE_DIR}/print.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sys_calls.h ${CMAKE_CURRENT_SOURCE_DIR}/thk_wrapper.h + ${CMAKE_CURRENT_SOURCE_DIR}/trim_callback.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_info.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_info.h + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory_create_with_fallback.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory.h + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory_dxcore.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory_dxcore.h + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory_dxgi.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm/adapter_factory_dxgi.h ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h ${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_defs.h diff --git a/shared/source/os_interface/windows/d3dkmthk_wrapper.h b/shared/source/os_interface/windows/d3dkmthk_wrapper.h new file mode 100644 index 0000000000..3160d65dfb --- /dev/null +++ b/shared/source/os_interface/windows/d3dkmthk_wrapper.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/os_interface/windows/windows_wrapper.h" + +#include + +struct _D3DDDICB_LOCKFLAGS; +typedef _D3DDDICB_LOCKFLAGS D3DDDICB_LOCKFLAGS; diff --git a/shared/source/os_interface/windows/dxcore_wrapper.h b/shared/source/os_interface/windows/dxcore_wrapper.h new file mode 100644 index 0000000000..ac59135465 --- /dev/null +++ b/shared/source/os_interface/windows/dxcore_wrapper.h @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include + +static const char *const dXCoreCreateAdapterFactoryFuncName = "DXCoreCreateAdapterFactory"; \ No newline at end of file diff --git a/shared/source/os_interface/windows/kmdaf_listener.cpp b/shared/source/os_interface/windows/kmdaf_listener.cpp index eba86f0d86..48c392c292 100644 --- a/shared/source/os_interface/windows/kmdaf_listener.cpp +++ b/shared/source/os_interface/windows/kmdaf_listener.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation * * SPDX-License-Identifier: MIT * diff --git a/shared/source/os_interface/windows/kmdaf_listener.h b/shared/source/os_interface/windows/kmdaf_listener.h index f7dc55982a..3ee546e3e0 100644 --- a/shared/source/os_interface/windows/kmdaf_listener.h +++ b/shared/source/os_interface/windows/kmdaf_listener.h @@ -1,14 +1,13 @@ /* - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation * * SPDX-License-Identifier: MIT * */ #pragma once -#include "shared/source/os_interface/windows/windows_wrapper.h" -#include +#include "shared/source/os_interface/windows/d3dkmthk_wrapper.h" namespace NEO { diff --git a/shared/source/os_interface/windows/kmdaf_listener_stub.cpp b/shared/source/os_interface/windows/kmdaf_listener_stub.cpp index d9577cc3b1..eb3a170a32 100644 --- a/shared/source/os_interface/windows/kmdaf_listener_stub.cpp +++ b/shared/source/os_interface/windows/kmdaf_listener_stub.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation * * SPDX-License-Identifier: MIT * diff --git a/shared/source/os_interface/windows/os_inc.h b/shared/source/os_interface/windows/os_inc.h index f8cec835c6..dbcbebd30a 100644 --- a/shared/source/os_interface/windows/os_inc.h +++ b/shared/source/os_interface/windows/os_inc.h @@ -14,4 +14,5 @@ namespace Os { // OS GDI name extern const char *gdiDllName; +extern const char *dxcoreDllName; }; // namespace Os diff --git a/shared/source/os_interface/windows/trim_callback.cpp b/shared/source/os_interface/windows/trim_callback.cpp new file mode 100644 index 0000000000..15384b6d8d --- /dev/null +++ b/shared/source/os_interface/windows/trim_callback.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/gdi_interface.h" +#include "shared/source/os_interface/windows/wddm/wddm.h" +#include "shared/source/os_interface/windows/wddm_allocation.h" +#include "shared/source/os_interface/windows/wddm_residency_controller.h" + +namespace NEO { + +VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { + if (DebugManager.flags.DoNotRegisterTrimCallback.get()) { + return nullptr; + } + D3DKMT_REGISTERTRIMNOTIFICATION registerTrimNotification; + registerTrimNotification.Callback = reinterpret_cast(callback); + registerTrimNotification.AdapterLuid = hwDeviceId->getAdapterLuid(); + registerTrimNotification.Context = &residencyController; + registerTrimNotification.hDevice = device; + + NTSTATUS status = getGdi()->registerTrimNotification(®isterTrimNotification); + if (status == STATUS_SUCCESS) { + return registerTrimNotification.Handle; + } + return nullptr; +} + +void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle) { + DEBUG_BREAK_IF(callback == nullptr); + if (trimCallbackHandle == nullptr || isShutdownInProgress()) { + return; + } + D3DKMT_UNREGISTERTRIMNOTIFICATION unregisterTrimNotification; + unregisterTrimNotification.Callback = callback; + unregisterTrimNotification.Handle = trimCallbackHandle; + + NTSTATUS status = getGdi()->unregisterTrimNotification(&unregisterTrimNotification); + DEBUG_BREAK_IF(status != STATUS_SUCCESS); +} + +void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) { + auto residencyController = static_cast(trimNotification->Context); + DEBUG_BREAK_IF(residencyController == nullptr); + + auto lock = residencyController->acquireTrimCallbackLock(); + residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); +} + +void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes) { + if (flags.PeriodicTrim) { + D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; + uint64_t sizeToTrim = 0; + auto lock = this->acquireLock(); + + WddmAllocation *wddmAllocation = nullptr; + while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) { + + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastTrimFenceValue); + + if (wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) { + break; + } + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId)); + + uint32_t fragmentsToEvict = 0; + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId)); + this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim); + } + + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId]; + if (!wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) { + auto osHandle = static_cast(wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", osHandle->handle, "lastFence =", + wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); + fragmentEvictHandles[fragmentsToEvict++] = static_cast(fragmentStorageData.osHandleStorage)->handle; + fragmentStorageData.residency->resident[osContextId] = false; + } + } + if (fragmentsToEvict != 0) { + this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); + } + wddmAllocation->getResidencyData().resident[osContextId] = false; + + this->removeFromTrimCandidateList(wddmAllocation, false); + } + + if (this->checkTrimCandidateListCompaction()) { + this->compactTrimCandidateList(); + } + } + + if (flags.TrimToBudget) { + auto lock = this->acquireLock(); + trimResidencyToBudget(bytes); + } + + if (flags.PeriodicTrim || flags.RestartPeriodicTrim) { + this->updateLastTrimFenceValue(); + DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastTrimFenceValue); + } +} + +bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { + D3DKMT_HANDLE fragmentEvictHandles[maxFragmentsCount] = {0}; + uint64_t numberOfBytesToTrim = bytes; + WddmAllocation *wddmAllocation = nullptr; + + while (numberOfBytesToTrim > 0 && (wddmAllocation = this->getTrimCandidateHead()) != nullptr) { + uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); + auto &monitoredFence = this->getMonitoredFence(); + + if (lastFence > monitoredFence.lastSubmittedFence) { + break; + } + + uint32_t fragmentsToEvict = 0; + uint64_t sizeEvicted = 0; + uint64_t sizeToTrim = 0; + + if (lastFence > *monitoredFence.cpuAddress) { + this->wddm.waitFromCpu(lastFence, this->getMonitoredFence()); + } + + if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { + this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim); + sizeEvicted = wddmAllocation->getAlignedSize(); + } else { + auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { + fragmentEvictHandles[fragmentsToEvict++] = static_cast(fragmentStorageData[allocationId].osHandleStorage)->handle; + } + } + + if (fragmentsToEvict != 0) { + this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); + + for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { + if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { + fragmentStorageData[allocationId].residency->resident[osContextId] = false; + sizeEvicted += fragmentStorageData[allocationId].fragmentSize; + } + } + } + } + + if (sizeEvicted >= numberOfBytesToTrim) { + numberOfBytesToTrim = 0; + } else { + numberOfBytesToTrim -= sizeEvicted; + } + + wddmAllocation->getResidencyData().resident[osContextId] = false; + this->removeFromTrimCandidateList(wddmAllocation, false); + } + + if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) { + this->compactTrimCandidateList(); + } + + return numberOfBytesToTrim == 0; +} + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/os_interface/windows/trim_callback_stub.cpp b/shared/source/os_interface/windows/trim_callback_stub.cpp new file mode 100644 index 0000000000..a4284fd4e2 --- /dev/null +++ b/shared/source/os_interface/windows/trim_callback_stub.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/gdi_interface.h" +#include "shared/source/os_interface/windows/wddm/wddm.h" +#include "shared/source/os_interface/windows/wddm_residency_controller.h" + +namespace NEO { + +VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { + return nullptr; +} + +void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle) { +} + +void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) { +} + +void WddmResidencyController::trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes) { +} + +bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { + return false; +} + +} // namespace NEO \ No newline at end of file diff --git a/shared/source/os_interface/windows/wddm/adapter_factory.cpp b/shared/source/os_interface/windows/wddm/adapter_factory.cpp new file mode 100644 index 0000000000..d9d37cc156 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" + +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/helpers/debug_helpers.h" +#include "shared/source/utilities/stackvec.h" + +#include +#include +#include + +namespace NEO { + +bool canUseAdapterBasedOnDriverDesc(const char *driverDescription) { + return (strstr(driverDescription, "Intel") != nullptr) || + (strstr(driverDescription, "Citrix") != nullptr) || + (strstr(driverDescription, "Virtual Render") != nullptr); +} + +bool isAllowedDeviceId(uint32_t deviceId) { + if (DebugManager.flags.ForceDeviceId.get() == "unk") { + return true; + } + + char *endptr = nullptr; + auto reqDeviceId = strtoul(DebugManager.flags.ForceDeviceId.get().c_str(), &endptr, 16); + return (static_cast(reqDeviceId) == deviceId); +} + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory.h b/shared/source/os_interface/windows/wddm/adapter_factory.h new file mode 100644 index 0000000000..da7fa0f903 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/os_interface/windows/windows_wrapper.h" + +#include +#include +#include + +typedef unsigned int D3DKMT_HANDLE; + +namespace NEO { + +class Gdi; +class OsLibrary; + +bool canUseAdapterBasedOnDriverDesc(const char *driverDescription); + +bool isAllowedDeviceId(uint32_t deviceId); + +class AdapterFactory { + public: + using CreateAdapterFactoryFcn = HRESULT(WINAPI *)(REFIID riid, void **ppFactory); + + struct AdapterDesc { + enum class Type { + Unknown, + Hardware, + NotHardware + }; + + Type type = Type::Unknown; + std::string driverDescription; + uint32_t deviceId = {}; + LUID luid = {}; + }; + + virtual ~AdapterFactory() = default; + + virtual bool createSnapshotOfAvailableAdapters() = 0; + virtual uint32_t getNumAdaptersInSnapshot() = 0; + virtual bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) = 0; + virtual bool isSupported() = 0; + + static std::unique_ptr create(AdapterFactory::CreateAdapterFactoryFcn dxCoreCreateAdapterFactoryF, + AdapterFactory::CreateAdapterFactoryFcn dxgiCreateAdapterFactoryF); +}; + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory_create_dxcore.cpp b/shared/source/os_interface/windows/wddm/adapter_factory_create_dxcore.cpp new file mode 100644 index 0000000000..f21c74dae3 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory_create_dxcore.cpp @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h" + +namespace NEO { + +std::unique_ptr AdapterFactory::create(AdapterFactory::CreateAdapterFactoryFcn dxCoreCreateAdapterFactoryF, + AdapterFactory::CreateAdapterFactoryFcn dxgiCreateAdapterFactoryF) { + return std::make_unique(dxCoreCreateAdapterFactoryF); +} + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory_create_with_fallback.cpp b/shared/source/os_interface/windows/wddm/adapter_factory_create_with_fallback.cpp new file mode 100644 index 0000000000..c9d4068240 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory_create_with_fallback.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxgi.h" + +namespace NEO { + +std::unique_ptr AdapterFactory::create(AdapterFactory::CreateAdapterFactoryFcn dxCoreCreateAdapterFactoryF, + AdapterFactory::CreateAdapterFactoryFcn dxgiCreateAdapterFactoryF) { + std::unique_ptr ret; + ret = std::make_unique(dxCoreCreateAdapterFactoryF); + if (false == ret->isSupported()) { + ret = std::make_unique(dxgiCreateAdapterFactoryF); + } + return ret; +} + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory_dxcore.cpp b/shared/source/os_interface/windows/wddm/adapter_factory_dxcore.cpp new file mode 100644 index 0000000000..745fcfcc94 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory_dxcore.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h" + +#include "shared/source/helpers/debug_helpers.h" +#include "shared/source/os_interface/os_library.h" +#include "shared/source/os_interface/windows/os_inc.h" +#include "shared/source/utilities/stackvec.h" + +#include + +namespace NEO { + +DxCoreAdapterFactory::DxCoreAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn) : createAdapterFactoryFcn(createAdapterFactoryFcn) { + if (nullptr == createAdapterFactoryFcn) { + dxCoreLibrary.reset(OsLibrary::load(Os::dxcoreDllName)); + if (dxCoreLibrary && dxCoreLibrary->isLoaded()) { + auto func = dxCoreLibrary->getProcAddress(dXCoreCreateAdapterFactoryFuncName); + createAdapterFactoryFcn = reinterpret_cast(func); + } + if (nullptr == createAdapterFactoryFcn) { + return; + } + } + + HRESULT hr = createAdapterFactoryFcn(__uuidof(adapterFactory), (void **)(&adapterFactory)); + if (hr != S_OK) { + adapterFactory = nullptr; + } +} + +DxCoreAdapterFactory::~DxCoreAdapterFactory() { + destroyCurrentSnapshot(); + + if (adapterFactory) { + adapterFactory->Release(); + adapterFactory = nullptr; + } +} + +bool DxCoreAdapterFactory::createSnapshotOfAvailableAdapters() { + if (false == this->isSupported()) { + DEBUG_BREAK_IF(true); + return false; + } + destroyCurrentSnapshot(); + + GUID attributes[]{DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE}; + HRESULT hr = adapterFactory->CreateAdapterList(1, attributes, __uuidof(adaptersInSnapshot), (void **)(&adaptersInSnapshot)); + if ((hr != S_OK) || (adaptersInSnapshot == nullptr)) { + DEBUG_BREAK_IF(true); + destroyCurrentSnapshot(); + return false; + } + + return true; +} + +uint32_t DxCoreAdapterFactory::getNumAdaptersInSnapshot() { + if (nullptr == adaptersInSnapshot) { + return 0U; + } + return adaptersInSnapshot->GetAdapterCount(); +} + +bool DxCoreAdapterFactory::getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) { + if (ordinal >= getNumAdaptersInSnapshot()) { + DEBUG_BREAK_IF(true); + return false; + } + + IDXCoreAdapter *adapter = nullptr; + HRESULT hr = adaptersInSnapshot->GetAdapter(ordinal, __uuidof(adapter), (void **)&adapter); + if ((hr != S_OK) || (adapter == nullptr)) { + return false; + } + + outAdapter = {}; + bool isHardware = false; + hr = adapter->GetProperty(DXCoreAdapterProperty::IsHardware, &isHardware); + DEBUG_BREAK_IF(S_OK != hr); + outAdapter.type = isHardware ? AdapterDesc::Type::Hardware : AdapterDesc::Type::NotHardware; + + static constexpr uint32_t maxDriverDescriptionStaticSize = 512; + StackVec driverDescription; + size_t driverDescSize = 0; + hr = adapter->GetPropertySize(DXCoreAdapterProperty::DriverDescription, &driverDescSize); + if (S_OK == hr) { + driverDescription.resize(driverDescSize); + } + hr = adapter->GetProperty(DXCoreAdapterProperty::DriverDescription, driverDescription.size(), driverDescription.data()); + if (S_OK != hr) { + adapter->Release(); + DEBUG_BREAK_IF(true); + return false; + } + outAdapter.driverDescription = driverDescription.data(); + + DXCoreHardwareID hwId = {}; + hr = adapter->GetProperty(DXCoreAdapterProperty::HardwareID, sizeof(hwId), &hwId); + DEBUG_BREAK_IF(S_OK != hr); + outAdapter.deviceId = hwId.deviceID; + + LUID luid = {}; + hr = adapter->GetProperty(DXCoreAdapterProperty::InstanceLuid, &luid); + if (S_OK != hr) { + adapter->Release(); + DEBUG_BREAK_IF(true); + return false; + } + outAdapter.luid = luid; + adapter->Release(); + return true; +} + +void DxCoreAdapterFactory::destroyCurrentSnapshot() { + if (adaptersInSnapshot) { + adaptersInSnapshot->Release(); + adaptersInSnapshot = nullptr; + } +} + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h b/shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h new file mode 100644 index 0000000000..131b079335 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/os_interface/os_library.h" +#include "shared/source/os_interface/windows/dxcore_wrapper.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" + +#include + +namespace NEO { + +class DxCoreAdapterFactory : public AdapterFactory { + public: + DxCoreAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn); + + ~DxCoreAdapterFactory() override; + + bool createSnapshotOfAvailableAdapters() override; + + bool isSupported() { + return nullptr != adapterFactory; + } + + uint32_t getNumAdaptersInSnapshot() override; + + bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) override; + + protected: + void destroyCurrentSnapshot(); + + void loadDxCore(DxCoreAdapterFactory::CreateAdapterFactoryFcn &outDxCoreCreateAdapterFactoryF); + std::unique_ptr dxCoreLibrary; + + AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn = nullptr; + IDXCoreAdapterFactory *adapterFactory = nullptr; + IDXCoreAdapterList *adaptersInSnapshot = nullptr; +}; + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory_dxgi.cpp b/shared/source/os_interface/windows/wddm/adapter_factory_dxgi.cpp new file mode 100644 index 0000000000..f092804383 --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory_dxgi.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxgi.h" + +#include "shared/source/helpers/debug_helpers.h" + +#include + +namespace NEO { +DxgiAdapterFactory::DxgiAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn) : createAdapterFactoryFcn(createAdapterFactoryFcn) { + if (nullptr == createAdapterFactoryFcn) { + return; + } + + HRESULT hr = createAdapterFactoryFcn(__uuidof(adapterFactory), (void **)(&adapterFactory)); + if (hr != S_OK) { + adapterFactory = nullptr; + } +} + +bool DxgiAdapterFactory::getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) { + if (ordinal >= getNumAdaptersInSnapshot()) { + DEBUG_BREAK_IF(true); + return false; + } + + outAdapter = adaptersInSnapshot[ordinal]; + return true; +} + +bool DxgiAdapterFactory::createSnapshotOfAvailableAdapters() { + if (false == this->isSupported()) { + DEBUG_BREAK_IF(true); + return false; + } + destroyCurrentSnapshot(); + + uint32_t ordinal = 0; + IDXGIAdapter1 *adapter = nullptr; + while (adapterFactory->EnumAdapters1(ordinal++, &adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC1 adapterDesc = {{0}}; + HRESULT hr = adapter->GetDesc1(&adapterDesc); + if (hr != S_OK) { + adapter->Release(); + DEBUG_BREAK_IF(true); + continue; + } + + adaptersInSnapshot.resize(adaptersInSnapshot.size() + 1); + auto &dstAdapterDesc = *adaptersInSnapshot.rbegin(); + dstAdapterDesc.luid = adapterDesc.AdapterLuid; + dstAdapterDesc.deviceId = adapterDesc.DeviceId; + static constexpr auto driverDescMaxChars = sizeof(adapterDesc.Description) / sizeof(adapterDesc.Description[0]); + dstAdapterDesc.driverDescription.reserve(driverDescMaxChars); + for (auto wchar : std::wstring(std::wstring(adapterDesc.Description, adapterDesc.Description + driverDescMaxChars).c_str())) { + dstAdapterDesc.driverDescription.push_back(static_cast(wchar)); + } + + adapter->Release(); + } + + return true; +} + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_factory_dxgi.h b/shared/source/os_interface/windows/wddm/adapter_factory_dxgi.h new file mode 100644 index 0000000000..ead886424c --- /dev/null +++ b/shared/source/os_interface/windows/wddm/adapter_factory_dxgi.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" + +#include +#include +#include + +namespace NEO { + +class DxgiAdapterFactory : public AdapterFactory { + public: + DxgiAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn); + + ~DxgiAdapterFactory() override { + destroyCurrentSnapshot(); + if (adapterFactory) { + adapterFactory->Release(); + } + } + + bool createSnapshotOfAvailableAdapters() override; + + bool isSupported() { + return nullptr != adapterFactory; + } + + uint32_t getNumAdaptersInSnapshot() { + return static_cast(adaptersInSnapshot.size()); + } + + bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) override; + + protected: + void destroyCurrentSnapshot() { + adaptersInSnapshot.clear(); + } + + AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn = nullptr; + IDXGIFactory1 *adapterFactory = nullptr; + std::vector adaptersInSnapshot; +}; + +} // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_info.cpp b/shared/source/os_interface/windows/wddm/adapter_info.cpp index 42f65ad674..6092bf23bd 100644 --- a/shared/source/os_interface/windows/wddm/adapter_info.cpp +++ b/shared/source/os_interface/windows/wddm/adapter_info.cpp @@ -7,13 +7,8 @@ #include "shared/source/os_interface/windows/wddm/adapter_info.h" -#include "shared/source/debug_settings/debug_settings_manager.h" #include "shared/source/helpers/debug_helpers.h" -#include "shared/source/os_interface/os_library.h" #include "shared/source/os_interface/windows/gdi_interface.h" -#include "shared/source/utilities/stackvec.h" - -#include namespace NEO { std::wstring queryAdapterDriverStorePath(const Gdi &gdi, D3DKMT_HANDLE adapter) { @@ -50,183 +45,4 @@ std::wstring queryAdapterDriverStorePath(const Gdi &gdi, D3DKMT_HANDLE adapter) return std::wstring(std::wstring(queryRegistryInfoValueDesc.OutputString, queryRegistryInfoValueDesc.OutputString + queryRegistryInfoValueDesc.OutputValueSize / sizeof(wchar_t)).c_str()); } -bool canUseAdapterBasedOnDriverDesc(const char *driverDescription) { - return (strstr(driverDescription, "Intel") != nullptr) || - (strstr(driverDescription, "Citrix") != nullptr) || - (strstr(driverDescription, "Virtual Render") != nullptr); -} - -bool isAllowedDeviceId(uint32_t deviceId) { - if (DebugManager.flags.ForceDeviceId.get() == "unk") { - return true; - } - - char *endptr = nullptr; - auto reqDeviceId = strtoul(DebugManager.flags.ForceDeviceId.get().c_str(), &endptr, 16); - return (static_cast(reqDeviceId) == deviceId); -} - -DxCoreAdapterFactory::DxCoreAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn) : createAdapterFactoryFcn(createAdapterFactoryFcn) { - if (nullptr == createAdapterFactoryFcn) { - return; - } - - HRESULT hr = createAdapterFactoryFcn(__uuidof(adapterFactory), (void **)(&adapterFactory)); - if (hr != S_OK) { - adapterFactory = nullptr; - } -} - -DxCoreAdapterFactory::~DxCoreAdapterFactory() { - destroyCurrentSnapshot(); - - if (adapterFactory) { - adapterFactory->Release(); - adapterFactory = nullptr; - } -} - -bool DxCoreAdapterFactory::createSnapshotOfAvailableAdapters() { - if (false == this->isSupported()) { - DEBUG_BREAK_IF(true); - return false; - } - destroyCurrentSnapshot(); - - GUID attributes[]{DXCORE_ADAPTER_ATTRIBUTE_D3D12_CORE_COMPUTE}; - HRESULT hr = adapterFactory->CreateAdapterList(1, attributes, __uuidof(adaptersInSnapshot), (void **)(&adaptersInSnapshot)); - if ((hr != S_OK) || (adaptersInSnapshot == nullptr)) { - DEBUG_BREAK_IF(true); - destroyCurrentSnapshot(); - return false; - } - - return true; -} - -uint32_t DxCoreAdapterFactory::getNumAdaptersInSnapshot() { - if (nullptr == adaptersInSnapshot) { - return 0U; - } - return adaptersInSnapshot->GetAdapterCount(); -} - -bool DxCoreAdapterFactory::getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) { - if (ordinal >= getNumAdaptersInSnapshot()) { - DEBUG_BREAK_IF(true); - return false; - } - - IDXCoreAdapter *adapter = nullptr; - HRESULT hr = adaptersInSnapshot->GetAdapter(ordinal, __uuidof(adapter), (void **)&adapter); - if ((hr != S_OK) || (adapter == nullptr)) { - return false; - } - - outAdapter = {}; - bool isHardware = false; - hr = adapter->GetProperty(DXCoreAdapterProperty::IsHardware, &isHardware); - DEBUG_BREAK_IF(S_OK != hr); - outAdapter.type = isHardware ? AdapterDesc::Type::Hardware : AdapterDesc::Type::NotHardware; - - static constexpr uint32_t maxDriverDescriptionStaticSize = 512; - StackVec driverDescription; - size_t driverDescSize = 0; - hr = adapter->GetPropertySize(DXCoreAdapterProperty::DriverDescription, &driverDescSize); - if (S_OK == hr) { - driverDescription.resize(driverDescSize); - } - hr = adapter->GetProperty(DXCoreAdapterProperty::DriverDescription, driverDescription.size(), driverDescription.data()); - if (S_OK != hr) { - adapter->Release(); - DEBUG_BREAK_IF(true); - return false; - } - outAdapter.driverDescription = driverDescription.data(); - - DXCoreHardwareID hwId = {}; - hr = adapter->GetProperty(DXCoreAdapterProperty::HardwareID, sizeof(hwId), &hwId); - DEBUG_BREAK_IF(S_OK != hr); - outAdapter.deviceId = hwId.deviceID; - - LUID luid = {}; - hr = adapter->GetProperty(DXCoreAdapterProperty::InstanceLuid, &luid); - if (S_OK != hr) { - adapter->Release(); - DEBUG_BREAK_IF(true); - return false; - } - outAdapter.luid = luid; - adapter->Release(); - return true; -} - -void DxCoreAdapterFactory::destroyCurrentSnapshot() { - if (adaptersInSnapshot) { - adaptersInSnapshot->Release(); - adaptersInSnapshot = nullptr; - } -} - -DxgiAdapterFactory::DxgiAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn) : createAdapterFactoryFcn(createAdapterFactoryFcn) { - if (nullptr == createAdapterFactoryFcn) { - return; - } - - HRESULT hr = createAdapterFactoryFcn(__uuidof(adapterFactory), (void **)(&adapterFactory)); - if (hr != S_OK) { - adapterFactory = nullptr; - } -} - -bool DxgiAdapterFactory::getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) { - if (ordinal >= getNumAdaptersInSnapshot()) { - DEBUG_BREAK_IF(true); - return false; - } - - outAdapter = adaptersInSnapshot[ordinal]; - return true; -} - -bool DxgiAdapterFactory::createSnapshotOfAvailableAdapters() { - if (false == this->isSupported()) { - DEBUG_BREAK_IF(true); - return false; - } - destroyCurrentSnapshot(); - - uint32_t ordinal = 0; - IDXGIAdapter1 *adapter = nullptr; - while (adapterFactory->EnumAdapters1(ordinal++, &adapter) != DXGI_ERROR_NOT_FOUND) { - DXGI_ADAPTER_DESC1 adapterDesc = {{0}}; - HRESULT hr = adapter->GetDesc1(&adapterDesc); - if (hr != S_OK) { - adapter->Release(); - DEBUG_BREAK_IF(true); - continue; - } - - adaptersInSnapshot.resize(adaptersInSnapshot.size() + 1); - auto &dstAdapterDesc = *adaptersInSnapshot.rbegin(); - dstAdapterDesc.luid = adapterDesc.AdapterLuid; - dstAdapterDesc.deviceId = adapterDesc.DeviceId; - static constexpr auto driverDescMaxChars = sizeof(adapterDesc.Description) / sizeof(adapterDesc.Description[0]); - dstAdapterDesc.driverDescription.reserve(driverDescMaxChars); - for (auto wchar : std::wstring(std::wstring(adapterDesc.Description, adapterDesc.Description + driverDescMaxChars).c_str())) { - dstAdapterDesc.driverDescription.push_back(static_cast(wchar)); - } - - adapter->Release(); - } - - return true; -} - -void WddmAdapterFactory::loadDxCore(DxCoreAdapterFactory::CreateAdapterFactoryFcn &outDxCoreCreateAdapterFactoryF) { - dxCoreLibrary.reset(OsLibrary::load("dxcore.dll")); - if (dxCoreLibrary && dxCoreLibrary->isLoaded()) { - outDxCoreCreateAdapterFactoryF = reinterpret_cast(dxCoreLibrary->getProcAddress("DXCoreCreateAdapterFactory")); - } -} } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/adapter_info.h b/shared/source/os_interface/windows/wddm/adapter_info.h index 844368c8fd..921c0a55d3 100644 --- a/shared/source/os_interface/windows/wddm/adapter_info.h +++ b/shared/source/os_interface/windows/wddm/adapter_info.h @@ -7,146 +7,12 @@ #pragma once -// clang-format off -#include -#include -#include -// clang-format on +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" #include -#include -#include - -typedef unsigned int D3DKMT_HANDLE; namespace NEO { -class Gdi; -class OsLibrary; - std::wstring queryAdapterDriverStorePath(const Gdi &gdi, D3DKMT_HANDLE adapter); -bool canUseAdapterBasedOnDriverDesc(const char *driverDescription); - -bool isAllowedDeviceId(uint32_t deviceId); - -class AdapterFactory { - public: - using CreateAdapterFactoryFcn = HRESULT(WINAPI *)(REFIID riid, void **ppFactory); - - struct AdapterDesc { - enum class Type { - Unknown, - Hardware, - NotHardware - }; - - Type type = Type::Unknown; - std::string driverDescription; - uint32_t deviceId = {}; - LUID luid = {}; - }; - - virtual ~AdapterFactory() = default; - - virtual bool createSnapshotOfAvailableAdapters() = 0; - virtual uint32_t getNumAdaptersInSnapshot() = 0; - virtual bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) = 0; - virtual bool isSupported() = 0; -}; - -class DxCoreAdapterFactory : public AdapterFactory { - public: - DxCoreAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn); - - ~DxCoreAdapterFactory() override; - - bool createSnapshotOfAvailableAdapters() override; - - bool isSupported() { - return nullptr != adapterFactory; - } - - uint32_t getNumAdaptersInSnapshot() override; - - bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) override; - - protected: - void destroyCurrentSnapshot(); - - AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn = nullptr; - IDXCoreAdapterFactory *adapterFactory = nullptr; - IDXCoreAdapterList *adaptersInSnapshot = nullptr; -}; - -class DxgiAdapterFactory : public AdapterFactory { - public: - DxgiAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn); - - ~DxgiAdapterFactory() override { - destroyCurrentSnapshot(); - if (adapterFactory) { - adapterFactory->Release(); - } - } - - bool createSnapshotOfAvailableAdapters() override; - - bool isSupported() { - return nullptr != adapterFactory; - } - - uint32_t getNumAdaptersInSnapshot() { - return static_cast(adaptersInSnapshot.size()); - } - - bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) override; - - protected: - void destroyCurrentSnapshot() { - adaptersInSnapshot.clear(); - } - - AdapterFactory::CreateAdapterFactoryFcn createAdapterFactoryFcn = nullptr; - IDXGIFactory1 *adapterFactory = nullptr; - std::vector adaptersInSnapshot; -}; - -class WddmAdapterFactory : public AdapterFactory { - public: - WddmAdapterFactory(AdapterFactory::CreateAdapterFactoryFcn dxCoreCreateAdapterFactoryF, - AdapterFactory::CreateAdapterFactoryFcn dxgiCreateAdapterFactoryF) { - if (nullptr == dxCoreCreateAdapterFactoryF) { - loadDxCore(dxCoreCreateAdapterFactoryF); - } - underlyingFactory = std::make_unique(dxCoreCreateAdapterFactoryF); - if (false == underlyingFactory->isSupported()) { - underlyingFactory = std::make_unique(dxgiCreateAdapterFactoryF); - } - } - - ~WddmAdapterFactory() override = default; - - bool createSnapshotOfAvailableAdapters() override { - return underlyingFactory->createSnapshotOfAvailableAdapters(); - } - - bool isSupported() override { - return underlyingFactory->isSupported(); - } - - uint32_t getNumAdaptersInSnapshot() override { - return underlyingFactory->getNumAdaptersInSnapshot(); - } - - bool getAdapterDesc(uint32_t ordinal, AdapterDesc &outAdapter) override { - return underlyingFactory->getAdapterDesc(ordinal, outAdapter); - } - - protected: - std::unique_ptr dxCoreLibrary; - std::unique_ptr underlyingFactory; - - void loadDxCore(DxCoreAdapterFactory::CreateAdapterFactoryFcn &outDxCoreCreateAdapterFactoryF); -}; } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index be2f42cad1..1199407de2 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -26,6 +26,7 @@ #include "shared/source/os_interface/windows/kmdaf_listener.h" #include "shared/source/os_interface/windows/os_context_win.h" #include "shared/source/os_interface/windows/os_environment_win.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" #include "shared/source/os_interface/windows/wddm/adapter_info.h" #include "shared/source/os_interface/windows/wddm/um_km_data_translator.h" #include "shared/source/os_interface/windows/wddm/wddm_interface.h" @@ -308,9 +309,9 @@ std::vector> OSInterface::discoverDevicesWddm(Execut return {}; } - WddmAdapterFactory adapterFactory{Wddm::dXCoreCreateAdapterFactory, Wddm::createDxgiFactory}; + auto adapterFactory = AdapterFactory::create(Wddm::dXCoreCreateAdapterFactory, Wddm::createDxgiFactory); - if (false == adapterFactory.isSupported()) { + if (false == adapterFactory->isSupported()) { return {}; } @@ -321,14 +322,14 @@ std::vector> OSInterface::discoverDevicesWddm(Execut std::vector> hwDeviceIds; do { - if (false == adapterFactory.createSnapshotOfAvailableAdapters()) { + if (false == adapterFactory->createSnapshotOfAvailableAdapters()) { return hwDeviceIds; } - auto adapterCount = adapterFactory.getNumAdaptersInSnapshot(); + auto adapterCount = adapterFactory->getNumAdaptersInSnapshot(); for (uint32_t i = 0; i < adapterCount; ++i) { AdapterFactory::AdapterDesc adapterDesc; - if (false == adapterFactory.getAdapterDesc(i, adapterDesc)) { + if (false == adapterFactory->getAdapterDesc(i, adapterDesc)) { DEBUG_BREAK_IF(true); continue; } @@ -1001,22 +1002,6 @@ LUID Wddm::getAdapterLuid() const { return hwDeviceId->getAdapterLuid(); } -VOID *Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController) { - if (DebugManager.flags.DoNotRegisterTrimCallback.get()) { - return nullptr; - } - D3DKMT_REGISTERTRIMNOTIFICATION registerTrimNotification; - registerTrimNotification.Callback = callback; - registerTrimNotification.AdapterLuid = hwDeviceId->getAdapterLuid(); - registerTrimNotification.Context = &residencyController; - registerTrimNotification.hDevice = device; - - NTSTATUS status = getGdi()->registerTrimNotification(®isterTrimNotification); - if (status == STATUS_SUCCESS) { - return registerTrimNotification.Handle; - } - return nullptr; -} bool Wddm::isShutdownInProgress() { auto handle = GetModuleHandleA("ntdll.dll"); @@ -1028,19 +1013,6 @@ bool Wddm::isShutdownInProgress() { return RtlDllShutdownInProgress(); } -void Wddm::unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle) { - DEBUG_BREAK_IF(callback == nullptr); - if (trimCallbackHandle == nullptr || isShutdownInProgress()) { - return; - } - D3DKMT_UNREGISTERTRIMNOTIFICATION unregisterTrimNotification; - unregisterTrimNotification.Callback = callback; - unregisterTrimNotification.Handle = trimCallbackHandle; - - NTSTATUS status = getGdi()->unregisterTrimNotification(&unregisterTrimNotification); - DEBUG_BREAK_IF(status != STATUS_SUCCESS); -} - void Wddm::releaseReservedAddress(void *reservedAddress) { if (reservedAddress) { [[maybe_unused]] auto status = virtualFree(reservedAddress, 0, MEM_RELEASE); @@ -1184,7 +1156,7 @@ PhysicalDevicePciBusInfo Wddm::getPciBusInfo() const { return PhysicalDevicePciBusInfo(0, adapterAddress.BusNumber, adapterAddress.DeviceNumber, adapterAddress.FunctionNumber); } - return PhysicalDevicePciBusInfo(PhysicalDevicePciBusInfo::InvalidValue, PhysicalDevicePciBusInfo::InvalidValue, PhysicalDevicePciBusInfo::InvalidValue, PhysicalDevicePciBusInfo::InvalidValue); + return PhysicalDevicePciBusInfo::invalid(); } } // namespace NEO diff --git a/shared/source/os_interface/windows/wddm/wddm.h b/shared/source/os_interface/windows/wddm/wddm.h index 27620eefa1..5aaa61fd16 100644 --- a/shared/source/os_interface/windows/wddm/wddm.h +++ b/shared/source/os_interface/windows/wddm/wddm.h @@ -6,6 +6,7 @@ */ #pragma once + #include "shared/source/command_stream/preemption_mode.h" #include "shared/source/gmm_helper/gmm_lib.h" #include "shared/source/helpers/debug_helpers.h" @@ -24,6 +25,13 @@ #include #include +struct _D3DKMT_TRIMNOTIFICATION; +typedef struct _D3DKMT_TRIMNOTIFICATION D3DKMT_TRIMNOTIFICATION; +typedef VOID(APIENTRY *PFND3DKMT_TRIMNOTIFICATIONCALLBACK)(_Inout_ D3DKMT_TRIMNOTIFICATION *); + +struct _SYSTEM_INFO; +typedef struct _SYSTEM_INFO SYSTEM_INFO; + namespace NEO { class Gdi; class Gmm; @@ -176,7 +184,7 @@ class Wddm : public DriverModel { uint32_t getTimestampFrequency() const { return timestampFrequency; } - PhysicalDevicePciBusInfo getPciBusInfo() const; + PhysicalDevicePciBusInfo getPciBusInfo() const override; protected: std::unique_ptr hwDeviceId; diff --git a/shared/source/os_interface/windows/wddm_residency_controller.cpp b/shared/source/os_interface/windows/wddm_residency_controller.cpp index 19b0f6d52c..d99aae787a 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.cpp +++ b/shared/source/os_interface/windows/wddm_residency_controller.cpp @@ -31,14 +31,6 @@ WddmResidencyController::~WddmResidencyController() { lock.lock(); } -void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) { - auto residencyController = static_cast(trimNotification->Context); - DEBUG_BREAK_IF(residencyController == nullptr); - - auto lock = residencyController->acquireTrimCallbackLock(); - residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim); -} - std::unique_lock WddmResidencyController::acquireLock() { return std::unique_lock{this->lock}; } @@ -181,124 +173,6 @@ void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, u monitoredFence.gpuAddress = gpuAddress; } -void WddmResidencyController::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) { - if (flags.PeriodicTrim) { - D3DKMT_HANDLE fragmentEvictHandles[3] = {0}; - uint64_t sizeToTrim = 0; - auto lock = this->acquireLock(); - - WddmAllocation *wddmAllocation = nullptr; - while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) { - - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastTrimFenceValue); - - if (wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) { - break; - } - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId)); - - uint32_t fragmentsToEvict = 0; - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId)); - this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim); - } - - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId]; - if (!wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) { - auto osHandle = static_cast(wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", osHandle->handle, "lastFence =", - wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId)); - fragmentEvictHandles[fragmentsToEvict++] = static_cast(fragmentStorageData.osHandleStorage)->handle; - fragmentStorageData.residency->resident[osContextId] = false; - } - } - if (fragmentsToEvict != 0) { - this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); - } - wddmAllocation->getResidencyData().resident[osContextId] = false; - - this->removeFromTrimCandidateList(wddmAllocation, false); - } - - if (this->checkTrimCandidateListCompaction()) { - this->compactTrimCandidateList(); - } - } - - if (flags.TrimToBudget) { - auto lock = this->acquireLock(); - trimResidencyToBudget(bytes); - } - - if (flags.PeriodicTrim || flags.RestartPeriodicTrim) { - this->updateLastTrimFenceValue(); - DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastTrimFenceValue); - } -} - -bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) { - D3DKMT_HANDLE fragmentEvictHandles[maxFragmentsCount] = {0}; - uint64_t numberOfBytesToTrim = bytes; - WddmAllocation *wddmAllocation = nullptr; - - while (numberOfBytesToTrim > 0 && (wddmAllocation = this->getTrimCandidateHead()) != nullptr) { - uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId); - auto &monitoredFence = this->getMonitoredFence(); - - if (lastFence > monitoredFence.lastSubmittedFence) { - break; - } - - uint32_t fragmentsToEvict = 0; - uint64_t sizeEvicted = 0; - uint64_t sizeToTrim = 0; - - if (lastFence > *monitoredFence.cpuAddress) { - this->wddm.waitFromCpu(lastFence, this->getMonitoredFence()); - } - - if (wddmAllocation->fragmentsStorage.fragmentCount == 0) { - this->wddm.evict(&wddmAllocation->getHandles()[0], wddmAllocation->getNumGmms(), sizeToTrim); - sizeEvicted = wddmAllocation->getAlignedSize(); - } else { - auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData; - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { - fragmentEvictHandles[fragmentsToEvict++] = static_cast(fragmentStorageData[allocationId].osHandleStorage)->handle; - } - } - - if (fragmentsToEvict != 0) { - this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim); - - for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) { - if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) { - fragmentStorageData[allocationId].residency->resident[osContextId] = false; - sizeEvicted += fragmentStorageData[allocationId].fragmentSize; - } - } - } - } - - if (sizeEvicted >= numberOfBytesToTrim) { - numberOfBytesToTrim = 0; - } else { - numberOfBytesToTrim -= sizeEvicted; - } - - wddmAllocation->getResidencyData().resident[osContextId] = false; - this->removeFromTrimCandidateList(wddmAllocation, false); - } - - if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) { - this->compactTrimCandidateList(); - } - - return numberOfBytesToTrim == 0; -} - bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency) { const size_t residencyCount = allocationsForResidency.size(); constexpr uint32_t stackAllocations = 64; diff --git a/shared/source/os_interface/windows/wddm_residency_controller.h b/shared/source/os_interface/windows/wddm_residency_controller.h index b512258546..4018a88497 100644 --- a/shared/source/os_interface/windows/wddm_residency_controller.h +++ b/shared/source/os_interface/windows/wddm_residency_controller.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -15,6 +15,10 @@ #include #include +struct _D3DKMT_TRIMNOTIFICATION; +typedef _D3DKMT_TRIMNOTIFICATION D3DKMT_TRIMNOTIFICATION; +struct D3DDDI_TRIMRESIDENCYSET_FLAGS; + namespace NEO { class GraphicsAllocation; @@ -50,7 +54,7 @@ class WddmResidencyController { void registerCallback(); - void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes); + void trimResidency(const D3DDDI_TRIMRESIDENCYSET_FLAGS &flags, uint64_t bytes); bool trimResidencyToBudget(uint64_t bytes); bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; } diff --git a/shared/test/unit_test/os_interface/windows/adapter_info_tests.cpp b/shared/test/unit_test/os_interface/windows/adapter_info_tests.cpp index bbc9165c44..b1beef906a 100644 --- a/shared/test/unit_test/os_interface/windows/adapter_info_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/adapter_info_tests.cpp @@ -7,6 +7,9 @@ #include "shared/test/unit_test/os_interface/windows/adapter_info_tests.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxcore.h" +#include "shared/source/os_interface/windows/wddm/adapter_factory_dxgi.h" #include "shared/source/os_interface/windows/wddm/adapter_info.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" @@ -124,40 +127,6 @@ TEST(DxgiAdapterFactory, whenSupportedThenGiveAccessToUnderlyingAdapterDesc) { EXPECT_STREQ("Intel", adapterDesc.driverDescription.c_str()); } -TEST(WddmAdapterFactory, whenCouldNotCreateAdapterFactoryThenReturnsUnsupported) { - { - NEO::WddmAdapterFactory adapterFactory{nullptr, nullptr}; - EXPECT_FALSE(adapterFactory.isSupported()); - } - - { - NEO::WddmAdapterFactory adapterFactory{failCreateTopLevelInterface, failCreateTopLevelInterface}; - EXPECT_FALSE(adapterFactory.isSupported()); - } -} - -TEST(WddmAdapterFactory, whenDxCoreAdapterFactoryNotAvailableThenFallbacksToDxgiAdapterFactory) { - NEO::WddmAdapterFactory adapterFactory{failCreateTopLevelInterface, &createTopLevelInterface}; - EXPECT_TRUE(adapterFactory.isSupported()); -} - -TEST(WddmAdapterFactory, whenSupportedThenGiveAccessToUnderlyingAdapterDesc) { - NEO::WddmAdapterFactory adapterFactory{&createTopLevelInterface, &createTopLevelInterface}; - - bool createdSnapshot = adapterFactory.createSnapshotOfAvailableAdapters(); - EXPECT_TRUE(createdSnapshot); - EXPECT_EQ(1U, adapterFactory.getNumAdaptersInSnapshot()); - - NEO::AdapterFactory::AdapterDesc adapterDesc; - bool retreivedAdapterDesc = adapterFactory.getAdapterDesc(0, adapterDesc); - EXPECT_TRUE(retreivedAdapterDesc); - EXPECT_EQ(NEO::AdapterFactory::AdapterDesc::Type::Hardware, adapterDesc.type); - EXPECT_EQ(0x1234U, adapterDesc.deviceId); - EXPECT_EQ(0x1234U, adapterDesc.luid.HighPart); - EXPECT_EQ(0U, adapterDesc.luid.LowPart); - EXPECT_STREQ("Intel", adapterDesc.driverDescription.c_str()); -} - TEST(IsAllowedDeviceId, whenDebugKeyNotSetThenReturnTrue) { EXPECT_TRUE(NEO::isAllowedDeviceId(0xdeadbeef)); }