mirror of
https://github.com/intel/compute-runtime.git
synced 2026-01-04 07:14:10 +08:00
Move wddm files to core folder
Change-Id: I28cb7ee6f63275e5b44697e22259023354b9311e
This commit is contained in:
kamdiedrich
@@ -8,9 +8,6 @@ set(RUNTIME_SRCS_OS_INTERFACE_BASE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/device_factory.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/device_factory.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.inl
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config_bdw_plus.inl
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/metrics_library.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/metrics_library.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ocl_reg_path.h
|
||||
|
||||
@@ -10,9 +10,9 @@
|
||||
#include "core/helpers/hw_helper.h"
|
||||
#include "core/helpers/options.h"
|
||||
#include "core/os_interface/aub_memory_operations_handler.h"
|
||||
#include "core/os_interface/hw_info_config.h"
|
||||
#include "runtime/aub/aub_center.h"
|
||||
#include "runtime/device/device.h"
|
||||
#include "runtime/os_interface/hw_info_config.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/helpers/hw_info.h"
|
||||
#include "core/unified_memory/usm_memory_support.h"
|
||||
|
||||
#include "igfxfmid.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
struct HardwareInfo;
|
||||
class OSInterface;
|
||||
class HwInfoConfig;
|
||||
|
||||
extern HwInfoConfig *hwInfoConfigFactory[IGFX_MAX_PRODUCT];
|
||||
|
||||
class HwInfoConfig {
|
||||
public:
|
||||
static HwInfoConfig *get(PRODUCT_FAMILY product) {
|
||||
return hwInfoConfigFactory[product];
|
||||
}
|
||||
int configureHwInfo(const HardwareInfo *inHwInfo, HardwareInfo *outHwInfo, OSInterface *osIface);
|
||||
virtual int configureHardwareCustom(HardwareInfo *hwInfo, OSInterface *osIface) = 0;
|
||||
virtual void adjustPlatformForProductFamily(HardwareInfo *hwInfo) = 0;
|
||||
virtual uint64_t getHostMemCapabilities() = 0;
|
||||
virtual uint64_t getDeviceMemCapabilities() = 0;
|
||||
virtual uint64_t getSingleDeviceSharedMemCapabilities() = 0;
|
||||
virtual uint64_t getCrossDeviceSharedMemCapabilities() = 0;
|
||||
virtual uint64_t getSharedSystemMemCapabilities() = 0;
|
||||
uint32_t threadsPerEu;
|
||||
};
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
class HwInfoConfigHw : public HwInfoConfig {
|
||||
public:
|
||||
static HwInfoConfig *get() {
|
||||
static HwInfoConfigHw<gfxProduct> instance;
|
||||
return &instance;
|
||||
}
|
||||
int configureHardwareCustom(HardwareInfo *hwInfo, OSInterface *osIface) override;
|
||||
void adjustPlatformForProductFamily(HardwareInfo *hwInfo) override;
|
||||
uint64_t getHostMemCapabilities() override;
|
||||
uint64_t getDeviceMemCapabilities() override;
|
||||
uint64_t getSingleDeviceSharedMemCapabilities() override;
|
||||
uint64_t getCrossDeviceSharedMemCapabilities() override;
|
||||
uint64_t getSharedSystemMemCapabilities() override;
|
||||
|
||||
protected:
|
||||
HwInfoConfigHw() {}
|
||||
};
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
struct EnableProductHwInfoConfig {
|
||||
typedef typename HwMapper<gfxProduct>::GfxProduct GfxProduct;
|
||||
|
||||
EnableProductHwInfoConfig() {
|
||||
HwInfoConfig *pHwInfoConfig = HwInfoConfigHw<gfxProduct>::get();
|
||||
hwInfoConfigFactory[gfxProduct] = pHwInfoConfig;
|
||||
pHwInfoConfig->threadsPerEu = GfxProduct::threadsPerEu;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,16 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/hw_info_config.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
void HwInfoConfigHw<gfxProduct>::adjustPlatformForProductFamily(HardwareInfo *hwInfo) {
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,36 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/hw_info_config.h"
|
||||
|
||||
namespace NEO {
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
uint64_t HwInfoConfigHw<gfxProduct>::getHostMemCapabilities() {
|
||||
return (UNIFIED_SHARED_MEMORY_ACCESS | UNIFIED_SHARED_MEMORY_ATOMIC_ACCESS);
|
||||
}
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
uint64_t HwInfoConfigHw<gfxProduct>::getDeviceMemCapabilities() {
|
||||
return (UNIFIED_SHARED_MEMORY_ACCESS | UNIFIED_SHARED_MEMORY_ATOMIC_ACCESS);
|
||||
}
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
uint64_t HwInfoConfigHw<gfxProduct>::getSingleDeviceSharedMemCapabilities() {
|
||||
return (UNIFIED_SHARED_MEMORY_ACCESS | UNIFIED_SHARED_MEMORY_ATOMIC_ACCESS);
|
||||
}
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
uint64_t HwInfoConfigHw<gfxProduct>::getCrossDeviceSharedMemCapabilities() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <PRODUCT_FAMILY gfxProduct>
|
||||
uint64_t HwInfoConfigHw<gfxProduct>::getSharedSystemMemCapabilities() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -8,12 +8,12 @@
|
||||
#include "core/debug_settings/debug_settings_manager.h"
|
||||
#include "core/execution_environment/root_device_environment.h"
|
||||
#include "core/helpers/hw_info.h"
|
||||
#include "core/os_interface/hw_info_config.h"
|
||||
#include "core/os_interface/linux/drm_neo.h"
|
||||
#include "core/os_interface/linux/os_interface.h"
|
||||
#include "runtime/device/cl_device.h"
|
||||
#include "runtime/execution_environment/execution_environment.h"
|
||||
#include "runtime/os_interface/device_factory.h"
|
||||
#include "runtime/os_interface/hw_info_config.h"
|
||||
#include "runtime/os_interface/linux/drm_memory_operations_handler.h"
|
||||
|
||||
#include "drm/i915_drm.h"
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/hw_info_config.h"
|
||||
#include "core/os_interface/hw_info_config.h"
|
||||
|
||||
#include "core/command_stream/preemption.h"
|
||||
#include "core/debug_settings/debug_settings_manager.h"
|
||||
|
||||
@@ -10,42 +10,21 @@ set(RUNTIME_SRCS_OS_INTERFACE_WINDOWS
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/d3d10_11_sharing_functions.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/d3d9_sharing_functions.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/d3d_sharing_functions.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/device_command_stream.inl
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/device_factory_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/driver_info.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/driver_info.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/environment_variables.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ocl_reg_path.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_context_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_context_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_interface.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_interface.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_metrics_library.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_socket.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_time_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_time_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/performance_counters_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/performance_counters_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_additional_context_flags.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_device_command_stream.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_device_command_stream.inl
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_allocations_container.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_allocations_container.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_operations_handler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_operations_handler.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/windows_inc.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_interface.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_interface.cpp
|
||||
)
|
||||
|
||||
if(WIN32)
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/deferrable_deletion_win.h"
|
||||
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
template <typename... Args>
|
||||
DeferrableDeletion *DeferrableDeletion::create(Args... args) {
|
||||
return new DeferrableDeletionImpl(std::forward<Args>(args)...);
|
||||
}
|
||||
template DeferrableDeletion *DeferrableDeletion::create(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
|
||||
|
||||
DeferrableDeletionImpl::DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle)
|
||||
: wddm(wddm), allocationCount(allocationCount), resourceHandle(resourceHandle) {
|
||||
if (handles) {
|
||||
this->handles = new D3DKMT_HANDLE[allocationCount];
|
||||
for (uint32_t i = 0; i < allocationCount; i++) {
|
||||
this->handles[i] = handles[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
bool DeferrableDeletionImpl::apply() {
|
||||
bool destroyStatus = wddm->destroyAllocations(handles, allocationCount, resourceHandle);
|
||||
DEBUG_BREAK_IF(!destroyStatus);
|
||||
return true;
|
||||
}
|
||||
DeferrableDeletionImpl::~DeferrableDeletionImpl() {
|
||||
if (handles) {
|
||||
delete[] handles;
|
||||
}
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -1,35 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/memory_manager/deferrable_deletion.h"
|
||||
#include "core/os_interface/os_context.h"
|
||||
#include "core/os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class OsContextWin;
|
||||
class Wddm;
|
||||
|
||||
class DeferrableDeletionImpl : public DeferrableDeletion {
|
||||
public:
|
||||
DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
|
||||
bool apply() override;
|
||||
~DeferrableDeletionImpl();
|
||||
|
||||
DeferrableDeletionImpl(const DeferrableDeletionImpl &) = delete;
|
||||
DeferrableDeletionImpl &operator=(const DeferrableDeletionImpl &) = delete;
|
||||
|
||||
protected:
|
||||
Wddm *wddm;
|
||||
D3DKMT_HANDLE *handles = nullptr;
|
||||
uint32_t allocationCount;
|
||||
D3DKMT_HANDLE resourceHandle;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -9,12 +9,12 @@
|
||||
|
||||
#include "core/debug_settings/debug_settings_manager.h"
|
||||
#include "core/execution_environment/root_device_environment.h"
|
||||
#include "core/os_interface/windows/os_interface.h"
|
||||
#include "core/os_interface/windows/wddm/wddm.h"
|
||||
#include "core/os_interface/windows/wddm_memory_operations_handler.h"
|
||||
#include "runtime/device/cl_device.h"
|
||||
#include "runtime/device/device.h"
|
||||
#include "runtime/os_interface/device_factory.h"
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/os_interface/windows/wddm_memory_operations_handler.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
@@ -8,9 +8,9 @@
|
||||
#include "runtime/device/driver_info.h"
|
||||
|
||||
#include "core/os_interface/windows/debug_registry_reader.h"
|
||||
#include "core/os_interface/windows/os_interface.h"
|
||||
#include "core/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/os_interface/windows/driver_info.h"
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
|
||||
@@ -8,12 +8,12 @@
|
||||
#include "core/helpers/timestamp_packet.h"
|
||||
#include "core/os_interface/os_interface.h"
|
||||
#include "core/os_interface/windows/gdi_interface.h"
|
||||
#include "core/os_interface/windows/os_context_win.h"
|
||||
#include "core/os_interface/windows/os_interface.h"
|
||||
#include "core/os_interface/windows/wddm/wddm.h"
|
||||
#include "public/cl_gl_private_intel.h"
|
||||
#include "runtime/command_stream/command_stream_receiver.h"
|
||||
#include "runtime/context/context.h"
|
||||
#include "runtime/os_interface/windows/os_context_win.h"
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/sharings/gl/gl_arb_sync_event.h"
|
||||
#include "runtime/sharings/gl/windows/gl_sharing_windows.h"
|
||||
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/hw_info_config.h"
|
||||
|
||||
#include "core/command_stream/preemption.h"
|
||||
#include "core/debug_settings/debug_settings_manager.h"
|
||||
#include "core/helpers/hw_cmds.h"
|
||||
#include "core/helpers/hw_helper.h"
|
||||
#include "core/helpers/hw_info.h"
|
||||
#include "core/memory_manager/memory_constants.h"
|
||||
|
||||
#include "instrumentation.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
HwInfoConfig *hwInfoConfigFactory[IGFX_MAX_PRODUCT] = {};
|
||||
|
||||
int HwInfoConfig::configureHwInfo(const HardwareInfo *inHwInfo, HardwareInfo *outHwInfo, OSInterface *osIface) {
|
||||
HwHelper &hwHelper = HwHelper::get(outHwInfo->platform.eRenderCoreFamily);
|
||||
|
||||
outHwInfo->capabilityTable.ftrSvm = outHwInfo->featureTable.ftrSVM;
|
||||
|
||||
hwHelper.adjustDefaultEngineType(outHwInfo);
|
||||
outHwInfo->capabilityTable.defaultEngineType = getChosenEngineType(*outHwInfo);
|
||||
|
||||
hwHelper.setCapabilityCoherencyFlag(outHwInfo, outHwInfo->capabilityTable.ftrSupportsCoherency);
|
||||
outHwInfo->capabilityTable.ftrSupportsCoherency &= inHwInfo->featureTable.ftrL3IACoherency;
|
||||
|
||||
PreemptionHelper::adjustDefaultPreemptionMode(outHwInfo->capabilityTable,
|
||||
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuMidThreadLevelPreempt),
|
||||
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuThreadGroupLevelPreempt),
|
||||
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuMidBatchPreempt));
|
||||
outHwInfo->capabilityTable.requiredPreemptionSurfaceSize = outHwInfo->gtSystemInfo.CsrSizeInMb * MemoryConstants::megaByte;
|
||||
|
||||
outHwInfo->capabilityTable.instrumentationEnabled =
|
||||
(outHwInfo->capabilityTable.instrumentationEnabled && haveInstrumentation);
|
||||
|
||||
auto &kmdNotifyProperties = outHwInfo->capabilityTable.kmdNotifyProperties;
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableKmdNotify.get(), kmdNotifyProperties.enableKmdNotify);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideKmdNotifyDelayMicroseconds.get(), kmdNotifyProperties.delayKmdNotifyMicroseconds);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableQuickKmdSleep.get(), kmdNotifyProperties.enableQuickKmdSleep);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideQuickKmdSleepDelayMicroseconds.get(), kmdNotifyProperties.delayQuickKmdSleepMicroseconds);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableQuickKmdSleepForSporadicWaits.get(), kmdNotifyProperties.enableQuickKmdSleepForSporadicWaits);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideDelayQuickKmdSleepForSporadicWaitsMicroseconds.get(), kmdNotifyProperties.delayQuickKmdSleepForSporadicWaitsMicroseconds);
|
||||
|
||||
// Product specific config
|
||||
int ret = configureHardwareCustom(outHwInfo, osIface);
|
||||
if (ret != 0) {
|
||||
outHwInfo = {};
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,51 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/os_context_win.h"
|
||||
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm_interface.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
OsContext *OsContext::create(OSInterface *osInterface, uint32_t contextId, DeviceBitfield deviceBitfield,
|
||||
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority) {
|
||||
if (osInterface) {
|
||||
return new OsContextWin(*osInterface->get()->getWddm(), contextId, deviceBitfield, engineType, preemptionMode, lowPriority);
|
||||
}
|
||||
return new OsContext(contextId, deviceBitfield, engineType, preemptionMode, lowPriority);
|
||||
}
|
||||
|
||||
OsContextWin::OsContextWin(Wddm &wddm, uint32_t contextId, DeviceBitfield deviceBitfield,
|
||||
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority)
|
||||
: OsContext(contextId, deviceBitfield, engineType, preemptionMode, lowPriority), wddm(wddm), residencyController(wddm, contextId) {
|
||||
|
||||
auto wddmInterface = wddm.getWddmInterface();
|
||||
if (!wddm.createContext(*this)) {
|
||||
return;
|
||||
}
|
||||
if (wddmInterface->hwQueuesSupported()) {
|
||||
if (!wddmInterface->createHwQueue(*this)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
initialized = wddmInterface->createMonitoredFence(*this);
|
||||
residencyController.registerCallback();
|
||||
};
|
||||
|
||||
bool OsContextWin::isInitialized() const {
|
||||
return (initialized && residencyController.isInitialized());
|
||||
}
|
||||
|
||||
OsContextWin::~OsContextWin() {
|
||||
wddm.getWddmInterface()->destroyHwQueue(hardwareQueue.handle);
|
||||
wddm.getWddmInterface()->destroyMonitorFence(residencyController.getMonitoredFence());
|
||||
wddm.destroyContext(wddmContextHandle);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,46 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/os_interface/os_context.h"
|
||||
#include "runtime/os_interface/windows/wddm_residency_controller.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class Wddm;
|
||||
|
||||
struct HardwareQueue {
|
||||
D3DKMT_HANDLE handle = 0;
|
||||
D3DKMT_HANDLE progressFenceHandle = 0;
|
||||
VOID *progressFenceCpuVA = nullptr;
|
||||
D3DGPU_VIRTUAL_ADDRESS progressFenceGpuVA = 0;
|
||||
};
|
||||
|
||||
class OsContextWin : public OsContext {
|
||||
public:
|
||||
OsContextWin() = delete;
|
||||
~OsContextWin() override;
|
||||
|
||||
OsContextWin(Wddm &wddm, uint32_t contextId, DeviceBitfield deviceBitfield,
|
||||
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority);
|
||||
|
||||
D3DKMT_HANDLE getWddmContextHandle() const { return wddmContextHandle; }
|
||||
void setWddmContextHandle(D3DKMT_HANDLE wddmContextHandle) { this->wddmContextHandle = wddmContextHandle; }
|
||||
HardwareQueue getHwQueue() const { return hardwareQueue; }
|
||||
void setHwQueue(HardwareQueue hardwareQueue) { this->hardwareQueue = hardwareQueue; }
|
||||
bool isInitialized() const override;
|
||||
Wddm *getWddm() const { return &wddm; }
|
||||
MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController() { return residencyController; }
|
||||
|
||||
protected:
|
||||
bool initialized = false;
|
||||
D3DKMT_HANDLE wddmContextHandle = 0;
|
||||
HardwareQueue hardwareQueue;
|
||||
Wddm &wddm;
|
||||
WddmResidencyController residencyController;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -1,74 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface.h"
|
||||
|
||||
#include "core/os_interface/windows/sys_calls.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
bool OSInterface::osEnabled64kbPages = true;
|
||||
|
||||
OSInterface::OSInterface() {
|
||||
osInterfaceImpl = new OSInterfaceImpl();
|
||||
}
|
||||
|
||||
OSInterface::~OSInterface() {
|
||||
delete osInterfaceImpl;
|
||||
}
|
||||
|
||||
uint32_t OSInterface::getDeviceHandle() const {
|
||||
return static_cast<uint32_t>(osInterfaceImpl->getDeviceHandle());
|
||||
}
|
||||
|
||||
void OSInterface::setGmmInputArgs(void *args) {
|
||||
this->get()->getWddm()->setGmmInputArg(args);
|
||||
}
|
||||
|
||||
OSInterface::OSInterfaceImpl::OSInterfaceImpl() = default;
|
||||
|
||||
D3DKMT_HANDLE OSInterface::OSInterfaceImpl::getAdapterHandle() const {
|
||||
return wddm->getAdapter();
|
||||
}
|
||||
|
||||
D3DKMT_HANDLE OSInterface::OSInterfaceImpl::getDeviceHandle() const {
|
||||
return wddm->getDevice();
|
||||
}
|
||||
|
||||
PFND3DKMT_ESCAPE OSInterface::OSInterfaceImpl::getEscapeHandle() const {
|
||||
return wddm->getEscapeHandle();
|
||||
}
|
||||
|
||||
uint32_t OSInterface::OSInterfaceImpl::getHwContextId() const {
|
||||
if (wddm == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return wddm->getHwContextId();
|
||||
}
|
||||
|
||||
bool OSInterface::are64kbPagesEnabled() {
|
||||
return osEnabled64kbPages;
|
||||
}
|
||||
|
||||
Wddm *OSInterface::OSInterfaceImpl::getWddm() const {
|
||||
return wddm.get();
|
||||
}
|
||||
|
||||
void OSInterface::OSInterfaceImpl::setWddm(Wddm *wddm) {
|
||||
this->wddm.reset(wddm);
|
||||
}
|
||||
|
||||
HANDLE OSInterface::OSInterfaceImpl::createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState,
|
||||
LPCSTR lpName) {
|
||||
return SysCalls::createEvent(lpEventAttributes, bManualReset, bInitialState, lpName);
|
||||
}
|
||||
|
||||
BOOL OSInterface::OSInterfaceImpl::closeHandle(HANDLE hObject) {
|
||||
return SysCalls::closeHandle(hObject);
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/os_interface/os_interface.h"
|
||||
#include "core/os_interface/windows/windows_wrapper.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
#include "profileapi.h"
|
||||
#include "umKmInc/sharedata.h"
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace NEO {
|
||||
class Wddm;
|
||||
class WddmMemoryOperationsHandler;
|
||||
|
||||
class OSInterface::OSInterfaceImpl {
|
||||
public:
|
||||
OSInterfaceImpl();
|
||||
virtual ~OSInterfaceImpl() = default;
|
||||
Wddm *getWddm() const;
|
||||
void setWddm(Wddm *wddm);
|
||||
D3DKMT_HANDLE getAdapterHandle() const;
|
||||
D3DKMT_HANDLE getDeviceHandle() const;
|
||||
PFND3DKMT_ESCAPE getEscapeHandle() const;
|
||||
uint32_t getHwContextId() const;
|
||||
|
||||
MOCKABLE_VIRTUAL HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState,
|
||||
LPCSTR lpName);
|
||||
MOCKABLE_VIRTUAL BOOL closeHandle(HANDLE hObject);
|
||||
|
||||
protected:
|
||||
std::unique_ptr<Wddm> wddm;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -1,112 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/os_time_win.h"
|
||||
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#undef WIN32_NO_STATUS
|
||||
|
||||
namespace NEO {
|
||||
|
||||
bool runEscape(Wddm *wddm, TimeStampDataHeader &escapeInfo) {
|
||||
if (wddm) {
|
||||
D3DKMT_ESCAPE escapeCommand = {0};
|
||||
|
||||
GTDIGetGpuCpuTimestampsIn in = {GTDI_FNC_GET_GPU_CPU_TIMESTAMPS};
|
||||
uint32_t outSize = sizeof(GTDIGetGpuCpuTimestampsOut);
|
||||
|
||||
escapeInfo.m_Header.EscapeCode = GFX_ESCAPE_IGPA_INSTRUMENTATION_CONTROL;
|
||||
escapeInfo.m_Header.Size = outSize;
|
||||
escapeInfo.m_Data.m_In = in;
|
||||
|
||||
escapeCommand.Flags.Value = 0;
|
||||
escapeCommand.hAdapter = (D3DKMT_HANDLE)0;
|
||||
escapeCommand.hContext = (D3DKMT_HANDLE)0; // escape is not context specific
|
||||
escapeCommand.hDevice = (D3DKMT_HANDLE)wddm->getDevice(); // escape not device specific, passing only for instrumentation
|
||||
escapeCommand.pPrivateDriverData = &escapeInfo;
|
||||
escapeCommand.PrivateDriverDataSize = sizeof(escapeInfo);
|
||||
escapeCommand.Type = D3DKMT_ESCAPE_DRIVERPRIVATE;
|
||||
|
||||
auto status = wddm->escape(escapeCommand);
|
||||
|
||||
if (status == STATUS_SUCCESS) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool OSTimeWin::getCpuGpuTime(TimeStampData *pGpuCpuTime) {
|
||||
bool retVal = false;
|
||||
|
||||
pGpuCpuTime->CPUTimeinNS = 0;
|
||||
pGpuCpuTime->GPUTimeStamp = 0;
|
||||
|
||||
TimeStampDataHeader escapeInfo = {0};
|
||||
|
||||
if (runEscape(wddm, escapeInfo)) {
|
||||
double cpuNanoseconds = escapeInfo.m_Data.m_Out.cpuPerfTicks *
|
||||
(1000000000.0 / escapeInfo.m_Data.m_Out.cpuPerfFreq);
|
||||
|
||||
pGpuCpuTime->CPUTimeinNS = (unsigned long long)cpuNanoseconds;
|
||||
pGpuCpuTime->GPUTimeStamp = (unsigned long long)escapeInfo.m_Data.m_Out.gpuPerfTicks;
|
||||
retVal = true;
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
bool OSTimeWin::getCpuTime(uint64_t *timeStamp) {
|
||||
uint64_t time;
|
||||
this->QueryPerfomanceCounterFnc((LARGE_INTEGER *)&time);
|
||||
|
||||
*timeStamp = static_cast<uint64_t>((static_cast<double>(time) * NSEC_PER_SEC / frequency.QuadPart));
|
||||
return true;
|
||||
};
|
||||
|
||||
std::unique_ptr<OSTime> OSTime::create(OSInterface *osInterface) {
|
||||
return std::unique_ptr<OSTime>(new OSTimeWin(osInterface));
|
||||
}
|
||||
|
||||
OSTimeWin::OSTimeWin(OSInterface *osInterface) {
|
||||
this->osInterface = osInterface;
|
||||
if (osInterface) {
|
||||
wddm = osInterface->get()->getWddm();
|
||||
}
|
||||
QueryPerformanceFrequency(&frequency);
|
||||
}
|
||||
|
||||
double OSTimeWin::getHostTimerResolution() const {
|
||||
double retValue = 0;
|
||||
if (frequency.QuadPart) {
|
||||
retValue = 1e9 / frequency.QuadPart;
|
||||
}
|
||||
return retValue;
|
||||
}
|
||||
|
||||
double OSTimeWin::getDynamicDeviceTimerResolution(HardwareInfo const &hwInfo) const {
|
||||
double retVal = 0;
|
||||
TimeStampDataHeader escapeInfo = {0};
|
||||
|
||||
if (runEscape(wddm, escapeInfo)) {
|
||||
retVal = 1000000000.0 / (double)escapeInfo.m_Data.m_Out.gpuPerfFreq;
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
uint64_t OSTimeWin::getCpuRawTimestamp() {
|
||||
LARGE_INTEGER cpuRawTimestamp = {};
|
||||
this->QueryPerfomanceCounterFnc(&cpuRawTimestamp);
|
||||
return cpuRawTimestamp.QuadPart;
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -1,75 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/os_interface/os_time.h"
|
||||
#include "core/os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include "gfxEscape.h"
|
||||
|
||||
namespace NEO {
|
||||
class Wddm;
|
||||
|
||||
class OSTimeWin : public OSTime {
|
||||
|
||||
public:
|
||||
OSTimeWin(OSInterface *osInterface);
|
||||
bool getCpuTime(uint64_t *timeStamp) override;
|
||||
bool getCpuGpuTime(TimeStampData *pGpuCpuTime) override;
|
||||
double getHostTimerResolution() const override;
|
||||
double getDynamicDeviceTimerResolution(HardwareInfo const &hwInfo) const override;
|
||||
uint64_t getCpuRawTimestamp() override;
|
||||
|
||||
protected:
|
||||
Wddm *wddm = nullptr;
|
||||
LARGE_INTEGER frequency;
|
||||
OSTimeWin() = default;
|
||||
decltype(&QueryPerformanceCounter) QueryPerfomanceCounterFnc = QueryPerformanceCounter;
|
||||
};
|
||||
|
||||
typedef enum GTDI_ESCAPE_FUNCTION_ENUM {
|
||||
GTDI_FNC_GET_GPU_CPU_TIMESTAMPS = 25
|
||||
} GTDI_ESCAPE_FUNCTION;
|
||||
|
||||
typedef struct GTDIBaseInStruct {
|
||||
GTDI_ESCAPE_FUNCTION Function;
|
||||
} GTDIHeaderIn;
|
||||
|
||||
typedef GTDIHeaderIn GTDIGetGpuCpuTimestampsIn;
|
||||
|
||||
typedef enum GTDI_RETURN_CODE_ENUM {
|
||||
GTDI_RET_OK = 0,
|
||||
GTDI_RET_FAILED,
|
||||
GTDI_RET_NOT_CONNECTED,
|
||||
GTDI_RET_HW_METRICS_NOT_ENABLED,
|
||||
GTDI_RET_CONTEXT_ID_MISMATCH,
|
||||
GTDI_RET_NOT_SUPPORTED,
|
||||
GTDI_RET_PENDING,
|
||||
GTDI_RET_INVALID_CONFIGURATION,
|
||||
GTDI_RET_CONCURRENT_API_ENABLED,
|
||||
GTDI_RET_NO_INFORMATION, // for GTDI_FNC_GET_ERROR_INFO escape only
|
||||
// ...
|
||||
GTDI_RET_MAX = 0xFFFFFFFF
|
||||
} GTDI_RETURN_CODE;
|
||||
|
||||
typedef struct GTDIGetGpuCpuTimestampsOutStruct {
|
||||
GTDI_RETURN_CODE RetCode; // Result of the call
|
||||
uint64_t gpuPerfTicks; // in GPU_timestamp_ticks
|
||||
uint64_t cpuPerfTicks; // in CPU_timestamp_ticks
|
||||
uint64_t gpuPerfFreq; // in GPU_timestamp_ticks/s
|
||||
uint64_t cpuPerfFreq; // in CPU_timestamp_ticks/s
|
||||
} GTDIGetGpuCpuTimestampsOut;
|
||||
|
||||
struct TimeStampDataHeader {
|
||||
GFX_ESCAPE_HEADER_T m_Header;
|
||||
union {
|
||||
GTDIGetGpuCpuTimestampsIn m_In;
|
||||
GTDIGetGpuCpuTimestampsOut m_Out;
|
||||
} m_Data;
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
@@ -8,9 +8,9 @@
|
||||
#include "performance_counters_win.h"
|
||||
|
||||
#include "core/helpers/hw_helper.h"
|
||||
#include "core/os_interface/windows/os_interface.h"
|
||||
#include "core/os_interface/windows/os_time_win.h"
|
||||
#include "runtime/device/device.h"
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/os_time_win.h"
|
||||
|
||||
namespace NEO {
|
||||
/////////////////////////////////////////////////////
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,208 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/command_stream/preemption_mode.h"
|
||||
#include "core/gmm_helper/gmm_lib.h"
|
||||
#include "core/helpers/debug_helpers.h"
|
||||
#include "core/memory_manager/gfx_partition.h"
|
||||
#include "core/os_interface/os_context.h"
|
||||
#include "core/utilities/spinlock.h"
|
||||
|
||||
#include "sku_info.h"
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace NEO {
|
||||
class Gdi;
|
||||
class Gmm;
|
||||
class GmmMemory;
|
||||
class OsContextWin;
|
||||
class SettingsReader;
|
||||
class WddmAllocation;
|
||||
class WddmInterface;
|
||||
class WddmResidencyController;
|
||||
class WddmResidentAllocationsContainer;
|
||||
|
||||
struct AllocationStorageData;
|
||||
struct HardwareInfo;
|
||||
struct KmDafListener;
|
||||
struct RootDeviceEnvironment;
|
||||
struct MonitoredFence;
|
||||
struct OsHandleStorage;
|
||||
|
||||
enum class HeapIndex : uint32_t;
|
||||
|
||||
struct WddmSubmitArguments {
|
||||
MonitoredFence *monitorFence;
|
||||
D3DKMT_HANDLE contextHandle;
|
||||
D3DKMT_HANDLE hwQueueHandle;
|
||||
};
|
||||
|
||||
enum class WddmVersion : uint32_t {
|
||||
WDDM_2_0 = 0,
|
||||
WDDM_2_3
|
||||
};
|
||||
|
||||
class Wddm {
|
||||
public:
|
||||
typedef HRESULT(WINAPI *CreateDXGIFactoryFcn)(REFIID riid, void **ppFactory);
|
||||
typedef void(WINAPI *GetSystemInfoFcn)(SYSTEM_INFO *pSystemInfo);
|
||||
typedef BOOL(WINAPI *VirtualFreeFcn)(LPVOID ptr, SIZE_T size, DWORD flags);
|
||||
typedef LPVOID(WINAPI *VirtualAllocFcn)(LPVOID inPtr, SIZE_T size, DWORD flags, DWORD type);
|
||||
|
||||
virtual ~Wddm();
|
||||
|
||||
static Wddm *createWddm(RootDeviceEnvironment &rootDeviceEnvironment);
|
||||
bool init(HardwareInfo &outHardwareInfo);
|
||||
|
||||
MOCKABLE_VIRTUAL bool evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim);
|
||||
MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim);
|
||||
MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr);
|
||||
bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData);
|
||||
MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size);
|
||||
MOCKABLE_VIRTUAL bool createContext(OsContextWin &osContext);
|
||||
MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext);
|
||||
MOCKABLE_VIRTUAL bool freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
|
||||
MOCKABLE_VIRTUAL NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, uint32_t shareable);
|
||||
MOCKABLE_VIRTUAL bool createAllocation64k(const Gmm *gmm, D3DKMT_HANDLE &outHandle);
|
||||
MOCKABLE_VIRTUAL NTSTATUS createAllocationsAndMapGpuVa(OsHandleStorage &osHandles);
|
||||
MOCKABLE_VIRTUAL bool destroyAllocations(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
|
||||
MOCKABLE_VIRTUAL bool openSharedHandle(D3DKMT_HANDLE handle, WddmAllocation *alloc);
|
||||
bool openNTHandle(HANDLE handle, WddmAllocation *alloc);
|
||||
MOCKABLE_VIRTUAL void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock);
|
||||
MOCKABLE_VIRTUAL void unlockResource(const D3DKMT_HANDLE &handle);
|
||||
MOCKABLE_VIRTUAL void kmDafLock(D3DKMT_HANDLE handle);
|
||||
MOCKABLE_VIRTUAL bool isKmDafEnabled() const { return featureTable->ftrKmdDaf; }
|
||||
|
||||
MOCKABLE_VIRTUAL bool destroyContext(D3DKMT_HANDLE context);
|
||||
MOCKABLE_VIRTUAL bool queryAdapterInfo();
|
||||
|
||||
MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments);
|
||||
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence);
|
||||
|
||||
NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand);
|
||||
MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController);
|
||||
void unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle);
|
||||
MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress);
|
||||
MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem);
|
||||
|
||||
MOCKABLE_VIRTUAL void *virtualAlloc(void *inPtr, size_t size, unsigned long flags, unsigned long type);
|
||||
MOCKABLE_VIRTUAL int virtualFree(void *ptr, size_t size, unsigned long flags);
|
||||
|
||||
bool configureDeviceAddressSpace();
|
||||
|
||||
GT_SYSTEM_INFO *getGtSysInfo() const {
|
||||
DEBUG_BREAK_IF(!gtSystemInfo);
|
||||
return gtSystemInfo.get();
|
||||
}
|
||||
|
||||
const GMM_GFX_PARTITIONING &getGfxPartition() const {
|
||||
return gfxPartition;
|
||||
}
|
||||
|
||||
void initGfxPartition(GfxPartition &outGfxPartition, uint32_t rootDeviceIndex, size_t numRootDevices) const;
|
||||
|
||||
const std::string &getDeviceRegistryPath() const {
|
||||
return deviceRegistryPath;
|
||||
}
|
||||
|
||||
uint64_t getSystemSharedMemory() const;
|
||||
uint64_t getDedicatedVideoMemory() const;
|
||||
|
||||
uint64_t getMaxApplicationAddress() const;
|
||||
|
||||
D3DKMT_HANDLE getAdapter() const { return adapter; }
|
||||
D3DKMT_HANDLE getDevice() const { return device; }
|
||||
D3DKMT_HANDLE getPagingQueue() const { return pagingQueue; }
|
||||
D3DKMT_HANDLE getPagingQueueSyncObject() const { return pagingQueueSyncObject; }
|
||||
Gdi *getGdi() const { return gdi.get(); }
|
||||
|
||||
PFND3DKMT_ESCAPE getEscapeHandle() const;
|
||||
|
||||
uint32_t getHwContextId() const {
|
||||
return static_cast<uint32_t>(hwContextId);
|
||||
}
|
||||
|
||||
std::unique_ptr<SettingsReader> registryReader;
|
||||
|
||||
uintptr_t getWddmMinAddress() const {
|
||||
return this->minAddress;
|
||||
}
|
||||
WddmInterface *getWddmInterface() const {
|
||||
return wddmInterface.get();
|
||||
}
|
||||
|
||||
unsigned int readEnablePreemptionRegKey();
|
||||
MOCKABLE_VIRTUAL uint64_t *getPagingFenceAddress() {
|
||||
return pagingFenceAddress;
|
||||
}
|
||||
WddmResidentAllocationsContainer *getTemporaryResourcesContainer() {
|
||||
return temporaryResources.get();
|
||||
}
|
||||
void updatePagingFenceValue(uint64_t newPagingFenceValue);
|
||||
GmmMemory *getGmmMemory() const {
|
||||
return gmmMemory.get();
|
||||
}
|
||||
void waitOnPagingFenceFromCpu();
|
||||
|
||||
void setGmmInputArg(void *args);
|
||||
|
||||
WddmVersion getWddmVersion();
|
||||
|
||||
protected:
|
||||
std::unique_ptr<Gdi> gdi;
|
||||
D3DKMT_HANDLE adapter = 0;
|
||||
D3DKMT_HANDLE device = 0;
|
||||
D3DKMT_HANDLE pagingQueue = 0;
|
||||
D3DKMT_HANDLE pagingQueueSyncObject = 0;
|
||||
|
||||
uint64_t *pagingFenceAddress = nullptr;
|
||||
std::atomic<std::uint64_t> currentPagingFenceValue{0};
|
||||
|
||||
// Adapter information
|
||||
std::unique_ptr<PLATFORM> gfxPlatform;
|
||||
std::unique_ptr<GT_SYSTEM_INFO> gtSystemInfo;
|
||||
std::unique_ptr<FeatureTable> featureTable;
|
||||
std::unique_ptr<WorkaroundTable> workaroundTable;
|
||||
GMM_GFX_PARTITIONING gfxPartition;
|
||||
ADAPTER_BDF adapterBDF;
|
||||
uint64_t systemSharedMemory = 0;
|
||||
uint64_t dedicatedVideoMemory = 0;
|
||||
uint32_t maxRenderFrequency = 0;
|
||||
bool instrumentationEnabled = false;
|
||||
std::string deviceRegistryPath;
|
||||
RootDeviceEnvironment &rootDeviceEnvironment;
|
||||
|
||||
unsigned long hwContextId = 0;
|
||||
LUID adapterLuid;
|
||||
uintptr_t maximumApplicationAddress = 0;
|
||||
std::unique_ptr<GmmMemory> gmmMemory;
|
||||
uintptr_t minAddress = 0;
|
||||
|
||||
Wddm(RootDeviceEnvironment &rootDeviceEnvironment);
|
||||
MOCKABLE_VIRTUAL bool openAdapter();
|
||||
MOCKABLE_VIRTUAL bool waitOnGPU(D3DKMT_HANDLE context);
|
||||
bool createDevice(PreemptionMode preemptionMode);
|
||||
bool createPagingQueue();
|
||||
bool destroyPagingQueue();
|
||||
bool destroyDevice();
|
||||
bool closeAdapter();
|
||||
void getDeviceState();
|
||||
void handleCompletion(OsContextWin &osContext);
|
||||
|
||||
static CreateDXGIFactoryFcn createDxgiFactory;
|
||||
static GetSystemInfoFcn getSystemInfo;
|
||||
static VirtualFreeFcn virtualFreeFnc;
|
||||
static VirtualAllocFcn virtualAllocFnc;
|
||||
|
||||
std::unique_ptr<KmDafListener> kmDafListener;
|
||||
std::unique_ptr<WddmInterface> wddmInterface;
|
||||
std::unique_ptr<WddmResidentAllocationsContainer> temporaryResources;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -1,28 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
#include <dxgi.h>
|
||||
|
||||
namespace NEO {
|
||||
Wddm::CreateDXGIFactoryFcn getCreateDxgiFactory() {
|
||||
return CreateDXGIFactory;
|
||||
}
|
||||
|
||||
Wddm::GetSystemInfoFcn getGetSystemInfo() {
|
||||
return GetSystemInfo;
|
||||
}
|
||||
|
||||
Wddm::VirtualFreeFcn getVirtualFree() {
|
||||
return VirtualFree;
|
||||
}
|
||||
|
||||
Wddm::VirtualAllocFcn getVirtualAlloc() {
|
||||
return VirtualAlloc;
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -1,14 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
Wddm *Wddm::createWddm(RootDeviceEnvironment &rootDeviceEnvironment) {
|
||||
return new Wddm(rootDeviceEnvironment);
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -1,147 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm/wddm_interface.h"
|
||||
|
||||
#include "core/memory_manager/memory_constants.h"
|
||||
#include "core/os_interface/windows/gdi_interface.h"
|
||||
#include "runtime/os_interface/windows/os_context_win.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
using namespace NEO;
|
||||
|
||||
bool WddmInterface::createMonitoredFence(MonitoredFence &monitorFence) {
|
||||
NTSTATUS status = STATUS_SUCCESS;
|
||||
D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject = {0};
|
||||
CreateSynchronizationObject.hDevice = wddm.getDevice();
|
||||
CreateSynchronizationObject.Info.Type = D3DDDI_MONITORED_FENCE;
|
||||
CreateSynchronizationObject.Info.MonitoredFence.InitialFenceValue = 0;
|
||||
|
||||
status = wddm.getGdi()->createSynchronizationObject2(&CreateSynchronizationObject);
|
||||
DEBUG_BREAK_IF(STATUS_SUCCESS != status);
|
||||
|
||||
monitorFence.fenceHandle = CreateSynchronizationObject.hSyncObject;
|
||||
monitorFence.cpuAddress = reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress);
|
||||
monitorFence.gpuAddress = CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress;
|
||||
|
||||
return status == STATUS_SUCCESS;
|
||||
}
|
||||
void WddmInterface::destroyMonitorFence(D3DKMT_HANDLE fenceHandle) {
|
||||
NTSTATUS status = STATUS_SUCCESS;
|
||||
D3DKMT_DESTROYSYNCHRONIZATIONOBJECT destroySyncObject = {0};
|
||||
destroySyncObject.hSyncObject = fenceHandle;
|
||||
status = wddm.getGdi()->destroySynchronizationObject(&destroySyncObject);
|
||||
DEBUG_BREAK_IF(STATUS_SUCCESS != status);
|
||||
}
|
||||
|
||||
bool WddmInterface20::createHwQueue(OsContextWin &osContext) {
|
||||
return false;
|
||||
}
|
||||
void WddmInterface20::destroyHwQueue(D3DKMT_HANDLE hwQueue) {}
|
||||
|
||||
bool WddmInterface20::createMonitoredFence(OsContextWin &osContext) {
|
||||
auto &residencyController = osContext.getResidencyController();
|
||||
MonitoredFence &monitorFence = residencyController.getMonitoredFence();
|
||||
bool ret = WddmInterface::createMonitoredFence(monitorFence);
|
||||
|
||||
monitorFence.currentFenceValue = 1;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void WddmInterface20::destroyMonitorFence(MonitoredFence &monitorFence) {
|
||||
WddmInterface::destroyMonitorFence(monitorFence.fenceHandle);
|
||||
}
|
||||
|
||||
const bool WddmInterface20::hwQueuesSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WddmInterface20::submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) {
|
||||
D3DKMT_SUBMITCOMMAND SubmitCommand = {0};
|
||||
NTSTATUS status = STATUS_SUCCESS;
|
||||
|
||||
SubmitCommand.Commands = commandBuffer;
|
||||
SubmitCommand.CommandLength = static_cast<UINT>(size);
|
||||
SubmitCommand.BroadcastContextCount = 1;
|
||||
SubmitCommand.BroadcastContext[0] = submitArguments.contextHandle;
|
||||
SubmitCommand.Flags.NullRendering = (UINT)DebugManager.flags.EnableNullHardware.get();
|
||||
|
||||
COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandHeader);
|
||||
|
||||
pHeader->MonitorFenceVA = submitArguments.monitorFence->gpuAddress;
|
||||
pHeader->MonitorFenceValue = submitArguments.monitorFence->currentFenceValue;
|
||||
|
||||
// Note: Private data should be the CPU VA Address
|
||||
SubmitCommand.pPrivateDriverData = commandHeader;
|
||||
SubmitCommand.PrivateDriverDataSize = sizeof(COMMAND_BUFFER_HEADER);
|
||||
|
||||
status = wddm.getGdi()->submitCommand(&SubmitCommand);
|
||||
|
||||
return STATUS_SUCCESS == status;
|
||||
}
|
||||
|
||||
bool WddmInterface23::createHwQueue(OsContextWin &osContext) {
|
||||
D3DKMT_CREATEHWQUEUE createHwQueue = {};
|
||||
|
||||
if (!wddm.getGdi()->setupHwQueueProcAddresses()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
createHwQueue.hHwContext = osContext.getWddmContextHandle();
|
||||
if (osContext.getPreemptionMode() >= PreemptionMode::MidBatch) {
|
||||
createHwQueue.Flags.DisableGpuTimeout = wddm.readEnablePreemptionRegKey();
|
||||
}
|
||||
|
||||
auto status = wddm.getGdi()->createHwQueue(&createHwQueue);
|
||||
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
|
||||
osContext.setHwQueue({createHwQueue.hHwQueue, createHwQueue.hHwQueueProgressFence, createHwQueue.HwQueueProgressFenceCPUVirtualAddress,
|
||||
createHwQueue.HwQueueProgressFenceGPUVirtualAddress});
|
||||
|
||||
return status == STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
bool WddmInterface23::createMonitoredFence(OsContextWin &osContext) {
|
||||
auto &residencyController = osContext.getResidencyController();
|
||||
auto hwQueue = osContext.getHwQueue();
|
||||
residencyController.resetMonitoredFenceParams(hwQueue.progressFenceHandle,
|
||||
reinterpret_cast<uint64_t *>(hwQueue.progressFenceCpuVA),
|
||||
hwQueue.progressFenceGpuVA);
|
||||
return true;
|
||||
}
|
||||
|
||||
void WddmInterface23::destroyMonitorFence(MonitoredFence &monitorFence) {
|
||||
}
|
||||
|
||||
void WddmInterface23::destroyHwQueue(D3DKMT_HANDLE hwQueue) {
|
||||
if (hwQueue) {
|
||||
D3DKMT_DESTROYHWQUEUE destroyHwQueue = {};
|
||||
destroyHwQueue.hHwQueue = hwQueue;
|
||||
|
||||
auto status = wddm.getGdi()->destroyHwQueue(&destroyHwQueue);
|
||||
DEBUG_BREAK_IF(status != STATUS_SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
const bool WddmInterface23::hwQueuesSupported() {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WddmInterface23::submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) {
|
||||
D3DKMT_SUBMITCOMMANDTOHWQUEUE submitCommand = {};
|
||||
submitCommand.hHwQueue = submitArguments.hwQueueHandle;
|
||||
submitCommand.HwQueueProgressFenceId = submitArguments.monitorFence->currentFenceValue;
|
||||
submitCommand.CommandBuffer = commandBuffer;
|
||||
submitCommand.CommandLength = static_cast<UINT>(size);
|
||||
|
||||
submitCommand.pPrivateDriverData = commandHeader;
|
||||
submitCommand.PrivateDriverDataSize = MemoryConstants::pageSize;
|
||||
|
||||
auto status = wddm.getGdi()->submitCommandToHwQueue(&submitCommand);
|
||||
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
|
||||
return status == STATUS_SUCCESS;
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/helpers/hw_info.h"
|
||||
#include "core/os_interface/os_context.h"
|
||||
#include "core/os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace NEO {
|
||||
class Gdi;
|
||||
class Wddm;
|
||||
class OsContextWin;
|
||||
class WddmResidencyController;
|
||||
struct MonitoredFence;
|
||||
struct WddmSubmitArguments;
|
||||
|
||||
class WddmInterface {
|
||||
public:
|
||||
WddmInterface(Wddm &wddm) : wddm(wddm){};
|
||||
virtual ~WddmInterface() = default;
|
||||
WddmInterface() = delete;
|
||||
virtual bool createHwQueue(OsContextWin &osContext) = 0;
|
||||
virtual void destroyHwQueue(D3DKMT_HANDLE hwQueue) = 0;
|
||||
virtual bool createMonitoredFence(OsContextWin &osContext) = 0;
|
||||
MOCKABLE_VIRTUAL bool createMonitoredFence(MonitoredFence &monitorFence);
|
||||
void destroyMonitorFence(D3DKMT_HANDLE fenceHandle);
|
||||
virtual void destroyMonitorFence(MonitoredFence &monitorFence) = 0;
|
||||
virtual const bool hwQueuesSupported() = 0;
|
||||
virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) = 0;
|
||||
Wddm &wddm;
|
||||
};
|
||||
|
||||
class WddmInterface20 : public WddmInterface {
|
||||
public:
|
||||
using WddmInterface::WddmInterface;
|
||||
bool createHwQueue(OsContextWin &osContext) override;
|
||||
void destroyHwQueue(D3DKMT_HANDLE hwQueue) override;
|
||||
bool createMonitoredFence(OsContextWin &osContext) override;
|
||||
void destroyMonitorFence(MonitoredFence &monitorFence) override;
|
||||
const bool hwQueuesSupported() override;
|
||||
bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override;
|
||||
};
|
||||
|
||||
class WddmInterface23 : public WddmInterface {
|
||||
public:
|
||||
using WddmInterface::WddmInterface;
|
||||
bool createHwQueue(OsContextWin &osContext) override;
|
||||
void destroyHwQueue(D3DKMT_HANDLE hwQueue) override;
|
||||
bool createMonitoredFence(OsContextWin &osContext) override;
|
||||
void destroyMonitorFence(MonitoredFence &monitorFence) override;
|
||||
const bool hwQueuesSupported() override;
|
||||
bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -1,13 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
void Wddm::applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext) {
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -16,15 +16,15 @@
|
||||
#include "core/helpers/hw_cmds.h"
|
||||
#include "core/helpers/ptr_math.h"
|
||||
#include "core/helpers/windows/gmm_callbacks.h"
|
||||
#include "core/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/device/device.h"
|
||||
#include "runtime/mem_obj/mem_obj.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/os_interface/windows/wddm_device_command_stream.h"
|
||||
#pragma warning(pop)
|
||||
|
||||
#include "core/os_interface/windows/gdi_interface.h"
|
||||
#include "runtime/os_interface/windows/os_context_win.h"
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "core/os_interface/windows/os_context_win.h"
|
||||
#include "core/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm_memory_manager.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
@@ -20,15 +20,15 @@
|
||||
#include "core/memory_manager/host_ptr_manager.h"
|
||||
#include "core/memory_manager/memory_operations_handler.h"
|
||||
#include "core/os_interface/os_interface.h"
|
||||
#include "core/os_interface/windows/os_context_win.h"
|
||||
#include "core/os_interface/windows/os_interface.h"
|
||||
#include "core/os_interface/windows/wddm/wddm.h"
|
||||
#include "core/os_interface/windows/wddm_allocation.h"
|
||||
#include "core/os_interface/windows/wddm_residency_allocations_container.h"
|
||||
#include "core/os_interface/windows/wddm_residency_controller.h"
|
||||
#include "runtime/command_stream/command_stream_receiver_hw.h"
|
||||
#include "runtime/device/device.h"
|
||||
#include "runtime/helpers/surface_formats.h"
|
||||
#include "runtime/os_interface/windows/os_context_win.h"
|
||||
#include "runtime/os_interface/windows/os_interface.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/os_interface/windows/wddm_residency_allocations_container.h"
|
||||
#include "runtime/os_interface/windows/wddm_residency_controller.h"
|
||||
#include "runtime/platform/platform.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm_memory_operations_handler.h"
|
||||
|
||||
#include "core/memory_manager/host_ptr_defines.h"
|
||||
#include "core/os_interface/windows/wddm_allocation.h"
|
||||
#include "core/utilities/stackvec.h"
|
||||
#include "runtime/os_interface/windows/wddm_residency_allocations_container.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
WddmMemoryOperationsHandler::WddmMemoryOperationsHandler(Wddm *wddm) : wddm(wddm) {
|
||||
residentAllocations = std::make_unique<WddmResidentAllocationsContainer>(wddm);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRef<GraphicsAllocation *> gfxAllocations) {
|
||||
uint32_t totalHandlesCount = 0;
|
||||
constexpr uint32_t stackAllocations = 64;
|
||||
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount * stackAllocations;
|
||||
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForResidency;
|
||||
|
||||
for (const auto &allocation : gfxAllocations) {
|
||||
WddmAllocation *wddmAllocation = reinterpret_cast<WddmAllocation *>(allocation);
|
||||
|
||||
if (wddmAllocation->fragmentsStorage.fragmentCount > 0) {
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
handlesForResidency[totalHandlesCount++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
|
||||
}
|
||||
} else {
|
||||
memcpy_s(&handlesForResidency[totalHandlesCount],
|
||||
wddmAllocation->getNumHandles() * sizeof(D3DKMT_HANDLE),
|
||||
wddmAllocation->getHandles().data(),
|
||||
wddmAllocation->getNumHandles() * sizeof(D3DKMT_HANDLE));
|
||||
totalHandlesCount += wddmAllocation->getNumHandles();
|
||||
}
|
||||
}
|
||||
return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmMemoryOperationsHandler::evict(GraphicsAllocation &gfxAllocation) {
|
||||
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount;
|
||||
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForEviction;
|
||||
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
|
||||
uint32_t totalHandleCount = 0;
|
||||
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {
|
||||
OsHandleStorage &fragmentStorage = wddmAllocation.fragmentsStorage;
|
||||
|
||||
for (uint32_t allocId = 0; allocId < fragmentStorage.fragmentCount; allocId++) {
|
||||
handlesForEviction.push_back(fragmentStorage.fragmentStorageData[allocId].osHandleStorage->handle);
|
||||
totalHandleCount++;
|
||||
}
|
||||
} else {
|
||||
const D3DKMT_HANDLE *handlePtr = wddmAllocation.getHandles().data();
|
||||
size_t handleCount = wddmAllocation.getNumHandles();
|
||||
for (uint32_t i = 0; i < handleCount; i++, totalHandleCount++) {
|
||||
handlesForEviction.push_back(*handlePtr);
|
||||
handlePtr++;
|
||||
}
|
||||
}
|
||||
return residentAllocations->evictResources(handlesForEviction.begin(), totalHandleCount);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(GraphicsAllocation &gfxAllocation) {
|
||||
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
|
||||
D3DKMT_HANDLE defaultHandle = 0u;
|
||||
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {
|
||||
defaultHandle = wddmAllocation.fragmentsStorage.fragmentStorageData[0].osHandleStorage->handle;
|
||||
} else {
|
||||
defaultHandle = wddmAllocation.getDefaultHandle();
|
||||
}
|
||||
return residentAllocations->isAllocationResident(defaultHandle);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,31 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/memory_manager/memory_operations_handler.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class Wddm;
|
||||
class WddmResidentAllocationsContainer;
|
||||
|
||||
class WddmMemoryOperationsHandler : public MemoryOperationsHandler {
|
||||
public:
|
||||
WddmMemoryOperationsHandler(Wddm *wddm);
|
||||
~WddmMemoryOperationsHandler() override = default;
|
||||
|
||||
MemoryOperationsStatus makeResident(ArrayRef<GraphicsAllocation *> gfxAllocations) override;
|
||||
MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override;
|
||||
MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override;
|
||||
|
||||
protected:
|
||||
Wddm *wddm;
|
||||
std::unique_ptr<WddmResidentAllocationsContainer> residentAllocations;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -1,95 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm_residency_allocations_container.h"
|
||||
|
||||
#include "core/os_interface/windows/wddm_allocation.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace NEO {
|
||||
WddmResidentAllocationsContainer::~WddmResidentAllocationsContainer() {
|
||||
evictAllResources();
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::isAllocationResident(const D3DKMT_HANDLE &handle) {
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handle);
|
||||
return position != resourceHandles.end() ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::MEMORY_NOT_FOUND;
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::evictAllResources() {
|
||||
decltype(resourceHandles) resourcesToEvict;
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
resourceHandles.swap(resourcesToEvict);
|
||||
if (resourcesToEvict.empty()) {
|
||||
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
|
||||
}
|
||||
uint64_t sizeToTrim = 0;
|
||||
uint32_t evictedResources = static_cast<uint32_t>(resourcesToEvict.size());
|
||||
bool success = wddm->evict(resourcesToEvict.data(), evictedResources, sizeToTrim);
|
||||
return success ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED;
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::evictResource(const D3DKMT_HANDLE &handle) {
|
||||
return evictResources(&handle, 1u);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::evictResources(const D3DKMT_HANDLE *handles, const uint32_t count) {
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handles[0]);
|
||||
if (position == resourceHandles.end()) {
|
||||
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
|
||||
}
|
||||
auto distance = static_cast<size_t>(std::distance(resourceHandles.begin(), position));
|
||||
UNRECOVERABLE_IF(distance + count > resourceHandles.size());
|
||||
resourceHandles.erase(position, position + count);
|
||||
uint64_t sizeToTrim = 0;
|
||||
if (!wddm->evict(handles, count, sizeToTrim)) {
|
||||
return MemoryOperationsStatus::FAILED;
|
||||
}
|
||||
return MemoryOperationsStatus::SUCCESS;
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResource(const D3DKMT_HANDLE &handle) {
|
||||
return makeResidentResources(&handle, 1u);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count) {
|
||||
bool madeResident = false;
|
||||
while (!(madeResident = wddm->makeResident(handles, count, false, nullptr))) {
|
||||
if (evictAllResources() == MemoryOperationsStatus::SUCCESS) {
|
||||
continue;
|
||||
}
|
||||
if (!wddm->makeResident(handles, count, false, nullptr)) {
|
||||
DEBUG_BREAK_IF(true);
|
||||
return MemoryOperationsStatus::OUT_OF_MEMORY;
|
||||
};
|
||||
break;
|
||||
}
|
||||
DEBUG_BREAK_IF(!madeResident);
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
resourceHandles.push_back(handles[i]);
|
||||
}
|
||||
lock.unlock();
|
||||
wddm->waitOnPagingFenceFromCpu();
|
||||
return madeResident ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED;
|
||||
}
|
||||
|
||||
void WddmResidentAllocationsContainer::removeResource(const D3DKMT_HANDLE &handle) {
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handle);
|
||||
if (position == resourceHandles.end()) {
|
||||
return;
|
||||
}
|
||||
*position = resourceHandles.back();
|
||||
resourceHandles.pop_back();
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "core/memory_manager/memory_operations_status.h"
|
||||
#include "core/os_interface/windows/windows_defs.h"
|
||||
#include "core/utilities/spinlock.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
namespace NEO {
|
||||
class Wddm;
|
||||
|
||||
class WddmResidentAllocationsContainer {
|
||||
public:
|
||||
WddmResidentAllocationsContainer(Wddm *wddm) : wddm(wddm) {}
|
||||
virtual ~WddmResidentAllocationsContainer();
|
||||
|
||||
MemoryOperationsStatus isAllocationResident(const D3DKMT_HANDLE &handle);
|
||||
MOCKABLE_VIRTUAL MemoryOperationsStatus evictAllResources();
|
||||
MOCKABLE_VIRTUAL MemoryOperationsStatus evictResource(const D3DKMT_HANDLE &handle);
|
||||
MemoryOperationsStatus evictResources(const D3DKMT_HANDLE *handles, const uint32_t count);
|
||||
MOCKABLE_VIRTUAL MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle);
|
||||
MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count);
|
||||
MOCKABLE_VIRTUAL void removeResource(const D3DKMT_HANDLE &handle);
|
||||
|
||||
protected:
|
||||
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock(SpinLock &lock) {
|
||||
return std::unique_lock<SpinLock>{lock};
|
||||
}
|
||||
|
||||
Wddm *wddm;
|
||||
std::vector<D3DKMT_HANDLE> resourceHandles;
|
||||
SpinLock resourcesLock;
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,392 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "runtime/os_interface/windows/wddm_residency_controller.h"
|
||||
|
||||
#include "core/debug_settings/debug_settings_manager.h"
|
||||
#include "core/os_interface/windows/wddm_allocation.h"
|
||||
#include "core/utilities/spinlock.h"
|
||||
#include "runtime/os_interface/windows/wddm/wddm.h"
|
||||
#include "runtime/os_interface/windows/wddm_residency_allocations_container.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
WddmResidencyController::WddmResidencyController(Wddm &wddm, uint32_t osContextId) : wddm(wddm), osContextId(osContextId) {
|
||||
}
|
||||
|
||||
void WddmResidencyController::registerCallback() {
|
||||
this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback, *this);
|
||||
}
|
||||
|
||||
WddmResidencyController::~WddmResidencyController() {
|
||||
auto lock = this->acquireTrimCallbackLock();
|
||||
wddm.unregisterTrimCallback(WddmResidencyController::trimCallback, this->trimCallbackHandle);
|
||||
lock.unlock();
|
||||
|
||||
// Wait for lock to ensure trimCallback ended
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) {
|
||||
auto residencyController = static_cast<WddmResidencyController *>(trimNotification->Context);
|
||||
DEBUG_BREAK_IF(residencyController == nullptr);
|
||||
|
||||
auto lock = residencyController->acquireTrimCallbackLock();
|
||||
residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
|
||||
}
|
||||
|
||||
std::unique_lock<SpinLock> WddmResidencyController::acquireLock() {
|
||||
return std::unique_lock<SpinLock>{this->lock};
|
||||
}
|
||||
|
||||
std::unique_lock<SpinLock> WddmResidencyController::acquireTrimCallbackLock() {
|
||||
return std::unique_lock<SpinLock>{this->trimCallbackLock};
|
||||
}
|
||||
|
||||
WddmAllocation *WddmResidencyController::getTrimCandidateHead() {
|
||||
uint32_t i = 0;
|
||||
const size_t size = trimCandidateList.size();
|
||||
|
||||
if (size == 0) {
|
||||
return nullptr;
|
||||
}
|
||||
while ((trimCandidateList[i] == nullptr) && (i < size))
|
||||
i++;
|
||||
|
||||
return static_cast<WddmAllocation *>(trimCandidateList[i]);
|
||||
}
|
||||
|
||||
void WddmResidencyController::addToTrimCandidateList(GraphicsAllocation *allocation) {
|
||||
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
|
||||
size_t position = trimCandidateList.size();
|
||||
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
|
||||
if (wddmAllocation->getTrimCandidateListPosition(this->osContextId) == trimListUnusedPosition) {
|
||||
trimCandidatesCount++;
|
||||
trimCandidateList.push_back(allocation);
|
||||
wddmAllocation->setTrimCandidateListPosition(this->osContextId, position);
|
||||
}
|
||||
|
||||
checkTrimCandidateCount();
|
||||
}
|
||||
|
||||
void WddmResidencyController::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) {
|
||||
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
|
||||
size_t position = wddmAllocation->getTrimCandidateListPosition(this->osContextId);
|
||||
|
||||
DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1)));
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
|
||||
trimCandidatesCount--;
|
||||
|
||||
trimCandidateList[position] = nullptr;
|
||||
|
||||
checkTrimCandidateCount();
|
||||
|
||||
if (position == trimCandidateList.size() - 1) {
|
||||
size_t erasePosition = position;
|
||||
|
||||
if (position == 0) {
|
||||
trimCandidateList.resize(0);
|
||||
} else {
|
||||
while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) {
|
||||
erasePosition--;
|
||||
}
|
||||
|
||||
size_t sizeRemaining = erasePosition + 1;
|
||||
if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) {
|
||||
sizeRemaining = 0;
|
||||
}
|
||||
|
||||
trimCandidateList.resize(sizeRemaining);
|
||||
}
|
||||
}
|
||||
wddmAllocation->setTrimCandidateListPosition(this->osContextId, trimListUnusedPosition);
|
||||
|
||||
if (compactList && checkTrimCandidateListCompaction()) {
|
||||
compactTrimCandidateList();
|
||||
}
|
||||
|
||||
checkTrimCandidateCount();
|
||||
}
|
||||
|
||||
void WddmResidencyController::removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList) {
|
||||
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
|
||||
this->removeFromTrimCandidateList(allocation, true);
|
||||
}
|
||||
}
|
||||
|
||||
void WddmResidencyController::checkTrimCandidateCount() {
|
||||
if (DebugManager.flags.ResidencyDebugEnable.get()) {
|
||||
uint32_t sum = 0;
|
||||
for (auto trimCandidate : trimCandidateList) {
|
||||
if (trimCandidate != nullptr) {
|
||||
sum++;
|
||||
}
|
||||
}
|
||||
DEBUG_BREAK_IF(sum != trimCandidatesCount);
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmResidencyController::checkTrimCandidateListCompaction() {
|
||||
return 2 * trimCandidatesCount <= trimCandidateList.size();
|
||||
}
|
||||
|
||||
void WddmResidencyController::compactTrimCandidateList() {
|
||||
size_t size = trimCandidateList.size();
|
||||
size_t freePosition = 0;
|
||||
|
||||
if (size == 0 || size == trimCandidatesCount) {
|
||||
return;
|
||||
}
|
||||
|
||||
DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr));
|
||||
|
||||
uint32_t previousCount = trimCandidatesCount;
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
|
||||
while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr)
|
||||
freePosition++;
|
||||
|
||||
for (uint32_t i = 1; i < size; i++) {
|
||||
|
||||
if (trimCandidateList[i] != nullptr && freePosition < i) {
|
||||
trimCandidateList[freePosition] = trimCandidateList[i];
|
||||
trimCandidateList[i] = nullptr;
|
||||
static_cast<WddmAllocation *>(trimCandidateList[freePosition])->setTrimCandidateListPosition(this->osContextId, freePosition);
|
||||
freePosition++;
|
||||
|
||||
// Last element was moved, erase elements from freePosition
|
||||
if (i == size - 1) {
|
||||
trimCandidateList.resize(freePosition);
|
||||
}
|
||||
}
|
||||
}
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
DEBUG_BREAK_IF(trimCandidatesCount != previousCount);
|
||||
|
||||
checkTrimCandidateCount();
|
||||
}
|
||||
|
||||
void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress) {
|
||||
monitoredFence.lastSubmittedFence = 0;
|
||||
monitoredFence.currentFenceValue = 1;
|
||||
monitoredFence.fenceHandle = handle;
|
||||
monitoredFence.cpuAddress = cpuAddress;
|
||||
monitoredFence.gpuAddress = gpuAddress;
|
||||
}
|
||||
|
||||
void WddmResidencyController::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
|
||||
if (flags.PeriodicTrim) {
|
||||
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
|
||||
uint64_t sizeToTrim = 0;
|
||||
auto lock = this->acquireLock();
|
||||
|
||||
WddmAllocation *wddmAllocation = nullptr;
|
||||
while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
|
||||
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastTrimFenceValue);
|
||||
|
||||
if (wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) {
|
||||
break;
|
||||
}
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId));
|
||||
|
||||
uint32_t fragmentsToEvict = 0;
|
||||
|
||||
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId));
|
||||
this->wddm.evict(wddmAllocation->getHandles().data(), wddmAllocation->getNumHandles(), sizeToTrim);
|
||||
}
|
||||
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId];
|
||||
if (!wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) {
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId));
|
||||
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData.osHandleStorage->handle;
|
||||
fragmentStorageData.residency->resident[osContextId] = false;
|
||||
}
|
||||
}
|
||||
if (fragmentsToEvict != 0) {
|
||||
this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
|
||||
}
|
||||
wddmAllocation->getResidencyData().resident[osContextId] = false;
|
||||
|
||||
this->removeFromTrimCandidateList(wddmAllocation, false);
|
||||
}
|
||||
|
||||
if (this->checkTrimCandidateListCompaction()) {
|
||||
this->compactTrimCandidateList();
|
||||
}
|
||||
}
|
||||
|
||||
if (flags.TrimToBudget) {
|
||||
auto lock = this->acquireLock();
|
||||
trimResidencyToBudget(bytes);
|
||||
}
|
||||
|
||||
if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
|
||||
this->updateLastTrimFenceValue();
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastTrimFenceValue);
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) {
|
||||
D3DKMT_HANDLE fragmentEvictHandles[maxFragmentsCount] = {0};
|
||||
uint64_t numberOfBytesToTrim = bytes;
|
||||
WddmAllocation *wddmAllocation = nullptr;
|
||||
|
||||
while (numberOfBytesToTrim > 0 && (wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
|
||||
uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId);
|
||||
auto &monitoredFence = this->getMonitoredFence();
|
||||
|
||||
if (lastFence > monitoredFence.lastSubmittedFence) {
|
||||
break;
|
||||
}
|
||||
|
||||
uint32_t fragmentsToEvict = 0;
|
||||
uint64_t sizeEvicted = 0;
|
||||
uint64_t sizeToTrim = 0;
|
||||
|
||||
if (lastFence > *monitoredFence.cpuAddress) {
|
||||
this->wddm.waitFromCpu(lastFence, this->getMonitoredFence());
|
||||
}
|
||||
|
||||
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
|
||||
this->wddm.evict(wddmAllocation->getHandles().data(), wddmAllocation->getNumHandles(), sizeToTrim);
|
||||
sizeEvicted = wddmAllocation->getAlignedSize();
|
||||
} else {
|
||||
auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData;
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) {
|
||||
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle;
|
||||
}
|
||||
}
|
||||
|
||||
if (fragmentsToEvict != 0) {
|
||||
this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
|
||||
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) {
|
||||
fragmentStorageData[allocationId].residency->resident[osContextId] = false;
|
||||
sizeEvicted += fragmentStorageData[allocationId].fragmentSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (sizeEvicted >= numberOfBytesToTrim) {
|
||||
numberOfBytesToTrim = 0;
|
||||
} else {
|
||||
numberOfBytesToTrim -= sizeEvicted;
|
||||
}
|
||||
|
||||
wddmAllocation->getResidencyData().resident[osContextId] = false;
|
||||
this->removeFromTrimCandidateList(wddmAllocation, false);
|
||||
}
|
||||
|
||||
if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) {
|
||||
this->compactTrimCandidateList();
|
||||
}
|
||||
|
||||
return numberOfBytesToTrim == 0;
|
||||
}
|
||||
|
||||
bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency) {
|
||||
const size_t residencyCount = allocationsForResidency.size();
|
||||
std::unique_ptr<D3DKMT_HANDLE[]> handlesForResidency(new D3DKMT_HANDLE[residencyCount * maxFragmentsCount * EngineLimits::maxHandleCount]);
|
||||
uint32_t totalHandlesCount = 0;
|
||||
|
||||
auto lock = this->acquireLock();
|
||||
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", this->getMonitoredFence().currentFenceValue);
|
||||
|
||||
for (uint32_t i = 0; i < residencyCount; i++) {
|
||||
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
|
||||
ResidencyData &residencyData = allocation->getResidencyData();
|
||||
bool fragmentResidency[3] = {false, false, false};
|
||||
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, residencyData.resident[osContextId] ? "resident" : "not resident");
|
||||
|
||||
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, "on trimCandidateList");
|
||||
this->removeFromTrimCandidateList(allocation, false);
|
||||
} else {
|
||||
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
fragmentResidency[allocationId] = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident[osContextId];
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "fragment handle =", allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, fragmentResidency[allocationId] ? "resident" : "not resident");
|
||||
}
|
||||
}
|
||||
|
||||
if (allocation->fragmentsStorage.fragmentCount > 0) {
|
||||
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
if (!fragmentResidency[allocationId])
|
||||
handlesForResidency[totalHandlesCount++] = allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
|
||||
}
|
||||
} else if (!residencyData.resident[osContextId]) {
|
||||
memcpy_s(&handlesForResidency[totalHandlesCount], allocation->getNumHandles() * sizeof(D3DKMT_HANDLE), allocation->getHandles().data(), allocation->getNumHandles() * sizeof(D3DKMT_HANDLE));
|
||||
totalHandlesCount += allocation->getNumHandles();
|
||||
}
|
||||
}
|
||||
|
||||
bool result = true;
|
||||
if (totalHandlesCount) {
|
||||
uint64_t bytesToTrim = 0;
|
||||
while ((result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) {
|
||||
this->setMemoryBudgetExhausted();
|
||||
const bool trimmingDone = this->trimResidencyToBudget(bytesToTrim);
|
||||
if (!trimmingDone) {
|
||||
auto evictionStatus = wddm.getTemporaryResourcesContainer()->evictAllResources();
|
||||
if (evictionStatus == MemoryOperationsStatus::SUCCESS) {
|
||||
continue;
|
||||
}
|
||||
DEBUG_BREAK_IF(evictionStatus != MemoryOperationsStatus::MEMORY_NOT_FOUND);
|
||||
result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (result == true) {
|
||||
for (uint32_t i = 0; i < residencyCount; i++) {
|
||||
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
|
||||
// Update fence value not to early destroy / evict allocation
|
||||
const auto currentFence = this->getMonitoredFence().currentFenceValue;
|
||||
allocation->getResidencyData().updateCompletionData(currentFence, this->osContextId);
|
||||
allocation->getResidencyData().resident[osContextId] = true;
|
||||
|
||||
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
auto residencyData = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency;
|
||||
// Update fence value not to remove the fragment referenced by different GA in trimming callback
|
||||
residencyData->updateCompletionData(currentFence, this->osContextId);
|
||||
residencyData->resident[osContextId] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void WddmResidencyController::makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations) {
|
||||
auto lock = this->acquireLock();
|
||||
const size_t residencyCount = evictionAllocations.size();
|
||||
|
||||
for (uint32_t i = 0; i < residencyCount; i++) {
|
||||
WddmAllocation *allocation = static_cast<WddmAllocation *>(evictionAllocations[i]);
|
||||
this->addToTrimCandidateList(allocation);
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmResidencyController::isInitialized() const {
|
||||
if (!DebugManager.flags.DoNotRegisterTrimCallback.get()) {
|
||||
return trimCallbackHandle != nullptr;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -1,79 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/memory_manager/residency_container.h"
|
||||
#include "core/os_interface/windows/windows_defs.h"
|
||||
#include "core/os_interface/windows/windows_wrapper.h"
|
||||
#include "core/utilities/spinlock.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class GraphicsAllocation;
|
||||
class WddmAllocation;
|
||||
class Wddm;
|
||||
|
||||
class WddmResidencyController {
|
||||
public:
|
||||
WddmResidencyController(Wddm &wddm, uint32_t osContextId);
|
||||
MOCKABLE_VIRTUAL ~WddmResidencyController();
|
||||
|
||||
static void APIENTRY trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification);
|
||||
|
||||
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock();
|
||||
std::unique_lock<SpinLock> acquireTrimCallbackLock();
|
||||
|
||||
WddmAllocation *getTrimCandidateHead();
|
||||
void addToTrimCandidateList(GraphicsAllocation *allocation);
|
||||
void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList);
|
||||
void removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList);
|
||||
void checkTrimCandidateCount();
|
||||
|
||||
bool checkTrimCandidateListCompaction();
|
||||
void compactTrimCandidateList();
|
||||
|
||||
bool wasAllocationUsedSinceLastTrim(uint64_t fenceValue) { return fenceValue > lastTrimFenceValue; }
|
||||
void updateLastTrimFenceValue() { lastTrimFenceValue = *this->getMonitoredFence().cpuAddress; }
|
||||
const ResidencyContainer &peekTrimCandidateList() const { return trimCandidateList; }
|
||||
uint32_t peekTrimCandidatesCount() const { return trimCandidatesCount; }
|
||||
|
||||
MonitoredFence &getMonitoredFence() { return monitoredFence; }
|
||||
void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress);
|
||||
|
||||
void registerCallback();
|
||||
|
||||
void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes);
|
||||
bool trimResidencyToBudget(uint64_t bytes);
|
||||
|
||||
bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; }
|
||||
void setMemoryBudgetExhausted() { memoryBudgetExhausted = true; }
|
||||
|
||||
bool makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency);
|
||||
void makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations);
|
||||
|
||||
bool isInitialized() const;
|
||||
|
||||
protected:
|
||||
Wddm &wddm;
|
||||
uint32_t osContextId;
|
||||
MonitoredFence monitoredFence = {};
|
||||
|
||||
SpinLock lock;
|
||||
SpinLock trimCallbackLock;
|
||||
|
||||
bool memoryBudgetExhausted = false;
|
||||
uint64_t lastTrimFenceValue = 0u;
|
||||
ResidencyContainer trimCandidateList;
|
||||
uint32_t trimCandidatesCount = 0;
|
||||
|
||||
VOID *trimCallbackHandle = nullptr;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -1,10 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2019 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
namespace Os {
|
||||
const char *fileSeparator = "/";
|
||||
}
|
||||
Reference in New Issue
Block a user