mirror of
https://github.com/intel/compute-runtime.git
synced 2025-12-25 05:24:02 +08:00
Reorganization directory structure [3/n]
Change-Id: If3dfa3f6007f8810a6a1ae1a4f0c7da38544648d
This commit is contained in:
70
shared/source/os_interface/windows/CMakeLists.txt
Normal file
70
shared/source/os_interface/windows/CMakeLists.txt
Normal file
@@ -0,0 +1,70 @@
|
||||
#
|
||||
# Copyright (C) 2019-2020 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
|
||||
if(KMDAF_HEADERS_DIR)
|
||||
set(KMDAF_FILE_SUFFIX "")
|
||||
else()
|
||||
set(KMDAF_FILE_SUFFIX "_stub")
|
||||
endif()
|
||||
|
||||
set(NEO_CORE_OS_INTERFACE_WINDOWS
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_additional_context_flags.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_allocation.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_engine_mapper.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/debug_registry_reader.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/debug_registry_reader.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/gdi_interface.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/gdi_interface.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/kmdaf_listener${KMDAF_FILE_SUFFIX}.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/kmdaf_listener.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_context_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_context_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_inc.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_interface.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_interface.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_library_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_library_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_memory_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_memory_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_socket.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_thread_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_thread_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_time_win.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/os_time_win.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/page_table_manager_functions.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/print.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/sys_calls.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/thk_wrapper.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_defs.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_interface.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_interface.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_allocation.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_engine_mapper.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_operations_handler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_operations_handler.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_allocations_container.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_allocations_container.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/windows_defs.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/windows_inc.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/windows_wrapper.h
|
||||
)
|
||||
|
||||
if(WIN32)
|
||||
set_property(GLOBAL PROPERTY NEO_CORE_OS_INTERFACE_WINDOWS ${NEO_CORE_OS_INTERFACE_WINDOWS})
|
||||
endif()
|
||||
135
shared/source/os_interface/windows/debug_registry_reader.cpp
Normal file
135
shared/source/os_interface/windows/debug_registry_reader.cpp
Normal file
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/debug_registry_reader.h"
|
||||
|
||||
#include "debug_settings/debug_settings_manager.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
#include "utilities/debug_settings_reader.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
SettingsReader *SettingsReader::createOsReader(bool userScope, const std::string ®Key) {
|
||||
return new RegistryReader(userScope, regKey);
|
||||
}
|
||||
|
||||
RegistryReader::RegistryReader(bool userScope, const std::string ®Key) : registryReadRootKey(regKey) {
|
||||
igdrclHkeyType = userScope ? HKEY_CURRENT_USER : HKEY_LOCAL_MACHINE;
|
||||
setUpProcessName();
|
||||
}
|
||||
|
||||
void RegistryReader::setUpProcessName() {
|
||||
char buff[MAX_PATH];
|
||||
GetModuleFileNameA(nullptr, buff, MAX_PATH);
|
||||
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
|
||||
processName = "";
|
||||
}
|
||||
processName.assign(buff);
|
||||
}
|
||||
const char *RegistryReader::appSpecificLocation(const std::string &name) {
|
||||
if (processName.length() > 0)
|
||||
return processName.c_str();
|
||||
return name.c_str();
|
||||
}
|
||||
|
||||
bool RegistryReader::getSetting(const char *settingName, bool defaultValue) {
|
||||
return getSetting(settingName, static_cast<int32_t>(defaultValue)) ? true : false;
|
||||
}
|
||||
|
||||
int32_t RegistryReader::getSetting(const char *settingName, int32_t defaultValue) {
|
||||
HKEY Key;
|
||||
DWORD value = defaultValue;
|
||||
DWORD success = ERROR_SUCCESS;
|
||||
|
||||
success = RegOpenKeyExA(igdrclHkeyType,
|
||||
registryReadRootKey.c_str(),
|
||||
0,
|
||||
KEY_READ,
|
||||
&Key);
|
||||
|
||||
if (ERROR_SUCCESS == success) {
|
||||
DWORD regType;
|
||||
DWORD size = sizeof(ULONG);
|
||||
|
||||
success = RegQueryValueExA(Key,
|
||||
settingName,
|
||||
NULL,
|
||||
®Type,
|
||||
(LPBYTE)&value,
|
||||
&size);
|
||||
RegCloseKey(Key);
|
||||
value = ERROR_SUCCESS == success ? value : defaultValue;
|
||||
} else { // Check environment variables
|
||||
const char *envValue = getenv(settingName);
|
||||
if (envValue) {
|
||||
value = atoi(envValue);
|
||||
}
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
std::string RegistryReader::getSetting(const char *settingName, const std::string &value) {
|
||||
HKEY Key;
|
||||
DWORD success = ERROR_SUCCESS;
|
||||
std::string keyValue = value;
|
||||
|
||||
success = RegOpenKeyExA(igdrclHkeyType,
|
||||
registryReadRootKey.c_str(),
|
||||
0,
|
||||
KEY_READ,
|
||||
&Key);
|
||||
if (ERROR_SUCCESS == success) {
|
||||
DWORD regType = REG_NONE;
|
||||
DWORD regSize = 0;
|
||||
|
||||
success = RegQueryValueExA(Key,
|
||||
settingName,
|
||||
NULL,
|
||||
®Type,
|
||||
NULL,
|
||||
®Size);
|
||||
if (success == ERROR_SUCCESS && regType == REG_SZ) {
|
||||
char *regData = new char[regSize];
|
||||
success = RegQueryValueExA(Key,
|
||||
settingName,
|
||||
NULL,
|
||||
®Type,
|
||||
(LPBYTE)regData,
|
||||
®Size);
|
||||
keyValue.assign(regData);
|
||||
delete[] regData;
|
||||
} else if (success == ERROR_SUCCESS && regType == REG_BINARY) {
|
||||
std::unique_ptr<wchar_t[]> regData(new wchar_t[regSize]);
|
||||
success = RegQueryValueExA(Key,
|
||||
settingName,
|
||||
NULL,
|
||||
®Type,
|
||||
(LPBYTE)regData.get(),
|
||||
®Size);
|
||||
|
||||
size_t charsConverted = 0;
|
||||
std::unique_ptr<char[]> convertedData(new char[regSize]);
|
||||
|
||||
wcstombs_s(&charsConverted, convertedData.get(), regSize, regData.get(), regSize);
|
||||
|
||||
keyValue.assign(convertedData.get());
|
||||
}
|
||||
|
||||
RegCloseKey(Key);
|
||||
} else { // Check environment variables
|
||||
const char *envValue = getenv(settingName);
|
||||
if (envValue) {
|
||||
keyValue.assign(envValue);
|
||||
}
|
||||
}
|
||||
return keyValue;
|
||||
}
|
||||
|
||||
}; // namespace NEO
|
||||
33
shared/source/os_interface/windows/debug_registry_reader.h
Normal file
33
shared/source/os_interface/windows/debug_registry_reader.h
Normal file
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utilities/debug_settings_reader.h"
|
||||
|
||||
#include <Windows.h>
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
|
||||
namespace NEO {
|
||||
class RegistryReader : public SettingsReader {
|
||||
public:
|
||||
RegistryReader() = delete;
|
||||
RegistryReader(bool userScope, const std::string ®Key);
|
||||
int32_t getSetting(const char *settingName, int32_t defaultValue) override;
|
||||
bool getSetting(const char *settingName, bool defaultValue) override;
|
||||
std::string getSetting(const char *settingName, const std::string &value) override;
|
||||
const char *appSpecificLocation(const std::string &name) override;
|
||||
|
||||
protected:
|
||||
HKEY igdrclHkeyType;
|
||||
std::string registryReadRootKey;
|
||||
void setUpProcessName();
|
||||
std::string processName;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/deferrable_deletion_win.h"
|
||||
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
template <typename... Args>
|
||||
DeferrableDeletion *DeferrableDeletion::create(Args... args) {
|
||||
return new DeferrableDeletionImpl(std::forward<Args>(args)...);
|
||||
}
|
||||
template DeferrableDeletion *DeferrableDeletion::create(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
|
||||
|
||||
DeferrableDeletionImpl::DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle)
|
||||
: wddm(wddm), allocationCount(allocationCount), resourceHandle(resourceHandle) {
|
||||
if (handles) {
|
||||
this->handles = new D3DKMT_HANDLE[allocationCount];
|
||||
for (uint32_t i = 0; i < allocationCount; i++) {
|
||||
this->handles[i] = handles[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
bool DeferrableDeletionImpl::apply() {
|
||||
bool destroyStatus = wddm->destroyAllocations(handles, allocationCount, resourceHandle);
|
||||
DEBUG_BREAK_IF(!destroyStatus);
|
||||
return true;
|
||||
}
|
||||
DeferrableDeletionImpl::~DeferrableDeletionImpl() {
|
||||
if (handles) {
|
||||
delete[] handles;
|
||||
}
|
||||
}
|
||||
} // namespace NEO
|
||||
35
shared/source/os_interface/windows/deferrable_deletion_win.h
Normal file
35
shared/source/os_interface/windows/deferrable_deletion_win.h
Normal file
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "memory_manager/deferrable_deletion.h"
|
||||
#include "os_interface/os_context.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class OsContextWin;
|
||||
class Wddm;
|
||||
|
||||
class DeferrableDeletionImpl : public DeferrableDeletion {
|
||||
public:
|
||||
DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
|
||||
bool apply() override;
|
||||
~DeferrableDeletionImpl();
|
||||
|
||||
DeferrableDeletionImpl(const DeferrableDeletionImpl &) = delete;
|
||||
DeferrableDeletionImpl &operator=(const DeferrableDeletionImpl &) = delete;
|
||||
|
||||
protected:
|
||||
Wddm *wddm;
|
||||
D3DKMT_HANDLE *handles = nullptr;
|
||||
uint32_t allocationCount;
|
||||
D3DKMT_HANDLE resourceHandle;
|
||||
};
|
||||
} // namespace NEO
|
||||
104
shared/source/os_interface/windows/gdi_interface.cpp
Normal file
104
shared/source/os_interface/windows/gdi_interface.cpp
Normal file
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/gdi_interface.h"
|
||||
|
||||
#include "debug_settings/debug_settings_manager.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
inline const std::string getGdiName() {
|
||||
if (DebugManager.flags.OverrideGdiPath.get() != "unk") {
|
||||
return DebugManager.flags.OverrideGdiPath.get();
|
||||
} else {
|
||||
return Os::gdiDllName;
|
||||
}
|
||||
}
|
||||
|
||||
Gdi::Gdi() : gdiDll(getGdiName()),
|
||||
initialized(false) {
|
||||
if (gdiDll.isLoaded()) {
|
||||
initialized = getAllProcAddresses();
|
||||
}
|
||||
}
|
||||
|
||||
bool Gdi::setupHwQueueProcAddresses() {
|
||||
createHwQueue = reinterpret_cast<PFND3DKMT_CREATEHWQUEUE>(gdiDll.getProcAddress("D3DKMTCreateHwQueue"));
|
||||
destroyHwQueue = reinterpret_cast<PFND3DKMT_DESTROYHWQUEUE>(gdiDll.getProcAddress("D3DKMTDestroyHwQueue"));
|
||||
submitCommandToHwQueue = reinterpret_cast<PFND3DKMT_SUBMITCOMMANDTOHWQUEUE>(gdiDll.getProcAddress("D3DKMTSubmitCommandToHwQueue"));
|
||||
|
||||
if (!createHwQueue || !destroyHwQueue || !submitCommandToHwQueue) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Gdi::getAllProcAddresses() {
|
||||
openAdapterFromHdc = reinterpret_cast<PFND3DKMT_OPENADAPTERFROMHDC>(gdiDll.getProcAddress("D3DKMTOpenAdapterFromHdc"));
|
||||
openAdapterFromLuid = reinterpret_cast<PFND3DKMT_OPENADAPTERFROMLUID>(gdiDll.getProcAddress("D3DKMTOpenAdapterFromLuid"));
|
||||
createAllocation = reinterpret_cast<PFND3DKMT_CREATEALLOCATION>(gdiDll.getProcAddress("D3DKMTCreateAllocation"));
|
||||
destroyAllocation = reinterpret_cast<PFND3DKMT_DESTROYALLOCATION>(gdiDll.getProcAddress("D3DKMTDestroyAllocation"));
|
||||
destroyAllocation2 = reinterpret_cast<PFND3DKMT_DESTROYALLOCATION2>(gdiDll.getProcAddress("D3DKMTDestroyAllocation2"));
|
||||
queryAdapterInfo = reinterpret_cast<PFND3DKMT_QUERYADAPTERINFO>(gdiDll.getProcAddress("D3DKMTQueryAdapterInfo"));
|
||||
closeAdapter = reinterpret_cast<PFND3DKMT_CLOSEADAPTER>(gdiDll.getProcAddress("D3DKMTCloseAdapter"));
|
||||
createDevice = reinterpret_cast<PFND3DKMT_CREATEDEVICE>(gdiDll.getProcAddress("D3DKMTCreateDevice"));
|
||||
destroyDevice = reinterpret_cast<PFND3DKMT_DESTROYDEVICE>(gdiDll.getProcAddress("D3DKMTDestroyDevice"));
|
||||
escape = reinterpret_cast<PFND3DKMT_ESCAPE>(gdiDll.getProcAddress("D3DKMTEscape"));
|
||||
createContext = reinterpret_cast<PFND3DKMT_CREATECONTEXTVIRTUAL>(gdiDll.getProcAddress("D3DKMTCreateContextVirtual"));
|
||||
destroyContext = reinterpret_cast<PFND3DKMT_DESTROYCONTEXT>(gdiDll.getProcAddress("D3DKMTDestroyContext"));
|
||||
openResource = reinterpret_cast<PFND3DKMT_OPENRESOURCE>(gdiDll.getProcAddress("D3DKMTOpenResource"));
|
||||
openResourceFromNtHandle = reinterpret_cast<PFND3DKMT_OPENRESOURCEFROMNTHANDLE>(gdiDll.getProcAddress("D3DKMTOpenResourceFromNtHandle"));
|
||||
queryResourceInfo = reinterpret_cast<PFND3DKMT_QUERYRESOURCEINFO>(gdiDll.getProcAddress("D3DKMTQueryResourceInfo"));
|
||||
queryResourceInfoFromNtHandle = reinterpret_cast<PFND3DKMT_QUERYRESOURCEINFOFROMNTHANDLE>(gdiDll.getProcAddress("D3DKMTQueryResourceInfoFromNtHandle"));
|
||||
lock = reinterpret_cast<PFND3DKMT_LOCK>(gdiDll.getProcAddress("D3DKMTLock"));
|
||||
unlock = reinterpret_cast<PFND3DKMT_UNLOCK>(gdiDll.getProcAddress("D3DKMTUnlock"));
|
||||
render = reinterpret_cast<PFND3DKMT_RENDER>(gdiDll.getProcAddress("D3DKMTRender"));
|
||||
createSynchronizationObject = reinterpret_cast<PFND3DKMT_CREATESYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTCreateSynchronizationObject"));
|
||||
createSynchronizationObject2 = reinterpret_cast<PFND3DKMT_CREATESYNCHRONIZATIONOBJECT2>(gdiDll.getProcAddress("D3DKMTCreateSynchronizationObject2"));
|
||||
destroySynchronizationObject = reinterpret_cast<PFND3DKMT_DESTROYSYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTDestroySynchronizationObject"));
|
||||
signalSynchronizationObject = reinterpret_cast<PFND3DKMT_SIGNALSYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTSignalSynchronizationObject"));
|
||||
waitForSynchronizationObject = reinterpret_cast<PFND3DKMT_WAITFORSYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTWaitForSynchronizationObject"));
|
||||
waitForSynchronizationObjectFromCpu = reinterpret_cast<PFND3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU>(gdiDll.getProcAddress("D3DKMTWaitForSynchronizationObjectFromCpu"));
|
||||
signalSynchronizationObjectFromCpu = reinterpret_cast<PFND3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU>(gdiDll.getProcAddress("D3DKMTSignalSynchronizationObjectFromCpu"));
|
||||
waitForSynchronizationObjectFromGpu = reinterpret_cast<PFND3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU>(gdiDll.getProcAddress("D3DKMTWaitForSynchronizationObjectFromGpu"));
|
||||
signalSynchronizationObjectFromGpu = reinterpret_cast<PFND3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU>(gdiDll.getProcAddress("D3DKMTSignalSynchronizationObjectFromGpu"));
|
||||
createPagingQueue = reinterpret_cast<PFND3DKMT_CREATEPAGINGQUEUE>(gdiDll.getProcAddress("D3DKMTCreatePagingQueue"));
|
||||
destroyPagingQueue = reinterpret_cast<PFND3DKMT_DESTROYPAGINGQUEUE>(gdiDll.getProcAddress("D3DKMTDestroyPagingQueue"));
|
||||
lock2 = reinterpret_cast<PFND3DKMT_LOCK2>(gdiDll.getProcAddress("D3DKMTLock2"));
|
||||
unlock2 = reinterpret_cast<PFND3DKMT_UNLOCK2>(gdiDll.getProcAddress("D3DKMTUnlock2"));
|
||||
mapGpuVirtualAddress = reinterpret_cast<PFND3DKMT_MAPGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTMapGpuVirtualAddress"));
|
||||
reserveGpuVirtualAddress = reinterpret_cast<PFND3DKMT_RESERVEGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTReserveGpuVirtualAddress"));
|
||||
freeGpuVirtualAddress = reinterpret_cast<PFND3DKMT_FREEGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTFreeGpuVirtualAddress"));
|
||||
updateGpuVirtualAddress = reinterpret_cast<PFND3DKMT_UPDATEGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTUpdateGpuVirtualAddress"));
|
||||
submitCommand = reinterpret_cast<PFND3DKMT_SUBMITCOMMAND>(gdiDll.getProcAddress("D3DKMTSubmitCommand"));
|
||||
makeResident = reinterpret_cast<PFND3DKMT_MAKERESIDENT>(gdiDll.getProcAddress("D3DKMTMakeResident"));
|
||||
evict = reinterpret_cast<PFND3DKMT_EVICT>(gdiDll.getProcAddress("D3DKMTEvict"));
|
||||
registerTrimNotification = reinterpret_cast<PFND3DKMT_REGISTERTRIMNOTIFICATION>(gdiDll.getProcAddress("D3DKMTRegisterTrimNotification"));
|
||||
unregisterTrimNotification = reinterpret_cast<PFND3DKMT_UNREGISTERTRIMNOTIFICATION>(gdiDll.getProcAddress("D3DKMTUnregisterTrimNotification"));
|
||||
|
||||
// For debug purposes
|
||||
getDeviceState = reinterpret_cast<PFND3DKMT_GETDEVICESTATE>(gdiDll.getProcAddress("D3DKMTGetDeviceState"));
|
||||
|
||||
// clang-format off
|
||||
if (openAdapterFromHdc && openAdapterFromLuid && createAllocation && destroyAllocation
|
||||
&& destroyAllocation2 && queryAdapterInfo && closeAdapter && createDevice
|
||||
&& destroyDevice && escape && createContext && destroyContext
|
||||
&& openResource && queryResourceInfo && lock && unlock && render
|
||||
&& createSynchronizationObject && createSynchronizationObject2
|
||||
&& destroySynchronizationObject && signalSynchronizationObject
|
||||
&& waitForSynchronizationObject && waitForSynchronizationObjectFromCpu
|
||||
&& signalSynchronizationObjectFromCpu && waitForSynchronizationObjectFromGpu
|
||||
&& signalSynchronizationObjectFromGpu && createPagingQueue && destroyPagingQueue
|
||||
&& lock2 && unlock2 && mapGpuVirtualAddress && reserveGpuVirtualAddress
|
||||
&& freeGpuVirtualAddress && updateGpuVirtualAddress &&submitCommand
|
||||
&& makeResident && evict && registerTrimNotification && unregisterTrimNotification){
|
||||
return true;
|
||||
}
|
||||
// clang-format on
|
||||
return false;
|
||||
}
|
||||
} // namespace NEO
|
||||
87
shared/source/os_interface/windows/gdi_interface.h
Normal file
87
shared/source/os_interface/windows/gdi_interface.h
Normal file
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "os_interface/windows/os_inc.h"
|
||||
#include "os_interface/windows/os_library_win.h"
|
||||
#include "os_interface/windows/thk_wrapper.h"
|
||||
|
||||
#include <d3d9types.h>
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class Gdi {
|
||||
public:
|
||||
Gdi();
|
||||
~Gdi(){};
|
||||
|
||||
ThkWrapper<IN OUT D3DKMT_OPENADAPTERFROMHDC *> openAdapterFromHdc;
|
||||
ThkWrapper<IN OUT D3DKMT_OPENADAPTERFROMLUID *> openAdapterFromLuid;
|
||||
ThkWrapper<IN OUT D3DKMT_CREATEALLOCATION *> createAllocation;
|
||||
ThkWrapper<IN CONST D3DKMT_DESTROYALLOCATION *> destroyAllocation;
|
||||
ThkWrapper<IN CONST D3DKMT_DESTROYALLOCATION2 *> destroyAllocation2;
|
||||
ThkWrapper<IN CONST D3DKMT_QUERYADAPTERINFO *> queryAdapterInfo;
|
||||
ThkWrapper<IN CONST D3DKMT_CLOSEADAPTER *> closeAdapter;
|
||||
ThkWrapper<IN OUT D3DKMT_CREATEDEVICE *> createDevice;
|
||||
ThkWrapper<IN CONST D3DKMT_DESTROYDEVICE *> destroyDevice;
|
||||
ThkWrapper<IN CONST D3DKMT_ESCAPE *> escape;
|
||||
ThkWrapper<IN D3DKMT_CREATECONTEXTVIRTUAL *> createContext;
|
||||
ThkWrapper<IN CONST D3DKMT_DESTROYCONTEXT *> destroyContext;
|
||||
ThkWrapper<IN OUT D3DKMT_OPENRESOURCE *> openResource;
|
||||
ThkWrapper<IN OUT D3DKMT_OPENRESOURCEFROMNTHANDLE *> openResourceFromNtHandle;
|
||||
ThkWrapper<IN OUT D3DKMT_QUERYRESOURCEINFO *> queryResourceInfo;
|
||||
ThkWrapper<IN OUT D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *> queryResourceInfoFromNtHandle;
|
||||
ThkWrapper<IN OUT D3DKMT_LOCK *> lock;
|
||||
ThkWrapper<IN CONST D3DKMT_UNLOCK *> unlock;
|
||||
ThkWrapper<IN OUT D3DKMT_RENDER *> render;
|
||||
ThkWrapper<IN OUT D3DKMT_CREATESYNCHRONIZATIONOBJECT *> createSynchronizationObject;
|
||||
ThkWrapper<IN OUT D3DKMT_CREATESYNCHRONIZATIONOBJECT2 *> createSynchronizationObject2;
|
||||
ThkWrapper<IN CONST D3DKMT_DESTROYSYNCHRONIZATIONOBJECT *> destroySynchronizationObject;
|
||||
ThkWrapper<IN CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECT *> signalSynchronizationObject;
|
||||
ThkWrapper<IN CONST_FROM_WDK_10_0_18328_0 D3DKMT_WAITFORSYNCHRONIZATIONOBJECT *> waitForSynchronizationObject;
|
||||
ThkWrapper<IN CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *> waitForSynchronizationObjectFromCpu;
|
||||
ThkWrapper<IN CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU *> signalSynchronizationObjectFromCpu;
|
||||
ThkWrapper<IN CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU *> waitForSynchronizationObjectFromGpu;
|
||||
ThkWrapper<IN CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU *> signalSynchronizationObjectFromGpu;
|
||||
ThkWrapper<IN OUT D3DKMT_CREATEPAGINGQUEUE *> createPagingQueue;
|
||||
ThkWrapper<IN OUT D3DDDI_DESTROYPAGINGQUEUE *> destroyPagingQueue;
|
||||
ThkWrapper<IN OUT D3DKMT_LOCK2 *> lock2;
|
||||
ThkWrapper<IN CONST D3DKMT_UNLOCK2 *> unlock2;
|
||||
ThkWrapper<IN OUT D3DDDI_MAPGPUVIRTUALADDRESS *> mapGpuVirtualAddress;
|
||||
ThkWrapper<IN OUT D3DDDI_RESERVEGPUVIRTUALADDRESS *> reserveGpuVirtualAddress;
|
||||
ThkWrapper<IN CONST D3DKMT_FREEGPUVIRTUALADDRESS *> freeGpuVirtualAddress;
|
||||
ThkWrapper<IN CONST D3DKMT_UPDATEGPUVIRTUALADDRESS *> updateGpuVirtualAddress;
|
||||
ThkWrapper<IN CONST D3DKMT_SUBMITCOMMAND *> submitCommand;
|
||||
ThkWrapper<IN OUT D3DDDI_MAKERESIDENT *> makeResident;
|
||||
ThkWrapper<IN D3DKMT_EVICT *> evict;
|
||||
ThkWrapper<IN D3DKMT_REGISTERTRIMNOTIFICATION *> registerTrimNotification;
|
||||
ThkWrapper<IN D3DKMT_UNREGISTERTRIMNOTIFICATION *> unregisterTrimNotification;
|
||||
|
||||
// HW queue
|
||||
ThkWrapper<IN OUT D3DKMT_CREATEHWQUEUE *> createHwQueue;
|
||||
ThkWrapper<IN CONST D3DKMT_DESTROYHWQUEUE *> destroyHwQueue;
|
||||
ThkWrapper<IN CONST D3DKMT_SUBMITCOMMANDTOHWQUEUE *> submitCommandToHwQueue;
|
||||
|
||||
// For debug purposes
|
||||
ThkWrapper<IN OUT D3DKMT_GETDEVICESTATE *> getDeviceState;
|
||||
|
||||
bool isInitialized() {
|
||||
return initialized;
|
||||
}
|
||||
|
||||
MOCKABLE_VIRTUAL bool setupHwQueueProcAddresses();
|
||||
|
||||
protected:
|
||||
MOCKABLE_VIRTUAL bool getAllProcAddresses();
|
||||
bool initialized;
|
||||
NEO::Windows::OsLibrary gdiDll;
|
||||
};
|
||||
} // namespace NEO
|
||||
36
shared/source/os_interface/windows/gmm_interface_win.cpp
Normal file
36
shared/source/os_interface/windows/gmm_interface_win.cpp
Normal file
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "gmm_helper/gmm_interface.h"
|
||||
#include "helpers/debug_helpers.h"
|
||||
#include "os_interface/os_library.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
static std::unique_ptr<OsLibrary> gmmLib;
|
||||
|
||||
namespace GmmInterface {
|
||||
|
||||
GMM_STATUS initialize(GMM_INIT_IN_ARGS *pInArgs, GMM_INIT_OUT_ARGS *pOutArgs) {
|
||||
if (!gmmLib) {
|
||||
gmmLib.reset(OsLibrary::load(GMM_UMD_DLL));
|
||||
UNRECOVERABLE_IF(!gmmLib);
|
||||
}
|
||||
auto initGmmFunc = reinterpret_cast<decltype(&InitializeGmm)>(gmmLib->getProcAddress(GMM_ADAPTER_INIT_NAME));
|
||||
UNRECOVERABLE_IF(!initGmmFunc);
|
||||
return initGmmFunc(pInArgs, pOutArgs);
|
||||
}
|
||||
|
||||
void destroy(GMM_INIT_OUT_ARGS *pInArgs) {
|
||||
auto destroyGmmFunc = reinterpret_cast<decltype(&GmmAdapterDestroy)>(gmmLib->getProcAddress(GMM_ADAPTER_DESTROY_NAME));
|
||||
UNRECOVERABLE_IF(!destroyGmmFunc);
|
||||
destroyGmmFunc(pInArgs);
|
||||
}
|
||||
} // namespace GmmInterface
|
||||
} // namespace NEO
|
||||
37
shared/source/os_interface/windows/hw_device_id.h
Normal file
37
shared/source/os_interface/windows/hw_device_id.h
Normal file
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "helpers/non_copyable_or_moveable.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace NEO {
|
||||
class Gdi;
|
||||
class HwDeviceId : NonCopyableClass {
|
||||
public:
|
||||
HwDeviceId(D3DKMT_HANDLE adapterIn, LUID adapterLuidIn, std::unique_ptr<Gdi> gdiIn);
|
||||
inline Gdi *getGdi() const {
|
||||
return gdi.get();
|
||||
}
|
||||
constexpr D3DKMT_HANDLE getAdapter() const {
|
||||
return adapter;
|
||||
}
|
||||
constexpr LUID getAdapterLuid() const {
|
||||
return adapterLuid;
|
||||
}
|
||||
~HwDeviceId();
|
||||
|
||||
protected:
|
||||
const D3DKMT_HANDLE adapter;
|
||||
const LUID adapterLuid;
|
||||
const std::unique_ptr<Gdi> gdi;
|
||||
};
|
||||
} // namespace NEO
|
||||
23
shared/source/os_interface/windows/hw_device_id_win.cpp
Normal file
23
shared/source/os_interface/windows/hw_device_id_win.cpp
Normal file
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "helpers/debug_helpers.h"
|
||||
#include "os_interface/windows/gdi_interface.h"
|
||||
#include "os_interface/windows/hw_device_id.h"
|
||||
namespace NEO {
|
||||
|
||||
HwDeviceId::~HwDeviceId() {
|
||||
NTSTATUS status = STATUS_UNSUCCESSFUL;
|
||||
D3DKMT_CLOSEADAPTER CloseAdapter = {0};
|
||||
CloseAdapter.hAdapter = adapter;
|
||||
status = gdi->closeAdapter(&CloseAdapter);
|
||||
DEBUG_BREAK_IF(status != STATUS_SUCCESS);
|
||||
}
|
||||
HwDeviceId::HwDeviceId(D3DKMT_HANDLE adapterIn, LUID adapterLuidIn, std::unique_ptr<Gdi> gdiIn) : adapter(adapterIn),
|
||||
adapterLuid(adapterLuidIn),
|
||||
gdi(std::move(gdiIn)){};
|
||||
} // namespace NEO
|
||||
59
shared/source/os_interface/windows/hw_info_config.cpp
Normal file
59
shared/source/os_interface/windows/hw_info_config.cpp
Normal file
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/hw_info_config.h"
|
||||
|
||||
#include "command_stream/preemption.h"
|
||||
#include "debug_settings/debug_settings_manager.h"
|
||||
#include "helpers/hw_cmds.h"
|
||||
#include "helpers/hw_helper.h"
|
||||
#include "helpers/hw_info.h"
|
||||
#include "memory_manager/memory_constants.h"
|
||||
|
||||
#include "instrumentation.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
HwInfoConfig *hwInfoConfigFactory[IGFX_MAX_PRODUCT] = {};
|
||||
|
||||
int HwInfoConfig::configureHwInfo(const HardwareInfo *inHwInfo, HardwareInfo *outHwInfo, OSInterface *osIface) {
|
||||
HwHelper &hwHelper = HwHelper::get(outHwInfo->platform.eRenderCoreFamily);
|
||||
|
||||
outHwInfo->capabilityTable.ftrSvm = outHwInfo->featureTable.ftrSVM;
|
||||
|
||||
hwHelper.adjustDefaultEngineType(outHwInfo);
|
||||
outHwInfo->capabilityTable.defaultEngineType = getChosenEngineType(*outHwInfo);
|
||||
|
||||
hwHelper.setCapabilityCoherencyFlag(outHwInfo, outHwInfo->capabilityTable.ftrSupportsCoherency);
|
||||
outHwInfo->capabilityTable.ftrSupportsCoherency &= inHwInfo->featureTable.ftrL3IACoherency;
|
||||
|
||||
PreemptionHelper::adjustDefaultPreemptionMode(outHwInfo->capabilityTable,
|
||||
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuMidThreadLevelPreempt),
|
||||
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuThreadGroupLevelPreempt),
|
||||
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuMidBatchPreempt));
|
||||
outHwInfo->capabilityTable.requiredPreemptionSurfaceSize = outHwInfo->gtSystemInfo.CsrSizeInMb * MemoryConstants::megaByte;
|
||||
|
||||
outHwInfo->capabilityTable.instrumentationEnabled =
|
||||
(outHwInfo->capabilityTable.instrumentationEnabled && haveInstrumentation);
|
||||
|
||||
auto &kmdNotifyProperties = outHwInfo->capabilityTable.kmdNotifyProperties;
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableKmdNotify.get(), kmdNotifyProperties.enableKmdNotify);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideKmdNotifyDelayMicroseconds.get(), kmdNotifyProperties.delayKmdNotifyMicroseconds);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableQuickKmdSleep.get(), kmdNotifyProperties.enableQuickKmdSleep);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideQuickKmdSleepDelayMicroseconds.get(), kmdNotifyProperties.delayQuickKmdSleepMicroseconds);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableQuickKmdSleepForSporadicWaits.get(), kmdNotifyProperties.enableQuickKmdSleepForSporadicWaits);
|
||||
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideDelayQuickKmdSleepForSporadicWaitsMicroseconds.get(), kmdNotifyProperties.delayQuickKmdSleepForSporadicWaitsMicroseconds);
|
||||
|
||||
// Product specific config
|
||||
int ret = configureHardwareCustom(outHwInfo, osIface);
|
||||
if (ret != 0) {
|
||||
outHwInfo = {};
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
43
shared/source/os_interface/windows/kmdaf_listener.cpp
Normal file
43
shared/source/os_interface/windows/kmdaf_listener.cpp
Normal file
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/kmdaf_listener.h"
|
||||
#pragma warning(push) // save the current state
|
||||
#pragma warning(disable : 4189) // disable warning 4189 (unused local variable)
|
||||
#include "kmdaf.h"
|
||||
#pragma warning(pop) // restore state.
|
||||
|
||||
namespace NEO {
|
||||
|
||||
void KmDafListener::notifyLock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DDDICB_LOCKFLAGS *pLockFlags, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_LOCK(ftrKmdDaf, hAdapter, hDevice, hAllocation, pLockFlags, pfnEscape);
|
||||
}
|
||||
|
||||
void KmDafListener::notifyUnlock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_UNLOCK(ftrKmdDaf, hAdapter, hDevice, phAllocation, allocations, pfnEscape);
|
||||
}
|
||||
|
||||
void KmDafListener::notifyMapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_MAP_GPUVA(ftrKmdDaf, hAdapter, hDevice, hAllocation, GpuVirtualAddress, pfnEscape);
|
||||
}
|
||||
|
||||
void KmDafListener::notifyUnmapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_UNMAP_GPUVA(ftrKmdDaf, hAdapter, hDevice, GpuVirtualAddress, pfnEscape);
|
||||
}
|
||||
|
||||
void KmDafListener::notifyMakeResident(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_MAKERESIDENT(ftrKmdDaf, hAdapter, hDevice, phAllocation, allocations, pfnEscape);
|
||||
}
|
||||
|
||||
void KmDafListener::notifyEvict(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_EVICT(ftrKmdDaf, hAdapter, hDevice, phAllocation, allocations, pfnEscape);
|
||||
}
|
||||
|
||||
void KmDafListener::notifyWriteTarget(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
KM_DAF_NOTIFY_WRITE_TARGET(ftrKmdDaf, hAdapter, hDevice, hAllocation, pfnEscape);
|
||||
}
|
||||
} // namespace NEO
|
||||
31
shared/source/os_interface/windows/kmdaf_listener.h
Normal file
31
shared/source/os_interface/windows/kmdaf_listener.h
Normal file
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
struct KmDafListener {
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyLock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DDDICB_LOCKFLAGS *pLockFlags, PFND3DKMT_ESCAPE pfnEscape);
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyUnlock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape);
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyMapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape);
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyUnmapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape);
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyMakeResident(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape);
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyEvict(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape);
|
||||
|
||||
MOCKABLE_VIRTUAL void notifyWriteTarget(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, PFND3DKMT_ESCAPE pfnEscape);
|
||||
};
|
||||
} // namespace NEO
|
||||
32
shared/source/os_interface/windows/kmdaf_listener_stub.cpp
Normal file
32
shared/source/os_interface/windows/kmdaf_listener_stub.cpp
Normal file
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/kmdaf_listener.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
void KmDafListener::notifyLock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DDDICB_LOCKFLAGS *pLockFlags, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
|
||||
void KmDafListener::notifyUnlock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
|
||||
void KmDafListener::notifyMapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
|
||||
void KmDafListener::notifyUnmapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
|
||||
void KmDafListener::notifyMakeResident(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
|
||||
void KmDafListener::notifyEvict(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
|
||||
void KmDafListener::notifyWriteTarget(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, PFND3DKMT_ESCAPE pfnEscape) {
|
||||
}
|
||||
} // namespace NEO
|
||||
51
shared/source/os_interface/windows/os_context_win.cpp
Normal file
51
shared/source/os_interface/windows/os_context_win.cpp
Normal file
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/os_context_win.h"
|
||||
|
||||
#include "os_interface/windows/os_interface.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
#include "os_interface/windows/wddm/wddm_interface.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
OsContext *OsContext::create(OSInterface *osInterface, uint32_t contextId, DeviceBitfield deviceBitfield,
|
||||
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority) {
|
||||
if (osInterface) {
|
||||
return new OsContextWin(*osInterface->get()->getWddm(), contextId, deviceBitfield, engineType, preemptionMode, lowPriority);
|
||||
}
|
||||
return new OsContext(contextId, deviceBitfield, engineType, preemptionMode, lowPriority);
|
||||
}
|
||||
|
||||
OsContextWin::OsContextWin(Wddm &wddm, uint32_t contextId, DeviceBitfield deviceBitfield,
|
||||
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority)
|
||||
: OsContext(contextId, deviceBitfield, engineType, preemptionMode, lowPriority), wddm(wddm), residencyController(wddm, contextId) {
|
||||
|
||||
auto wddmInterface = wddm.getWddmInterface();
|
||||
if (!wddm.createContext(*this)) {
|
||||
return;
|
||||
}
|
||||
if (wddmInterface->hwQueuesSupported()) {
|
||||
if (!wddmInterface->createHwQueue(*this)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
initialized = wddmInterface->createMonitoredFence(*this);
|
||||
residencyController.registerCallback();
|
||||
};
|
||||
|
||||
bool OsContextWin::isInitialized() const {
|
||||
return (initialized && residencyController.isInitialized());
|
||||
}
|
||||
|
||||
OsContextWin::~OsContextWin() {
|
||||
wddm.getWddmInterface()->destroyHwQueue(hardwareQueue.handle);
|
||||
wddm.getWddmInterface()->destroyMonitorFence(residencyController.getMonitoredFence());
|
||||
wddm.destroyContext(wddmContextHandle);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
46
shared/source/os_interface/windows/os_context_win.h
Normal file
46
shared/source/os_interface/windows/os_context_win.h
Normal file
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "os_interface/os_context.h"
|
||||
#include "os_interface/windows/wddm_residency_controller.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class Wddm;
|
||||
|
||||
struct HardwareQueue {
|
||||
D3DKMT_HANDLE handle = 0;
|
||||
D3DKMT_HANDLE progressFenceHandle = 0;
|
||||
VOID *progressFenceCpuVA = nullptr;
|
||||
D3DGPU_VIRTUAL_ADDRESS progressFenceGpuVA = 0;
|
||||
};
|
||||
|
||||
class OsContextWin : public OsContext {
|
||||
public:
|
||||
OsContextWin() = delete;
|
||||
~OsContextWin() override;
|
||||
|
||||
OsContextWin(Wddm &wddm, uint32_t contextId, DeviceBitfield deviceBitfield,
|
||||
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority);
|
||||
|
||||
D3DKMT_HANDLE getWddmContextHandle() const { return wddmContextHandle; }
|
||||
void setWddmContextHandle(D3DKMT_HANDLE wddmContextHandle) { this->wddmContextHandle = wddmContextHandle; }
|
||||
HardwareQueue getHwQueue() const { return hardwareQueue; }
|
||||
void setHwQueue(HardwareQueue hardwareQueue) { this->hardwareQueue = hardwareQueue; }
|
||||
bool isInitialized() const override;
|
||||
Wddm *getWddm() const { return &wddm; }
|
||||
MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController() { return residencyController; }
|
||||
|
||||
protected:
|
||||
bool initialized = false;
|
||||
D3DKMT_HANDLE wddmContextHandle = 0;
|
||||
HardwareQueue hardwareQueue;
|
||||
Wddm &wddm;
|
||||
WddmResidencyController residencyController;
|
||||
};
|
||||
} // namespace NEO
|
||||
17
shared/source/os_interface/windows/os_inc.h
Normal file
17
shared/source/os_interface/windows/os_inc.h
Normal file
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#define PATH_SEPARATOR '\\'
|
||||
|
||||
// For now we need to keep this file clean of OS specific #includes.
|
||||
// Only issues to address portability should be covered here.
|
||||
|
||||
namespace Os {
|
||||
// OS GDI name
|
||||
extern const char *gdiDllName;
|
||||
}; // namespace Os
|
||||
86
shared/source/os_interface/windows/os_interface.cpp
Normal file
86
shared/source/os_interface/windows/os_interface.cpp
Normal file
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/os_interface.h"
|
||||
|
||||
#include "execution_environment/root_device_environment.h"
|
||||
#include "os_interface/windows/sys_calls.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
#include "os_interface/windows/wddm_memory_operations_handler.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
bool OSInterface::osEnabled64kbPages = true;
|
||||
|
||||
OSInterface::OSInterface() {
|
||||
osInterfaceImpl = new OSInterfaceImpl();
|
||||
}
|
||||
|
||||
OSInterface::~OSInterface() {
|
||||
delete osInterfaceImpl;
|
||||
}
|
||||
|
||||
uint32_t OSInterface::getDeviceHandle() const {
|
||||
return static_cast<uint32_t>(osInterfaceImpl->getDeviceHandle());
|
||||
}
|
||||
|
||||
void OSInterface::setGmmInputArgs(void *args) {
|
||||
this->get()->getWddm()->setGmmInputArg(args);
|
||||
}
|
||||
|
||||
OSInterface::OSInterfaceImpl::OSInterfaceImpl() = default;
|
||||
|
||||
D3DKMT_HANDLE OSInterface::OSInterfaceImpl::getAdapterHandle() const {
|
||||
return wddm->getAdapter();
|
||||
}
|
||||
|
||||
D3DKMT_HANDLE OSInterface::OSInterfaceImpl::getDeviceHandle() const {
|
||||
return wddm->getDevice();
|
||||
}
|
||||
|
||||
PFND3DKMT_ESCAPE OSInterface::OSInterfaceImpl::getEscapeHandle() const {
|
||||
return wddm->getEscapeHandle();
|
||||
}
|
||||
|
||||
uint32_t OSInterface::OSInterfaceImpl::getHwContextId() const {
|
||||
if (wddm == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return wddm->getHwContextId();
|
||||
}
|
||||
|
||||
bool OSInterface::are64kbPagesEnabled() {
|
||||
return osEnabled64kbPages;
|
||||
}
|
||||
|
||||
Wddm *OSInterface::OSInterfaceImpl::getWddm() const {
|
||||
return wddm.get();
|
||||
}
|
||||
|
||||
void OSInterface::OSInterfaceImpl::setWddm(Wddm *wddm) {
|
||||
this->wddm.reset(wddm);
|
||||
}
|
||||
|
||||
HANDLE OSInterface::OSInterfaceImpl::createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState,
|
||||
LPCSTR lpName) {
|
||||
return SysCalls::createEvent(lpEventAttributes, bManualReset, bInitialState, lpName);
|
||||
}
|
||||
|
||||
BOOL OSInterface::OSInterfaceImpl::closeHandle(HANDLE hObject) {
|
||||
return SysCalls::closeHandle(hObject);
|
||||
}
|
||||
bool RootDeviceEnvironment::initOsInterface(std::unique_ptr<HwDeviceId> &&hwDeviceId) {
|
||||
std::unique_ptr<Wddm> wddm(Wddm::createWddm(std::move(hwDeviceId), *this));
|
||||
if (!wddm->init()) {
|
||||
return false;
|
||||
}
|
||||
memoryOperationsInterface = std::make_unique<WddmMemoryOperationsHandler>(wddm.get());
|
||||
osInterface = std::make_unique<OSInterface>();
|
||||
osInterface->get()->setWddm(wddm.release());
|
||||
return true;
|
||||
}
|
||||
} // namespace NEO
|
||||
42
shared/source/os_interface/windows/os_interface.h
Normal file
42
shared/source/os_interface/windows/os_interface.h
Normal file
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "os_interface/os_interface.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include "profileapi.h"
|
||||
#include "umKmInc/sharedata.h"
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace NEO {
|
||||
class Wddm;
|
||||
class WddmMemoryOperationsHandler;
|
||||
|
||||
class OSInterface::OSInterfaceImpl {
|
||||
public:
|
||||
OSInterfaceImpl();
|
||||
virtual ~OSInterfaceImpl() = default;
|
||||
Wddm *getWddm() const;
|
||||
void setWddm(Wddm *wddm);
|
||||
D3DKMT_HANDLE getAdapterHandle() const;
|
||||
D3DKMT_HANDLE getDeviceHandle() const;
|
||||
PFND3DKMT_ESCAPE getEscapeHandle() const;
|
||||
uint32_t getHwContextId() const;
|
||||
|
||||
MOCKABLE_VIRTUAL HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState,
|
||||
LPCSTR lpName);
|
||||
MOCKABLE_VIRTUAL BOOL closeHandle(HANDLE hObject);
|
||||
|
||||
protected:
|
||||
std::unique_ptr<Wddm> wddm;
|
||||
};
|
||||
} // namespace NEO
|
||||
69
shared/source/os_interface/windows/os_library_win.cpp
Normal file
69
shared/source/os_interface/windows/os_library_win.cpp
Normal file
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/os_library_win.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
OsLibrary *OsLibrary::load(const std::string &name) {
|
||||
Windows::OsLibrary *ptr = new Windows::OsLibrary(name);
|
||||
|
||||
if (!ptr->isLoaded()) {
|
||||
delete ptr;
|
||||
return nullptr;
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
namespace Windows {
|
||||
decltype(&LoadLibraryExA) OsLibrary::loadLibraryExA = LoadLibraryExA;
|
||||
decltype(&GetModuleFileNameA) OsLibrary::getModuleFileNameA = GetModuleFileNameA;
|
||||
|
||||
extern "C" IMAGE_DOS_HEADER __ImageBase;
|
||||
__inline HINSTANCE GetModuleHINSTANCE() { return (HINSTANCE)&__ImageBase; }
|
||||
|
||||
HMODULE OsLibrary::loadDependency(const std::string &dependencyFileName) const {
|
||||
char dllPath[MAX_PATH];
|
||||
DWORD length = getModuleFileNameA(GetModuleHINSTANCE(), dllPath, MAX_PATH);
|
||||
for (DWORD idx = length; idx > 0; idx--) {
|
||||
if (dllPath[idx - 1] == '\\') {
|
||||
dllPath[idx] = '\0';
|
||||
break;
|
||||
}
|
||||
}
|
||||
strcat_s(dllPath, MAX_PATH, dependencyFileName.c_str());
|
||||
|
||||
return loadLibraryExA(dllPath, NULL, 0);
|
||||
}
|
||||
|
||||
OsLibrary::OsLibrary(const std::string &name) {
|
||||
if (name.empty()) {
|
||||
this->handle = GetModuleHandleA(nullptr);
|
||||
} else {
|
||||
this->handle = loadDependency(name);
|
||||
if (this->handle == nullptr) {
|
||||
this->handle = ::LoadLibraryA(name.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
OsLibrary::~OsLibrary() {
|
||||
if ((this->handle != nullptr) && (this->handle != GetModuleHandleA(nullptr))) {
|
||||
::FreeLibrary(this->handle);
|
||||
this->handle = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
bool OsLibrary::isLoaded() {
|
||||
return this->handle != nullptr;
|
||||
}
|
||||
|
||||
void *OsLibrary::getProcAddress(const std::string &procName) {
|
||||
return ::GetProcAddress(this->handle, procName.c_str());
|
||||
}
|
||||
} // namespace Windows
|
||||
} // namespace NEO
|
||||
35
shared/source/os_interface/windows/os_library_win.h
Normal file
35
shared/source/os_interface/windows/os_library_win.h
Normal file
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "os_interface/os_library.h"
|
||||
|
||||
#define UMDF_USING_NTSTATUS
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
namespace NEO {
|
||||
namespace Windows {
|
||||
|
||||
class OsLibrary : public NEO::OsLibrary {
|
||||
private:
|
||||
HMODULE handle;
|
||||
|
||||
public:
|
||||
OsLibrary(const std::string &name);
|
||||
~OsLibrary();
|
||||
|
||||
bool isLoaded();
|
||||
void *getProcAddress(const std::string &procName);
|
||||
|
||||
protected:
|
||||
HMODULE loadDependency(const std::string &dependencyFileName) const;
|
||||
|
||||
static decltype(&LoadLibraryExA) loadLibraryExA;
|
||||
static decltype(&GetModuleFileNameA) getModuleFileNameA;
|
||||
};
|
||||
} // namespace Windows
|
||||
} // namespace NEO
|
||||
32
shared/source/os_interface/windows/os_memory_win.cpp
Normal file
32
shared/source/os_interface/windows/os_memory_win.cpp
Normal file
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/os_memory_win.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
std::unique_ptr<OSMemory> OSMemory::create() {
|
||||
return std::make_unique<OSMemoryWindows>();
|
||||
}
|
||||
|
||||
void *OSMemoryWindows::reserveCpuAddressRange(size_t sizeToReserve) {
|
||||
return virtualAllocWrapper(0, sizeToReserve, MEM_RESERVE, PAGE_READWRITE);
|
||||
}
|
||||
|
||||
void OSMemoryWindows::releaseCpuAddressRange(void *reservedCpuAddressRange, size_t /* reservedSize */) {
|
||||
virtualFreeWrapper(reservedCpuAddressRange, 0, MEM_RELEASE);
|
||||
}
|
||||
|
||||
LPVOID OSMemoryWindows::virtualAllocWrapper(LPVOID lpAddress, SIZE_T dwSize, DWORD flAllocationType, DWORD flProtect) {
|
||||
return VirtualAlloc(lpAddress, dwSize, flAllocationType, flProtect);
|
||||
}
|
||||
|
||||
BOOL OSMemoryWindows::virtualFreeWrapper(LPVOID lpAddress, SIZE_T dwSize, DWORD dwFreeType) {
|
||||
return VirtualFree(lpAddress, dwSize, dwFreeType);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
25
shared/source/os_interface/windows/os_memory_win.h
Normal file
25
shared/source/os_interface/windows/os_memory_win.h
Normal file
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/os_memory.h"
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class OSMemoryWindows : public OSMemory {
|
||||
public:
|
||||
OSMemoryWindows() = default;
|
||||
void *reserveCpuAddressRange(size_t sizeToReserve) override;
|
||||
void releaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) override;
|
||||
|
||||
protected:
|
||||
MOCKABLE_VIRTUAL LPVOID virtualAllocWrapper(LPVOID, SIZE_T, DWORD, DWORD);
|
||||
MOCKABLE_VIRTUAL BOOL virtualFreeWrapper(LPVOID, SIZE_T, DWORD);
|
||||
};
|
||||
|
||||
}; // namespace NEO
|
||||
9
shared/source/os_interface/windows/os_socket.h
Normal file
9
shared/source/os_interface/windows/os_socket.h
Normal file
@@ -0,0 +1,9 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <winsock.h>
|
||||
21
shared/source/os_interface/windows/os_thread_win.cpp
Normal file
21
shared/source/os_interface/windows/os_thread_win.cpp
Normal file
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/os_thread_win.h"
|
||||
namespace NEO {
|
||||
ThreadWin::ThreadWin(std::thread *thread) {
|
||||
this->thread.reset(thread);
|
||||
};
|
||||
|
||||
std::unique_ptr<Thread> Thread::create(void *(*func)(void *), void *arg) {
|
||||
return std::unique_ptr<Thread>(new ThreadWin(new std::thread(func, arg)));
|
||||
}
|
||||
|
||||
void ThreadWin::join() {
|
||||
thread->join();
|
||||
}
|
||||
} // namespace NEO
|
||||
21
shared/source/os_interface/windows/os_thread_win.h
Normal file
21
shared/source/os_interface/windows/os_thread_win.h
Normal file
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/os_thread.h"
|
||||
|
||||
#include <thread>
|
||||
|
||||
namespace NEO {
|
||||
class ThreadWin : public Thread {
|
||||
public:
|
||||
ThreadWin(std::thread *thread);
|
||||
void join() override;
|
||||
|
||||
protected:
|
||||
std::unique_ptr<std::thread> thread;
|
||||
};
|
||||
} // namespace NEO
|
||||
112
shared/source/os_interface/windows/os_time_win.cpp
Normal file
112
shared/source/os_interface/windows/os_time_win.cpp
Normal file
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/os_time_win.h"
|
||||
|
||||
#include "os_interface/windows/os_interface.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#undef WIN32_NO_STATUS
|
||||
|
||||
namespace NEO {
|
||||
|
||||
bool runEscape(Wddm *wddm, TimeStampDataHeader &escapeInfo) {
|
||||
if (wddm) {
|
||||
D3DKMT_ESCAPE escapeCommand = {0};
|
||||
|
||||
GTDIGetGpuCpuTimestampsIn in = {GTDI_FNC_GET_GPU_CPU_TIMESTAMPS};
|
||||
uint32_t outSize = sizeof(GTDIGetGpuCpuTimestampsOut);
|
||||
|
||||
escapeInfo.m_Header.EscapeCode = GFX_ESCAPE_IGPA_INSTRUMENTATION_CONTROL;
|
||||
escapeInfo.m_Header.Size = outSize;
|
||||
escapeInfo.m_Data.m_In = in;
|
||||
|
||||
escapeCommand.Flags.Value = 0;
|
||||
escapeCommand.hAdapter = (D3DKMT_HANDLE)0;
|
||||
escapeCommand.hContext = (D3DKMT_HANDLE)0; // escape is not context specific
|
||||
escapeCommand.hDevice = (D3DKMT_HANDLE)wddm->getDevice(); // escape not device specific, passing only for instrumentation
|
||||
escapeCommand.pPrivateDriverData = &escapeInfo;
|
||||
escapeCommand.PrivateDriverDataSize = sizeof(escapeInfo);
|
||||
escapeCommand.Type = D3DKMT_ESCAPE_DRIVERPRIVATE;
|
||||
|
||||
auto status = wddm->escape(escapeCommand);
|
||||
|
||||
if (status == STATUS_SUCCESS) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool OSTimeWin::getCpuGpuTime(TimeStampData *pGpuCpuTime) {
|
||||
bool retVal = false;
|
||||
|
||||
pGpuCpuTime->CPUTimeinNS = 0;
|
||||
pGpuCpuTime->GPUTimeStamp = 0;
|
||||
|
||||
TimeStampDataHeader escapeInfo = {0};
|
||||
|
||||
if (runEscape(wddm, escapeInfo)) {
|
||||
double cpuNanoseconds = escapeInfo.m_Data.m_Out.cpuPerfTicks *
|
||||
(1000000000.0 / escapeInfo.m_Data.m_Out.cpuPerfFreq);
|
||||
|
||||
pGpuCpuTime->CPUTimeinNS = (unsigned long long)cpuNanoseconds;
|
||||
pGpuCpuTime->GPUTimeStamp = (unsigned long long)escapeInfo.m_Data.m_Out.gpuPerfTicks;
|
||||
retVal = true;
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
bool OSTimeWin::getCpuTime(uint64_t *timeStamp) {
|
||||
uint64_t time;
|
||||
this->QueryPerfomanceCounterFnc((LARGE_INTEGER *)&time);
|
||||
|
||||
*timeStamp = static_cast<uint64_t>((static_cast<double>(time) * NSEC_PER_SEC / frequency.QuadPart));
|
||||
return true;
|
||||
};
|
||||
|
||||
std::unique_ptr<OSTime> OSTime::create(OSInterface *osInterface) {
|
||||
return std::unique_ptr<OSTime>(new OSTimeWin(osInterface));
|
||||
}
|
||||
|
||||
OSTimeWin::OSTimeWin(OSInterface *osInterface) {
|
||||
this->osInterface = osInterface;
|
||||
if (osInterface) {
|
||||
wddm = osInterface->get()->getWddm();
|
||||
}
|
||||
QueryPerformanceFrequency(&frequency);
|
||||
}
|
||||
|
||||
double OSTimeWin::getHostTimerResolution() const {
|
||||
double retValue = 0;
|
||||
if (frequency.QuadPart) {
|
||||
retValue = 1e9 / frequency.QuadPart;
|
||||
}
|
||||
return retValue;
|
||||
}
|
||||
|
||||
double OSTimeWin::getDynamicDeviceTimerResolution(HardwareInfo const &hwInfo) const {
|
||||
double retVal = 0;
|
||||
TimeStampDataHeader escapeInfo = {0};
|
||||
|
||||
if (runEscape(wddm, escapeInfo)) {
|
||||
retVal = 1000000000.0 / (double)escapeInfo.m_Data.m_Out.gpuPerfFreq;
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
uint64_t OSTimeWin::getCpuRawTimestamp() {
|
||||
LARGE_INTEGER cpuRawTimestamp = {};
|
||||
this->QueryPerfomanceCounterFnc(&cpuRawTimestamp);
|
||||
return cpuRawTimestamp.QuadPart;
|
||||
}
|
||||
} // namespace NEO
|
||||
75
shared/source/os_interface/windows/os_time_win.h
Normal file
75
shared/source/os_interface/windows/os_time_win.h
Normal file
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "os_interface/os_time.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include "gfxEscape.h"
|
||||
|
||||
namespace NEO {
|
||||
class Wddm;
|
||||
|
||||
class OSTimeWin : public OSTime {
|
||||
|
||||
public:
|
||||
OSTimeWin(OSInterface *osInterface);
|
||||
bool getCpuTime(uint64_t *timeStamp) override;
|
||||
bool getCpuGpuTime(TimeStampData *pGpuCpuTime) override;
|
||||
double getHostTimerResolution() const override;
|
||||
double getDynamicDeviceTimerResolution(HardwareInfo const &hwInfo) const override;
|
||||
uint64_t getCpuRawTimestamp() override;
|
||||
|
||||
protected:
|
||||
Wddm *wddm = nullptr;
|
||||
LARGE_INTEGER frequency;
|
||||
OSTimeWin() = default;
|
||||
decltype(&QueryPerformanceCounter) QueryPerfomanceCounterFnc = QueryPerformanceCounter;
|
||||
};
|
||||
|
||||
typedef enum GTDI_ESCAPE_FUNCTION_ENUM {
|
||||
GTDI_FNC_GET_GPU_CPU_TIMESTAMPS = 25
|
||||
} GTDI_ESCAPE_FUNCTION;
|
||||
|
||||
typedef struct GTDIBaseInStruct {
|
||||
GTDI_ESCAPE_FUNCTION Function;
|
||||
} GTDIHeaderIn;
|
||||
|
||||
typedef GTDIHeaderIn GTDIGetGpuCpuTimestampsIn;
|
||||
|
||||
typedef enum GTDI_RETURN_CODE_ENUM {
|
||||
GTDI_RET_OK = 0,
|
||||
GTDI_RET_FAILED,
|
||||
GTDI_RET_NOT_CONNECTED,
|
||||
GTDI_RET_HW_METRICS_NOT_ENABLED,
|
||||
GTDI_RET_CONTEXT_ID_MISMATCH,
|
||||
GTDI_RET_NOT_SUPPORTED,
|
||||
GTDI_RET_PENDING,
|
||||
GTDI_RET_INVALID_CONFIGURATION,
|
||||
GTDI_RET_CONCURRENT_API_ENABLED,
|
||||
GTDI_RET_NO_INFORMATION, // for GTDI_FNC_GET_ERROR_INFO escape only
|
||||
// ...
|
||||
GTDI_RET_MAX = 0xFFFFFFFF
|
||||
} GTDI_RETURN_CODE;
|
||||
|
||||
typedef struct GTDIGetGpuCpuTimestampsOutStruct {
|
||||
GTDI_RETURN_CODE RetCode; // Result of the call
|
||||
uint64_t gpuPerfTicks; // in GPU_timestamp_ticks
|
||||
uint64_t cpuPerfTicks; // in CPU_timestamp_ticks
|
||||
uint64_t gpuPerfFreq; // in GPU_timestamp_ticks/s
|
||||
uint64_t cpuPerfFreq; // in CPU_timestamp_ticks/s
|
||||
} GTDIGetGpuCpuTimestampsOut;
|
||||
|
||||
struct TimeStampDataHeader {
|
||||
GFX_ESCAPE_HEADER_T m_Header;
|
||||
union {
|
||||
GTDIGetGpuCpuTimestampsIn m_In;
|
||||
GTDIGetGpuCpuTimestampsOut m_Out;
|
||||
} m_Data;
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "gmm_helper/gmm_helper.h"
|
||||
#include "gmm_helper/page_table_mngr.h"
|
||||
#include "opencl/source/platform/platform.h"
|
||||
|
||||
#include "gmm_client_context.h"
|
||||
|
||||
namespace NEO {
|
||||
GmmPageTableMngr::GmmPageTableMngr(GmmClientContext *gmmClientContext, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb) : clientContext(gmmClientContext->getHandle()) {
|
||||
pageTableManager = clientContext->CreatePageTblMgrObject(translationTableCb, translationTableFlags);
|
||||
}
|
||||
|
||||
void GmmPageTableMngr::setCsrHandle(void *csrHandle) {
|
||||
pageTableManager->GmmSetCsrHandle(csrHandle);
|
||||
}
|
||||
} // namespace NEO
|
||||
81
shared/source/os_interface/windows/print.cpp
Normal file
81
shared/source/os_interface/windows/print.cpp
Normal file
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/print.h"
|
||||
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <cstdint>
|
||||
#include <fcntl.h>
|
||||
#include <io.h>
|
||||
#include <iostream>
|
||||
|
||||
void printToSTDOUT(const char *str) {
|
||||
int fd = 0;
|
||||
HANDLE stdoutDuplicate = 0;
|
||||
FILE *pFile = nullptr;
|
||||
|
||||
if ((DuplicateHandle(GetCurrentProcess(), GetStdHandle(STD_OUTPUT_HANDLE),
|
||||
GetCurrentProcess(), &stdoutDuplicate, 0L, TRUE, DUPLICATE_SAME_ACCESS))) {
|
||||
if ((fd = _open_osfhandle((DWORD_PTR)stdoutDuplicate, _O_TEXT)) &&
|
||||
(pFile = _fdopen(fd, "w"))) {
|
||||
fprintf_s(pFile, "%s", str);
|
||||
|
||||
fflush(pFile);
|
||||
fclose(pFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
size_t simple_sprintf(char *output, size_t outputSize, const char *format, T value) {
|
||||
#if (_MSC_VER == 1800)
|
||||
_set_output_format(_TWO_DIGIT_EXPONENT);
|
||||
#endif
|
||||
size_t len = strlen(format);
|
||||
if (len == 0) {
|
||||
output[0] = '\0';
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (len > 3 && *(format + len - 2) == 'h' && *(format + len - 3) == 'h') {
|
||||
if (*(format + len - 1) == 'i' || *(format + len - 1) == 'd') {
|
||||
int32_t fixedValue = (char)value;
|
||||
return sprintf_s(output, outputSize, format, fixedValue);
|
||||
} else {
|
||||
uint32_t fixedValue = (unsigned char)value;
|
||||
return sprintf_s(output, outputSize, format, fixedValue);
|
||||
}
|
||||
} else if (format[len - 1] == 'F') {
|
||||
std::string formatCopy = format;
|
||||
*formatCopy.rbegin() = 'f';
|
||||
|
||||
size_t returnValue = sprintf_s(output, outputSize, formatCopy.c_str(), value);
|
||||
for (size_t i = 0; i < returnValue; i++)
|
||||
output[i] = std::toupper(output[i]);
|
||||
return returnValue;
|
||||
} else {
|
||||
return sprintf_s(output, outputSize, format, value);
|
||||
}
|
||||
}
|
||||
|
||||
size_t simple_sprintf(char *output, size_t outputSize, const char *format, const char *value) {
|
||||
return sprintf_s(output, outputSize, format, value);
|
||||
}
|
||||
|
||||
template size_t simple_sprintf<float>(char *output, size_t output_size, const char *format, float value);
|
||||
template size_t simple_sprintf<double>(char *output, size_t output_size, const char *format, double value);
|
||||
template size_t simple_sprintf<char>(char *output, size_t output_size, const char *format, char value);
|
||||
template size_t simple_sprintf<int8_t>(char *output, size_t output_size, const char *format, int8_t value);
|
||||
template size_t simple_sprintf<int16_t>(char *output, size_t output_size, const char *format, int16_t value);
|
||||
template size_t simple_sprintf<int32_t>(char *output, size_t output_size, const char *format, int32_t value);
|
||||
template size_t simple_sprintf<int64_t>(char *output, size_t output_size, const char *format, int64_t value);
|
||||
template size_t simple_sprintf<uint8_t>(char *output, size_t output_size, const char *format, uint8_t value);
|
||||
template size_t simple_sprintf<uint16_t>(char *output, size_t output_size, const char *format, uint16_t value);
|
||||
template size_t simple_sprintf<uint32_t>(char *output, size_t output_size, const char *format, uint32_t value);
|
||||
template size_t simple_sprintf<uint64_t>(char *output, size_t output_size, const char *format, uint64_t value);
|
||||
33
shared/source/os_interface/windows/sys_calls.cpp
Normal file
33
shared/source/os_interface/windows/sys_calls.cpp
Normal file
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/sys_calls.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
namespace SysCalls {
|
||||
|
||||
HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState, LPCSTR lpName) {
|
||||
return CreateEventA(lpEventAttributes, bManualReset, bInitialState, lpName);
|
||||
}
|
||||
|
||||
BOOL closeHandle(HANDLE hObject) {
|
||||
return CloseHandle(hObject);
|
||||
}
|
||||
|
||||
BOOL getSystemPowerStatus(LPSYSTEM_POWER_STATUS systemPowerStatusPtr) {
|
||||
return GetSystemPowerStatus(systemPowerStatusPtr);
|
||||
}
|
||||
BOOL getModuleHandle(DWORD dwFlags, LPCWSTR lpModuleName, HMODULE *phModule) {
|
||||
return GetModuleHandleEx(dwFlags, lpModuleName, phModule);
|
||||
}
|
||||
DWORD getModuleFileName(HMODULE hModule, LPWSTR lpFilename, DWORD nSize) {
|
||||
return GetModuleFileName(hModule, lpFilename, nSize);
|
||||
}
|
||||
} // namespace SysCalls
|
||||
|
||||
} // namespace NEO
|
||||
23
shared/source/os_interface/windows/sys_calls.h
Normal file
23
shared/source/os_interface/windows/sys_calls.h
Normal file
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <Windows.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
namespace SysCalls {
|
||||
|
||||
HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState, LPCSTR lpName);
|
||||
BOOL closeHandle(HANDLE hObject);
|
||||
BOOL getSystemPowerStatus(LPSYSTEM_POWER_STATUS systemPowerStatusPtr);
|
||||
BOOL getModuleHandle(DWORD dwFlags, LPCWSTR lpModuleName, HMODULE *phModule);
|
||||
DWORD getModuleFileName(HMODULE hModule, LPWSTR lpFilename, DWORD nSize);
|
||||
|
||||
} // namespace SysCalls
|
||||
|
||||
} // namespace NEO
|
||||
110
shared/source/os_interface/windows/thk_wrapper.h
Normal file
110
shared/source/os_interface/windows/thk_wrapper.h
Normal file
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "helpers/options.h"
|
||||
#include "utilities/api_intercept.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
namespace NEO {
|
||||
template <typename Param>
|
||||
class ThkWrapper {
|
||||
typedef NTSTATUS(APIENTRY *Func)(Param);
|
||||
|
||||
public:
|
||||
Func mFunc = nullptr;
|
||||
|
||||
inline NTSTATUS operator()(Param param) const {
|
||||
if (KMD_PROFILING) {
|
||||
SYSTEM_ENTER()
|
||||
NTSTATUS Status;
|
||||
Status = mFunc(param);
|
||||
SYSTEM_LEAVE(getId<Param>());
|
||||
return Status;
|
||||
} else {
|
||||
return mFunc(param);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
inline T &operator=(T func) {
|
||||
return mFunc = func;
|
||||
}
|
||||
|
||||
// This operator overload is for implicit casting ThkWrapper struct to Function Pointer in GetPfn methods like GetEscapePfn() or for comparing against NULL function pointer
|
||||
operator Func() const {
|
||||
return mFunc;
|
||||
}
|
||||
|
||||
private:
|
||||
// Default template for GetID( ) for Thk function, causing compilation error !!
|
||||
// Returns ID for specific ThkWrapper type
|
||||
template <class Param>
|
||||
unsigned int getId() const {
|
||||
static_assert(0, "Template specialization for GetID is required for each new THKWrapper");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Template specializations of GetID(), required for every new Thk function
|
||||
#define GET_ID(TYPE, VALUE) \
|
||||
template <> \
|
||||
unsigned int getId<TYPE>() const { \
|
||||
return VALUE; \
|
||||
}
|
||||
|
||||
GET_ID(D3DKMT_OPENADAPTERFROMHDC *, SYSTIMER_ID_OPENADAPTERFROMHDC)
|
||||
GET_ID(D3DKMT_OPENADAPTERFROMLUID *, SYSTIMER_ID_OPENADAPTERFROMLUID)
|
||||
GET_ID(CONST D3DKMT_CLOSEADAPTER *, SYSTIMER_ID_CLOSEADAPTER)
|
||||
GET_ID(CONST D3DKMT_QUERYADAPTERINFO *, SYSTIMER_ID_QUERYADAPTERINFO)
|
||||
GET_ID(CONST D3DKMT_ESCAPE *, SYSTIMER_ID_ESCAPE)
|
||||
GET_ID(D3DKMT_CREATEDEVICE *, SYSTIMER_ID_CREATEDEVICE)
|
||||
GET_ID(CONST D3DKMT_DESTROYDEVICE *, SYSTIMER_ID_DESTROYDEVICE)
|
||||
GET_ID(D3DKMT_CREATECONTEXT *, SYSTIMER_ID_CREATECONTEXT)
|
||||
GET_ID(CONST D3DKMT_DESTROYCONTEXT *, SYSTIMER_ID_DESTROYCONTEXT)
|
||||
GET_ID(D3DKMT_CREATEALLOCATION *, SYSTIMER_ID_CREATEALLOCATION)
|
||||
GET_ID(CONST D3DKMT_DESTROYALLOCATION *, SYSTIMER_ID_DESTROYALLOCATION)
|
||||
GET_ID(D3DKMT_OPENRESOURCE *, SYSTIMER_ID_OPENRESOURCE)
|
||||
GET_ID(D3DKMT_QUERYRESOURCEINFO *, SYSTIMER_ID_QUERYRESOURCEINFO)
|
||||
GET_ID(D3DKMT_LOCK *, SYSTIMER_ID_LOCK)
|
||||
GET_ID(CONST D3DKMT_UNLOCK *, SYSTIMER_ID_UNLOCK)
|
||||
GET_ID(D3DKMT_RENDER *, SYSTIMER_ID_RENDER)
|
||||
GET_ID(D3DKMT_CREATESYNCHRONIZATIONOBJECT *, SYSTIMER_ID_CREATESYNCHRONIZATIONOBJECT)
|
||||
GET_ID(CONST D3DKMT_DESTROYSYNCHRONIZATIONOBJECT *, SYSTIMER_ID_DESTROYSYNCHRONIZATIONOBJECT)
|
||||
GET_ID(CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECT *, SYSTIMER_ID_SIGNALSYNCHRONIZATIONOBJECT)
|
||||
GET_ID(CONST_FROM_WDK_10_0_18328_0 D3DKMT_WAITFORSYNCHRONIZATIONOBJECT *, SYSTIMER_ID_WAITFORSYNCHRONIZATIONOBJECT)
|
||||
GET_ID(D3DKMT_CREATESYNCHRONIZATIONOBJECT2 *, SYSTIMER_ID_CREATESYNCHRONIZATIONOBJECT2)
|
||||
GET_ID(D3DKMT_GETDEVICESTATE *, SYSTIMER_ID_GETDEVICESTATE)
|
||||
GET_ID(D3DDDI_MAKERESIDENT *, SYSTIMER_ID_MAKERESIDENT)
|
||||
GET_ID(D3DKMT_EVICT *, SYSTIMER_ID_EVICT)
|
||||
GET_ID(CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *, SYSTIMER_ID_WAITFORSYNCHRONIZATIONOBJECTFROMCPU)
|
||||
GET_ID(CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU *, SYSTIMER_ID_SIGNALSYNCHRONIZATIONOBJECTFROMCPU)
|
||||
GET_ID(CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU *, SYSTIMER_ID_WAITFORSYNCHRONIZATIONOBJECTFROMGPU)
|
||||
GET_ID(CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU *, SYSTIMER_ID_SIGNALSYNCHRONIZATIONOBJECTFROMGPU)
|
||||
GET_ID(D3DKMT_CREATEPAGINGQUEUE *, SYSTIMER_ID_CREATEPAGINGQUEUE)
|
||||
GET_ID(D3DDDI_DESTROYPAGINGQUEUE *, SYSTIMER_ID_D3DDDI_DESTROYPAGINGQUEUE)
|
||||
GET_ID(D3DKMT_LOCK2 *, SYSTIMER_ID_LOCK2)
|
||||
GET_ID(CONST D3DKMT_UNLOCK2 *, SYSTIMER_ID_UNLOCK2)
|
||||
GET_ID(CONST D3DKMT_INVALIDATECACHE *, SYSTIMER_ID_INVALIDATECACHE)
|
||||
GET_ID(D3DDDI_MAPGPUVIRTUALADDRESS *, SYSTIMER_ID_D3DDDI_MAPGPUVIRTUALADDRESS)
|
||||
GET_ID(D3DDDI_RESERVEGPUVIRTUALADDRESS *, SYSTIMER_ID_D3DDDI_RESERVEGPUVIRTUALADDRESS)
|
||||
GET_ID(CONST D3DKMT_FREEGPUVIRTUALADDRESS *, SYSTIMER_ID_FREEGPUVIRTUALADDRESS)
|
||||
GET_ID(CONST D3DKMT_UPDATEGPUVIRTUALADDRESS *, SYSTIMER_ID_UPDATEGPUVIRTUALADDRESS)
|
||||
GET_ID(D3DKMT_CREATECONTEXTVIRTUAL *, SYSTIMER_ID_CREATECONTEXTVIRTUAL)
|
||||
GET_ID(CONST D3DKMT_SUBMITCOMMAND *, SYSTIMER_ID_SUBMITCOMMAND)
|
||||
GET_ID(D3DKMT_OPENSYNCOBJECTFROMNTHANDLE2 *, SYSTIMER_ID_OPENSYNCOBJECTFROMNTHANDLE2)
|
||||
GET_ID(D3DKMT_OPENSYNCOBJECTNTHANDLEFROMNAME *, SYSTIMER_ID_OPENSYNCOBJECTNTHANDLEFROMNAME)
|
||||
GET_ID(CONST D3DKMT_DESTROYALLOCATION2 *, SYSTIMER_ID_DESTROYALLOCATION2)
|
||||
GET_ID(D3DKMT_REGISTERTRIMNOTIFICATION *, SYSTIMER_ID_REGISTERTRIMNOTIFICATION)
|
||||
GET_ID(D3DKMT_UNREGISTERTRIMNOTIFICATION *, SYSTIMER_ID_UNREGISTERTRIMNOTIFICATION)
|
||||
GET_ID(D3DKMT_OPENRESOURCEFROMNTHANDLE *, SYSTIMER_ID_OPENRESOURCEFROMNTHANDLE)
|
||||
GET_ID(D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *, SYSTIMER_ID_QUERYRESOURCEINFOFROMNTHANDLE)
|
||||
GET_ID(D3DKMT_CREATEHWQUEUE *, SYSTIMER_ID_CREATEHWQUEUE)
|
||||
GET_ID(CONST D3DKMT_DESTROYHWQUEUE *, SYSTIMER_ID_DESTROYHWQUEUE)
|
||||
GET_ID(CONST D3DKMT_SUBMITCOMMANDTOHWQUEUE *, SYSTIMER_ID_SUBMITCOMMANDTOHWQUEUE)
|
||||
};
|
||||
} // namespace NEO
|
||||
1021
shared/source/os_interface/windows/wddm/wddm.cpp
Normal file
1021
shared/source/os_interface/windows/wddm/wddm.cpp
Normal file
File diff suppressed because it is too large
Load Diff
196
shared/source/os_interface/windows/wddm/wddm.h
Normal file
196
shared/source/os_interface/windows/wddm/wddm.h
Normal file
@@ -0,0 +1,196 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "command_stream/preemption_mode.h"
|
||||
#include "gmm_helper/gmm_lib.h"
|
||||
#include "helpers/debug_helpers.h"
|
||||
#include "memory_manager/gfx_partition.h"
|
||||
#include "os_interface/os_context.h"
|
||||
#include "os_interface/windows/hw_device_id.h"
|
||||
#include "os_interface/windows/wddm/wddm_defs.h"
|
||||
#include "utilities/spinlock.h"
|
||||
|
||||
#include "sku_info.h"
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace NEO {
|
||||
class Gdi;
|
||||
class Gmm;
|
||||
class GmmMemory;
|
||||
class OsContextWin;
|
||||
class SettingsReader;
|
||||
class WddmAllocation;
|
||||
class WddmInterface;
|
||||
class WddmResidencyController;
|
||||
class WddmResidentAllocationsContainer;
|
||||
class HwDeviceId;
|
||||
|
||||
struct AllocationStorageData;
|
||||
struct HardwareInfo;
|
||||
struct KmDafListener;
|
||||
struct RootDeviceEnvironment;
|
||||
struct MonitoredFence;
|
||||
struct OsHandleStorage;
|
||||
|
||||
enum class HeapIndex : uint32_t;
|
||||
|
||||
class Wddm {
|
||||
public:
|
||||
typedef HRESULT(WINAPI *CreateDXGIFactoryFcn)(REFIID riid, void **ppFactory);
|
||||
typedef void(WINAPI *GetSystemInfoFcn)(SYSTEM_INFO *pSystemInfo);
|
||||
typedef BOOL(WINAPI *VirtualFreeFcn)(LPVOID ptr, SIZE_T size, DWORD flags);
|
||||
typedef LPVOID(WINAPI *VirtualAllocFcn)(LPVOID inPtr, SIZE_T size, DWORD flags, DWORD type);
|
||||
|
||||
virtual ~Wddm();
|
||||
|
||||
static Wddm *createWddm(std::unique_ptr<HwDeviceId> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment);
|
||||
bool init();
|
||||
|
||||
MOCKABLE_VIRTUAL bool evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim);
|
||||
MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim);
|
||||
MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr);
|
||||
bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData);
|
||||
MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size);
|
||||
MOCKABLE_VIRTUAL bool createContext(OsContextWin &osContext);
|
||||
MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext);
|
||||
MOCKABLE_VIRTUAL bool freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
|
||||
MOCKABLE_VIRTUAL NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, uint32_t shareable);
|
||||
MOCKABLE_VIRTUAL bool createAllocation64k(const Gmm *gmm, D3DKMT_HANDLE &outHandle);
|
||||
MOCKABLE_VIRTUAL NTSTATUS createAllocationsAndMapGpuVa(OsHandleStorage &osHandles);
|
||||
MOCKABLE_VIRTUAL bool destroyAllocations(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
|
||||
MOCKABLE_VIRTUAL bool openSharedHandle(D3DKMT_HANDLE handle, WddmAllocation *alloc);
|
||||
bool openNTHandle(HANDLE handle, WddmAllocation *alloc);
|
||||
MOCKABLE_VIRTUAL void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock);
|
||||
MOCKABLE_VIRTUAL void unlockResource(const D3DKMT_HANDLE &handle);
|
||||
MOCKABLE_VIRTUAL void kmDafLock(D3DKMT_HANDLE handle);
|
||||
MOCKABLE_VIRTUAL bool isKmDafEnabled() const { return featureTable->ftrKmdDaf; }
|
||||
|
||||
MOCKABLE_VIRTUAL bool destroyContext(D3DKMT_HANDLE context);
|
||||
MOCKABLE_VIRTUAL bool queryAdapterInfo();
|
||||
|
||||
MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments);
|
||||
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence);
|
||||
|
||||
NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand);
|
||||
MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController);
|
||||
void unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle);
|
||||
MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress);
|
||||
MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem);
|
||||
|
||||
MOCKABLE_VIRTUAL void *virtualAlloc(void *inPtr, size_t size, unsigned long flags, unsigned long type);
|
||||
MOCKABLE_VIRTUAL int virtualFree(void *ptr, size_t size, unsigned long flags);
|
||||
|
||||
bool configureDeviceAddressSpace();
|
||||
|
||||
GT_SYSTEM_INFO *getGtSysInfo() const {
|
||||
DEBUG_BREAK_IF(!gtSystemInfo);
|
||||
return gtSystemInfo.get();
|
||||
}
|
||||
|
||||
const GMM_GFX_PARTITIONING &getGfxPartition() const {
|
||||
return gfxPartition;
|
||||
}
|
||||
|
||||
void initGfxPartition(GfxPartition &outGfxPartition, uint32_t rootDeviceIndex, size_t numRootDevices) const;
|
||||
|
||||
const std::string &getDeviceRegistryPath() const {
|
||||
return deviceRegistryPath;
|
||||
}
|
||||
|
||||
uint64_t getSystemSharedMemory() const;
|
||||
uint64_t getDedicatedVideoMemory() const;
|
||||
|
||||
uint64_t getMaxApplicationAddress() const;
|
||||
|
||||
inline D3DKMT_HANDLE getAdapter() const { return hwDeviceId->getAdapter(); }
|
||||
D3DKMT_HANDLE getDevice() const { return device; }
|
||||
D3DKMT_HANDLE getPagingQueue() const { return pagingQueue; }
|
||||
D3DKMT_HANDLE getPagingQueueSyncObject() const { return pagingQueueSyncObject; }
|
||||
inline Gdi *getGdi() const { return hwDeviceId->getGdi(); }
|
||||
|
||||
PFND3DKMT_ESCAPE getEscapeHandle() const;
|
||||
|
||||
uint32_t getHwContextId() const {
|
||||
return static_cast<uint32_t>(hwContextId);
|
||||
}
|
||||
|
||||
std::unique_ptr<SettingsReader> registryReader;
|
||||
|
||||
uintptr_t getWddmMinAddress() const {
|
||||
return this->minAddress;
|
||||
}
|
||||
WddmInterface *getWddmInterface() const {
|
||||
return wddmInterface.get();
|
||||
}
|
||||
|
||||
unsigned int readEnablePreemptionRegKey();
|
||||
MOCKABLE_VIRTUAL uint64_t *getPagingFenceAddress() {
|
||||
return pagingFenceAddress;
|
||||
}
|
||||
WddmResidentAllocationsContainer *getTemporaryResourcesContainer() {
|
||||
return temporaryResources.get();
|
||||
}
|
||||
void updatePagingFenceValue(uint64_t newPagingFenceValue);
|
||||
GmmMemory *getGmmMemory() const {
|
||||
return gmmMemory.get();
|
||||
}
|
||||
MOCKABLE_VIRTUAL void waitOnPagingFenceFromCpu();
|
||||
|
||||
void setGmmInputArg(void *args);
|
||||
|
||||
WddmVersion getWddmVersion();
|
||||
static CreateDXGIFactoryFcn createDxgiFactory;
|
||||
|
||||
protected:
|
||||
std::unique_ptr<HwDeviceId> hwDeviceId;
|
||||
D3DKMT_HANDLE device = 0;
|
||||
D3DKMT_HANDLE pagingQueue = 0;
|
||||
D3DKMT_HANDLE pagingQueueSyncObject = 0;
|
||||
|
||||
uint64_t *pagingFenceAddress = nullptr;
|
||||
std::atomic<std::uint64_t> currentPagingFenceValue{0};
|
||||
|
||||
// Adapter information
|
||||
std::unique_ptr<PLATFORM> gfxPlatform;
|
||||
std::unique_ptr<GT_SYSTEM_INFO> gtSystemInfo;
|
||||
std::unique_ptr<FeatureTable> featureTable;
|
||||
std::unique_ptr<WorkaroundTable> workaroundTable;
|
||||
GMM_GFX_PARTITIONING gfxPartition{};
|
||||
ADAPTER_BDF adapterBDF{};
|
||||
uint64_t systemSharedMemory = 0;
|
||||
uint64_t dedicatedVideoMemory = 0;
|
||||
uint32_t maxRenderFrequency = 0;
|
||||
bool instrumentationEnabled = false;
|
||||
std::string deviceRegistryPath;
|
||||
RootDeviceEnvironment &rootDeviceEnvironment;
|
||||
|
||||
unsigned long hwContextId = 0;
|
||||
uintptr_t maximumApplicationAddress = 0;
|
||||
std::unique_ptr<GmmMemory> gmmMemory;
|
||||
uintptr_t minAddress = 0;
|
||||
|
||||
Wddm(std::unique_ptr<HwDeviceId> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment);
|
||||
MOCKABLE_VIRTUAL bool waitOnGPU(D3DKMT_HANDLE context);
|
||||
bool createDevice(PreemptionMode preemptionMode);
|
||||
bool createPagingQueue();
|
||||
bool destroyPagingQueue();
|
||||
bool destroyDevice();
|
||||
void getDeviceState();
|
||||
void handleCompletion(OsContextWin &osContext);
|
||||
|
||||
static GetSystemInfoFcn getSystemInfo;
|
||||
static VirtualFreeFcn virtualFreeFnc;
|
||||
static VirtualAllocFcn virtualAllocFnc;
|
||||
|
||||
std::unique_ptr<KmDafListener> kmDafListener;
|
||||
std::unique_ptr<WddmInterface> wddmInterface;
|
||||
std::unique_ptr<WddmResidentAllocationsContainer> temporaryResources;
|
||||
};
|
||||
} // namespace NEO
|
||||
28
shared/source/os_interface/windows/wddm/wddm_calls.cpp
Normal file
28
shared/source/os_interface/windows/wddm/wddm_calls.cpp
Normal file
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
|
||||
#include <dxgi.h>
|
||||
|
||||
namespace NEO {
|
||||
Wddm::CreateDXGIFactoryFcn getCreateDxgiFactory() {
|
||||
return CreateDXGIFactory;
|
||||
}
|
||||
|
||||
Wddm::GetSystemInfoFcn getGetSystemInfo() {
|
||||
return GetSystemInfo;
|
||||
}
|
||||
|
||||
Wddm::VirtualFreeFcn getVirtualFree() {
|
||||
return VirtualFree;
|
||||
}
|
||||
|
||||
Wddm::VirtualAllocFcn getVirtualAlloc() {
|
||||
return VirtualAlloc;
|
||||
}
|
||||
} // namespace NEO
|
||||
14
shared/source/os_interface/windows/wddm/wddm_create.cpp
Normal file
14
shared/source/os_interface/windows/wddm/wddm_create.cpp
Normal file
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
Wddm *Wddm::createWddm(std::unique_ptr<HwDeviceId> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment) {
|
||||
return new Wddm(std::move(hwDeviceId), rootDeviceEnvironment);
|
||||
}
|
||||
} // namespace NEO
|
||||
25
shared/source/os_interface/windows/wddm/wddm_defs.h
Normal file
25
shared/source/os_interface/windows/wddm/wddm_defs.h
Normal file
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "os_interface/windows/windows_defs.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
struct WddmSubmitArguments {
|
||||
MonitoredFence *monitorFence;
|
||||
D3DKMT_HANDLE contextHandle;
|
||||
D3DKMT_HANDLE hwQueueHandle;
|
||||
};
|
||||
|
||||
enum class WddmVersion : uint32_t {
|
||||
WDDM_2_0 = 0,
|
||||
WDDM_2_3
|
||||
};
|
||||
} // namespace NEO
|
||||
147
shared/source/os_interface/windows/wddm/wddm_interface.cpp
Normal file
147
shared/source/os_interface/windows/wddm/wddm_interface.cpp
Normal file
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm/wddm_interface.h"
|
||||
|
||||
#include "memory_manager/memory_constants.h"
|
||||
#include "os_interface/windows/gdi_interface.h"
|
||||
#include "os_interface/windows/os_context_win.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
|
||||
using namespace NEO;
|
||||
|
||||
bool WddmInterface::createMonitoredFence(MonitoredFence &monitorFence) {
|
||||
NTSTATUS status = STATUS_SUCCESS;
|
||||
D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject = {0};
|
||||
CreateSynchronizationObject.hDevice = wddm.getDevice();
|
||||
CreateSynchronizationObject.Info.Type = D3DDDI_MONITORED_FENCE;
|
||||
CreateSynchronizationObject.Info.MonitoredFence.InitialFenceValue = 0;
|
||||
|
||||
status = wddm.getGdi()->createSynchronizationObject2(&CreateSynchronizationObject);
|
||||
DEBUG_BREAK_IF(STATUS_SUCCESS != status);
|
||||
|
||||
monitorFence.fenceHandle = CreateSynchronizationObject.hSyncObject;
|
||||
monitorFence.cpuAddress = reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress);
|
||||
monitorFence.gpuAddress = CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress;
|
||||
|
||||
return status == STATUS_SUCCESS;
|
||||
}
|
||||
void WddmInterface::destroyMonitorFence(D3DKMT_HANDLE fenceHandle) {
|
||||
NTSTATUS status = STATUS_SUCCESS;
|
||||
D3DKMT_DESTROYSYNCHRONIZATIONOBJECT destroySyncObject = {0};
|
||||
destroySyncObject.hSyncObject = fenceHandle;
|
||||
status = wddm.getGdi()->destroySynchronizationObject(&destroySyncObject);
|
||||
DEBUG_BREAK_IF(STATUS_SUCCESS != status);
|
||||
}
|
||||
|
||||
bool WddmInterface20::createHwQueue(OsContextWin &osContext) {
|
||||
return false;
|
||||
}
|
||||
void WddmInterface20::destroyHwQueue(D3DKMT_HANDLE hwQueue) {}
|
||||
|
||||
bool WddmInterface20::createMonitoredFence(OsContextWin &osContext) {
|
||||
auto &residencyController = osContext.getResidencyController();
|
||||
MonitoredFence &monitorFence = residencyController.getMonitoredFence();
|
||||
bool ret = WddmInterface::createMonitoredFence(monitorFence);
|
||||
|
||||
monitorFence.currentFenceValue = 1;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void WddmInterface20::destroyMonitorFence(MonitoredFence &monitorFence) {
|
||||
WddmInterface::destroyMonitorFence(monitorFence.fenceHandle);
|
||||
}
|
||||
|
||||
const bool WddmInterface20::hwQueuesSupported() {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WddmInterface20::submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) {
|
||||
D3DKMT_SUBMITCOMMAND SubmitCommand = {0};
|
||||
NTSTATUS status = STATUS_SUCCESS;
|
||||
|
||||
SubmitCommand.Commands = commandBuffer;
|
||||
SubmitCommand.CommandLength = static_cast<UINT>(size);
|
||||
SubmitCommand.BroadcastContextCount = 1;
|
||||
SubmitCommand.BroadcastContext[0] = submitArguments.contextHandle;
|
||||
SubmitCommand.Flags.NullRendering = (UINT)DebugManager.flags.EnableNullHardware.get();
|
||||
|
||||
COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandHeader);
|
||||
|
||||
pHeader->MonitorFenceVA = submitArguments.monitorFence->gpuAddress;
|
||||
pHeader->MonitorFenceValue = submitArguments.monitorFence->currentFenceValue;
|
||||
|
||||
// Note: Private data should be the CPU VA Address
|
||||
SubmitCommand.pPrivateDriverData = commandHeader;
|
||||
SubmitCommand.PrivateDriverDataSize = sizeof(COMMAND_BUFFER_HEADER);
|
||||
|
||||
status = wddm.getGdi()->submitCommand(&SubmitCommand);
|
||||
|
||||
return STATUS_SUCCESS == status;
|
||||
}
|
||||
|
||||
bool WddmInterface23::createHwQueue(OsContextWin &osContext) {
|
||||
D3DKMT_CREATEHWQUEUE createHwQueue = {};
|
||||
|
||||
if (!wddm.getGdi()->setupHwQueueProcAddresses()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
createHwQueue.hHwContext = osContext.getWddmContextHandle();
|
||||
if (osContext.getPreemptionMode() >= PreemptionMode::MidBatch) {
|
||||
createHwQueue.Flags.DisableGpuTimeout = wddm.readEnablePreemptionRegKey();
|
||||
}
|
||||
|
||||
auto status = wddm.getGdi()->createHwQueue(&createHwQueue);
|
||||
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
|
||||
osContext.setHwQueue({createHwQueue.hHwQueue, createHwQueue.hHwQueueProgressFence, createHwQueue.HwQueueProgressFenceCPUVirtualAddress,
|
||||
createHwQueue.HwQueueProgressFenceGPUVirtualAddress});
|
||||
|
||||
return status == STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
bool WddmInterface23::createMonitoredFence(OsContextWin &osContext) {
|
||||
auto &residencyController = osContext.getResidencyController();
|
||||
auto hwQueue = osContext.getHwQueue();
|
||||
residencyController.resetMonitoredFenceParams(hwQueue.progressFenceHandle,
|
||||
reinterpret_cast<uint64_t *>(hwQueue.progressFenceCpuVA),
|
||||
hwQueue.progressFenceGpuVA);
|
||||
return true;
|
||||
}
|
||||
|
||||
void WddmInterface23::destroyMonitorFence(MonitoredFence &monitorFence) {
|
||||
}
|
||||
|
||||
void WddmInterface23::destroyHwQueue(D3DKMT_HANDLE hwQueue) {
|
||||
if (hwQueue) {
|
||||
D3DKMT_DESTROYHWQUEUE destroyHwQueue = {};
|
||||
destroyHwQueue.hHwQueue = hwQueue;
|
||||
|
||||
auto status = wddm.getGdi()->destroyHwQueue(&destroyHwQueue);
|
||||
DEBUG_BREAK_IF(status != STATUS_SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
const bool WddmInterface23::hwQueuesSupported() {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WddmInterface23::submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) {
|
||||
D3DKMT_SUBMITCOMMANDTOHWQUEUE submitCommand = {};
|
||||
submitCommand.hHwQueue = submitArguments.hwQueueHandle;
|
||||
submitCommand.HwQueueProgressFenceId = submitArguments.monitorFence->currentFenceValue;
|
||||
submitCommand.CommandBuffer = commandBuffer;
|
||||
submitCommand.CommandLength = static_cast<UINT>(size);
|
||||
|
||||
submitCommand.pPrivateDriverData = commandHeader;
|
||||
submitCommand.PrivateDriverDataSize = MemoryConstants::pageSize;
|
||||
|
||||
auto status = wddm.getGdi()->submitCommandToHwQueue(&submitCommand);
|
||||
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
|
||||
return status == STATUS_SUCCESS;
|
||||
}
|
||||
62
shared/source/os_interface/windows/wddm/wddm_interface.h
Normal file
62
shared/source/os_interface/windows/wddm/wddm_interface.h
Normal file
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "helpers/hw_info.h"
|
||||
#include "os_interface/os_context.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace NEO {
|
||||
class Gdi;
|
||||
class Wddm;
|
||||
class OsContextWin;
|
||||
class WddmResidencyController;
|
||||
struct MonitoredFence;
|
||||
struct WddmSubmitArguments;
|
||||
|
||||
class WddmInterface {
|
||||
public:
|
||||
WddmInterface(Wddm &wddm) : wddm(wddm){};
|
||||
virtual ~WddmInterface() = default;
|
||||
WddmInterface() = delete;
|
||||
virtual bool createHwQueue(OsContextWin &osContext) = 0;
|
||||
virtual void destroyHwQueue(D3DKMT_HANDLE hwQueue) = 0;
|
||||
virtual bool createMonitoredFence(OsContextWin &osContext) = 0;
|
||||
MOCKABLE_VIRTUAL bool createMonitoredFence(MonitoredFence &monitorFence);
|
||||
void destroyMonitorFence(D3DKMT_HANDLE fenceHandle);
|
||||
virtual void destroyMonitorFence(MonitoredFence &monitorFence) = 0;
|
||||
virtual const bool hwQueuesSupported() = 0;
|
||||
virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) = 0;
|
||||
Wddm &wddm;
|
||||
};
|
||||
|
||||
class WddmInterface20 : public WddmInterface {
|
||||
public:
|
||||
using WddmInterface::WddmInterface;
|
||||
bool createHwQueue(OsContextWin &osContext) override;
|
||||
void destroyHwQueue(D3DKMT_HANDLE hwQueue) override;
|
||||
bool createMonitoredFence(OsContextWin &osContext) override;
|
||||
void destroyMonitorFence(MonitoredFence &monitorFence) override;
|
||||
const bool hwQueuesSupported() override;
|
||||
bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override;
|
||||
};
|
||||
|
||||
class WddmInterface23 : public WddmInterface {
|
||||
public:
|
||||
using WddmInterface::WddmInterface;
|
||||
bool createHwQueue(OsContextWin &osContext) override;
|
||||
void destroyHwQueue(D3DKMT_HANDLE hwQueue) override;
|
||||
bool createMonitoredFence(OsContextWin &osContext) override;
|
||||
void destroyMonitorFence(MonitoredFence &monitorFence) override;
|
||||
const bool hwQueuesSupported() override;
|
||||
bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
|
||||
namespace NEO {
|
||||
void Wddm::applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext) {
|
||||
}
|
||||
} // namespace NEO
|
||||
14
shared/source/os_interface/windows/wddm_allocation.cpp
Normal file
14
shared/source/os_interface/windows/wddm_allocation.cpp
Normal file
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm_allocation.h"
|
||||
|
||||
namespace NEO {
|
||||
std::string WddmAllocation::getAllocationInfoString() const {
|
||||
return getHandleInfoString();
|
||||
}
|
||||
} // namespace NEO
|
||||
98
shared/source/os_interface/windows/wddm_allocation.h
Normal file
98
shared/source/os_interface/windows/wddm_allocation.h
Normal file
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#define UMDF_USING_NTSTATUS
|
||||
#include "helpers/aligned_memory.h"
|
||||
#include "memory_manager/graphics_allocation.h"
|
||||
#include "memory_manager/residency.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
struct OsHandle {
|
||||
D3DKMT_HANDLE handle;
|
||||
D3DGPU_VIRTUAL_ADDRESS gpuPtr;
|
||||
Gmm *gmm;
|
||||
};
|
||||
|
||||
constexpr size_t trimListUnusedPosition = std::numeric_limits<size_t>::max();
|
||||
|
||||
class WddmAllocation : public GraphicsAllocation {
|
||||
public:
|
||||
WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool)
|
||||
: GraphicsAllocation(rootDeviceIndex, allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool), trimCandidateListPositions(MemoryManager::maxOsContextCount, trimListUnusedPosition) {
|
||||
reservedAddressRangeInfo.addressPtr = reservedAddr;
|
||||
reservedAddressRangeInfo.rangeSize = sizeIn;
|
||||
}
|
||||
|
||||
WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool, uint32_t shareable)
|
||||
: GraphicsAllocation(rootDeviceIndex, allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool), shareable(shareable), trimCandidateListPositions(MemoryManager::maxOsContextCount, trimListUnusedPosition) {
|
||||
reservedAddressRangeInfo.addressPtr = reservedAddr;
|
||||
reservedAddressRangeInfo.rangeSize = sizeIn;
|
||||
}
|
||||
|
||||
WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandle, MemoryPool::Type pool)
|
||||
: GraphicsAllocation(rootDeviceIndex, allocationType, cpuPtrIn, sizeIn, sharedHandle, pool), trimCandidateListPositions(MemoryManager::maxOsContextCount, trimListUnusedPosition) {}
|
||||
|
||||
void *getAlignedCpuPtr() const {
|
||||
return alignDown(this->cpuPtr, MemoryConstants::pageSize);
|
||||
}
|
||||
|
||||
size_t getAlignedSize() const {
|
||||
return alignSizeWholePage(this->cpuPtr, this->size);
|
||||
}
|
||||
|
||||
ResidencyData &getResidencyData() {
|
||||
return residency;
|
||||
}
|
||||
const std::array<D3DKMT_HANDLE, EngineLimits::maxHandleCount> &getHandles() const { return handles; }
|
||||
D3DKMT_HANDLE &getHandleToModify(uint32_t handleIndex) { return handles[handleIndex]; }
|
||||
D3DKMT_HANDLE getDefaultHandle() const { return handles[0]; }
|
||||
void setDefaultHandle(D3DKMT_HANDLE handle) {
|
||||
handles[0] = handle;
|
||||
}
|
||||
|
||||
void setTrimCandidateListPosition(uint32_t osContextId, size_t position) {
|
||||
trimCandidateListPositions[osContextId] = position;
|
||||
}
|
||||
|
||||
size_t getTrimCandidateListPosition(uint32_t osContextId) const {
|
||||
if (osContextId < trimCandidateListPositions.size()) {
|
||||
return trimCandidateListPositions[osContextId];
|
||||
}
|
||||
return trimListUnusedPosition;
|
||||
}
|
||||
|
||||
void setGpuAddress(uint64_t graphicsAddress) { this->gpuAddress = graphicsAddress; }
|
||||
void setCpuAddress(void *cpuPtr) { this->cpuPtr = cpuPtr; }
|
||||
|
||||
std::string getAllocationInfoString() const override;
|
||||
uint64_t &getGpuAddressToModify() { return gpuAddress; }
|
||||
|
||||
// OS assigned fields
|
||||
D3DKMT_HANDLE resourceHandle = 0u; // used by shared resources
|
||||
bool needsMakeResidentBeforeLock = false;
|
||||
D3DGPU_VIRTUAL_ADDRESS reservedGpuVirtualAddress = 0u;
|
||||
uint64_t reservedSizeForGpuVirtualAddress = 0u;
|
||||
uint32_t shareable = 0u;
|
||||
|
||||
protected:
|
||||
std::string getHandleInfoString() const {
|
||||
std::stringstream ss;
|
||||
for (auto &handle : handles) {
|
||||
ss << " Handle: " << handle;
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
std::array<D3DKMT_HANDLE, EngineLimits::maxHandleCount> handles{};
|
||||
ResidencyData residency;
|
||||
std::vector<size_t> trimCandidateListPositions;
|
||||
};
|
||||
} // namespace NEO
|
||||
25
shared/source/os_interface/windows/wddm_engine_mapper.cpp
Normal file
25
shared/source/os_interface/windows/wddm_engine_mapper.cpp
Normal file
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm_engine_mapper.h"
|
||||
|
||||
#include "helpers/debug_helpers.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
GPUNODE_ORDINAL WddmEngineMapper::engineNodeMap(aub_stream::EngineType engineType) {
|
||||
if (aub_stream::ENGINE_RCS == engineType) {
|
||||
return GPUNODE_3D;
|
||||
} else if (aub_stream::ENGINE_BCS == engineType) {
|
||||
return GPUNODE_BLT;
|
||||
} else if (aub_stream::ENGINE_CCS == engineType) {
|
||||
return GPUNODE_CCS0;
|
||||
}
|
||||
UNRECOVERABLE_IF(true);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
21
shared/source/os_interface/windows/wddm_engine_mapper.h
Normal file
21
shared/source/os_interface/windows/wddm_engine_mapper.h
Normal file
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "gmm_helper/gmm_lib.h"
|
||||
|
||||
#include "engine_node.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class WddmEngineMapper {
|
||||
public:
|
||||
static GPUNODE_ORDINAL engineNodeMap(aub_stream::EngineType engineType);
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
562
shared/source/os_interface/windows/wddm_memory_manager.cpp
Normal file
562
shared/source/os_interface/windows/wddm_memory_manager.cpp
Normal file
@@ -0,0 +1,562 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm_memory_manager.h"
|
||||
|
||||
#include "command_stream/command_stream_receiver_hw.h"
|
||||
#include "execution_environment/root_device_environment.h"
|
||||
#include "gmm_helper/gmm.h"
|
||||
#include "gmm_helper/gmm_helper.h"
|
||||
#include "gmm_helper/page_table_mngr.h"
|
||||
#include "gmm_helper/resource_info.h"
|
||||
#include "helpers/aligned_memory.h"
|
||||
#include "helpers/deferred_deleter_helper.h"
|
||||
#include "helpers/ptr_math.h"
|
||||
#include "helpers/surface_format_info.h"
|
||||
#include "memory_manager/deferrable_deletion.h"
|
||||
#include "memory_manager/deferred_deleter.h"
|
||||
#include "memory_manager/host_ptr_manager.h"
|
||||
#include "memory_manager/memory_operations_handler.h"
|
||||
#include "os_interface/os_interface.h"
|
||||
#include "os_interface/windows/os_context_win.h"
|
||||
#include "os_interface/windows/os_interface.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
#include "os_interface/windows/wddm_allocation.h"
|
||||
#include "os_interface/windows/wddm_residency_allocations_container.h"
|
||||
#include "os_interface/windows/wddm_residency_controller.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
WddmMemoryManager::~WddmMemoryManager() = default;
|
||||
|
||||
WddmMemoryManager::WddmMemoryManager(ExecutionEnvironment &executionEnvironment) : MemoryManager(executionEnvironment) {
|
||||
asyncDeleterEnabled = isDeferredDeleterEnabled();
|
||||
if (asyncDeleterEnabled)
|
||||
deferredDeleter = createDeferredDeleter();
|
||||
mallocRestrictions.minAddress = 0u;
|
||||
|
||||
for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < gfxPartitions.size(); ++rootDeviceIndex) {
|
||||
getWddm(rootDeviceIndex).initGfxPartition(*getGfxPartition(rootDeviceIndex), rootDeviceIndex, gfxPartitions.size());
|
||||
mallocRestrictions.minAddress = std::max(mallocRestrictions.minAddress, getWddm(rootDeviceIndex).getWddmMinAddress());
|
||||
}
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocateShareableMemory(const AllocationData &allocationData) {
|
||||
auto gmm = std::make_unique<Gmm>(executionEnvironment.getGmmClientContext(), allocationData.hostPtr, allocationData.size, false);
|
||||
auto allocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, nullptr, allocationData.size, nullptr, MemoryPool::SystemCpuInaccessible, allocationData.flags.shareable);
|
||||
allocation->setDefaultGmm(gmm.get());
|
||||
if (!createWddmAllocation(allocation.get(), nullptr)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
gmm.release();
|
||||
|
||||
return allocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) {
|
||||
if (allocationData.imgInfo->linearStorage && allocationData.imgInfo->mipCount == 0) {
|
||||
return allocateGraphicsMemoryWithAlignment(allocationData);
|
||||
}
|
||||
|
||||
auto allocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, nullptr, allocationData.imgInfo->size, nullptr, MemoryPool::SystemCpuInaccessible);
|
||||
allocation->setDefaultGmm(gmm.get());
|
||||
if (!createWddmAllocation(allocation.get(), nullptr)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
gmm.release();
|
||||
|
||||
return allocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory64kb(const AllocationData &allocationData) {
|
||||
size_t sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize64k);
|
||||
|
||||
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, nullptr, sizeAligned, nullptr, MemoryPool::System64KBPages);
|
||||
|
||||
auto gmm = new Gmm(executionEnvironment.getGmmClientContext(), nullptr, sizeAligned, false, allocationData.flags.preferRenderCompressed, true, {});
|
||||
wddmAllocation->setDefaultGmm(gmm);
|
||||
|
||||
if (!getWddm(allocationData.rootDeviceIndex).createAllocation64k(gmm, wddmAllocation->getHandleToModify(0u))) {
|
||||
delete gmm;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto cpuPtr = lockResource(wddmAllocation.get());
|
||||
|
||||
// 64kb map is not needed
|
||||
auto status = mapGpuVirtualAddress(wddmAllocation.get(), cpuPtr);
|
||||
DEBUG_BREAK_IF(!status);
|
||||
wddmAllocation->setCpuAddress(cpuPtr);
|
||||
|
||||
return wddmAllocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) {
|
||||
size_t newAlignment = allocationData.alignment ? alignUp(allocationData.alignment, MemoryConstants::pageSize) : MemoryConstants::pageSize;
|
||||
size_t sizeAligned = allocationData.size ? alignUp(allocationData.size, MemoryConstants::pageSize) : MemoryConstants::pageSize;
|
||||
void *pSysMem = allocateSystemMemory(sizeAligned, newAlignment);
|
||||
Gmm *gmm = nullptr;
|
||||
|
||||
if (pSysMem == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, pSysMem, sizeAligned, nullptr, MemoryPool::System4KBPages);
|
||||
wddmAllocation->setDriverAllocatedCpuPtr(pSysMem);
|
||||
|
||||
gmm = new Gmm(executionEnvironment.getGmmClientContext(), pSysMem, sizeAligned, allocationData.flags.uncacheable);
|
||||
|
||||
wddmAllocation->setDefaultGmm(gmm);
|
||||
void *mapPtr = wddmAllocation->getAlignedCpuPtr();
|
||||
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_CPU) {
|
||||
//add 2MB padding in case mapPtr is not 2MB aligned
|
||||
size_t reserveSizeAligned = sizeAligned + allocationData.alignment;
|
||||
bool ret = getWddm(wddmAllocation->getRootDeviceIndex()).reserveValidAddressRange(reserveSizeAligned, mapPtr);
|
||||
if (!ret) {
|
||||
delete gmm;
|
||||
freeSystemMemory(pSysMem);
|
||||
return nullptr;
|
||||
}
|
||||
wddmAllocation->setReservedAddressRange(mapPtr, reserveSizeAligned);
|
||||
mapPtr = alignUp(mapPtr, newAlignment);
|
||||
}
|
||||
|
||||
if (!createWddmAllocation(wddmAllocation.get(), mapPtr)) {
|
||||
delete gmm;
|
||||
freeSystemMemory(pSysMem);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return wddmAllocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) {
|
||||
auto alignedPtr = alignDown(allocationData.hostPtr, MemoryConstants::pageSize);
|
||||
auto offsetInPage = ptrDiff(allocationData.hostPtr, alignedPtr);
|
||||
auto alignedSize = alignSizeWholePage(allocationData.hostPtr, allocationData.size);
|
||||
|
||||
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(allocationData.hostPtr),
|
||||
allocationData.size, nullptr, MemoryPool::System4KBPages);
|
||||
wddmAllocation->setAllocationOffset(offsetInPage);
|
||||
|
||||
auto gmm = new Gmm(executionEnvironment.getGmmClientContext(), alignedPtr, alignedSize, false);
|
||||
|
||||
wddmAllocation->setDefaultGmm(gmm);
|
||||
|
||||
if (!createWddmAllocation(wddmAllocation.get(), nullptr)) {
|
||||
delete gmm;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return wddmAllocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) {
|
||||
if (mallocRestrictions.minAddress > reinterpret_cast<uintptr_t>(allocationData.hostPtr)) {
|
||||
auto inputPtr = allocationData.hostPtr;
|
||||
void *reserve = nullptr;
|
||||
auto ptrAligned = alignDown(inputPtr, MemoryConstants::allocationAlignment);
|
||||
size_t sizeAligned = alignSizeWholePage(inputPtr, allocationData.size);
|
||||
size_t offset = ptrDiff(inputPtr, ptrAligned);
|
||||
|
||||
if (!getWddm(allocationData.rootDeviceIndex).reserveValidAddressRange(sizeAligned, reserve)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto allocation = new WddmAllocation(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(inputPtr), allocationData.size, reserve, MemoryPool::System4KBPages);
|
||||
allocation->setAllocationOffset(offset);
|
||||
|
||||
Gmm *gmm = new Gmm(executionEnvironment.getGmmClientContext(), ptrAligned, sizeAligned, false);
|
||||
allocation->setDefaultGmm(gmm);
|
||||
if (createWddmAllocation(allocation, reserve)) {
|
||||
return allocation;
|
||||
}
|
||||
freeGraphicsMemory(allocation);
|
||||
return nullptr;
|
||||
}
|
||||
return MemoryManager::allocateGraphicsMemoryWithHostPtr(allocationData);
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) {
|
||||
Gmm *gmm = nullptr;
|
||||
const void *ptrAligned = nullptr;
|
||||
size_t sizeAligned = allocationData.size;
|
||||
void *pSysMem = nullptr;
|
||||
size_t offset = 0;
|
||||
|
||||
if (allocationData.hostPtr) {
|
||||
ptrAligned = alignDown(allocationData.hostPtr, MemoryConstants::allocationAlignment);
|
||||
sizeAligned = alignSizeWholePage(allocationData.hostPtr, sizeAligned);
|
||||
offset = ptrDiff(allocationData.hostPtr, ptrAligned);
|
||||
} else {
|
||||
sizeAligned = alignUp(sizeAligned, MemoryConstants::allocationAlignment);
|
||||
pSysMem = allocateSystemMemory(sizeAligned, MemoryConstants::allocationAlignment);
|
||||
if (pSysMem == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
ptrAligned = pSysMem;
|
||||
}
|
||||
|
||||
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(ptrAligned), sizeAligned, nullptr,
|
||||
MemoryPool::System4KBPagesWith32BitGpuAddressing);
|
||||
wddmAllocation->setDriverAllocatedCpuPtr(pSysMem);
|
||||
wddmAllocation->set32BitAllocation(true);
|
||||
wddmAllocation->setAllocationOffset(offset);
|
||||
|
||||
gmm = new Gmm(executionEnvironment.getGmmClientContext(), ptrAligned, sizeAligned, false);
|
||||
wddmAllocation->setDefaultGmm(gmm);
|
||||
|
||||
if (!createWddmAllocation(wddmAllocation.get(), nullptr)) {
|
||||
delete gmm;
|
||||
freeSystemMemory(pSysMem);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto baseAddress = useInternal32BitAllocator(allocationData.type) ? getInternalHeapBaseAddress(allocationData.rootDeviceIndex) : getExternalHeapBaseAddress(allocationData.rootDeviceIndex);
|
||||
wddmAllocation->setGpuBaseAddress(GmmHelper::canonize(baseAddress));
|
||||
|
||||
return wddmAllocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle, GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex) {
|
||||
auto allocation = std::make_unique<WddmAllocation>(rootDeviceIndex, allocationType, nullptr, 0, handle, MemoryPool::SystemCpuInaccessible);
|
||||
|
||||
bool status = ntHandle ? getWddm(rootDeviceIndex).openNTHandle(reinterpret_cast<HANDLE>(static_cast<uintptr_t>(handle)), allocation.get())
|
||||
: getWddm(rootDeviceIndex).openSharedHandle(handle, allocation.get());
|
||||
|
||||
if (!status) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Shared objects are passed without size
|
||||
size_t size = allocation->getDefaultGmm()->gmmResourceInfo->getSizeAllocation();
|
||||
allocation->setSize(size);
|
||||
|
||||
if (is32bit) {
|
||||
void *ptr = nullptr;
|
||||
if (!getWddm(rootDeviceIndex).reserveValidAddressRange(size, ptr)) {
|
||||
return nullptr;
|
||||
}
|
||||
allocation->setReservedAddressRange(ptr, size);
|
||||
} else if (requireSpecificBitness && this->force32bitAllocations) {
|
||||
allocation->set32BitAllocation(true);
|
||||
allocation->setGpuBaseAddress(GmmHelper::canonize(getExternalHeapBaseAddress(allocation->getRootDeviceIndex())));
|
||||
}
|
||||
status = mapGpuVirtualAddress(allocation.get(), allocation->getReservedAddressPtr());
|
||||
DEBUG_BREAK_IF(!status);
|
||||
if (!status) {
|
||||
freeGraphicsMemoryImpl(allocation.release());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
FileLoggerInstance().logAllocation(allocation.get());
|
||||
return allocation.release();
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) {
|
||||
return createAllocationFromHandle(handle, requireSpecificBitness, false, properties.allocationType, properties.rootDeviceIndex);
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) {
|
||||
return createAllocationFromHandle(toOsHandle(handle), false, true, GraphicsAllocation::AllocationType::SHARED_IMAGE, rootDeviceIndex);
|
||||
}
|
||||
|
||||
void WddmMemoryManager::addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) {
|
||||
WddmAllocation *wddmMemory = static_cast<WddmAllocation *>(gfxAllocation);
|
||||
FragmentStorage fragment = {};
|
||||
fragment.driverAllocation = true;
|
||||
fragment.fragmentCpuPointer = gfxAllocation->getUnderlyingBuffer();
|
||||
fragment.fragmentSize = alignUp(gfxAllocation->getUnderlyingBufferSize(), MemoryConstants::pageSize);
|
||||
|
||||
fragment.osInternalStorage = new OsHandle();
|
||||
fragment.osInternalStorage->gpuPtr = gfxAllocation->getGpuAddress();
|
||||
fragment.osInternalStorage->handle = wddmMemory->getDefaultHandle();
|
||||
fragment.osInternalStorage->gmm = gfxAllocation->getDefaultGmm();
|
||||
fragment.residency = &wddmMemory->getResidencyData();
|
||||
hostPtrManager->storeFragment(fragment);
|
||||
}
|
||||
|
||||
void WddmMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) {
|
||||
auto buffer = gfxAllocation->getUnderlyingBuffer();
|
||||
auto fragment = hostPtrManager->getFragment(buffer);
|
||||
if (fragment && fragment->driverAllocation) {
|
||||
OsHandle *osStorageToRelease = fragment->osInternalStorage;
|
||||
if (hostPtrManager->releaseHostPtr(buffer)) {
|
||||
delete osStorageToRelease;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void *WddmMemoryManager::lockResourceImpl(GraphicsAllocation &graphicsAllocation) {
|
||||
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
|
||||
return getWddm(graphicsAllocation.getRootDeviceIndex()).lockResource(wddmAllocation.getDefaultHandle(), wddmAllocation.needsMakeResidentBeforeLock);
|
||||
}
|
||||
void WddmMemoryManager::unlockResourceImpl(GraphicsAllocation &graphicsAllocation) {
|
||||
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
|
||||
getWddm(graphicsAllocation.getRootDeviceIndex()).unlockResource(wddmAllocation.getDefaultHandle());
|
||||
if (wddmAllocation.needsMakeResidentBeforeLock) {
|
||||
auto evictionStatus = getWddm(graphicsAllocation.getRootDeviceIndex()).getTemporaryResourcesContainer()->evictResource(wddmAllocation.getDefaultHandle());
|
||||
DEBUG_BREAK_IF(evictionStatus == MemoryOperationsStatus::FAILED);
|
||||
}
|
||||
}
|
||||
void WddmMemoryManager::freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) {
|
||||
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
|
||||
if (wddmAllocation.needsMakeResidentBeforeLock) {
|
||||
for (auto i = 0u; i < wddmAllocation.getNumHandles(); i++) {
|
||||
getWddm(graphicsAllocation.getRootDeviceIndex()).getTemporaryResourcesContainer()->removeResource(wddmAllocation.getHandles()[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) {
|
||||
WddmAllocation *input = static_cast<WddmAllocation *>(gfxAllocation);
|
||||
DEBUG_BREAK_IF(!validateAllocation(input));
|
||||
|
||||
for (auto &engine : this->registeredEngines) {
|
||||
auto &residencyController = static_cast<OsContextWin *>(engine.osContext)->getResidencyController();
|
||||
auto lock = residencyController.acquireLock();
|
||||
residencyController.removeFromTrimCandidateListIfUsed(input, true);
|
||||
}
|
||||
|
||||
executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface->evict(*input);
|
||||
|
||||
auto defaultGmm = gfxAllocation->getDefaultGmm();
|
||||
if (defaultGmm) {
|
||||
auto index = gfxAllocation->getRootDeviceIndex();
|
||||
if (defaultGmm->isRenderCompressed && executionEnvironment.rootDeviceEnvironments[index]->pageTableManager.get()) {
|
||||
auto status = executionEnvironment.rootDeviceEnvironments[index]->pageTableManager->updateAuxTable(input->getGpuAddress(), defaultGmm, false);
|
||||
DEBUG_BREAK_IF(!status);
|
||||
}
|
||||
}
|
||||
for (auto handleId = 0u; handleId < EngineLimits::maxHandleCount; handleId++) {
|
||||
delete gfxAllocation->getGmm(handleId);
|
||||
}
|
||||
|
||||
if (input->peekSharedHandle() == false &&
|
||||
input->getDriverAllocatedCpuPtr() == nullptr &&
|
||||
input->fragmentsStorage.fragmentCount > 0) {
|
||||
cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
|
||||
} else {
|
||||
const D3DKMT_HANDLE *allocationHandles = nullptr;
|
||||
uint32_t allocationCount = 0;
|
||||
D3DKMT_HANDLE resourceHandle = 0;
|
||||
if (input->peekSharedHandle()) {
|
||||
resourceHandle = input->resourceHandle;
|
||||
} else {
|
||||
allocationHandles = input->getHandles().data();
|
||||
allocationCount = input->getNumHandles();
|
||||
}
|
||||
auto status = tryDeferDeletions(allocationHandles, allocationCount, resourceHandle, gfxAllocation->getRootDeviceIndex());
|
||||
DEBUG_BREAK_IF(!status);
|
||||
alignedFreeWrapper(input->getDriverAllocatedCpuPtr());
|
||||
}
|
||||
if (input->getReservedAddressPtr()) {
|
||||
releaseReservedCpuAddressRange(input->getReservedAddressPtr(), input->getReservedAddressSize(), gfxAllocation->getRootDeviceIndex());
|
||||
}
|
||||
if (input->reservedGpuVirtualAddress) {
|
||||
getWddm(gfxAllocation->getRootDeviceIndex()).freeGpuVirtualAddress(input->reservedGpuVirtualAddress, input->reservedSizeForGpuVirtualAddress);
|
||||
}
|
||||
delete gfxAllocation;
|
||||
}
|
||||
|
||||
void WddmMemoryManager::handleFenceCompletion(GraphicsAllocation *allocation) {
|
||||
auto wddmAllocation = static_cast<WddmAllocation *>(allocation);
|
||||
for (auto &engine : this->registeredEngines) {
|
||||
const auto lastFenceValue = wddmAllocation->getResidencyData().getFenceValueForContextId(engine.osContext->getContextId());
|
||||
if (lastFenceValue != 0u) {
|
||||
const auto &monitoredFence = static_cast<OsContextWin *>(engine.osContext)->getResidencyController().getMonitoredFence();
|
||||
const auto wddm = static_cast<OsContextWin *>(engine.osContext)->getWddm();
|
||||
wddm->waitFromCpu(lastFenceValue, monitoredFence);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmMemoryManager::tryDeferDeletions(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, uint32_t rootDeviceIndex) {
|
||||
bool status = true;
|
||||
if (deferredDeleter) {
|
||||
deferredDeleter->deferDeletion(DeferrableDeletion::create(&getWddm(rootDeviceIndex), handles, allocationCount, resourceHandle));
|
||||
} else {
|
||||
status = getWddm(rootDeviceIndex).destroyAllocations(handles, allocationCount, resourceHandle);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
bool WddmMemoryManager::isMemoryBudgetExhausted() const {
|
||||
for (auto &engine : this->registeredEngines) {
|
||||
if (static_cast<OsContextWin *>(engine.osContext)->getResidencyController().isMemoryBudgetExhausted()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WddmMemoryManager::validateAllocation(WddmAllocation *alloc) {
|
||||
if (alloc == nullptr)
|
||||
return false;
|
||||
auto size = alloc->getUnderlyingBufferSize();
|
||||
if (alloc->getGpuAddress() == 0u || size == 0 || (alloc->getDefaultHandle() == 0 && alloc->fragmentsStorage.fragmentCount == 0))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
MemoryManager::AllocationStatus WddmMemoryManager::populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) {
|
||||
uint32_t allocatedFragmentIndexes[maxFragmentsCount];
|
||||
uint32_t allocatedFragmentsCounter = 0;
|
||||
|
||||
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
||||
// If no fragment is present it means it already exists.
|
||||
if (!handleStorage.fragmentStorageData[i].osHandleStorage && handleStorage.fragmentStorageData[i].cpuPtr) {
|
||||
handleStorage.fragmentStorageData[i].osHandleStorage = new OsHandle();
|
||||
handleStorage.fragmentStorageData[i].residency = new ResidencyData();
|
||||
|
||||
handleStorage.fragmentStorageData[i].osHandleStorage->gmm = new Gmm(executionEnvironment.getGmmClientContext(), handleStorage.fragmentStorageData[i].cpuPtr,
|
||||
handleStorage.fragmentStorageData[i].fragmentSize, false);
|
||||
allocatedFragmentIndexes[allocatedFragmentsCounter] = i;
|
||||
allocatedFragmentsCounter++;
|
||||
}
|
||||
}
|
||||
auto result = getWddm(rootDeviceIndex).createAllocationsAndMapGpuVa(handleStorage);
|
||||
|
||||
if (result == STATUS_GRAPHICS_NO_VIDEO_MEMORY) {
|
||||
return AllocationStatus::InvalidHostPointer;
|
||||
}
|
||||
|
||||
UNRECOVERABLE_IF(result != STATUS_SUCCESS);
|
||||
|
||||
for (uint32_t i = 0; i < allocatedFragmentsCounter; i++) {
|
||||
hostPtrManager->storeFragment(handleStorage.fragmentStorageData[allocatedFragmentIndexes[i]]);
|
||||
}
|
||||
|
||||
return AllocationStatus::Success;
|
||||
}
|
||||
|
||||
void WddmMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) {
|
||||
|
||||
D3DKMT_HANDLE handles[maxFragmentsCount] = {0};
|
||||
auto allocationCount = 0;
|
||||
|
||||
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
||||
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
|
||||
handles[allocationCount++] = handleStorage.fragmentStorageData[i].osHandleStorage->handle;
|
||||
std::fill(handleStorage.fragmentStorageData[i].residency->resident.begin(), handleStorage.fragmentStorageData[i].residency->resident.end(), false);
|
||||
}
|
||||
}
|
||||
|
||||
bool success = tryDeferDeletions(handles, allocationCount, 0, rootDeviceIndex);
|
||||
|
||||
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
|
||||
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
|
||||
if (success) {
|
||||
handleStorage.fragmentStorageData[i].osHandleStorage->handle = 0;
|
||||
}
|
||||
delete handleStorage.fragmentStorageData[i].osHandleStorage->gmm;
|
||||
delete handleStorage.fragmentStorageData[i].osHandleStorage;
|
||||
delete handleStorage.fragmentStorageData[i].residency;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WddmMemoryManager::obtainGpuAddressFromFragments(WddmAllocation *allocation, OsHandleStorage &handleStorage) {
|
||||
if (this->force32bitAllocations && (handleStorage.fragmentCount > 0)) {
|
||||
auto hostPtr = allocation->getUnderlyingBuffer();
|
||||
auto fragment = hostPtrManager->getFragment(hostPtr);
|
||||
if (fragment && fragment->driverAllocation) {
|
||||
auto gpuPtr = handleStorage.fragmentStorageData[0].osHandleStorage->gpuPtr;
|
||||
for (uint32_t i = 1; i < handleStorage.fragmentCount; i++) {
|
||||
if (handleStorage.fragmentStorageData[i].osHandleStorage->gpuPtr < gpuPtr) {
|
||||
gpuPtr = handleStorage.fragmentStorageData[i].osHandleStorage->gpuPtr;
|
||||
}
|
||||
}
|
||||
allocation->setAllocationOffset(reinterpret_cast<uint64_t>(hostPtr) & MemoryConstants::pageMask);
|
||||
allocation->setGpuAddress(gpuPtr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) {
|
||||
auto allocation = new WddmAllocation(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(allocationData.hostPtr), allocationData.size, nullptr, MemoryPool::System4KBPages);
|
||||
allocation->fragmentsStorage = handleStorage;
|
||||
obtainGpuAddressFromFragments(allocation, handleStorage);
|
||||
return allocation;
|
||||
}
|
||||
|
||||
uint64_t WddmMemoryManager::getSystemSharedMemory(uint32_t rootDeviceIndex) {
|
||||
return getWddm(rootDeviceIndex).getSystemSharedMemory();
|
||||
}
|
||||
|
||||
AlignedMallocRestrictions *WddmMemoryManager::getAlignedMallocRestrictions() {
|
||||
return &mallocRestrictions;
|
||||
}
|
||||
|
||||
bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr) {
|
||||
auto status = createGpuAllocationsWithRetry(allocation);
|
||||
if (!status) {
|
||||
return false;
|
||||
}
|
||||
return mapGpuVirtualAddress(allocation, requiredGpuPtr);
|
||||
}
|
||||
|
||||
bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocation, const void *preferredGpuVirtualAddress) {
|
||||
D3DGPU_VIRTUAL_ADDRESS addressToMap = castToUint64(preferredGpuVirtualAddress);
|
||||
auto heapIndex = selectHeap(allocation, preferredGpuVirtualAddress != nullptr, is32bit || executionEnvironment.isFullRangeSvm());
|
||||
if (!executionEnvironment.isFullRangeSvm()) {
|
||||
addressToMap = 0u;
|
||||
}
|
||||
if (allocation->reservedGpuVirtualAddress) {
|
||||
addressToMap = allocation->reservedGpuVirtualAddress;
|
||||
}
|
||||
auto gfxPartition = getGfxPartition(allocation->getRootDeviceIndex());
|
||||
auto status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, allocation->getGpuAddressToModify());
|
||||
|
||||
if (!status && deferredDeleter) {
|
||||
deferredDeleter->drain(true);
|
||||
status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, allocation->getGpuAddressToModify());
|
||||
}
|
||||
if (!status) {
|
||||
if (allocation->reservedGpuVirtualAddress) {
|
||||
getWddm(allocation->getRootDeviceIndex()).freeGpuVirtualAddress(allocation->reservedGpuVirtualAddress, allocation->reservedSizeForGpuVirtualAddress);
|
||||
}
|
||||
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(allocation->getHandles().data(), allocation->getNumHandles(), allocation->resourceHandle);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WddmMemoryManager::createGpuAllocationsWithRetry(WddmAllocation *allocation) {
|
||||
for (auto handleId = 0u; handleId < allocation->getNumHandles(); handleId++) {
|
||||
auto status = getWddm(allocation->getRootDeviceIndex()).createAllocation(allocation->getAlignedCpuPtr(), allocation->getGmm(handleId), allocation->getHandleToModify(handleId), allocation->shareable);
|
||||
if (status == STATUS_GRAPHICS_NO_VIDEO_MEMORY && deferredDeleter) {
|
||||
deferredDeleter->drain(true);
|
||||
status = getWddm(allocation->getRootDeviceIndex()).createAllocation(allocation->getAlignedCpuPtr(), allocation->getGmm(handleId), allocation->getHandleToModify(handleId), allocation->shareable);
|
||||
}
|
||||
if (status != STATUS_SUCCESS) {
|
||||
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(allocation->getHandles().data(), handleId, allocation->resourceHandle);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Wddm &WddmMemoryManager::getWddm(uint32_t rootDeviceIndex) const {
|
||||
return *this->executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface->get()->getWddm();
|
||||
}
|
||||
|
||||
void *WddmMemoryManager::reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) {
|
||||
void *reservePtr = nullptr;
|
||||
getWddm(rootDeviceIndex).reserveValidAddressRange(size, reservePtr);
|
||||
return reservePtr;
|
||||
}
|
||||
|
||||
void WddmMemoryManager::releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) {
|
||||
getWddm(rootDeviceIndex).releaseReservedAddress(reserved);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
88
shared/source/os_interface/windows/wddm_memory_manager.h
Normal file
88
shared/source/os_interface/windows/wddm_memory_manager.h
Normal file
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "helpers/aligned_memory.h"
|
||||
#include "memory_manager/memory_manager.h"
|
||||
#include "os_interface/os_context.h"
|
||||
#include "os_interface/windows/wddm_allocation.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
namespace NEO {
|
||||
class Gmm;
|
||||
class OsContextWin;
|
||||
class Wddm;
|
||||
|
||||
class WddmMemoryManager : public MemoryManager {
|
||||
public:
|
||||
using MemoryManager::allocateGraphicsMemoryWithProperties;
|
||||
|
||||
~WddmMemoryManager() override;
|
||||
WddmMemoryManager(ExecutionEnvironment &executionEnvironment);
|
||||
|
||||
WddmMemoryManager(const WddmMemoryManager &) = delete;
|
||||
WddmMemoryManager &operator=(const WddmMemoryManager &) = delete;
|
||||
|
||||
void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override;
|
||||
void handleFenceCompletion(GraphicsAllocation *allocation) override;
|
||||
|
||||
GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) override;
|
||||
GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) override;
|
||||
|
||||
void addAllocationToHostPtrManager(GraphicsAllocation *memory) override;
|
||||
void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) override;
|
||||
|
||||
AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
|
||||
void cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
|
||||
|
||||
void obtainGpuAddressFromFragments(WddmAllocation *allocation, OsHandleStorage &handleStorage);
|
||||
|
||||
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override;
|
||||
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex) override;
|
||||
|
||||
bool tryDeferDeletions(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, uint32_t rootDeviceIndex);
|
||||
|
||||
bool isMemoryBudgetExhausted() const override;
|
||||
|
||||
AlignedMallocRestrictions *getAlignedMallocRestrictions() override;
|
||||
|
||||
bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, const void *memoryToCopy, size_t sizeToCopy) override;
|
||||
void *reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) override;
|
||||
void releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) override;
|
||||
|
||||
protected:
|
||||
GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateShareableMemory(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override;
|
||||
|
||||
void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
|
||||
void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
|
||||
void freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) override;
|
||||
GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override;
|
||||
GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
|
||||
|
||||
GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle, GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex);
|
||||
static bool validateAllocation(WddmAllocation *alloc);
|
||||
bool createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr);
|
||||
bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr);
|
||||
bool mapGpuVaForOneHandleAllocation(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr);
|
||||
bool createGpuAllocationsWithRetry(WddmAllocation *graphicsAllocation);
|
||||
AlignedMallocRestrictions mallocRestrictions;
|
||||
|
||||
Wddm &getWddm(uint32_t rootDeviceIndex) const;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "helpers/basic_math.h"
|
||||
#include "os_interface/windows/wddm_memory_manager.h"
|
||||
|
||||
namespace NEO {
|
||||
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) {
|
||||
status = AllocationStatus::RetryInNonDevicePool;
|
||||
return nullptr;
|
||||
}
|
||||
bool WddmMemoryManager::copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, const void *memoryToCopy, size_t sizeToCopy) {
|
||||
return MemoryManager::copyMemoryToAllocation(graphicsAllocation, memoryToCopy, sizeToCopy);
|
||||
}
|
||||
bool WddmMemoryManager::mapGpuVirtualAddress(WddmAllocation *allocation, const void *requiredPtr) {
|
||||
return mapGpuVaForOneHandleAllocation(allocation, requiredPtr);
|
||||
}
|
||||
uint64_t WddmMemoryManager::getLocalMemorySize(uint32_t rootDeviceIndex) {
|
||||
return 0 * GB;
|
||||
}
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm_memory_operations_handler.h"
|
||||
|
||||
#include "memory_manager/host_ptr_defines.h"
|
||||
#include "os_interface/windows/wddm_allocation.h"
|
||||
#include "os_interface/windows/wddm_residency_allocations_container.h"
|
||||
#include "utilities/stackvec.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
WddmMemoryOperationsHandler::WddmMemoryOperationsHandler(Wddm *wddm) : wddm(wddm) {
|
||||
residentAllocations = std::make_unique<WddmResidentAllocationsContainer>(wddm);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRef<GraphicsAllocation *> gfxAllocations) {
|
||||
uint32_t totalHandlesCount = 0;
|
||||
constexpr uint32_t stackAllocations = 64;
|
||||
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount * stackAllocations;
|
||||
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForResidency;
|
||||
|
||||
for (const auto &allocation : gfxAllocations) {
|
||||
WddmAllocation *wddmAllocation = reinterpret_cast<WddmAllocation *>(allocation);
|
||||
|
||||
if (wddmAllocation->fragmentsStorage.fragmentCount > 0) {
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
handlesForResidency[totalHandlesCount++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
|
||||
}
|
||||
} else {
|
||||
memcpy_s(&handlesForResidency[totalHandlesCount],
|
||||
wddmAllocation->getNumHandles() * sizeof(D3DKMT_HANDLE),
|
||||
wddmAllocation->getHandles().data(),
|
||||
wddmAllocation->getNumHandles() * sizeof(D3DKMT_HANDLE));
|
||||
totalHandlesCount += wddmAllocation->getNumHandles();
|
||||
}
|
||||
}
|
||||
return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmMemoryOperationsHandler::evict(GraphicsAllocation &gfxAllocation) {
|
||||
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount;
|
||||
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForEviction;
|
||||
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
|
||||
uint32_t totalHandleCount = 0;
|
||||
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {
|
||||
OsHandleStorage &fragmentStorage = wddmAllocation.fragmentsStorage;
|
||||
|
||||
for (uint32_t allocId = 0; allocId < fragmentStorage.fragmentCount; allocId++) {
|
||||
handlesForEviction.push_back(fragmentStorage.fragmentStorageData[allocId].osHandleStorage->handle);
|
||||
totalHandleCount++;
|
||||
}
|
||||
} else {
|
||||
const D3DKMT_HANDLE *handlePtr = wddmAllocation.getHandles().data();
|
||||
size_t handleCount = wddmAllocation.getNumHandles();
|
||||
for (uint32_t i = 0; i < handleCount; i++, totalHandleCount++) {
|
||||
handlesForEviction.push_back(*handlePtr);
|
||||
handlePtr++;
|
||||
}
|
||||
}
|
||||
return residentAllocations->evictResources(handlesForEviction.begin(), totalHandleCount);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(GraphicsAllocation &gfxAllocation) {
|
||||
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
|
||||
D3DKMT_HANDLE defaultHandle = 0u;
|
||||
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {
|
||||
defaultHandle = wddmAllocation.fragmentsStorage.fragmentStorageData[0].osHandleStorage->handle;
|
||||
} else {
|
||||
defaultHandle = wddmAllocation.getDefaultHandle();
|
||||
}
|
||||
return residentAllocations->isAllocationResident(defaultHandle);
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "memory_manager/memory_operations_handler.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class Wddm;
|
||||
class WddmResidentAllocationsContainer;
|
||||
|
||||
class WddmMemoryOperationsHandler : public MemoryOperationsHandler {
|
||||
public:
|
||||
WddmMemoryOperationsHandler(Wddm *wddm);
|
||||
~WddmMemoryOperationsHandler() override = default;
|
||||
|
||||
MemoryOperationsStatus makeResident(ArrayRef<GraphicsAllocation *> gfxAllocations) override;
|
||||
MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override;
|
||||
MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override;
|
||||
|
||||
protected:
|
||||
Wddm *wddm;
|
||||
std::unique_ptr<WddmResidentAllocationsContainer> residentAllocations;
|
||||
};
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,95 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm_residency_allocations_container.h"
|
||||
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
#include "os_interface/windows/wddm_allocation.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace NEO {
|
||||
WddmResidentAllocationsContainer::~WddmResidentAllocationsContainer() {
|
||||
evictAllResources();
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::isAllocationResident(const D3DKMT_HANDLE &handle) {
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handle);
|
||||
return position != resourceHandles.end() ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::MEMORY_NOT_FOUND;
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::evictAllResources() {
|
||||
decltype(resourceHandles) resourcesToEvict;
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
resourceHandles.swap(resourcesToEvict);
|
||||
if (resourcesToEvict.empty()) {
|
||||
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
|
||||
}
|
||||
uint64_t sizeToTrim = 0;
|
||||
uint32_t evictedResources = static_cast<uint32_t>(resourcesToEvict.size());
|
||||
bool success = wddm->evict(resourcesToEvict.data(), evictedResources, sizeToTrim);
|
||||
return success ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED;
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::evictResource(const D3DKMT_HANDLE &handle) {
|
||||
return evictResources(&handle, 1u);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::evictResources(const D3DKMT_HANDLE *handles, const uint32_t count) {
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handles[0]);
|
||||
if (position == resourceHandles.end()) {
|
||||
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
|
||||
}
|
||||
auto distance = static_cast<size_t>(std::distance(resourceHandles.begin(), position));
|
||||
UNRECOVERABLE_IF(distance + count > resourceHandles.size());
|
||||
resourceHandles.erase(position, position + count);
|
||||
uint64_t sizeToTrim = 0;
|
||||
if (!wddm->evict(handles, count, sizeToTrim)) {
|
||||
return MemoryOperationsStatus::FAILED;
|
||||
}
|
||||
return MemoryOperationsStatus::SUCCESS;
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResource(const D3DKMT_HANDLE &handle) {
|
||||
return makeResidentResources(&handle, 1u);
|
||||
}
|
||||
|
||||
MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count) {
|
||||
bool madeResident = false;
|
||||
while (!(madeResident = wddm->makeResident(handles, count, false, nullptr))) {
|
||||
if (evictAllResources() == MemoryOperationsStatus::SUCCESS) {
|
||||
continue;
|
||||
}
|
||||
if (!wddm->makeResident(handles, count, false, nullptr)) {
|
||||
DEBUG_BREAK_IF(true);
|
||||
return MemoryOperationsStatus::OUT_OF_MEMORY;
|
||||
};
|
||||
break;
|
||||
}
|
||||
DEBUG_BREAK_IF(!madeResident);
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
resourceHandles.push_back(handles[i]);
|
||||
}
|
||||
lock.unlock();
|
||||
wddm->waitOnPagingFenceFromCpu();
|
||||
return madeResident ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED;
|
||||
}
|
||||
|
||||
void WddmResidentAllocationsContainer::removeResource(const D3DKMT_HANDLE &handle) {
|
||||
auto lock = acquireLock(resourcesLock);
|
||||
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handle);
|
||||
if (position == resourceHandles.end()) {
|
||||
return;
|
||||
}
|
||||
*position = resourceHandles.back();
|
||||
resourceHandles.pop_back();
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (C) 2019-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "memory_manager/memory_operations_status.h"
|
||||
#include "os_interface/windows/windows_defs.h"
|
||||
#include "utilities/spinlock.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
namespace NEO {
|
||||
class Wddm;
|
||||
|
||||
class WddmResidentAllocationsContainer {
|
||||
public:
|
||||
WddmResidentAllocationsContainer(Wddm *wddm) : wddm(wddm) {}
|
||||
virtual ~WddmResidentAllocationsContainer();
|
||||
|
||||
MemoryOperationsStatus isAllocationResident(const D3DKMT_HANDLE &handle);
|
||||
MOCKABLE_VIRTUAL MemoryOperationsStatus evictAllResources();
|
||||
MOCKABLE_VIRTUAL MemoryOperationsStatus evictResource(const D3DKMT_HANDLE &handle);
|
||||
MemoryOperationsStatus evictResources(const D3DKMT_HANDLE *handles, const uint32_t count);
|
||||
MOCKABLE_VIRTUAL MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle);
|
||||
MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count);
|
||||
MOCKABLE_VIRTUAL void removeResource(const D3DKMT_HANDLE &handle);
|
||||
|
||||
protected:
|
||||
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock(SpinLock &lock) {
|
||||
return std::unique_lock<SpinLock>{lock};
|
||||
}
|
||||
|
||||
Wddm *wddm;
|
||||
std::vector<D3DKMT_HANDLE> resourceHandles;
|
||||
SpinLock resourcesLock;
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
392
shared/source/os_interface/windows/wddm_residency_controller.cpp
Normal file
392
shared/source/os_interface/windows/wddm_residency_controller.cpp
Normal file
@@ -0,0 +1,392 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#include "os_interface/windows/wddm_residency_controller.h"
|
||||
|
||||
#include "debug_settings/debug_settings_manager.h"
|
||||
#include "os_interface/windows/wddm/wddm.h"
|
||||
#include "os_interface/windows/wddm_allocation.h"
|
||||
#include "os_interface/windows/wddm_residency_allocations_container.h"
|
||||
#include "utilities/spinlock.h"
|
||||
|
||||
namespace NEO {
|
||||
|
||||
WddmResidencyController::WddmResidencyController(Wddm &wddm, uint32_t osContextId) : wddm(wddm), osContextId(osContextId) {
|
||||
}
|
||||
|
||||
void WddmResidencyController::registerCallback() {
|
||||
this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback, *this);
|
||||
}
|
||||
|
||||
WddmResidencyController::~WddmResidencyController() {
|
||||
auto lock = this->acquireTrimCallbackLock();
|
||||
wddm.unregisterTrimCallback(WddmResidencyController::trimCallback, this->trimCallbackHandle);
|
||||
lock.unlock();
|
||||
|
||||
// Wait for lock to ensure trimCallback ended
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) {
|
||||
auto residencyController = static_cast<WddmResidencyController *>(trimNotification->Context);
|
||||
DEBUG_BREAK_IF(residencyController == nullptr);
|
||||
|
||||
auto lock = residencyController->acquireTrimCallbackLock();
|
||||
residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
|
||||
}
|
||||
|
||||
std::unique_lock<SpinLock> WddmResidencyController::acquireLock() {
|
||||
return std::unique_lock<SpinLock>{this->lock};
|
||||
}
|
||||
|
||||
std::unique_lock<SpinLock> WddmResidencyController::acquireTrimCallbackLock() {
|
||||
return std::unique_lock<SpinLock>{this->trimCallbackLock};
|
||||
}
|
||||
|
||||
WddmAllocation *WddmResidencyController::getTrimCandidateHead() {
|
||||
uint32_t i = 0;
|
||||
const size_t size = trimCandidateList.size();
|
||||
|
||||
if (size == 0) {
|
||||
return nullptr;
|
||||
}
|
||||
while ((trimCandidateList[i] == nullptr) && (i < size))
|
||||
i++;
|
||||
|
||||
return static_cast<WddmAllocation *>(trimCandidateList[i]);
|
||||
}
|
||||
|
||||
void WddmResidencyController::addToTrimCandidateList(GraphicsAllocation *allocation) {
|
||||
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
|
||||
size_t position = trimCandidateList.size();
|
||||
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
|
||||
if (wddmAllocation->getTrimCandidateListPosition(this->osContextId) == trimListUnusedPosition) {
|
||||
trimCandidatesCount++;
|
||||
trimCandidateList.push_back(allocation);
|
||||
wddmAllocation->setTrimCandidateListPosition(this->osContextId, position);
|
||||
}
|
||||
|
||||
checkTrimCandidateCount();
|
||||
}
|
||||
|
||||
void WddmResidencyController::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) {
|
||||
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
|
||||
size_t position = wddmAllocation->getTrimCandidateListPosition(this->osContextId);
|
||||
|
||||
DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1)));
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
|
||||
trimCandidatesCount--;
|
||||
|
||||
trimCandidateList[position] = nullptr;
|
||||
|
||||
checkTrimCandidateCount();
|
||||
|
||||
if (position == trimCandidateList.size() - 1) {
|
||||
size_t erasePosition = position;
|
||||
|
||||
if (position == 0) {
|
||||
trimCandidateList.resize(0);
|
||||
} else {
|
||||
while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) {
|
||||
erasePosition--;
|
||||
}
|
||||
|
||||
size_t sizeRemaining = erasePosition + 1;
|
||||
if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) {
|
||||
sizeRemaining = 0;
|
||||
}
|
||||
|
||||
trimCandidateList.resize(sizeRemaining);
|
||||
}
|
||||
}
|
||||
wddmAllocation->setTrimCandidateListPosition(this->osContextId, trimListUnusedPosition);
|
||||
|
||||
if (compactList && checkTrimCandidateListCompaction()) {
|
||||
compactTrimCandidateList();
|
||||
}
|
||||
|
||||
checkTrimCandidateCount();
|
||||
}
|
||||
|
||||
void WddmResidencyController::removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList) {
|
||||
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
|
||||
this->removeFromTrimCandidateList(allocation, true);
|
||||
}
|
||||
}
|
||||
|
||||
void WddmResidencyController::checkTrimCandidateCount() {
|
||||
if (DebugManager.flags.ResidencyDebugEnable.get()) {
|
||||
uint32_t sum = 0;
|
||||
for (auto trimCandidate : trimCandidateList) {
|
||||
if (trimCandidate != nullptr) {
|
||||
sum++;
|
||||
}
|
||||
}
|
||||
DEBUG_BREAK_IF(sum != trimCandidatesCount);
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmResidencyController::checkTrimCandidateListCompaction() {
|
||||
return 2 * trimCandidatesCount <= trimCandidateList.size();
|
||||
}
|
||||
|
||||
void WddmResidencyController::compactTrimCandidateList() {
|
||||
size_t size = trimCandidateList.size();
|
||||
size_t freePosition = 0;
|
||||
|
||||
if (size == 0 || size == trimCandidatesCount) {
|
||||
return;
|
||||
}
|
||||
|
||||
DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr));
|
||||
|
||||
uint32_t previousCount = trimCandidatesCount;
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
|
||||
while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr)
|
||||
freePosition++;
|
||||
|
||||
for (uint32_t i = 1; i < size; i++) {
|
||||
|
||||
if (trimCandidateList[i] != nullptr && freePosition < i) {
|
||||
trimCandidateList[freePosition] = trimCandidateList[i];
|
||||
trimCandidateList[i] = nullptr;
|
||||
static_cast<WddmAllocation *>(trimCandidateList[freePosition])->setTrimCandidateListPosition(this->osContextId, freePosition);
|
||||
freePosition++;
|
||||
|
||||
// Last element was moved, erase elements from freePosition
|
||||
if (i == size - 1) {
|
||||
trimCandidateList.resize(freePosition);
|
||||
}
|
||||
}
|
||||
}
|
||||
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
|
||||
DEBUG_BREAK_IF(trimCandidatesCount != previousCount);
|
||||
|
||||
checkTrimCandidateCount();
|
||||
}
|
||||
|
||||
void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress) {
|
||||
monitoredFence.lastSubmittedFence = 0;
|
||||
monitoredFence.currentFenceValue = 1;
|
||||
monitoredFence.fenceHandle = handle;
|
||||
monitoredFence.cpuAddress = cpuAddress;
|
||||
monitoredFence.gpuAddress = gpuAddress;
|
||||
}
|
||||
|
||||
void WddmResidencyController::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
|
||||
if (flags.PeriodicTrim) {
|
||||
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
|
||||
uint64_t sizeToTrim = 0;
|
||||
auto lock = this->acquireLock();
|
||||
|
||||
WddmAllocation *wddmAllocation = nullptr;
|
||||
while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
|
||||
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastTrimFenceValue);
|
||||
|
||||
if (wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) {
|
||||
break;
|
||||
}
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId));
|
||||
|
||||
uint32_t fragmentsToEvict = 0;
|
||||
|
||||
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId));
|
||||
this->wddm.evict(wddmAllocation->getHandles().data(), wddmAllocation->getNumHandles(), sizeToTrim);
|
||||
}
|
||||
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId];
|
||||
if (!wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) {
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId));
|
||||
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData.osHandleStorage->handle;
|
||||
fragmentStorageData.residency->resident[osContextId] = false;
|
||||
}
|
||||
}
|
||||
if (fragmentsToEvict != 0) {
|
||||
this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
|
||||
}
|
||||
wddmAllocation->getResidencyData().resident[osContextId] = false;
|
||||
|
||||
this->removeFromTrimCandidateList(wddmAllocation, false);
|
||||
}
|
||||
|
||||
if (this->checkTrimCandidateListCompaction()) {
|
||||
this->compactTrimCandidateList();
|
||||
}
|
||||
}
|
||||
|
||||
if (flags.TrimToBudget) {
|
||||
auto lock = this->acquireLock();
|
||||
trimResidencyToBudget(bytes);
|
||||
}
|
||||
|
||||
if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
|
||||
this->updateLastTrimFenceValue();
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastTrimFenceValue);
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) {
|
||||
D3DKMT_HANDLE fragmentEvictHandles[maxFragmentsCount] = {0};
|
||||
uint64_t numberOfBytesToTrim = bytes;
|
||||
WddmAllocation *wddmAllocation = nullptr;
|
||||
|
||||
while (numberOfBytesToTrim > 0 && (wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
|
||||
uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId);
|
||||
auto &monitoredFence = this->getMonitoredFence();
|
||||
|
||||
if (lastFence > monitoredFence.lastSubmittedFence) {
|
||||
break;
|
||||
}
|
||||
|
||||
uint32_t fragmentsToEvict = 0;
|
||||
uint64_t sizeEvicted = 0;
|
||||
uint64_t sizeToTrim = 0;
|
||||
|
||||
if (lastFence > *monitoredFence.cpuAddress) {
|
||||
this->wddm.waitFromCpu(lastFence, this->getMonitoredFence());
|
||||
}
|
||||
|
||||
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
|
||||
this->wddm.evict(wddmAllocation->getHandles().data(), wddmAllocation->getNumHandles(), sizeToTrim);
|
||||
sizeEvicted = wddmAllocation->getAlignedSize();
|
||||
} else {
|
||||
auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData;
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) {
|
||||
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle;
|
||||
}
|
||||
}
|
||||
|
||||
if (fragmentsToEvict != 0) {
|
||||
this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
|
||||
|
||||
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) {
|
||||
fragmentStorageData[allocationId].residency->resident[osContextId] = false;
|
||||
sizeEvicted += fragmentStorageData[allocationId].fragmentSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (sizeEvicted >= numberOfBytesToTrim) {
|
||||
numberOfBytesToTrim = 0;
|
||||
} else {
|
||||
numberOfBytesToTrim -= sizeEvicted;
|
||||
}
|
||||
|
||||
wddmAllocation->getResidencyData().resident[osContextId] = false;
|
||||
this->removeFromTrimCandidateList(wddmAllocation, false);
|
||||
}
|
||||
|
||||
if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) {
|
||||
this->compactTrimCandidateList();
|
||||
}
|
||||
|
||||
return numberOfBytesToTrim == 0;
|
||||
}
|
||||
|
||||
bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency) {
|
||||
const size_t residencyCount = allocationsForResidency.size();
|
||||
std::unique_ptr<D3DKMT_HANDLE[]> handlesForResidency(new D3DKMT_HANDLE[residencyCount * maxFragmentsCount * EngineLimits::maxHandleCount]);
|
||||
uint32_t totalHandlesCount = 0;
|
||||
|
||||
auto lock = this->acquireLock();
|
||||
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", this->getMonitoredFence().currentFenceValue);
|
||||
|
||||
for (uint32_t i = 0; i < residencyCount; i++) {
|
||||
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
|
||||
ResidencyData &residencyData = allocation->getResidencyData();
|
||||
bool fragmentResidency[3] = {false, false, false};
|
||||
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, residencyData.resident[osContextId] ? "resident" : "not resident");
|
||||
|
||||
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, "on trimCandidateList");
|
||||
this->removeFromTrimCandidateList(allocation, false);
|
||||
} else {
|
||||
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
fragmentResidency[allocationId] = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident[osContextId];
|
||||
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "fragment handle =", allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, fragmentResidency[allocationId] ? "resident" : "not resident");
|
||||
}
|
||||
}
|
||||
|
||||
if (allocation->fragmentsStorage.fragmentCount > 0) {
|
||||
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
if (!fragmentResidency[allocationId])
|
||||
handlesForResidency[totalHandlesCount++] = allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
|
||||
}
|
||||
} else if (!residencyData.resident[osContextId]) {
|
||||
memcpy_s(&handlesForResidency[totalHandlesCount], allocation->getNumHandles() * sizeof(D3DKMT_HANDLE), allocation->getHandles().data(), allocation->getNumHandles() * sizeof(D3DKMT_HANDLE));
|
||||
totalHandlesCount += allocation->getNumHandles();
|
||||
}
|
||||
}
|
||||
|
||||
bool result = true;
|
||||
if (totalHandlesCount) {
|
||||
uint64_t bytesToTrim = 0;
|
||||
while ((result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) {
|
||||
this->setMemoryBudgetExhausted();
|
||||
const bool trimmingDone = this->trimResidencyToBudget(bytesToTrim);
|
||||
if (!trimmingDone) {
|
||||
auto evictionStatus = wddm.getTemporaryResourcesContainer()->evictAllResources();
|
||||
if (evictionStatus == MemoryOperationsStatus::SUCCESS) {
|
||||
continue;
|
||||
}
|
||||
DEBUG_BREAK_IF(evictionStatus != MemoryOperationsStatus::MEMORY_NOT_FOUND);
|
||||
result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (result == true) {
|
||||
for (uint32_t i = 0; i < residencyCount; i++) {
|
||||
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
|
||||
// Update fence value not to early destroy / evict allocation
|
||||
const auto currentFence = this->getMonitoredFence().currentFenceValue;
|
||||
allocation->getResidencyData().updateCompletionData(currentFence, this->osContextId);
|
||||
allocation->getResidencyData().resident[osContextId] = true;
|
||||
|
||||
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
|
||||
auto residencyData = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency;
|
||||
// Update fence value not to remove the fragment referenced by different GA in trimming callback
|
||||
residencyData->updateCompletionData(currentFence, this->osContextId);
|
||||
residencyData->resident[osContextId] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void WddmResidencyController::makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations) {
|
||||
auto lock = this->acquireLock();
|
||||
const size_t residencyCount = evictionAllocations.size();
|
||||
|
||||
for (uint32_t i = 0; i < residencyCount; i++) {
|
||||
WddmAllocation *allocation = static_cast<WddmAllocation *>(evictionAllocations[i]);
|
||||
this->addToTrimCandidateList(allocation);
|
||||
}
|
||||
}
|
||||
|
||||
bool WddmResidencyController::isInitialized() const {
|
||||
if (!DebugManager.flags.DoNotRegisterTrimCallback.get()) {
|
||||
return trimCallbackHandle != nullptr;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace NEO
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "memory_manager/residency_container.h"
|
||||
#include "os_interface/windows/windows_defs.h"
|
||||
#include "os_interface/windows/windows_wrapper.h"
|
||||
#include "utilities/spinlock.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
class GraphicsAllocation;
|
||||
class WddmAllocation;
|
||||
class Wddm;
|
||||
|
||||
class WddmResidencyController {
|
||||
public:
|
||||
WddmResidencyController(Wddm &wddm, uint32_t osContextId);
|
||||
MOCKABLE_VIRTUAL ~WddmResidencyController();
|
||||
|
||||
static void APIENTRY trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification);
|
||||
|
||||
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock();
|
||||
std::unique_lock<SpinLock> acquireTrimCallbackLock();
|
||||
|
||||
WddmAllocation *getTrimCandidateHead();
|
||||
void addToTrimCandidateList(GraphicsAllocation *allocation);
|
||||
void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList);
|
||||
void removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList);
|
||||
void checkTrimCandidateCount();
|
||||
|
||||
bool checkTrimCandidateListCompaction();
|
||||
void compactTrimCandidateList();
|
||||
|
||||
bool wasAllocationUsedSinceLastTrim(uint64_t fenceValue) { return fenceValue > lastTrimFenceValue; }
|
||||
void updateLastTrimFenceValue() { lastTrimFenceValue = *this->getMonitoredFence().cpuAddress; }
|
||||
const ResidencyContainer &peekTrimCandidateList() const { return trimCandidateList; }
|
||||
uint32_t peekTrimCandidatesCount() const { return trimCandidatesCount; }
|
||||
|
||||
MonitoredFence &getMonitoredFence() { return monitoredFence; }
|
||||
void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress);
|
||||
|
||||
void registerCallback();
|
||||
|
||||
void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes);
|
||||
bool trimResidencyToBudget(uint64_t bytes);
|
||||
|
||||
bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; }
|
||||
void setMemoryBudgetExhausted() { memoryBudgetExhausted = true; }
|
||||
|
||||
bool makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency);
|
||||
void makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations);
|
||||
|
||||
bool isInitialized() const;
|
||||
|
||||
protected:
|
||||
Wddm &wddm;
|
||||
uint32_t osContextId;
|
||||
MonitoredFence monitoredFence = {};
|
||||
|
||||
SpinLock lock;
|
||||
SpinLock trimCallbackLock;
|
||||
|
||||
bool memoryBudgetExhausted = false;
|
||||
uint64_t lastTrimFenceValue = 0u;
|
||||
ResidencyContainer trimCandidateList;
|
||||
uint32_t trimCandidatesCount = 0;
|
||||
|
||||
VOID *trimCallbackHandle = nullptr;
|
||||
};
|
||||
} // namespace NEO
|
||||
27
shared/source/os_interface/windows/windows_defs.h
Normal file
27
shared/source/os_interface/windows/windows_defs.h
Normal file
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (C) 2018-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Windows.h"
|
||||
#include <d3dkmthk.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace NEO {
|
||||
|
||||
constexpr uintptr_t windowsMinAddress = 0x200000;
|
||||
|
||||
struct MonitoredFence {
|
||||
D3DKMT_HANDLE fenceHandle = 0;
|
||||
D3DGPU_VIRTUAL_ADDRESS gpuAddress = 0;
|
||||
volatile uint64_t *cpuAddress = nullptr;
|
||||
volatile uint64_t currentFenceValue = 0;
|
||||
uint64_t lastSubmittedFence = 0;
|
||||
};
|
||||
|
||||
} // namespace NEO
|
||||
10
shared/source/os_interface/windows/windows_inc.cpp
Normal file
10
shared/source/os_interface/windows/windows_inc.cpp
Normal file
@@ -0,0 +1,10 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
namespace Os {
|
||||
const char *fileSeparator = "/";
|
||||
}
|
||||
32
shared/source/os_interface/windows/windows_wrapper.h
Normal file
32
shared/source/os_interface/windows/windows_wrapper.h
Normal file
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright (C) 2017-2020 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <Windows.h>
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable : 4005)
|
||||
#include <ntstatus.h>
|
||||
#pragma warning(pop)
|
||||
// There is a conflict with max/min defined as macro in windows headers with std::max/std::min
|
||||
#undef min
|
||||
#undef max
|
||||
#undef RegOpenKeyExA
|
||||
#undef RegQueryValueExA
|
||||
#pragma warning(disable : 4273)
|
||||
LSTATUS APIENTRY RegOpenKeyExA(
|
||||
HKEY hKey,
|
||||
LPCSTR lpSubKey,
|
||||
DWORD ulOptions,
|
||||
REGSAM samDesired,
|
||||
PHKEY phkResult);
|
||||
LSTATUS APIENTRY RegQueryValueExA(
|
||||
HKEY hKey,
|
||||
LPCSTR lpValueName,
|
||||
LPDWORD lpReserved,
|
||||
LPDWORD lpType,
|
||||
LPBYTE lpData,
|
||||
LPDWORD lpcbData);
|
||||
Reference in New Issue
Block a user