Reorganization directory structure [3/n]

Change-Id: If3dfa3f6007f8810a6a1ae1a4f0c7da38544648d
This commit is contained in:
kamdiedrich
2020-02-23 21:00:51 +01:00
committed by sys_ocldev
parent e177b4fc0f
commit e072275ae6
711 changed files with 94 additions and 94 deletions

View File

@@ -0,0 +1,70 @@
#
# Copyright (C) 2019-2020 Intel Corporation
#
# SPDX-License-Identifier: MIT
#
if(KMDAF_HEADERS_DIR)
set(KMDAF_FILE_SUFFIX "")
else()
set(KMDAF_FILE_SUFFIX "_stub")
endif()
set(NEO_CORE_OS_INTERFACE_WINDOWS
${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_additional_context_flags.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_allocation.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_engine_mapper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/debug_registry_reader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/debug_registry_reader.h
${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/deferrable_deletion_win.h
${CMAKE_CURRENT_SOURCE_DIR}/gdi_interface.cpp
${CMAKE_CURRENT_SOURCE_DIR}/gdi_interface.h
${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h
${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.cpp
${CMAKE_CURRENT_SOURCE_DIR}/kmdaf_listener${KMDAF_FILE_SUFFIX}.cpp
${CMAKE_CURRENT_SOURCE_DIR}/kmdaf_listener.h
${CMAKE_CURRENT_SOURCE_DIR}/os_context_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_context_win.h
${CMAKE_CURRENT_SOURCE_DIR}/os_inc.h
${CMAKE_CURRENT_SOURCE_DIR}/os_interface.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_interface.h
${CMAKE_CURRENT_SOURCE_DIR}/os_library_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_library_win.h
${CMAKE_CURRENT_SOURCE_DIR}/os_memory_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_memory_win.h
${CMAKE_CURRENT_SOURCE_DIR}/os_socket.h
${CMAKE_CURRENT_SOURCE_DIR}/os_thread_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_thread_win.h
${CMAKE_CURRENT_SOURCE_DIR}/os_time_win.cpp
${CMAKE_CURRENT_SOURCE_DIR}/os_time_win.h
${CMAKE_CURRENT_SOURCE_DIR}/page_table_manager_functions.cpp
${CMAKE_CURRENT_SOURCE_DIR}/print.cpp
${CMAKE_CURRENT_SOURCE_DIR}/sys_calls.h
${CMAKE_CURRENT_SOURCE_DIR}/thk_wrapper.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_defs.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_manager.h
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/wddm_memory_manager_allocate_in_device_pool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_interface.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm/wddm_interface.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm_allocation.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm_engine_mapper.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_operations_handler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_memory_operations_handler.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_allocations_container.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_allocations_container.h
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.cpp
${CMAKE_CURRENT_SOURCE_DIR}/wddm_residency_controller.h
${CMAKE_CURRENT_SOURCE_DIR}/windows_defs.h
${CMAKE_CURRENT_SOURCE_DIR}/windows_inc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/windows_wrapper.h
)
if(WIN32)
set_property(GLOBAL PROPERTY NEO_CORE_OS_INTERFACE_WINDOWS ${NEO_CORE_OS_INTERFACE_WINDOWS})
endif()

View File

@@ -0,0 +1,135 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/debug_registry_reader.h"
#include "debug_settings/debug_settings_manager.h"
#include "os_interface/windows/windows_wrapper.h"
#include "utilities/debug_settings_reader.h"
#include <stdint.h>
namespace NEO {
SettingsReader *SettingsReader::createOsReader(bool userScope, const std::string &regKey) {
return new RegistryReader(userScope, regKey);
}
RegistryReader::RegistryReader(bool userScope, const std::string &regKey) : registryReadRootKey(regKey) {
igdrclHkeyType = userScope ? HKEY_CURRENT_USER : HKEY_LOCAL_MACHINE;
setUpProcessName();
}
void RegistryReader::setUpProcessName() {
char buff[MAX_PATH];
GetModuleFileNameA(nullptr, buff, MAX_PATH);
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
processName = "";
}
processName.assign(buff);
}
const char *RegistryReader::appSpecificLocation(const std::string &name) {
if (processName.length() > 0)
return processName.c_str();
return name.c_str();
}
bool RegistryReader::getSetting(const char *settingName, bool defaultValue) {
return getSetting(settingName, static_cast<int32_t>(defaultValue)) ? true : false;
}
int32_t RegistryReader::getSetting(const char *settingName, int32_t defaultValue) {
HKEY Key;
DWORD value = defaultValue;
DWORD success = ERROR_SUCCESS;
success = RegOpenKeyExA(igdrclHkeyType,
registryReadRootKey.c_str(),
0,
KEY_READ,
&Key);
if (ERROR_SUCCESS == success) {
DWORD regType;
DWORD size = sizeof(ULONG);
success = RegQueryValueExA(Key,
settingName,
NULL,
&regType,
(LPBYTE)&value,
&size);
RegCloseKey(Key);
value = ERROR_SUCCESS == success ? value : defaultValue;
} else { // Check environment variables
const char *envValue = getenv(settingName);
if (envValue) {
value = atoi(envValue);
}
}
return value;
}
std::string RegistryReader::getSetting(const char *settingName, const std::string &value) {
HKEY Key;
DWORD success = ERROR_SUCCESS;
std::string keyValue = value;
success = RegOpenKeyExA(igdrclHkeyType,
registryReadRootKey.c_str(),
0,
KEY_READ,
&Key);
if (ERROR_SUCCESS == success) {
DWORD regType = REG_NONE;
DWORD regSize = 0;
success = RegQueryValueExA(Key,
settingName,
NULL,
&regType,
NULL,
&regSize);
if (success == ERROR_SUCCESS && regType == REG_SZ) {
char *regData = new char[regSize];
success = RegQueryValueExA(Key,
settingName,
NULL,
&regType,
(LPBYTE)regData,
&regSize);
keyValue.assign(regData);
delete[] regData;
} else if (success == ERROR_SUCCESS && regType == REG_BINARY) {
std::unique_ptr<wchar_t[]> regData(new wchar_t[regSize]);
success = RegQueryValueExA(Key,
settingName,
NULL,
&regType,
(LPBYTE)regData.get(),
&regSize);
size_t charsConverted = 0;
std::unique_ptr<char[]> convertedData(new char[regSize]);
wcstombs_s(&charsConverted, convertedData.get(), regSize, regData.get(), regSize);
keyValue.assign(convertedData.get());
}
RegCloseKey(Key);
} else { // Check environment variables
const char *envValue = getenv(settingName);
if (envValue) {
keyValue.assign(envValue);
}
}
return keyValue;
}
}; // namespace NEO

View File

@@ -0,0 +1,33 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "utilities/debug_settings_reader.h"
#include <Windows.h>
#include <stdint.h>
#include <string>
namespace NEO {
class RegistryReader : public SettingsReader {
public:
RegistryReader() = delete;
RegistryReader(bool userScope, const std::string &regKey);
int32_t getSetting(const char *settingName, int32_t defaultValue) override;
bool getSetting(const char *settingName, bool defaultValue) override;
std::string getSetting(const char *settingName, const std::string &value) override;
const char *appSpecificLocation(const std::string &name) override;
protected:
HKEY igdrclHkeyType;
std::string registryReadRootKey;
void setUpProcessName();
std::string processName;
};
} // namespace NEO

View File

@@ -0,0 +1,39 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/deferrable_deletion_win.h"
#include "os_interface/windows/wddm/wddm.h"
namespace NEO {
template <typename... Args>
DeferrableDeletion *DeferrableDeletion::create(Args... args) {
return new DeferrableDeletionImpl(std::forward<Args>(args)...);
}
template DeferrableDeletion *DeferrableDeletion::create(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
DeferrableDeletionImpl::DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle)
: wddm(wddm), allocationCount(allocationCount), resourceHandle(resourceHandle) {
if (handles) {
this->handles = new D3DKMT_HANDLE[allocationCount];
for (uint32_t i = 0; i < allocationCount; i++) {
this->handles[i] = handles[i];
}
}
}
bool DeferrableDeletionImpl::apply() {
bool destroyStatus = wddm->destroyAllocations(handles, allocationCount, resourceHandle);
DEBUG_BREAK_IF(!destroyStatus);
return true;
}
DeferrableDeletionImpl::~DeferrableDeletionImpl() {
if (handles) {
delete[] handles;
}
}
} // namespace NEO

View File

@@ -0,0 +1,35 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "memory_manager/deferrable_deletion.h"
#include "os_interface/os_context.h"
#include "os_interface/windows/windows_wrapper.h"
#include <d3dkmthk.h>
namespace NEO {
class OsContextWin;
class Wddm;
class DeferrableDeletionImpl : public DeferrableDeletion {
public:
DeferrableDeletionImpl(Wddm *wddm, const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
bool apply() override;
~DeferrableDeletionImpl();
DeferrableDeletionImpl(const DeferrableDeletionImpl &) = delete;
DeferrableDeletionImpl &operator=(const DeferrableDeletionImpl &) = delete;
protected:
Wddm *wddm;
D3DKMT_HANDLE *handles = nullptr;
uint32_t allocationCount;
D3DKMT_HANDLE resourceHandle;
};
} // namespace NEO

View File

@@ -0,0 +1,104 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/gdi_interface.h"
#include "debug_settings/debug_settings_manager.h"
namespace NEO {
inline const std::string getGdiName() {
if (DebugManager.flags.OverrideGdiPath.get() != "unk") {
return DebugManager.flags.OverrideGdiPath.get();
} else {
return Os::gdiDllName;
}
}
Gdi::Gdi() : gdiDll(getGdiName()),
initialized(false) {
if (gdiDll.isLoaded()) {
initialized = getAllProcAddresses();
}
}
bool Gdi::setupHwQueueProcAddresses() {
createHwQueue = reinterpret_cast<PFND3DKMT_CREATEHWQUEUE>(gdiDll.getProcAddress("D3DKMTCreateHwQueue"));
destroyHwQueue = reinterpret_cast<PFND3DKMT_DESTROYHWQUEUE>(gdiDll.getProcAddress("D3DKMTDestroyHwQueue"));
submitCommandToHwQueue = reinterpret_cast<PFND3DKMT_SUBMITCOMMANDTOHWQUEUE>(gdiDll.getProcAddress("D3DKMTSubmitCommandToHwQueue"));
if (!createHwQueue || !destroyHwQueue || !submitCommandToHwQueue) {
return false;
}
return true;
}
bool Gdi::getAllProcAddresses() {
openAdapterFromHdc = reinterpret_cast<PFND3DKMT_OPENADAPTERFROMHDC>(gdiDll.getProcAddress("D3DKMTOpenAdapterFromHdc"));
openAdapterFromLuid = reinterpret_cast<PFND3DKMT_OPENADAPTERFROMLUID>(gdiDll.getProcAddress("D3DKMTOpenAdapterFromLuid"));
createAllocation = reinterpret_cast<PFND3DKMT_CREATEALLOCATION>(gdiDll.getProcAddress("D3DKMTCreateAllocation"));
destroyAllocation = reinterpret_cast<PFND3DKMT_DESTROYALLOCATION>(gdiDll.getProcAddress("D3DKMTDestroyAllocation"));
destroyAllocation2 = reinterpret_cast<PFND3DKMT_DESTROYALLOCATION2>(gdiDll.getProcAddress("D3DKMTDestroyAllocation2"));
queryAdapterInfo = reinterpret_cast<PFND3DKMT_QUERYADAPTERINFO>(gdiDll.getProcAddress("D3DKMTQueryAdapterInfo"));
closeAdapter = reinterpret_cast<PFND3DKMT_CLOSEADAPTER>(gdiDll.getProcAddress("D3DKMTCloseAdapter"));
createDevice = reinterpret_cast<PFND3DKMT_CREATEDEVICE>(gdiDll.getProcAddress("D3DKMTCreateDevice"));
destroyDevice = reinterpret_cast<PFND3DKMT_DESTROYDEVICE>(gdiDll.getProcAddress("D3DKMTDestroyDevice"));
escape = reinterpret_cast<PFND3DKMT_ESCAPE>(gdiDll.getProcAddress("D3DKMTEscape"));
createContext = reinterpret_cast<PFND3DKMT_CREATECONTEXTVIRTUAL>(gdiDll.getProcAddress("D3DKMTCreateContextVirtual"));
destroyContext = reinterpret_cast<PFND3DKMT_DESTROYCONTEXT>(gdiDll.getProcAddress("D3DKMTDestroyContext"));
openResource = reinterpret_cast<PFND3DKMT_OPENRESOURCE>(gdiDll.getProcAddress("D3DKMTOpenResource"));
openResourceFromNtHandle = reinterpret_cast<PFND3DKMT_OPENRESOURCEFROMNTHANDLE>(gdiDll.getProcAddress("D3DKMTOpenResourceFromNtHandle"));
queryResourceInfo = reinterpret_cast<PFND3DKMT_QUERYRESOURCEINFO>(gdiDll.getProcAddress("D3DKMTQueryResourceInfo"));
queryResourceInfoFromNtHandle = reinterpret_cast<PFND3DKMT_QUERYRESOURCEINFOFROMNTHANDLE>(gdiDll.getProcAddress("D3DKMTQueryResourceInfoFromNtHandle"));
lock = reinterpret_cast<PFND3DKMT_LOCK>(gdiDll.getProcAddress("D3DKMTLock"));
unlock = reinterpret_cast<PFND3DKMT_UNLOCK>(gdiDll.getProcAddress("D3DKMTUnlock"));
render = reinterpret_cast<PFND3DKMT_RENDER>(gdiDll.getProcAddress("D3DKMTRender"));
createSynchronizationObject = reinterpret_cast<PFND3DKMT_CREATESYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTCreateSynchronizationObject"));
createSynchronizationObject2 = reinterpret_cast<PFND3DKMT_CREATESYNCHRONIZATIONOBJECT2>(gdiDll.getProcAddress("D3DKMTCreateSynchronizationObject2"));
destroySynchronizationObject = reinterpret_cast<PFND3DKMT_DESTROYSYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTDestroySynchronizationObject"));
signalSynchronizationObject = reinterpret_cast<PFND3DKMT_SIGNALSYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTSignalSynchronizationObject"));
waitForSynchronizationObject = reinterpret_cast<PFND3DKMT_WAITFORSYNCHRONIZATIONOBJECT>(gdiDll.getProcAddress("D3DKMTWaitForSynchronizationObject"));
waitForSynchronizationObjectFromCpu = reinterpret_cast<PFND3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU>(gdiDll.getProcAddress("D3DKMTWaitForSynchronizationObjectFromCpu"));
signalSynchronizationObjectFromCpu = reinterpret_cast<PFND3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU>(gdiDll.getProcAddress("D3DKMTSignalSynchronizationObjectFromCpu"));
waitForSynchronizationObjectFromGpu = reinterpret_cast<PFND3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU>(gdiDll.getProcAddress("D3DKMTWaitForSynchronizationObjectFromGpu"));
signalSynchronizationObjectFromGpu = reinterpret_cast<PFND3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU>(gdiDll.getProcAddress("D3DKMTSignalSynchronizationObjectFromGpu"));
createPagingQueue = reinterpret_cast<PFND3DKMT_CREATEPAGINGQUEUE>(gdiDll.getProcAddress("D3DKMTCreatePagingQueue"));
destroyPagingQueue = reinterpret_cast<PFND3DKMT_DESTROYPAGINGQUEUE>(gdiDll.getProcAddress("D3DKMTDestroyPagingQueue"));
lock2 = reinterpret_cast<PFND3DKMT_LOCK2>(gdiDll.getProcAddress("D3DKMTLock2"));
unlock2 = reinterpret_cast<PFND3DKMT_UNLOCK2>(gdiDll.getProcAddress("D3DKMTUnlock2"));
mapGpuVirtualAddress = reinterpret_cast<PFND3DKMT_MAPGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTMapGpuVirtualAddress"));
reserveGpuVirtualAddress = reinterpret_cast<PFND3DKMT_RESERVEGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTReserveGpuVirtualAddress"));
freeGpuVirtualAddress = reinterpret_cast<PFND3DKMT_FREEGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTFreeGpuVirtualAddress"));
updateGpuVirtualAddress = reinterpret_cast<PFND3DKMT_UPDATEGPUVIRTUALADDRESS>(gdiDll.getProcAddress("D3DKMTUpdateGpuVirtualAddress"));
submitCommand = reinterpret_cast<PFND3DKMT_SUBMITCOMMAND>(gdiDll.getProcAddress("D3DKMTSubmitCommand"));
makeResident = reinterpret_cast<PFND3DKMT_MAKERESIDENT>(gdiDll.getProcAddress("D3DKMTMakeResident"));
evict = reinterpret_cast<PFND3DKMT_EVICT>(gdiDll.getProcAddress("D3DKMTEvict"));
registerTrimNotification = reinterpret_cast<PFND3DKMT_REGISTERTRIMNOTIFICATION>(gdiDll.getProcAddress("D3DKMTRegisterTrimNotification"));
unregisterTrimNotification = reinterpret_cast<PFND3DKMT_UNREGISTERTRIMNOTIFICATION>(gdiDll.getProcAddress("D3DKMTUnregisterTrimNotification"));
// For debug purposes
getDeviceState = reinterpret_cast<PFND3DKMT_GETDEVICESTATE>(gdiDll.getProcAddress("D3DKMTGetDeviceState"));
// clang-format off
if (openAdapterFromHdc && openAdapterFromLuid && createAllocation && destroyAllocation
&& destroyAllocation2 && queryAdapterInfo && closeAdapter && createDevice
&& destroyDevice && escape && createContext && destroyContext
&& openResource && queryResourceInfo && lock && unlock && render
&& createSynchronizationObject && createSynchronizationObject2
&& destroySynchronizationObject && signalSynchronizationObject
&& waitForSynchronizationObject && waitForSynchronizationObjectFromCpu
&& signalSynchronizationObjectFromCpu && waitForSynchronizationObjectFromGpu
&& signalSynchronizationObjectFromGpu && createPagingQueue && destroyPagingQueue
&& lock2 && unlock2 && mapGpuVirtualAddress && reserveGpuVirtualAddress
&& freeGpuVirtualAddress && updateGpuVirtualAddress &&submitCommand
&& makeResident && evict && registerTrimNotification && unregisterTrimNotification){
return true;
}
// clang-format on
return false;
}
} // namespace NEO

View File

@@ -0,0 +1,87 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/windows/os_inc.h"
#include "os_interface/windows/os_library_win.h"
#include "os_interface/windows/thk_wrapper.h"
#include <d3d9types.h>
#include <d3dkmthk.h>
#include <string>
namespace NEO {
class Gdi {
public:
Gdi();
~Gdi(){};
ThkWrapper<IN OUT D3DKMT_OPENADAPTERFROMHDC *> openAdapterFromHdc;
ThkWrapper<IN OUT D3DKMT_OPENADAPTERFROMLUID *> openAdapterFromLuid;
ThkWrapper<IN OUT D3DKMT_CREATEALLOCATION *> createAllocation;
ThkWrapper<IN CONST D3DKMT_DESTROYALLOCATION *> destroyAllocation;
ThkWrapper<IN CONST D3DKMT_DESTROYALLOCATION2 *> destroyAllocation2;
ThkWrapper<IN CONST D3DKMT_QUERYADAPTERINFO *> queryAdapterInfo;
ThkWrapper<IN CONST D3DKMT_CLOSEADAPTER *> closeAdapter;
ThkWrapper<IN OUT D3DKMT_CREATEDEVICE *> createDevice;
ThkWrapper<IN CONST D3DKMT_DESTROYDEVICE *> destroyDevice;
ThkWrapper<IN CONST D3DKMT_ESCAPE *> escape;
ThkWrapper<IN D3DKMT_CREATECONTEXTVIRTUAL *> createContext;
ThkWrapper<IN CONST D3DKMT_DESTROYCONTEXT *> destroyContext;
ThkWrapper<IN OUT D3DKMT_OPENRESOURCE *> openResource;
ThkWrapper<IN OUT D3DKMT_OPENRESOURCEFROMNTHANDLE *> openResourceFromNtHandle;
ThkWrapper<IN OUT D3DKMT_QUERYRESOURCEINFO *> queryResourceInfo;
ThkWrapper<IN OUT D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *> queryResourceInfoFromNtHandle;
ThkWrapper<IN OUT D3DKMT_LOCK *> lock;
ThkWrapper<IN CONST D3DKMT_UNLOCK *> unlock;
ThkWrapper<IN OUT D3DKMT_RENDER *> render;
ThkWrapper<IN OUT D3DKMT_CREATESYNCHRONIZATIONOBJECT *> createSynchronizationObject;
ThkWrapper<IN OUT D3DKMT_CREATESYNCHRONIZATIONOBJECT2 *> createSynchronizationObject2;
ThkWrapper<IN CONST D3DKMT_DESTROYSYNCHRONIZATIONOBJECT *> destroySynchronizationObject;
ThkWrapper<IN CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECT *> signalSynchronizationObject;
ThkWrapper<IN CONST_FROM_WDK_10_0_18328_0 D3DKMT_WAITFORSYNCHRONIZATIONOBJECT *> waitForSynchronizationObject;
ThkWrapper<IN CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *> waitForSynchronizationObjectFromCpu;
ThkWrapper<IN CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU *> signalSynchronizationObjectFromCpu;
ThkWrapper<IN CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU *> waitForSynchronizationObjectFromGpu;
ThkWrapper<IN CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU *> signalSynchronizationObjectFromGpu;
ThkWrapper<IN OUT D3DKMT_CREATEPAGINGQUEUE *> createPagingQueue;
ThkWrapper<IN OUT D3DDDI_DESTROYPAGINGQUEUE *> destroyPagingQueue;
ThkWrapper<IN OUT D3DKMT_LOCK2 *> lock2;
ThkWrapper<IN CONST D3DKMT_UNLOCK2 *> unlock2;
ThkWrapper<IN OUT D3DDDI_MAPGPUVIRTUALADDRESS *> mapGpuVirtualAddress;
ThkWrapper<IN OUT D3DDDI_RESERVEGPUVIRTUALADDRESS *> reserveGpuVirtualAddress;
ThkWrapper<IN CONST D3DKMT_FREEGPUVIRTUALADDRESS *> freeGpuVirtualAddress;
ThkWrapper<IN CONST D3DKMT_UPDATEGPUVIRTUALADDRESS *> updateGpuVirtualAddress;
ThkWrapper<IN CONST D3DKMT_SUBMITCOMMAND *> submitCommand;
ThkWrapper<IN OUT D3DDDI_MAKERESIDENT *> makeResident;
ThkWrapper<IN D3DKMT_EVICT *> evict;
ThkWrapper<IN D3DKMT_REGISTERTRIMNOTIFICATION *> registerTrimNotification;
ThkWrapper<IN D3DKMT_UNREGISTERTRIMNOTIFICATION *> unregisterTrimNotification;
// HW queue
ThkWrapper<IN OUT D3DKMT_CREATEHWQUEUE *> createHwQueue;
ThkWrapper<IN CONST D3DKMT_DESTROYHWQUEUE *> destroyHwQueue;
ThkWrapper<IN CONST D3DKMT_SUBMITCOMMANDTOHWQUEUE *> submitCommandToHwQueue;
// For debug purposes
ThkWrapper<IN OUT D3DKMT_GETDEVICESTATE *> getDeviceState;
bool isInitialized() {
return initialized;
}
MOCKABLE_VIRTUAL bool setupHwQueueProcAddresses();
protected:
MOCKABLE_VIRTUAL bool getAllProcAddresses();
bool initialized;
NEO::Windows::OsLibrary gdiDll;
};
} // namespace NEO

View File

@@ -0,0 +1,36 @@
/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "gmm_helper/gmm_interface.h"
#include "helpers/debug_helpers.h"
#include "os_interface/os_library.h"
#include <memory>
namespace NEO {
static std::unique_ptr<OsLibrary> gmmLib;
namespace GmmInterface {
GMM_STATUS initialize(GMM_INIT_IN_ARGS *pInArgs, GMM_INIT_OUT_ARGS *pOutArgs) {
if (!gmmLib) {
gmmLib.reset(OsLibrary::load(GMM_UMD_DLL));
UNRECOVERABLE_IF(!gmmLib);
}
auto initGmmFunc = reinterpret_cast<decltype(&InitializeGmm)>(gmmLib->getProcAddress(GMM_ADAPTER_INIT_NAME));
UNRECOVERABLE_IF(!initGmmFunc);
return initGmmFunc(pInArgs, pOutArgs);
}
void destroy(GMM_INIT_OUT_ARGS *pInArgs) {
auto destroyGmmFunc = reinterpret_cast<decltype(&GmmAdapterDestroy)>(gmmLib->getProcAddress(GMM_ADAPTER_DESTROY_NAME));
UNRECOVERABLE_IF(!destroyGmmFunc);
destroyGmmFunc(pInArgs);
}
} // namespace GmmInterface
} // namespace NEO

View File

@@ -0,0 +1,37 @@
/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "helpers/non_copyable_or_moveable.h"
#include "os_interface/windows/windows_wrapper.h"
#include <d3dkmthk.h>
#include <memory>
namespace NEO {
class Gdi;
class HwDeviceId : NonCopyableClass {
public:
HwDeviceId(D3DKMT_HANDLE adapterIn, LUID adapterLuidIn, std::unique_ptr<Gdi> gdiIn);
inline Gdi *getGdi() const {
return gdi.get();
}
constexpr D3DKMT_HANDLE getAdapter() const {
return adapter;
}
constexpr LUID getAdapterLuid() const {
return adapterLuid;
}
~HwDeviceId();
protected:
const D3DKMT_HANDLE adapter;
const LUID adapterLuid;
const std::unique_ptr<Gdi> gdi;
};
} // namespace NEO

View File

@@ -0,0 +1,23 @@
/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "helpers/debug_helpers.h"
#include "os_interface/windows/gdi_interface.h"
#include "os_interface/windows/hw_device_id.h"
namespace NEO {
HwDeviceId::~HwDeviceId() {
NTSTATUS status = STATUS_UNSUCCESSFUL;
D3DKMT_CLOSEADAPTER CloseAdapter = {0};
CloseAdapter.hAdapter = adapter;
status = gdi->closeAdapter(&CloseAdapter);
DEBUG_BREAK_IF(status != STATUS_SUCCESS);
}
HwDeviceId::HwDeviceId(D3DKMT_HANDLE adapterIn, LUID adapterLuidIn, std::unique_ptr<Gdi> gdiIn) : adapter(adapterIn),
adapterLuid(adapterLuidIn),
gdi(std::move(gdiIn)){};
} // namespace NEO

View File

@@ -0,0 +1,59 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/hw_info_config.h"
#include "command_stream/preemption.h"
#include "debug_settings/debug_settings_manager.h"
#include "helpers/hw_cmds.h"
#include "helpers/hw_helper.h"
#include "helpers/hw_info.h"
#include "memory_manager/memory_constants.h"
#include "instrumentation.h"
namespace NEO {
HwInfoConfig *hwInfoConfigFactory[IGFX_MAX_PRODUCT] = {};
int HwInfoConfig::configureHwInfo(const HardwareInfo *inHwInfo, HardwareInfo *outHwInfo, OSInterface *osIface) {
HwHelper &hwHelper = HwHelper::get(outHwInfo->platform.eRenderCoreFamily);
outHwInfo->capabilityTable.ftrSvm = outHwInfo->featureTable.ftrSVM;
hwHelper.adjustDefaultEngineType(outHwInfo);
outHwInfo->capabilityTable.defaultEngineType = getChosenEngineType(*outHwInfo);
hwHelper.setCapabilityCoherencyFlag(outHwInfo, outHwInfo->capabilityTable.ftrSupportsCoherency);
outHwInfo->capabilityTable.ftrSupportsCoherency &= inHwInfo->featureTable.ftrL3IACoherency;
PreemptionHelper::adjustDefaultPreemptionMode(outHwInfo->capabilityTable,
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuMidThreadLevelPreempt),
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuThreadGroupLevelPreempt),
static_cast<bool>(outHwInfo->featureTable.ftrGpGpuMidBatchPreempt));
outHwInfo->capabilityTable.requiredPreemptionSurfaceSize = outHwInfo->gtSystemInfo.CsrSizeInMb * MemoryConstants::megaByte;
outHwInfo->capabilityTable.instrumentationEnabled =
(outHwInfo->capabilityTable.instrumentationEnabled && haveInstrumentation);
auto &kmdNotifyProperties = outHwInfo->capabilityTable.kmdNotifyProperties;
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableKmdNotify.get(), kmdNotifyProperties.enableKmdNotify);
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideKmdNotifyDelayMicroseconds.get(), kmdNotifyProperties.delayKmdNotifyMicroseconds);
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableQuickKmdSleep.get(), kmdNotifyProperties.enableQuickKmdSleep);
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideQuickKmdSleepDelayMicroseconds.get(), kmdNotifyProperties.delayQuickKmdSleepMicroseconds);
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideEnableQuickKmdSleepForSporadicWaits.get(), kmdNotifyProperties.enableQuickKmdSleepForSporadicWaits);
KmdNotifyHelper::overrideFromDebugVariable(DebugManager.flags.OverrideDelayQuickKmdSleepForSporadicWaitsMicroseconds.get(), kmdNotifyProperties.delayQuickKmdSleepForSporadicWaitsMicroseconds);
// Product specific config
int ret = configureHardwareCustom(outHwInfo, osIface);
if (ret != 0) {
outHwInfo = {};
}
return ret;
}
} // namespace NEO

View File

@@ -0,0 +1,43 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/kmdaf_listener.h"
#pragma warning(push) // save the current state
#pragma warning(disable : 4189) // disable warning 4189 (unused local variable)
#include "kmdaf.h"
#pragma warning(pop) // restore state.
namespace NEO {
void KmDafListener::notifyLock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DDDICB_LOCKFLAGS *pLockFlags, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_LOCK(ftrKmdDaf, hAdapter, hDevice, hAllocation, pLockFlags, pfnEscape);
}
void KmDafListener::notifyUnlock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_UNLOCK(ftrKmdDaf, hAdapter, hDevice, phAllocation, allocations, pfnEscape);
}
void KmDafListener::notifyMapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_MAP_GPUVA(ftrKmdDaf, hAdapter, hDevice, hAllocation, GpuVirtualAddress, pfnEscape);
}
void KmDafListener::notifyUnmapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_UNMAP_GPUVA(ftrKmdDaf, hAdapter, hDevice, GpuVirtualAddress, pfnEscape);
}
void KmDafListener::notifyMakeResident(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_MAKERESIDENT(ftrKmdDaf, hAdapter, hDevice, phAllocation, allocations, pfnEscape);
}
void KmDafListener::notifyEvict(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_EVICT(ftrKmdDaf, hAdapter, hDevice, phAllocation, allocations, pfnEscape);
}
void KmDafListener::notifyWriteTarget(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, PFND3DKMT_ESCAPE pfnEscape) {
KM_DAF_NOTIFY_WRITE_TARGET(ftrKmdDaf, hAdapter, hDevice, hAllocation, pfnEscape);
}
} // namespace NEO

View File

@@ -0,0 +1,31 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/windows/windows_wrapper.h"
#include <d3dkmthk.h>
namespace NEO {
struct KmDafListener {
MOCKABLE_VIRTUAL void notifyLock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DDDICB_LOCKFLAGS *pLockFlags, PFND3DKMT_ESCAPE pfnEscape);
MOCKABLE_VIRTUAL void notifyUnlock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape);
MOCKABLE_VIRTUAL void notifyMapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape);
MOCKABLE_VIRTUAL void notifyUnmapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape);
MOCKABLE_VIRTUAL void notifyMakeResident(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape);
MOCKABLE_VIRTUAL void notifyEvict(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape);
MOCKABLE_VIRTUAL void notifyWriteTarget(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, PFND3DKMT_ESCAPE pfnEscape);
};
} // namespace NEO

View File

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/kmdaf_listener.h"
namespace NEO {
void KmDafListener::notifyLock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DDDICB_LOCKFLAGS *pLockFlags, PFND3DKMT_ESCAPE pfnEscape) {
}
void KmDafListener::notifyUnlock(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
}
void KmDafListener::notifyMapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
}
void KmDafListener::notifyUnmapGpuVA(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, D3DGPU_VIRTUAL_ADDRESS GpuVirtualAddress, PFND3DKMT_ESCAPE pfnEscape) {
}
void KmDafListener::notifyMakeResident(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
}
void KmDafListener::notifyEvict(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE *phAllocation, ULONG allocations, PFND3DKMT_ESCAPE pfnEscape) {
}
void KmDafListener::notifyWriteTarget(bool ftrKmdDaf, D3DKMT_HANDLE hAdapter, D3DKMT_HANDLE hDevice, const D3DKMT_HANDLE hAllocation, PFND3DKMT_ESCAPE pfnEscape) {
}
} // namespace NEO

View File

@@ -0,0 +1,51 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/os_context_win.h"
#include "os_interface/windows/os_interface.h"
#include "os_interface/windows/wddm/wddm.h"
#include "os_interface/windows/wddm/wddm_interface.h"
namespace NEO {
OsContext *OsContext::create(OSInterface *osInterface, uint32_t contextId, DeviceBitfield deviceBitfield,
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority) {
if (osInterface) {
return new OsContextWin(*osInterface->get()->getWddm(), contextId, deviceBitfield, engineType, preemptionMode, lowPriority);
}
return new OsContext(contextId, deviceBitfield, engineType, preemptionMode, lowPriority);
}
OsContextWin::OsContextWin(Wddm &wddm, uint32_t contextId, DeviceBitfield deviceBitfield,
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority)
: OsContext(contextId, deviceBitfield, engineType, preemptionMode, lowPriority), wddm(wddm), residencyController(wddm, contextId) {
auto wddmInterface = wddm.getWddmInterface();
if (!wddm.createContext(*this)) {
return;
}
if (wddmInterface->hwQueuesSupported()) {
if (!wddmInterface->createHwQueue(*this)) {
return;
}
}
initialized = wddmInterface->createMonitoredFence(*this);
residencyController.registerCallback();
};
bool OsContextWin::isInitialized() const {
return (initialized && residencyController.isInitialized());
}
OsContextWin::~OsContextWin() {
wddm.getWddmInterface()->destroyHwQueue(hardwareQueue.handle);
wddm.getWddmInterface()->destroyMonitorFence(residencyController.getMonitoredFence());
wddm.destroyContext(wddmContextHandle);
}
} // namespace NEO

View File

@@ -0,0 +1,46 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/os_context.h"
#include "os_interface/windows/wddm_residency_controller.h"
namespace NEO {
class Wddm;
struct HardwareQueue {
D3DKMT_HANDLE handle = 0;
D3DKMT_HANDLE progressFenceHandle = 0;
VOID *progressFenceCpuVA = nullptr;
D3DGPU_VIRTUAL_ADDRESS progressFenceGpuVA = 0;
};
class OsContextWin : public OsContext {
public:
OsContextWin() = delete;
~OsContextWin() override;
OsContextWin(Wddm &wddm, uint32_t contextId, DeviceBitfield deviceBitfield,
aub_stream::EngineType engineType, PreemptionMode preemptionMode, bool lowPriority);
D3DKMT_HANDLE getWddmContextHandle() const { return wddmContextHandle; }
void setWddmContextHandle(D3DKMT_HANDLE wddmContextHandle) { this->wddmContextHandle = wddmContextHandle; }
HardwareQueue getHwQueue() const { return hardwareQueue; }
void setHwQueue(HardwareQueue hardwareQueue) { this->hardwareQueue = hardwareQueue; }
bool isInitialized() const override;
Wddm *getWddm() const { return &wddm; }
MOCKABLE_VIRTUAL WddmResidencyController &getResidencyController() { return residencyController; }
protected:
bool initialized = false;
D3DKMT_HANDLE wddmContextHandle = 0;
HardwareQueue hardwareQueue;
Wddm &wddm;
WddmResidencyController residencyController;
};
} // namespace NEO

View File

@@ -0,0 +1,17 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#define PATH_SEPARATOR '\\'
// For now we need to keep this file clean of OS specific #includes.
// Only issues to address portability should be covered here.
namespace Os {
// OS GDI name
extern const char *gdiDllName;
}; // namespace Os

View File

@@ -0,0 +1,86 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/os_interface.h"
#include "execution_environment/root_device_environment.h"
#include "os_interface/windows/sys_calls.h"
#include "os_interface/windows/wddm/wddm.h"
#include "os_interface/windows/wddm_memory_operations_handler.h"
namespace NEO {
bool OSInterface::osEnabled64kbPages = true;
OSInterface::OSInterface() {
osInterfaceImpl = new OSInterfaceImpl();
}
OSInterface::~OSInterface() {
delete osInterfaceImpl;
}
uint32_t OSInterface::getDeviceHandle() const {
return static_cast<uint32_t>(osInterfaceImpl->getDeviceHandle());
}
void OSInterface::setGmmInputArgs(void *args) {
this->get()->getWddm()->setGmmInputArg(args);
}
OSInterface::OSInterfaceImpl::OSInterfaceImpl() = default;
D3DKMT_HANDLE OSInterface::OSInterfaceImpl::getAdapterHandle() const {
return wddm->getAdapter();
}
D3DKMT_HANDLE OSInterface::OSInterfaceImpl::getDeviceHandle() const {
return wddm->getDevice();
}
PFND3DKMT_ESCAPE OSInterface::OSInterfaceImpl::getEscapeHandle() const {
return wddm->getEscapeHandle();
}
uint32_t OSInterface::OSInterfaceImpl::getHwContextId() const {
if (wddm == nullptr) {
return 0;
}
return wddm->getHwContextId();
}
bool OSInterface::are64kbPagesEnabled() {
return osEnabled64kbPages;
}
Wddm *OSInterface::OSInterfaceImpl::getWddm() const {
return wddm.get();
}
void OSInterface::OSInterfaceImpl::setWddm(Wddm *wddm) {
this->wddm.reset(wddm);
}
HANDLE OSInterface::OSInterfaceImpl::createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState,
LPCSTR lpName) {
return SysCalls::createEvent(lpEventAttributes, bManualReset, bInitialState, lpName);
}
BOOL OSInterface::OSInterfaceImpl::closeHandle(HANDLE hObject) {
return SysCalls::closeHandle(hObject);
}
bool RootDeviceEnvironment::initOsInterface(std::unique_ptr<HwDeviceId> &&hwDeviceId) {
std::unique_ptr<Wddm> wddm(Wddm::createWddm(std::move(hwDeviceId), *this));
if (!wddm->init()) {
return false;
}
memoryOperationsInterface = std::make_unique<WddmMemoryOperationsHandler>(wddm.get());
osInterface = std::make_unique<OSInterface>();
osInterface->get()->setWddm(wddm.release());
return true;
}
} // namespace NEO

View File

@@ -0,0 +1,42 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/os_interface.h"
#include "os_interface/windows/wddm/wddm.h"
#include "os_interface/windows/windows_wrapper.h"
#include "profileapi.h"
#include "umKmInc/sharedata.h"
#include <d3dkmthk.h>
#include <memory>
namespace NEO {
class Wddm;
class WddmMemoryOperationsHandler;
class OSInterface::OSInterfaceImpl {
public:
OSInterfaceImpl();
virtual ~OSInterfaceImpl() = default;
Wddm *getWddm() const;
void setWddm(Wddm *wddm);
D3DKMT_HANDLE getAdapterHandle() const;
D3DKMT_HANDLE getDeviceHandle() const;
PFND3DKMT_ESCAPE getEscapeHandle() const;
uint32_t getHwContextId() const;
MOCKABLE_VIRTUAL HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState,
LPCSTR lpName);
MOCKABLE_VIRTUAL BOOL closeHandle(HANDLE hObject);
protected:
std::unique_ptr<Wddm> wddm;
};
} // namespace NEO

View File

@@ -0,0 +1,69 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/os_library_win.h"
namespace NEO {
OsLibrary *OsLibrary::load(const std::string &name) {
Windows::OsLibrary *ptr = new Windows::OsLibrary(name);
if (!ptr->isLoaded()) {
delete ptr;
return nullptr;
}
return ptr;
}
namespace Windows {
decltype(&LoadLibraryExA) OsLibrary::loadLibraryExA = LoadLibraryExA;
decltype(&GetModuleFileNameA) OsLibrary::getModuleFileNameA = GetModuleFileNameA;
extern "C" IMAGE_DOS_HEADER __ImageBase;
__inline HINSTANCE GetModuleHINSTANCE() { return (HINSTANCE)&__ImageBase; }
HMODULE OsLibrary::loadDependency(const std::string &dependencyFileName) const {
char dllPath[MAX_PATH];
DWORD length = getModuleFileNameA(GetModuleHINSTANCE(), dllPath, MAX_PATH);
for (DWORD idx = length; idx > 0; idx--) {
if (dllPath[idx - 1] == '\\') {
dllPath[idx] = '\0';
break;
}
}
strcat_s(dllPath, MAX_PATH, dependencyFileName.c_str());
return loadLibraryExA(dllPath, NULL, 0);
}
OsLibrary::OsLibrary(const std::string &name) {
if (name.empty()) {
this->handle = GetModuleHandleA(nullptr);
} else {
this->handle = loadDependency(name);
if (this->handle == nullptr) {
this->handle = ::LoadLibraryA(name.c_str());
}
}
}
OsLibrary::~OsLibrary() {
if ((this->handle != nullptr) && (this->handle != GetModuleHandleA(nullptr))) {
::FreeLibrary(this->handle);
this->handle = nullptr;
}
}
bool OsLibrary::isLoaded() {
return this->handle != nullptr;
}
void *OsLibrary::getProcAddress(const std::string &procName) {
return ::GetProcAddress(this->handle, procName.c_str());
}
} // namespace Windows
} // namespace NEO

View File

@@ -0,0 +1,35 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/os_library.h"
#define UMDF_USING_NTSTATUS
#include "os_interface/windows/windows_wrapper.h"
namespace NEO {
namespace Windows {
class OsLibrary : public NEO::OsLibrary {
private:
HMODULE handle;
public:
OsLibrary(const std::string &name);
~OsLibrary();
bool isLoaded();
void *getProcAddress(const std::string &procName);
protected:
HMODULE loadDependency(const std::string &dependencyFileName) const;
static decltype(&LoadLibraryExA) loadLibraryExA;
static decltype(&GetModuleFileNameA) getModuleFileNameA;
};
} // namespace Windows
} // namespace NEO

View File

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/os_memory_win.h"
namespace NEO {
std::unique_ptr<OSMemory> OSMemory::create() {
return std::make_unique<OSMemoryWindows>();
}
void *OSMemoryWindows::reserveCpuAddressRange(size_t sizeToReserve) {
return virtualAllocWrapper(0, sizeToReserve, MEM_RESERVE, PAGE_READWRITE);
}
void OSMemoryWindows::releaseCpuAddressRange(void *reservedCpuAddressRange, size_t /* reservedSize */) {
virtualFreeWrapper(reservedCpuAddressRange, 0, MEM_RELEASE);
}
LPVOID OSMemoryWindows::virtualAllocWrapper(LPVOID lpAddress, SIZE_T dwSize, DWORD flAllocationType, DWORD flProtect) {
return VirtualAlloc(lpAddress, dwSize, flAllocationType, flProtect);
}
BOOL OSMemoryWindows::virtualFreeWrapper(LPVOID lpAddress, SIZE_T dwSize, DWORD dwFreeType) {
return VirtualFree(lpAddress, dwSize, dwFreeType);
}
} // namespace NEO

View File

@@ -0,0 +1,25 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/os_memory.h"
#include <windows.h>
namespace NEO {
class OSMemoryWindows : public OSMemory {
public:
OSMemoryWindows() = default;
void *reserveCpuAddressRange(size_t sizeToReserve) override;
void releaseCpuAddressRange(void *reservedCpuAddressRange, size_t reservedSize) override;
protected:
MOCKABLE_VIRTUAL LPVOID virtualAllocWrapper(LPVOID, SIZE_T, DWORD, DWORD);
MOCKABLE_VIRTUAL BOOL virtualFreeWrapper(LPVOID, SIZE_T, DWORD);
};
}; // namespace NEO

View File

@@ -0,0 +1,9 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <winsock.h>

View File

@@ -0,0 +1,21 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/os_thread_win.h"
namespace NEO {
ThreadWin::ThreadWin(std::thread *thread) {
this->thread.reset(thread);
};
std::unique_ptr<Thread> Thread::create(void *(*func)(void *), void *arg) {
return std::unique_ptr<Thread>(new ThreadWin(new std::thread(func, arg)));
}
void ThreadWin::join() {
thread->join();
}
} // namespace NEO

View File

@@ -0,0 +1,21 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/os_thread.h"
#include <thread>
namespace NEO {
class ThreadWin : public Thread {
public:
ThreadWin(std::thread *thread);
void join() override;
protected:
std::unique_ptr<std::thread> thread;
};
} // namespace NEO

View File

@@ -0,0 +1,112 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/os_time_win.h"
#include "os_interface/windows/os_interface.h"
#include "os_interface/windows/wddm/wddm.h"
#include <memory>
#undef WIN32_NO_STATUS
namespace NEO {
bool runEscape(Wddm *wddm, TimeStampDataHeader &escapeInfo) {
if (wddm) {
D3DKMT_ESCAPE escapeCommand = {0};
GTDIGetGpuCpuTimestampsIn in = {GTDI_FNC_GET_GPU_CPU_TIMESTAMPS};
uint32_t outSize = sizeof(GTDIGetGpuCpuTimestampsOut);
escapeInfo.m_Header.EscapeCode = GFX_ESCAPE_IGPA_INSTRUMENTATION_CONTROL;
escapeInfo.m_Header.Size = outSize;
escapeInfo.m_Data.m_In = in;
escapeCommand.Flags.Value = 0;
escapeCommand.hAdapter = (D3DKMT_HANDLE)0;
escapeCommand.hContext = (D3DKMT_HANDLE)0; // escape is not context specific
escapeCommand.hDevice = (D3DKMT_HANDLE)wddm->getDevice(); // escape not device specific, passing only for instrumentation
escapeCommand.pPrivateDriverData = &escapeInfo;
escapeCommand.PrivateDriverDataSize = sizeof(escapeInfo);
escapeCommand.Type = D3DKMT_ESCAPE_DRIVERPRIVATE;
auto status = wddm->escape(escapeCommand);
if (status == STATUS_SUCCESS) {
return true;
}
}
return false;
}
bool OSTimeWin::getCpuGpuTime(TimeStampData *pGpuCpuTime) {
bool retVal = false;
pGpuCpuTime->CPUTimeinNS = 0;
pGpuCpuTime->GPUTimeStamp = 0;
TimeStampDataHeader escapeInfo = {0};
if (runEscape(wddm, escapeInfo)) {
double cpuNanoseconds = escapeInfo.m_Data.m_Out.cpuPerfTicks *
(1000000000.0 / escapeInfo.m_Data.m_Out.cpuPerfFreq);
pGpuCpuTime->CPUTimeinNS = (unsigned long long)cpuNanoseconds;
pGpuCpuTime->GPUTimeStamp = (unsigned long long)escapeInfo.m_Data.m_Out.gpuPerfTicks;
retVal = true;
}
return retVal;
}
bool OSTimeWin::getCpuTime(uint64_t *timeStamp) {
uint64_t time;
this->QueryPerfomanceCounterFnc((LARGE_INTEGER *)&time);
*timeStamp = static_cast<uint64_t>((static_cast<double>(time) * NSEC_PER_SEC / frequency.QuadPart));
return true;
};
std::unique_ptr<OSTime> OSTime::create(OSInterface *osInterface) {
return std::unique_ptr<OSTime>(new OSTimeWin(osInterface));
}
OSTimeWin::OSTimeWin(OSInterface *osInterface) {
this->osInterface = osInterface;
if (osInterface) {
wddm = osInterface->get()->getWddm();
}
QueryPerformanceFrequency(&frequency);
}
double OSTimeWin::getHostTimerResolution() const {
double retValue = 0;
if (frequency.QuadPart) {
retValue = 1e9 / frequency.QuadPart;
}
return retValue;
}
double OSTimeWin::getDynamicDeviceTimerResolution(HardwareInfo const &hwInfo) const {
double retVal = 0;
TimeStampDataHeader escapeInfo = {0};
if (runEscape(wddm, escapeInfo)) {
retVal = 1000000000.0 / (double)escapeInfo.m_Data.m_Out.gpuPerfFreq;
}
return retVal;
}
uint64_t OSTimeWin::getCpuRawTimestamp() {
LARGE_INTEGER cpuRawTimestamp = {};
this->QueryPerfomanceCounterFnc(&cpuRawTimestamp);
return cpuRawTimestamp.QuadPart;
}
} // namespace NEO

View File

@@ -0,0 +1,75 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/os_time.h"
#include "os_interface/windows/windows_wrapper.h"
#include "gfxEscape.h"
namespace NEO {
class Wddm;
class OSTimeWin : public OSTime {
public:
OSTimeWin(OSInterface *osInterface);
bool getCpuTime(uint64_t *timeStamp) override;
bool getCpuGpuTime(TimeStampData *pGpuCpuTime) override;
double getHostTimerResolution() const override;
double getDynamicDeviceTimerResolution(HardwareInfo const &hwInfo) const override;
uint64_t getCpuRawTimestamp() override;
protected:
Wddm *wddm = nullptr;
LARGE_INTEGER frequency;
OSTimeWin() = default;
decltype(&QueryPerformanceCounter) QueryPerfomanceCounterFnc = QueryPerformanceCounter;
};
typedef enum GTDI_ESCAPE_FUNCTION_ENUM {
GTDI_FNC_GET_GPU_CPU_TIMESTAMPS = 25
} GTDI_ESCAPE_FUNCTION;
typedef struct GTDIBaseInStruct {
GTDI_ESCAPE_FUNCTION Function;
} GTDIHeaderIn;
typedef GTDIHeaderIn GTDIGetGpuCpuTimestampsIn;
typedef enum GTDI_RETURN_CODE_ENUM {
GTDI_RET_OK = 0,
GTDI_RET_FAILED,
GTDI_RET_NOT_CONNECTED,
GTDI_RET_HW_METRICS_NOT_ENABLED,
GTDI_RET_CONTEXT_ID_MISMATCH,
GTDI_RET_NOT_SUPPORTED,
GTDI_RET_PENDING,
GTDI_RET_INVALID_CONFIGURATION,
GTDI_RET_CONCURRENT_API_ENABLED,
GTDI_RET_NO_INFORMATION, // for GTDI_FNC_GET_ERROR_INFO escape only
// ...
GTDI_RET_MAX = 0xFFFFFFFF
} GTDI_RETURN_CODE;
typedef struct GTDIGetGpuCpuTimestampsOutStruct {
GTDI_RETURN_CODE RetCode; // Result of the call
uint64_t gpuPerfTicks; // in GPU_timestamp_ticks
uint64_t cpuPerfTicks; // in CPU_timestamp_ticks
uint64_t gpuPerfFreq; // in GPU_timestamp_ticks/s
uint64_t cpuPerfFreq; // in CPU_timestamp_ticks/s
} GTDIGetGpuCpuTimestampsOut;
struct TimeStampDataHeader {
GFX_ESCAPE_HEADER_T m_Header;
union {
GTDIGetGpuCpuTimestampsIn m_In;
GTDIGetGpuCpuTimestampsOut m_Out;
} m_Data;
};
} // namespace NEO

View File

@@ -0,0 +1,22 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "gmm_helper/gmm_helper.h"
#include "gmm_helper/page_table_mngr.h"
#include "opencl/source/platform/platform.h"
#include "gmm_client_context.h"
namespace NEO {
GmmPageTableMngr::GmmPageTableMngr(GmmClientContext *gmmClientContext, unsigned int translationTableFlags, GMM_TRANSLATIONTABLE_CALLBACKS *translationTableCb) : clientContext(gmmClientContext->getHandle()) {
pageTableManager = clientContext->CreatePageTblMgrObject(translationTableCb, translationTableFlags);
}
void GmmPageTableMngr::setCsrHandle(void *csrHandle) {
pageTableManager->GmmSetCsrHandle(csrHandle);
}
} // namespace NEO

View File

@@ -0,0 +1,81 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/print.h"
#include "os_interface/windows/windows_wrapper.h"
#include <cctype>
#include <cstdint>
#include <fcntl.h>
#include <io.h>
#include <iostream>
void printToSTDOUT(const char *str) {
int fd = 0;
HANDLE stdoutDuplicate = 0;
FILE *pFile = nullptr;
if ((DuplicateHandle(GetCurrentProcess(), GetStdHandle(STD_OUTPUT_HANDLE),
GetCurrentProcess(), &stdoutDuplicate, 0L, TRUE, DUPLICATE_SAME_ACCESS))) {
if ((fd = _open_osfhandle((DWORD_PTR)stdoutDuplicate, _O_TEXT)) &&
(pFile = _fdopen(fd, "w"))) {
fprintf_s(pFile, "%s", str);
fflush(pFile);
fclose(pFile);
}
}
}
template <class T>
size_t simple_sprintf(char *output, size_t outputSize, const char *format, T value) {
#if (_MSC_VER == 1800)
_set_output_format(_TWO_DIGIT_EXPONENT);
#endif
size_t len = strlen(format);
if (len == 0) {
output[0] = '\0';
return 0;
}
if (len > 3 && *(format + len - 2) == 'h' && *(format + len - 3) == 'h') {
if (*(format + len - 1) == 'i' || *(format + len - 1) == 'd') {
int32_t fixedValue = (char)value;
return sprintf_s(output, outputSize, format, fixedValue);
} else {
uint32_t fixedValue = (unsigned char)value;
return sprintf_s(output, outputSize, format, fixedValue);
}
} else if (format[len - 1] == 'F') {
std::string formatCopy = format;
*formatCopy.rbegin() = 'f';
size_t returnValue = sprintf_s(output, outputSize, formatCopy.c_str(), value);
for (size_t i = 0; i < returnValue; i++)
output[i] = std::toupper(output[i]);
return returnValue;
} else {
return sprintf_s(output, outputSize, format, value);
}
}
size_t simple_sprintf(char *output, size_t outputSize, const char *format, const char *value) {
return sprintf_s(output, outputSize, format, value);
}
template size_t simple_sprintf<float>(char *output, size_t output_size, const char *format, float value);
template size_t simple_sprintf<double>(char *output, size_t output_size, const char *format, double value);
template size_t simple_sprintf<char>(char *output, size_t output_size, const char *format, char value);
template size_t simple_sprintf<int8_t>(char *output, size_t output_size, const char *format, int8_t value);
template size_t simple_sprintf<int16_t>(char *output, size_t output_size, const char *format, int16_t value);
template size_t simple_sprintf<int32_t>(char *output, size_t output_size, const char *format, int32_t value);
template size_t simple_sprintf<int64_t>(char *output, size_t output_size, const char *format, int64_t value);
template size_t simple_sprintf<uint8_t>(char *output, size_t output_size, const char *format, uint8_t value);
template size_t simple_sprintf<uint16_t>(char *output, size_t output_size, const char *format, uint16_t value);
template size_t simple_sprintf<uint32_t>(char *output, size_t output_size, const char *format, uint32_t value);
template size_t simple_sprintf<uint64_t>(char *output, size_t output_size, const char *format, uint64_t value);

View File

@@ -0,0 +1,33 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/sys_calls.h"
namespace NEO {
namespace SysCalls {
HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState, LPCSTR lpName) {
return CreateEventA(lpEventAttributes, bManualReset, bInitialState, lpName);
}
BOOL closeHandle(HANDLE hObject) {
return CloseHandle(hObject);
}
BOOL getSystemPowerStatus(LPSYSTEM_POWER_STATUS systemPowerStatusPtr) {
return GetSystemPowerStatus(systemPowerStatusPtr);
}
BOOL getModuleHandle(DWORD dwFlags, LPCWSTR lpModuleName, HMODULE *phModule) {
return GetModuleHandleEx(dwFlags, lpModuleName, phModule);
}
DWORD getModuleFileName(HMODULE hModule, LPWSTR lpFilename, DWORD nSize) {
return GetModuleFileName(hModule, lpFilename, nSize);
}
} // namespace SysCalls
} // namespace NEO

View File

@@ -0,0 +1,23 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <Windows.h>
namespace NEO {
namespace SysCalls {
HANDLE createEvent(LPSECURITY_ATTRIBUTES lpEventAttributes, BOOL bManualReset, BOOL bInitialState, LPCSTR lpName);
BOOL closeHandle(HANDLE hObject);
BOOL getSystemPowerStatus(LPSYSTEM_POWER_STATUS systemPowerStatusPtr);
BOOL getModuleHandle(DWORD dwFlags, LPCWSTR lpModuleName, HMODULE *phModule);
DWORD getModuleFileName(HMODULE hModule, LPWSTR lpFilename, DWORD nSize);
} // namespace SysCalls
} // namespace NEO

View File

@@ -0,0 +1,110 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "helpers/options.h"
#include "utilities/api_intercept.h"
#include <d3dkmthk.h>
namespace NEO {
template <typename Param>
class ThkWrapper {
typedef NTSTATUS(APIENTRY *Func)(Param);
public:
Func mFunc = nullptr;
inline NTSTATUS operator()(Param param) const {
if (KMD_PROFILING) {
SYSTEM_ENTER()
NTSTATUS Status;
Status = mFunc(param);
SYSTEM_LEAVE(getId<Param>());
return Status;
} else {
return mFunc(param);
}
}
template <class T>
inline T &operator=(T func) {
return mFunc = func;
}
// This operator overload is for implicit casting ThkWrapper struct to Function Pointer in GetPfn methods like GetEscapePfn() or for comparing against NULL function pointer
operator Func() const {
return mFunc;
}
private:
// Default template for GetID( ) for Thk function, causing compilation error !!
// Returns ID for specific ThkWrapper type
template <class Param>
unsigned int getId() const {
static_assert(0, "Template specialization for GetID is required for each new THKWrapper");
return 0;
}
// Template specializations of GetID(), required for every new Thk function
#define GET_ID(TYPE, VALUE) \
template <> \
unsigned int getId<TYPE>() const { \
return VALUE; \
}
GET_ID(D3DKMT_OPENADAPTERFROMHDC *, SYSTIMER_ID_OPENADAPTERFROMHDC)
GET_ID(D3DKMT_OPENADAPTERFROMLUID *, SYSTIMER_ID_OPENADAPTERFROMLUID)
GET_ID(CONST D3DKMT_CLOSEADAPTER *, SYSTIMER_ID_CLOSEADAPTER)
GET_ID(CONST D3DKMT_QUERYADAPTERINFO *, SYSTIMER_ID_QUERYADAPTERINFO)
GET_ID(CONST D3DKMT_ESCAPE *, SYSTIMER_ID_ESCAPE)
GET_ID(D3DKMT_CREATEDEVICE *, SYSTIMER_ID_CREATEDEVICE)
GET_ID(CONST D3DKMT_DESTROYDEVICE *, SYSTIMER_ID_DESTROYDEVICE)
GET_ID(D3DKMT_CREATECONTEXT *, SYSTIMER_ID_CREATECONTEXT)
GET_ID(CONST D3DKMT_DESTROYCONTEXT *, SYSTIMER_ID_DESTROYCONTEXT)
GET_ID(D3DKMT_CREATEALLOCATION *, SYSTIMER_ID_CREATEALLOCATION)
GET_ID(CONST D3DKMT_DESTROYALLOCATION *, SYSTIMER_ID_DESTROYALLOCATION)
GET_ID(D3DKMT_OPENRESOURCE *, SYSTIMER_ID_OPENRESOURCE)
GET_ID(D3DKMT_QUERYRESOURCEINFO *, SYSTIMER_ID_QUERYRESOURCEINFO)
GET_ID(D3DKMT_LOCK *, SYSTIMER_ID_LOCK)
GET_ID(CONST D3DKMT_UNLOCK *, SYSTIMER_ID_UNLOCK)
GET_ID(D3DKMT_RENDER *, SYSTIMER_ID_RENDER)
GET_ID(D3DKMT_CREATESYNCHRONIZATIONOBJECT *, SYSTIMER_ID_CREATESYNCHRONIZATIONOBJECT)
GET_ID(CONST D3DKMT_DESTROYSYNCHRONIZATIONOBJECT *, SYSTIMER_ID_DESTROYSYNCHRONIZATIONOBJECT)
GET_ID(CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECT *, SYSTIMER_ID_SIGNALSYNCHRONIZATIONOBJECT)
GET_ID(CONST_FROM_WDK_10_0_18328_0 D3DKMT_WAITFORSYNCHRONIZATIONOBJECT *, SYSTIMER_ID_WAITFORSYNCHRONIZATIONOBJECT)
GET_ID(D3DKMT_CREATESYNCHRONIZATIONOBJECT2 *, SYSTIMER_ID_CREATESYNCHRONIZATIONOBJECT2)
GET_ID(D3DKMT_GETDEVICESTATE *, SYSTIMER_ID_GETDEVICESTATE)
GET_ID(D3DDDI_MAKERESIDENT *, SYSTIMER_ID_MAKERESIDENT)
GET_ID(D3DKMT_EVICT *, SYSTIMER_ID_EVICT)
GET_ID(CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU *, SYSTIMER_ID_WAITFORSYNCHRONIZATIONOBJECTFROMCPU)
GET_ID(CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU *, SYSTIMER_ID_SIGNALSYNCHRONIZATIONOBJECTFROMCPU)
GET_ID(CONST D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU *, SYSTIMER_ID_WAITFORSYNCHRONIZATIONOBJECTFROMGPU)
GET_ID(CONST D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU *, SYSTIMER_ID_SIGNALSYNCHRONIZATIONOBJECTFROMGPU)
GET_ID(D3DKMT_CREATEPAGINGQUEUE *, SYSTIMER_ID_CREATEPAGINGQUEUE)
GET_ID(D3DDDI_DESTROYPAGINGQUEUE *, SYSTIMER_ID_D3DDDI_DESTROYPAGINGQUEUE)
GET_ID(D3DKMT_LOCK2 *, SYSTIMER_ID_LOCK2)
GET_ID(CONST D3DKMT_UNLOCK2 *, SYSTIMER_ID_UNLOCK2)
GET_ID(CONST D3DKMT_INVALIDATECACHE *, SYSTIMER_ID_INVALIDATECACHE)
GET_ID(D3DDDI_MAPGPUVIRTUALADDRESS *, SYSTIMER_ID_D3DDDI_MAPGPUVIRTUALADDRESS)
GET_ID(D3DDDI_RESERVEGPUVIRTUALADDRESS *, SYSTIMER_ID_D3DDDI_RESERVEGPUVIRTUALADDRESS)
GET_ID(CONST D3DKMT_FREEGPUVIRTUALADDRESS *, SYSTIMER_ID_FREEGPUVIRTUALADDRESS)
GET_ID(CONST D3DKMT_UPDATEGPUVIRTUALADDRESS *, SYSTIMER_ID_UPDATEGPUVIRTUALADDRESS)
GET_ID(D3DKMT_CREATECONTEXTVIRTUAL *, SYSTIMER_ID_CREATECONTEXTVIRTUAL)
GET_ID(CONST D3DKMT_SUBMITCOMMAND *, SYSTIMER_ID_SUBMITCOMMAND)
GET_ID(D3DKMT_OPENSYNCOBJECTFROMNTHANDLE2 *, SYSTIMER_ID_OPENSYNCOBJECTFROMNTHANDLE2)
GET_ID(D3DKMT_OPENSYNCOBJECTNTHANDLEFROMNAME *, SYSTIMER_ID_OPENSYNCOBJECTNTHANDLEFROMNAME)
GET_ID(CONST D3DKMT_DESTROYALLOCATION2 *, SYSTIMER_ID_DESTROYALLOCATION2)
GET_ID(D3DKMT_REGISTERTRIMNOTIFICATION *, SYSTIMER_ID_REGISTERTRIMNOTIFICATION)
GET_ID(D3DKMT_UNREGISTERTRIMNOTIFICATION *, SYSTIMER_ID_UNREGISTERTRIMNOTIFICATION)
GET_ID(D3DKMT_OPENRESOURCEFROMNTHANDLE *, SYSTIMER_ID_OPENRESOURCEFROMNTHANDLE)
GET_ID(D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *, SYSTIMER_ID_QUERYRESOURCEINFOFROMNTHANDLE)
GET_ID(D3DKMT_CREATEHWQUEUE *, SYSTIMER_ID_CREATEHWQUEUE)
GET_ID(CONST D3DKMT_DESTROYHWQUEUE *, SYSTIMER_ID_DESTROYHWQUEUE)
GET_ID(CONST D3DKMT_SUBMITCOMMANDTOHWQUEUE *, SYSTIMER_ID_SUBMITCOMMANDTOHWQUEUE)
};
} // namespace NEO

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,196 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "command_stream/preemption_mode.h"
#include "gmm_helper/gmm_lib.h"
#include "helpers/debug_helpers.h"
#include "memory_manager/gfx_partition.h"
#include "os_interface/os_context.h"
#include "os_interface/windows/hw_device_id.h"
#include "os_interface/windows/wddm/wddm_defs.h"
#include "utilities/spinlock.h"
#include "sku_info.h"
#include <memory>
#include <mutex>
namespace NEO {
class Gdi;
class Gmm;
class GmmMemory;
class OsContextWin;
class SettingsReader;
class WddmAllocation;
class WddmInterface;
class WddmResidencyController;
class WddmResidentAllocationsContainer;
class HwDeviceId;
struct AllocationStorageData;
struct HardwareInfo;
struct KmDafListener;
struct RootDeviceEnvironment;
struct MonitoredFence;
struct OsHandleStorage;
enum class HeapIndex : uint32_t;
class Wddm {
public:
typedef HRESULT(WINAPI *CreateDXGIFactoryFcn)(REFIID riid, void **ppFactory);
typedef void(WINAPI *GetSystemInfoFcn)(SYSTEM_INFO *pSystemInfo);
typedef BOOL(WINAPI *VirtualFreeFcn)(LPVOID ptr, SIZE_T size, DWORD flags);
typedef LPVOID(WINAPI *VirtualAllocFcn)(LPVOID inPtr, SIZE_T size, DWORD flags, DWORD type);
virtual ~Wddm();
static Wddm *createWddm(std::unique_ptr<HwDeviceId> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment);
bool init();
MOCKABLE_VIRTUAL bool evict(const D3DKMT_HANDLE *handleList, uint32_t numOfHandles, uint64_t &sizeToTrim);
MOCKABLE_VIRTUAL bool makeResident(const D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim);
MOCKABLE_VIRTUAL bool mapGpuVirtualAddress(Gmm *gmm, D3DKMT_HANDLE handle, D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_VIRTUAL_ADDRESS preferredAddress, D3DGPU_VIRTUAL_ADDRESS &gpuPtr);
bool mapGpuVirtualAddress(AllocationStorageData *allocationStorageData);
MOCKABLE_VIRTUAL D3DGPU_VIRTUAL_ADDRESS reserveGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS minimumAddress, D3DGPU_VIRTUAL_ADDRESS maximumAddress, D3DGPU_SIZE_T size);
MOCKABLE_VIRTUAL bool createContext(OsContextWin &osContext);
MOCKABLE_VIRTUAL void applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext);
MOCKABLE_VIRTUAL bool freeGpuVirtualAddress(D3DGPU_VIRTUAL_ADDRESS &gpuPtr, uint64_t size);
MOCKABLE_VIRTUAL NTSTATUS createAllocation(const void *alignedCpuPtr, const Gmm *gmm, D3DKMT_HANDLE &outHandle, uint32_t shareable);
MOCKABLE_VIRTUAL bool createAllocation64k(const Gmm *gmm, D3DKMT_HANDLE &outHandle);
MOCKABLE_VIRTUAL NTSTATUS createAllocationsAndMapGpuVa(OsHandleStorage &osHandles);
MOCKABLE_VIRTUAL bool destroyAllocations(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle);
MOCKABLE_VIRTUAL bool openSharedHandle(D3DKMT_HANDLE handle, WddmAllocation *alloc);
bool openNTHandle(HANDLE handle, WddmAllocation *alloc);
MOCKABLE_VIRTUAL void *lockResource(const D3DKMT_HANDLE &handle, bool applyMakeResidentPriorToLock);
MOCKABLE_VIRTUAL void unlockResource(const D3DKMT_HANDLE &handle);
MOCKABLE_VIRTUAL void kmDafLock(D3DKMT_HANDLE handle);
MOCKABLE_VIRTUAL bool isKmDafEnabled() const { return featureTable->ftrKmdDaf; }
MOCKABLE_VIRTUAL bool destroyContext(D3DKMT_HANDLE context);
MOCKABLE_VIRTUAL bool queryAdapterInfo();
MOCKABLE_VIRTUAL bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments);
MOCKABLE_VIRTUAL bool waitFromCpu(uint64_t lastFenceValue, const MonitoredFence &monitoredFence);
NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand);
MOCKABLE_VIRTUAL VOID *registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmResidencyController &residencyController);
void unregisterTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, VOID *trimCallbackHandle);
MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress);
MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem);
MOCKABLE_VIRTUAL void *virtualAlloc(void *inPtr, size_t size, unsigned long flags, unsigned long type);
MOCKABLE_VIRTUAL int virtualFree(void *ptr, size_t size, unsigned long flags);
bool configureDeviceAddressSpace();
GT_SYSTEM_INFO *getGtSysInfo() const {
DEBUG_BREAK_IF(!gtSystemInfo);
return gtSystemInfo.get();
}
const GMM_GFX_PARTITIONING &getGfxPartition() const {
return gfxPartition;
}
void initGfxPartition(GfxPartition &outGfxPartition, uint32_t rootDeviceIndex, size_t numRootDevices) const;
const std::string &getDeviceRegistryPath() const {
return deviceRegistryPath;
}
uint64_t getSystemSharedMemory() const;
uint64_t getDedicatedVideoMemory() const;
uint64_t getMaxApplicationAddress() const;
inline D3DKMT_HANDLE getAdapter() const { return hwDeviceId->getAdapter(); }
D3DKMT_HANDLE getDevice() const { return device; }
D3DKMT_HANDLE getPagingQueue() const { return pagingQueue; }
D3DKMT_HANDLE getPagingQueueSyncObject() const { return pagingQueueSyncObject; }
inline Gdi *getGdi() const { return hwDeviceId->getGdi(); }
PFND3DKMT_ESCAPE getEscapeHandle() const;
uint32_t getHwContextId() const {
return static_cast<uint32_t>(hwContextId);
}
std::unique_ptr<SettingsReader> registryReader;
uintptr_t getWddmMinAddress() const {
return this->minAddress;
}
WddmInterface *getWddmInterface() const {
return wddmInterface.get();
}
unsigned int readEnablePreemptionRegKey();
MOCKABLE_VIRTUAL uint64_t *getPagingFenceAddress() {
return pagingFenceAddress;
}
WddmResidentAllocationsContainer *getTemporaryResourcesContainer() {
return temporaryResources.get();
}
void updatePagingFenceValue(uint64_t newPagingFenceValue);
GmmMemory *getGmmMemory() const {
return gmmMemory.get();
}
MOCKABLE_VIRTUAL void waitOnPagingFenceFromCpu();
void setGmmInputArg(void *args);
WddmVersion getWddmVersion();
static CreateDXGIFactoryFcn createDxgiFactory;
protected:
std::unique_ptr<HwDeviceId> hwDeviceId;
D3DKMT_HANDLE device = 0;
D3DKMT_HANDLE pagingQueue = 0;
D3DKMT_HANDLE pagingQueueSyncObject = 0;
uint64_t *pagingFenceAddress = nullptr;
std::atomic<std::uint64_t> currentPagingFenceValue{0};
// Adapter information
std::unique_ptr<PLATFORM> gfxPlatform;
std::unique_ptr<GT_SYSTEM_INFO> gtSystemInfo;
std::unique_ptr<FeatureTable> featureTable;
std::unique_ptr<WorkaroundTable> workaroundTable;
GMM_GFX_PARTITIONING gfxPartition{};
ADAPTER_BDF adapterBDF{};
uint64_t systemSharedMemory = 0;
uint64_t dedicatedVideoMemory = 0;
uint32_t maxRenderFrequency = 0;
bool instrumentationEnabled = false;
std::string deviceRegistryPath;
RootDeviceEnvironment &rootDeviceEnvironment;
unsigned long hwContextId = 0;
uintptr_t maximumApplicationAddress = 0;
std::unique_ptr<GmmMemory> gmmMemory;
uintptr_t minAddress = 0;
Wddm(std::unique_ptr<HwDeviceId> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment);
MOCKABLE_VIRTUAL bool waitOnGPU(D3DKMT_HANDLE context);
bool createDevice(PreemptionMode preemptionMode);
bool createPagingQueue();
bool destroyPagingQueue();
bool destroyDevice();
void getDeviceState();
void handleCompletion(OsContextWin &osContext);
static GetSystemInfoFcn getSystemInfo;
static VirtualFreeFcn virtualFreeFnc;
static VirtualAllocFcn virtualAllocFnc;
std::unique_ptr<KmDafListener> kmDafListener;
std::unique_ptr<WddmInterface> wddmInterface;
std::unique_ptr<WddmResidentAllocationsContainer> temporaryResources;
};
} // namespace NEO

View File

@@ -0,0 +1,28 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm/wddm.h"
#include <dxgi.h>
namespace NEO {
Wddm::CreateDXGIFactoryFcn getCreateDxgiFactory() {
return CreateDXGIFactory;
}
Wddm::GetSystemInfoFcn getGetSystemInfo() {
return GetSystemInfo;
}
Wddm::VirtualFreeFcn getVirtualFree() {
return VirtualFree;
}
Wddm::VirtualAllocFcn getVirtualAlloc() {
return VirtualAlloc;
}
} // namespace NEO

View File

@@ -0,0 +1,14 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm/wddm.h"
namespace NEO {
Wddm *Wddm::createWddm(std::unique_ptr<HwDeviceId> hwDeviceId, RootDeviceEnvironment &rootDeviceEnvironment) {
return new Wddm(std::move(hwDeviceId), rootDeviceEnvironment);
}
} // namespace NEO

View File

@@ -0,0 +1,25 @@
/*
* Copyright (C) 2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "os_interface/windows/windows_defs.h"
#include <cstdint>
namespace NEO {
struct WddmSubmitArguments {
MonitoredFence *monitorFence;
D3DKMT_HANDLE contextHandle;
D3DKMT_HANDLE hwQueueHandle;
};
enum class WddmVersion : uint32_t {
WDDM_2_0 = 0,
WDDM_2_3
};
} // namespace NEO

View File

@@ -0,0 +1,147 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm/wddm_interface.h"
#include "memory_manager/memory_constants.h"
#include "os_interface/windows/gdi_interface.h"
#include "os_interface/windows/os_context_win.h"
#include "os_interface/windows/wddm/wddm.h"
using namespace NEO;
bool WddmInterface::createMonitoredFence(MonitoredFence &monitorFence) {
NTSTATUS status = STATUS_SUCCESS;
D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject = {0};
CreateSynchronizationObject.hDevice = wddm.getDevice();
CreateSynchronizationObject.Info.Type = D3DDDI_MONITORED_FENCE;
CreateSynchronizationObject.Info.MonitoredFence.InitialFenceValue = 0;
status = wddm.getGdi()->createSynchronizationObject2(&CreateSynchronizationObject);
DEBUG_BREAK_IF(STATUS_SUCCESS != status);
monitorFence.fenceHandle = CreateSynchronizationObject.hSyncObject;
monitorFence.cpuAddress = reinterpret_cast<uint64_t *>(CreateSynchronizationObject.Info.MonitoredFence.FenceValueCPUVirtualAddress);
monitorFence.gpuAddress = CreateSynchronizationObject.Info.MonitoredFence.FenceValueGPUVirtualAddress;
return status == STATUS_SUCCESS;
}
void WddmInterface::destroyMonitorFence(D3DKMT_HANDLE fenceHandle) {
NTSTATUS status = STATUS_SUCCESS;
D3DKMT_DESTROYSYNCHRONIZATIONOBJECT destroySyncObject = {0};
destroySyncObject.hSyncObject = fenceHandle;
status = wddm.getGdi()->destroySynchronizationObject(&destroySyncObject);
DEBUG_BREAK_IF(STATUS_SUCCESS != status);
}
bool WddmInterface20::createHwQueue(OsContextWin &osContext) {
return false;
}
void WddmInterface20::destroyHwQueue(D3DKMT_HANDLE hwQueue) {}
bool WddmInterface20::createMonitoredFence(OsContextWin &osContext) {
auto &residencyController = osContext.getResidencyController();
MonitoredFence &monitorFence = residencyController.getMonitoredFence();
bool ret = WddmInterface::createMonitoredFence(monitorFence);
monitorFence.currentFenceValue = 1;
return ret;
}
void WddmInterface20::destroyMonitorFence(MonitoredFence &monitorFence) {
WddmInterface::destroyMonitorFence(monitorFence.fenceHandle);
}
const bool WddmInterface20::hwQueuesSupported() {
return false;
}
bool WddmInterface20::submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) {
D3DKMT_SUBMITCOMMAND SubmitCommand = {0};
NTSTATUS status = STATUS_SUCCESS;
SubmitCommand.Commands = commandBuffer;
SubmitCommand.CommandLength = static_cast<UINT>(size);
SubmitCommand.BroadcastContextCount = 1;
SubmitCommand.BroadcastContext[0] = submitArguments.contextHandle;
SubmitCommand.Flags.NullRendering = (UINT)DebugManager.flags.EnableNullHardware.get();
COMMAND_BUFFER_HEADER *pHeader = reinterpret_cast<COMMAND_BUFFER_HEADER *>(commandHeader);
pHeader->MonitorFenceVA = submitArguments.monitorFence->gpuAddress;
pHeader->MonitorFenceValue = submitArguments.monitorFence->currentFenceValue;
// Note: Private data should be the CPU VA Address
SubmitCommand.pPrivateDriverData = commandHeader;
SubmitCommand.PrivateDriverDataSize = sizeof(COMMAND_BUFFER_HEADER);
status = wddm.getGdi()->submitCommand(&SubmitCommand);
return STATUS_SUCCESS == status;
}
bool WddmInterface23::createHwQueue(OsContextWin &osContext) {
D3DKMT_CREATEHWQUEUE createHwQueue = {};
if (!wddm.getGdi()->setupHwQueueProcAddresses()) {
return false;
}
createHwQueue.hHwContext = osContext.getWddmContextHandle();
if (osContext.getPreemptionMode() >= PreemptionMode::MidBatch) {
createHwQueue.Flags.DisableGpuTimeout = wddm.readEnablePreemptionRegKey();
}
auto status = wddm.getGdi()->createHwQueue(&createHwQueue);
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
osContext.setHwQueue({createHwQueue.hHwQueue, createHwQueue.hHwQueueProgressFence, createHwQueue.HwQueueProgressFenceCPUVirtualAddress,
createHwQueue.HwQueueProgressFenceGPUVirtualAddress});
return status == STATUS_SUCCESS;
}
bool WddmInterface23::createMonitoredFence(OsContextWin &osContext) {
auto &residencyController = osContext.getResidencyController();
auto hwQueue = osContext.getHwQueue();
residencyController.resetMonitoredFenceParams(hwQueue.progressFenceHandle,
reinterpret_cast<uint64_t *>(hwQueue.progressFenceCpuVA),
hwQueue.progressFenceGpuVA);
return true;
}
void WddmInterface23::destroyMonitorFence(MonitoredFence &monitorFence) {
}
void WddmInterface23::destroyHwQueue(D3DKMT_HANDLE hwQueue) {
if (hwQueue) {
D3DKMT_DESTROYHWQUEUE destroyHwQueue = {};
destroyHwQueue.hHwQueue = hwQueue;
auto status = wddm.getGdi()->destroyHwQueue(&destroyHwQueue);
DEBUG_BREAK_IF(status != STATUS_SUCCESS);
}
}
const bool WddmInterface23::hwQueuesSupported() {
return true;
}
bool WddmInterface23::submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) {
D3DKMT_SUBMITCOMMANDTOHWQUEUE submitCommand = {};
submitCommand.hHwQueue = submitArguments.hwQueueHandle;
submitCommand.HwQueueProgressFenceId = submitArguments.monitorFence->currentFenceValue;
submitCommand.CommandBuffer = commandBuffer;
submitCommand.CommandLength = static_cast<UINT>(size);
submitCommand.pPrivateDriverData = commandHeader;
submitCommand.PrivateDriverDataSize = MemoryConstants::pageSize;
auto status = wddm.getGdi()->submitCommandToHwQueue(&submitCommand);
UNRECOVERABLE_IF(status != STATUS_SUCCESS);
return status == STATUS_SUCCESS;
}

View File

@@ -0,0 +1,62 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "helpers/hw_info.h"
#include "os_interface/os_context.h"
#include "os_interface/windows/windows_wrapper.h"
#include <d3dkmthk.h>
#include <cstdint>
namespace NEO {
class Gdi;
class Wddm;
class OsContextWin;
class WddmResidencyController;
struct MonitoredFence;
struct WddmSubmitArguments;
class WddmInterface {
public:
WddmInterface(Wddm &wddm) : wddm(wddm){};
virtual ~WddmInterface() = default;
WddmInterface() = delete;
virtual bool createHwQueue(OsContextWin &osContext) = 0;
virtual void destroyHwQueue(D3DKMT_HANDLE hwQueue) = 0;
virtual bool createMonitoredFence(OsContextWin &osContext) = 0;
MOCKABLE_VIRTUAL bool createMonitoredFence(MonitoredFence &monitorFence);
void destroyMonitorFence(D3DKMT_HANDLE fenceHandle);
virtual void destroyMonitorFence(MonitoredFence &monitorFence) = 0;
virtual const bool hwQueuesSupported() = 0;
virtual bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) = 0;
Wddm &wddm;
};
class WddmInterface20 : public WddmInterface {
public:
using WddmInterface::WddmInterface;
bool createHwQueue(OsContextWin &osContext) override;
void destroyHwQueue(D3DKMT_HANDLE hwQueue) override;
bool createMonitoredFence(OsContextWin &osContext) override;
void destroyMonitorFence(MonitoredFence &monitorFence) override;
const bool hwQueuesSupported() override;
bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override;
};
class WddmInterface23 : public WddmInterface {
public:
using WddmInterface::WddmInterface;
bool createHwQueue(OsContextWin &osContext) override;
void destroyHwQueue(D3DKMT_HANDLE hwQueue) override;
bool createMonitoredFence(OsContextWin &osContext) override;
void destroyMonitorFence(MonitoredFence &monitorFence) override;
const bool hwQueuesSupported() override;
bool submit(uint64_t commandBuffer, size_t size, void *commandHeader, WddmSubmitArguments &submitArguments) override;
};
} // namespace NEO

View File

@@ -0,0 +1,13 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm/wddm.h"
namespace NEO {
void Wddm::applyAdditionalContextFlags(CREATECONTEXT_PVTDATA &privateData, OsContextWin &osContext) {
}
} // namespace NEO

View File

@@ -0,0 +1,14 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm_allocation.h"
namespace NEO {
std::string WddmAllocation::getAllocationInfoString() const {
return getHandleInfoString();
}
} // namespace NEO

View File

@@ -0,0 +1,98 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#define UMDF_USING_NTSTATUS
#include "helpers/aligned_memory.h"
#include "memory_manager/graphics_allocation.h"
#include "memory_manager/residency.h"
#include "os_interface/windows/windows_wrapper.h"
#include <d3dkmthk.h>
namespace NEO {
struct OsHandle {
D3DKMT_HANDLE handle;
D3DGPU_VIRTUAL_ADDRESS gpuPtr;
Gmm *gmm;
};
constexpr size_t trimListUnusedPosition = std::numeric_limits<size_t>::max();
class WddmAllocation : public GraphicsAllocation {
public:
WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool)
: GraphicsAllocation(rootDeviceIndex, allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool), trimCandidateListPositions(MemoryManager::maxOsContextCount, trimListUnusedPosition) {
reservedAddressRangeInfo.addressPtr = reservedAddr;
reservedAddressRangeInfo.rangeSize = sizeIn;
}
WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, void *reservedAddr, MemoryPool::Type pool, uint32_t shareable)
: GraphicsAllocation(rootDeviceIndex, allocationType, cpuPtrIn, castToUint64(cpuPtrIn), 0llu, sizeIn, pool), shareable(shareable), trimCandidateListPositions(MemoryManager::maxOsContextCount, trimListUnusedPosition) {
reservedAddressRangeInfo.addressPtr = reservedAddr;
reservedAddressRangeInfo.rangeSize = sizeIn;
}
WddmAllocation(uint32_t rootDeviceIndex, AllocationType allocationType, void *cpuPtrIn, size_t sizeIn, osHandle sharedHandle, MemoryPool::Type pool)
: GraphicsAllocation(rootDeviceIndex, allocationType, cpuPtrIn, sizeIn, sharedHandle, pool), trimCandidateListPositions(MemoryManager::maxOsContextCount, trimListUnusedPosition) {}
void *getAlignedCpuPtr() const {
return alignDown(this->cpuPtr, MemoryConstants::pageSize);
}
size_t getAlignedSize() const {
return alignSizeWholePage(this->cpuPtr, this->size);
}
ResidencyData &getResidencyData() {
return residency;
}
const std::array<D3DKMT_HANDLE, EngineLimits::maxHandleCount> &getHandles() const { return handles; }
D3DKMT_HANDLE &getHandleToModify(uint32_t handleIndex) { return handles[handleIndex]; }
D3DKMT_HANDLE getDefaultHandle() const { return handles[0]; }
void setDefaultHandle(D3DKMT_HANDLE handle) {
handles[0] = handle;
}
void setTrimCandidateListPosition(uint32_t osContextId, size_t position) {
trimCandidateListPositions[osContextId] = position;
}
size_t getTrimCandidateListPosition(uint32_t osContextId) const {
if (osContextId < trimCandidateListPositions.size()) {
return trimCandidateListPositions[osContextId];
}
return trimListUnusedPosition;
}
void setGpuAddress(uint64_t graphicsAddress) { this->gpuAddress = graphicsAddress; }
void setCpuAddress(void *cpuPtr) { this->cpuPtr = cpuPtr; }
std::string getAllocationInfoString() const override;
uint64_t &getGpuAddressToModify() { return gpuAddress; }
// OS assigned fields
D3DKMT_HANDLE resourceHandle = 0u; // used by shared resources
bool needsMakeResidentBeforeLock = false;
D3DGPU_VIRTUAL_ADDRESS reservedGpuVirtualAddress = 0u;
uint64_t reservedSizeForGpuVirtualAddress = 0u;
uint32_t shareable = 0u;
protected:
std::string getHandleInfoString() const {
std::stringstream ss;
for (auto &handle : handles) {
ss << " Handle: " << handle;
}
return ss.str();
}
std::array<D3DKMT_HANDLE, EngineLimits::maxHandleCount> handles{};
ResidencyData residency;
std::vector<size_t> trimCandidateListPositions;
};
} // namespace NEO

View File

@@ -0,0 +1,25 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm_engine_mapper.h"
#include "helpers/debug_helpers.h"
namespace NEO {
GPUNODE_ORDINAL WddmEngineMapper::engineNodeMap(aub_stream::EngineType engineType) {
if (aub_stream::ENGINE_RCS == engineType) {
return GPUNODE_3D;
} else if (aub_stream::ENGINE_BCS == engineType) {
return GPUNODE_BLT;
} else if (aub_stream::ENGINE_CCS == engineType) {
return GPUNODE_CCS0;
}
UNRECOVERABLE_IF(true);
}
} // namespace NEO

View File

@@ -0,0 +1,21 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "gmm_helper/gmm_lib.h"
#include "engine_node.h"
namespace NEO {
class WddmEngineMapper {
public:
static GPUNODE_ORDINAL engineNodeMap(aub_stream::EngineType engineType);
};
} // namespace NEO

View File

@@ -0,0 +1,562 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm_memory_manager.h"
#include "command_stream/command_stream_receiver_hw.h"
#include "execution_environment/root_device_environment.h"
#include "gmm_helper/gmm.h"
#include "gmm_helper/gmm_helper.h"
#include "gmm_helper/page_table_mngr.h"
#include "gmm_helper/resource_info.h"
#include "helpers/aligned_memory.h"
#include "helpers/deferred_deleter_helper.h"
#include "helpers/ptr_math.h"
#include "helpers/surface_format_info.h"
#include "memory_manager/deferrable_deletion.h"
#include "memory_manager/deferred_deleter.h"
#include "memory_manager/host_ptr_manager.h"
#include "memory_manager/memory_operations_handler.h"
#include "os_interface/os_interface.h"
#include "os_interface/windows/os_context_win.h"
#include "os_interface/windows/os_interface.h"
#include "os_interface/windows/wddm/wddm.h"
#include "os_interface/windows/wddm_allocation.h"
#include "os_interface/windows/wddm_residency_allocations_container.h"
#include "os_interface/windows/wddm_residency_controller.h"
#include <algorithm>
namespace NEO {
WddmMemoryManager::~WddmMemoryManager() = default;
WddmMemoryManager::WddmMemoryManager(ExecutionEnvironment &executionEnvironment) : MemoryManager(executionEnvironment) {
asyncDeleterEnabled = isDeferredDeleterEnabled();
if (asyncDeleterEnabled)
deferredDeleter = createDeferredDeleter();
mallocRestrictions.minAddress = 0u;
for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < gfxPartitions.size(); ++rootDeviceIndex) {
getWddm(rootDeviceIndex).initGfxPartition(*getGfxPartition(rootDeviceIndex), rootDeviceIndex, gfxPartitions.size());
mallocRestrictions.minAddress = std::max(mallocRestrictions.minAddress, getWddm(rootDeviceIndex).getWddmMinAddress());
}
}
GraphicsAllocation *WddmMemoryManager::allocateShareableMemory(const AllocationData &allocationData) {
auto gmm = std::make_unique<Gmm>(executionEnvironment.getGmmClientContext(), allocationData.hostPtr, allocationData.size, false);
auto allocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, nullptr, allocationData.size, nullptr, MemoryPool::SystemCpuInaccessible, allocationData.flags.shareable);
allocation->setDefaultGmm(gmm.get());
if (!createWddmAllocation(allocation.get(), nullptr)) {
return nullptr;
}
gmm.release();
return allocation.release();
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) {
if (allocationData.imgInfo->linearStorage && allocationData.imgInfo->mipCount == 0) {
return allocateGraphicsMemoryWithAlignment(allocationData);
}
auto allocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, nullptr, allocationData.imgInfo->size, nullptr, MemoryPool::SystemCpuInaccessible);
allocation->setDefaultGmm(gmm.get());
if (!createWddmAllocation(allocation.get(), nullptr)) {
return nullptr;
}
gmm.release();
return allocation.release();
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory64kb(const AllocationData &allocationData) {
size_t sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize64k);
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, nullptr, sizeAligned, nullptr, MemoryPool::System64KBPages);
auto gmm = new Gmm(executionEnvironment.getGmmClientContext(), nullptr, sizeAligned, false, allocationData.flags.preferRenderCompressed, true, {});
wddmAllocation->setDefaultGmm(gmm);
if (!getWddm(allocationData.rootDeviceIndex).createAllocation64k(gmm, wddmAllocation->getHandleToModify(0u))) {
delete gmm;
return nullptr;
}
auto cpuPtr = lockResource(wddmAllocation.get());
// 64kb map is not needed
auto status = mapGpuVirtualAddress(wddmAllocation.get(), cpuPtr);
DEBUG_BREAK_IF(!status);
wddmAllocation->setCpuAddress(cpuPtr);
return wddmAllocation.release();
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) {
size_t newAlignment = allocationData.alignment ? alignUp(allocationData.alignment, MemoryConstants::pageSize) : MemoryConstants::pageSize;
size_t sizeAligned = allocationData.size ? alignUp(allocationData.size, MemoryConstants::pageSize) : MemoryConstants::pageSize;
void *pSysMem = allocateSystemMemory(sizeAligned, newAlignment);
Gmm *gmm = nullptr;
if (pSysMem == nullptr) {
return nullptr;
}
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, pSysMem, sizeAligned, nullptr, MemoryPool::System4KBPages);
wddmAllocation->setDriverAllocatedCpuPtr(pSysMem);
gmm = new Gmm(executionEnvironment.getGmmClientContext(), pSysMem, sizeAligned, allocationData.flags.uncacheable);
wddmAllocation->setDefaultGmm(gmm);
void *mapPtr = wddmAllocation->getAlignedCpuPtr();
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_CPU) {
//add 2MB padding in case mapPtr is not 2MB aligned
size_t reserveSizeAligned = sizeAligned + allocationData.alignment;
bool ret = getWddm(wddmAllocation->getRootDeviceIndex()).reserveValidAddressRange(reserveSizeAligned, mapPtr);
if (!ret) {
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;
}
wddmAllocation->setReservedAddressRange(mapPtr, reserveSizeAligned);
mapPtr = alignUp(mapPtr, newAlignment);
}
if (!createWddmAllocation(wddmAllocation.get(), mapPtr)) {
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;
}
return wddmAllocation.release();
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) {
auto alignedPtr = alignDown(allocationData.hostPtr, MemoryConstants::pageSize);
auto offsetInPage = ptrDiff(allocationData.hostPtr, alignedPtr);
auto alignedSize = alignSizeWholePage(allocationData.hostPtr, allocationData.size);
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(allocationData.hostPtr),
allocationData.size, nullptr, MemoryPool::System4KBPages);
wddmAllocation->setAllocationOffset(offsetInPage);
auto gmm = new Gmm(executionEnvironment.getGmmClientContext(), alignedPtr, alignedSize, false);
wddmAllocation->setDefaultGmm(gmm);
if (!createWddmAllocation(wddmAllocation.get(), nullptr)) {
delete gmm;
return nullptr;
}
return wddmAllocation.release();
}
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) {
if (mallocRestrictions.minAddress > reinterpret_cast<uintptr_t>(allocationData.hostPtr)) {
auto inputPtr = allocationData.hostPtr;
void *reserve = nullptr;
auto ptrAligned = alignDown(inputPtr, MemoryConstants::allocationAlignment);
size_t sizeAligned = alignSizeWholePage(inputPtr, allocationData.size);
size_t offset = ptrDiff(inputPtr, ptrAligned);
if (!getWddm(allocationData.rootDeviceIndex).reserveValidAddressRange(sizeAligned, reserve)) {
return nullptr;
}
auto allocation = new WddmAllocation(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(inputPtr), allocationData.size, reserve, MemoryPool::System4KBPages);
allocation->setAllocationOffset(offset);
Gmm *gmm = new Gmm(executionEnvironment.getGmmClientContext(), ptrAligned, sizeAligned, false);
allocation->setDefaultGmm(gmm);
if (createWddmAllocation(allocation, reserve)) {
return allocation;
}
freeGraphicsMemory(allocation);
return nullptr;
}
return MemoryManager::allocateGraphicsMemoryWithHostPtr(allocationData);
}
GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) {
Gmm *gmm = nullptr;
const void *ptrAligned = nullptr;
size_t sizeAligned = allocationData.size;
void *pSysMem = nullptr;
size_t offset = 0;
if (allocationData.hostPtr) {
ptrAligned = alignDown(allocationData.hostPtr, MemoryConstants::allocationAlignment);
sizeAligned = alignSizeWholePage(allocationData.hostPtr, sizeAligned);
offset = ptrDiff(allocationData.hostPtr, ptrAligned);
} else {
sizeAligned = alignUp(sizeAligned, MemoryConstants::allocationAlignment);
pSysMem = allocateSystemMemory(sizeAligned, MemoryConstants::allocationAlignment);
if (pSysMem == nullptr) {
return nullptr;
}
ptrAligned = pSysMem;
}
auto wddmAllocation = std::make_unique<WddmAllocation>(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(ptrAligned), sizeAligned, nullptr,
MemoryPool::System4KBPagesWith32BitGpuAddressing);
wddmAllocation->setDriverAllocatedCpuPtr(pSysMem);
wddmAllocation->set32BitAllocation(true);
wddmAllocation->setAllocationOffset(offset);
gmm = new Gmm(executionEnvironment.getGmmClientContext(), ptrAligned, sizeAligned, false);
wddmAllocation->setDefaultGmm(gmm);
if (!createWddmAllocation(wddmAllocation.get(), nullptr)) {
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;
}
auto baseAddress = useInternal32BitAllocator(allocationData.type) ? getInternalHeapBaseAddress(allocationData.rootDeviceIndex) : getExternalHeapBaseAddress(allocationData.rootDeviceIndex);
wddmAllocation->setGpuBaseAddress(GmmHelper::canonize(baseAddress));
return wddmAllocation.release();
}
GraphicsAllocation *WddmMemoryManager::createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle, GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex) {
auto allocation = std::make_unique<WddmAllocation>(rootDeviceIndex, allocationType, nullptr, 0, handle, MemoryPool::SystemCpuInaccessible);
bool status = ntHandle ? getWddm(rootDeviceIndex).openNTHandle(reinterpret_cast<HANDLE>(static_cast<uintptr_t>(handle)), allocation.get())
: getWddm(rootDeviceIndex).openSharedHandle(handle, allocation.get());
if (!status) {
return nullptr;
}
// Shared objects are passed without size
size_t size = allocation->getDefaultGmm()->gmmResourceInfo->getSizeAllocation();
allocation->setSize(size);
if (is32bit) {
void *ptr = nullptr;
if (!getWddm(rootDeviceIndex).reserveValidAddressRange(size, ptr)) {
return nullptr;
}
allocation->setReservedAddressRange(ptr, size);
} else if (requireSpecificBitness && this->force32bitAllocations) {
allocation->set32BitAllocation(true);
allocation->setGpuBaseAddress(GmmHelper::canonize(getExternalHeapBaseAddress(allocation->getRootDeviceIndex())));
}
status = mapGpuVirtualAddress(allocation.get(), allocation->getReservedAddressPtr());
DEBUG_BREAK_IF(!status);
if (!status) {
freeGraphicsMemoryImpl(allocation.release());
return nullptr;
}
FileLoggerInstance().logAllocation(allocation.get());
return allocation.release();
}
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) {
return createAllocationFromHandle(handle, requireSpecificBitness, false, properties.allocationType, properties.rootDeviceIndex);
}
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) {
return createAllocationFromHandle(toOsHandle(handle), false, true, GraphicsAllocation::AllocationType::SHARED_IMAGE, rootDeviceIndex);
}
void WddmMemoryManager::addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) {
WddmAllocation *wddmMemory = static_cast<WddmAllocation *>(gfxAllocation);
FragmentStorage fragment = {};
fragment.driverAllocation = true;
fragment.fragmentCpuPointer = gfxAllocation->getUnderlyingBuffer();
fragment.fragmentSize = alignUp(gfxAllocation->getUnderlyingBufferSize(), MemoryConstants::pageSize);
fragment.osInternalStorage = new OsHandle();
fragment.osInternalStorage->gpuPtr = gfxAllocation->getGpuAddress();
fragment.osInternalStorage->handle = wddmMemory->getDefaultHandle();
fragment.osInternalStorage->gmm = gfxAllocation->getDefaultGmm();
fragment.residency = &wddmMemory->getResidencyData();
hostPtrManager->storeFragment(fragment);
}
void WddmMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) {
auto buffer = gfxAllocation->getUnderlyingBuffer();
auto fragment = hostPtrManager->getFragment(buffer);
if (fragment && fragment->driverAllocation) {
OsHandle *osStorageToRelease = fragment->osInternalStorage;
if (hostPtrManager->releaseHostPtr(buffer)) {
delete osStorageToRelease;
}
}
}
void *WddmMemoryManager::lockResourceImpl(GraphicsAllocation &graphicsAllocation) {
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
return getWddm(graphicsAllocation.getRootDeviceIndex()).lockResource(wddmAllocation.getDefaultHandle(), wddmAllocation.needsMakeResidentBeforeLock);
}
void WddmMemoryManager::unlockResourceImpl(GraphicsAllocation &graphicsAllocation) {
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
getWddm(graphicsAllocation.getRootDeviceIndex()).unlockResource(wddmAllocation.getDefaultHandle());
if (wddmAllocation.needsMakeResidentBeforeLock) {
auto evictionStatus = getWddm(graphicsAllocation.getRootDeviceIndex()).getTemporaryResourcesContainer()->evictResource(wddmAllocation.getDefaultHandle());
DEBUG_BREAK_IF(evictionStatus == MemoryOperationsStatus::FAILED);
}
}
void WddmMemoryManager::freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) {
auto &wddmAllocation = static_cast<WddmAllocation &>(graphicsAllocation);
if (wddmAllocation.needsMakeResidentBeforeLock) {
for (auto i = 0u; i < wddmAllocation.getNumHandles(); i++) {
getWddm(graphicsAllocation.getRootDeviceIndex()).getTemporaryResourcesContainer()->removeResource(wddmAllocation.getHandles()[i]);
}
}
}
void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) {
WddmAllocation *input = static_cast<WddmAllocation *>(gfxAllocation);
DEBUG_BREAK_IF(!validateAllocation(input));
for (auto &engine : this->registeredEngines) {
auto &residencyController = static_cast<OsContextWin *>(engine.osContext)->getResidencyController();
auto lock = residencyController.acquireLock();
residencyController.removeFromTrimCandidateListIfUsed(input, true);
}
executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface->evict(*input);
auto defaultGmm = gfxAllocation->getDefaultGmm();
if (defaultGmm) {
auto index = gfxAllocation->getRootDeviceIndex();
if (defaultGmm->isRenderCompressed && executionEnvironment.rootDeviceEnvironments[index]->pageTableManager.get()) {
auto status = executionEnvironment.rootDeviceEnvironments[index]->pageTableManager->updateAuxTable(input->getGpuAddress(), defaultGmm, false);
DEBUG_BREAK_IF(!status);
}
}
for (auto handleId = 0u; handleId < EngineLimits::maxHandleCount; handleId++) {
delete gfxAllocation->getGmm(handleId);
}
if (input->peekSharedHandle() == false &&
input->getDriverAllocatedCpuPtr() == nullptr &&
input->fragmentsStorage.fragmentCount > 0) {
cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation);
} else {
const D3DKMT_HANDLE *allocationHandles = nullptr;
uint32_t allocationCount = 0;
D3DKMT_HANDLE resourceHandle = 0;
if (input->peekSharedHandle()) {
resourceHandle = input->resourceHandle;
} else {
allocationHandles = input->getHandles().data();
allocationCount = input->getNumHandles();
}
auto status = tryDeferDeletions(allocationHandles, allocationCount, resourceHandle, gfxAllocation->getRootDeviceIndex());
DEBUG_BREAK_IF(!status);
alignedFreeWrapper(input->getDriverAllocatedCpuPtr());
}
if (input->getReservedAddressPtr()) {
releaseReservedCpuAddressRange(input->getReservedAddressPtr(), input->getReservedAddressSize(), gfxAllocation->getRootDeviceIndex());
}
if (input->reservedGpuVirtualAddress) {
getWddm(gfxAllocation->getRootDeviceIndex()).freeGpuVirtualAddress(input->reservedGpuVirtualAddress, input->reservedSizeForGpuVirtualAddress);
}
delete gfxAllocation;
}
void WddmMemoryManager::handleFenceCompletion(GraphicsAllocation *allocation) {
auto wddmAllocation = static_cast<WddmAllocation *>(allocation);
for (auto &engine : this->registeredEngines) {
const auto lastFenceValue = wddmAllocation->getResidencyData().getFenceValueForContextId(engine.osContext->getContextId());
if (lastFenceValue != 0u) {
const auto &monitoredFence = static_cast<OsContextWin *>(engine.osContext)->getResidencyController().getMonitoredFence();
const auto wddm = static_cast<OsContextWin *>(engine.osContext)->getWddm();
wddm->waitFromCpu(lastFenceValue, monitoredFence);
}
}
}
bool WddmMemoryManager::tryDeferDeletions(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, uint32_t rootDeviceIndex) {
bool status = true;
if (deferredDeleter) {
deferredDeleter->deferDeletion(DeferrableDeletion::create(&getWddm(rootDeviceIndex), handles, allocationCount, resourceHandle));
} else {
status = getWddm(rootDeviceIndex).destroyAllocations(handles, allocationCount, resourceHandle);
}
return status;
}
bool WddmMemoryManager::isMemoryBudgetExhausted() const {
for (auto &engine : this->registeredEngines) {
if (static_cast<OsContextWin *>(engine.osContext)->getResidencyController().isMemoryBudgetExhausted()) {
return true;
}
}
return false;
}
bool WddmMemoryManager::validateAllocation(WddmAllocation *alloc) {
if (alloc == nullptr)
return false;
auto size = alloc->getUnderlyingBufferSize();
if (alloc->getGpuAddress() == 0u || size == 0 || (alloc->getDefaultHandle() == 0 && alloc->fragmentsStorage.fragmentCount == 0))
return false;
return true;
}
MemoryManager::AllocationStatus WddmMemoryManager::populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) {
uint32_t allocatedFragmentIndexes[maxFragmentsCount];
uint32_t allocatedFragmentsCounter = 0;
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
// If no fragment is present it means it already exists.
if (!handleStorage.fragmentStorageData[i].osHandleStorage && handleStorage.fragmentStorageData[i].cpuPtr) {
handleStorage.fragmentStorageData[i].osHandleStorage = new OsHandle();
handleStorage.fragmentStorageData[i].residency = new ResidencyData();
handleStorage.fragmentStorageData[i].osHandleStorage->gmm = new Gmm(executionEnvironment.getGmmClientContext(), handleStorage.fragmentStorageData[i].cpuPtr,
handleStorage.fragmentStorageData[i].fragmentSize, false);
allocatedFragmentIndexes[allocatedFragmentsCounter] = i;
allocatedFragmentsCounter++;
}
}
auto result = getWddm(rootDeviceIndex).createAllocationsAndMapGpuVa(handleStorage);
if (result == STATUS_GRAPHICS_NO_VIDEO_MEMORY) {
return AllocationStatus::InvalidHostPointer;
}
UNRECOVERABLE_IF(result != STATUS_SUCCESS);
for (uint32_t i = 0; i < allocatedFragmentsCounter; i++) {
hostPtrManager->storeFragment(handleStorage.fragmentStorageData[allocatedFragmentIndexes[i]]);
}
return AllocationStatus::Success;
}
void WddmMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) {
D3DKMT_HANDLE handles[maxFragmentsCount] = {0};
auto allocationCount = 0;
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
handles[allocationCount++] = handleStorage.fragmentStorageData[i].osHandleStorage->handle;
std::fill(handleStorage.fragmentStorageData[i].residency->resident.begin(), handleStorage.fragmentStorageData[i].residency->resident.end(), false);
}
}
bool success = tryDeferDeletions(handles, allocationCount, 0, rootDeviceIndex);
for (unsigned int i = 0; i < maxFragmentsCount; i++) {
if (handleStorage.fragmentStorageData[i].freeTheFragment) {
if (success) {
handleStorage.fragmentStorageData[i].osHandleStorage->handle = 0;
}
delete handleStorage.fragmentStorageData[i].osHandleStorage->gmm;
delete handleStorage.fragmentStorageData[i].osHandleStorage;
delete handleStorage.fragmentStorageData[i].residency;
}
}
}
void WddmMemoryManager::obtainGpuAddressFromFragments(WddmAllocation *allocation, OsHandleStorage &handleStorage) {
if (this->force32bitAllocations && (handleStorage.fragmentCount > 0)) {
auto hostPtr = allocation->getUnderlyingBuffer();
auto fragment = hostPtrManager->getFragment(hostPtr);
if (fragment && fragment->driverAllocation) {
auto gpuPtr = handleStorage.fragmentStorageData[0].osHandleStorage->gpuPtr;
for (uint32_t i = 1; i < handleStorage.fragmentCount; i++) {
if (handleStorage.fragmentStorageData[i].osHandleStorage->gpuPtr < gpuPtr) {
gpuPtr = handleStorage.fragmentStorageData[i].osHandleStorage->gpuPtr;
}
}
allocation->setAllocationOffset(reinterpret_cast<uint64_t>(hostPtr) & MemoryConstants::pageMask);
allocation->setGpuAddress(gpuPtr);
}
}
}
GraphicsAllocation *WddmMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) {
auto allocation = new WddmAllocation(allocationData.rootDeviceIndex, allocationData.type, const_cast<void *>(allocationData.hostPtr), allocationData.size, nullptr, MemoryPool::System4KBPages);
allocation->fragmentsStorage = handleStorage;
obtainGpuAddressFromFragments(allocation, handleStorage);
return allocation;
}
uint64_t WddmMemoryManager::getSystemSharedMemory(uint32_t rootDeviceIndex) {
return getWddm(rootDeviceIndex).getSystemSharedMemory();
}
AlignedMallocRestrictions *WddmMemoryManager::getAlignedMallocRestrictions() {
return &mallocRestrictions;
}
bool WddmMemoryManager::createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr) {
auto status = createGpuAllocationsWithRetry(allocation);
if (!status) {
return false;
}
return mapGpuVirtualAddress(allocation, requiredGpuPtr);
}
bool WddmMemoryManager::mapGpuVaForOneHandleAllocation(WddmAllocation *allocation, const void *preferredGpuVirtualAddress) {
D3DGPU_VIRTUAL_ADDRESS addressToMap = castToUint64(preferredGpuVirtualAddress);
auto heapIndex = selectHeap(allocation, preferredGpuVirtualAddress != nullptr, is32bit || executionEnvironment.isFullRangeSvm());
if (!executionEnvironment.isFullRangeSvm()) {
addressToMap = 0u;
}
if (allocation->reservedGpuVirtualAddress) {
addressToMap = allocation->reservedGpuVirtualAddress;
}
auto gfxPartition = getGfxPartition(allocation->getRootDeviceIndex());
auto status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, allocation->getGpuAddressToModify());
if (!status && deferredDeleter) {
deferredDeleter->drain(true);
status = getWddm(allocation->getRootDeviceIndex()).mapGpuVirtualAddress(allocation->getDefaultGmm(), allocation->getDefaultHandle(), gfxPartition->getHeapMinimalAddress(heapIndex), gfxPartition->getHeapLimit(heapIndex), addressToMap, allocation->getGpuAddressToModify());
}
if (!status) {
if (allocation->reservedGpuVirtualAddress) {
getWddm(allocation->getRootDeviceIndex()).freeGpuVirtualAddress(allocation->reservedGpuVirtualAddress, allocation->reservedSizeForGpuVirtualAddress);
}
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(allocation->getHandles().data(), allocation->getNumHandles(), allocation->resourceHandle);
return false;
}
return true;
}
bool WddmMemoryManager::createGpuAllocationsWithRetry(WddmAllocation *allocation) {
for (auto handleId = 0u; handleId < allocation->getNumHandles(); handleId++) {
auto status = getWddm(allocation->getRootDeviceIndex()).createAllocation(allocation->getAlignedCpuPtr(), allocation->getGmm(handleId), allocation->getHandleToModify(handleId), allocation->shareable);
if (status == STATUS_GRAPHICS_NO_VIDEO_MEMORY && deferredDeleter) {
deferredDeleter->drain(true);
status = getWddm(allocation->getRootDeviceIndex()).createAllocation(allocation->getAlignedCpuPtr(), allocation->getGmm(handleId), allocation->getHandleToModify(handleId), allocation->shareable);
}
if (status != STATUS_SUCCESS) {
getWddm(allocation->getRootDeviceIndex()).destroyAllocations(allocation->getHandles().data(), handleId, allocation->resourceHandle);
return false;
}
}
return true;
}
Wddm &WddmMemoryManager::getWddm(uint32_t rootDeviceIndex) const {
return *this->executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface->get()->getWddm();
}
void *WddmMemoryManager::reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) {
void *reservePtr = nullptr;
getWddm(rootDeviceIndex).reserveValidAddressRange(size, reservePtr);
return reservePtr;
}
void WddmMemoryManager::releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) {
getWddm(rootDeviceIndex).releaseReservedAddress(reserved);
}
} // namespace NEO

View File

@@ -0,0 +1,88 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "helpers/aligned_memory.h"
#include "memory_manager/memory_manager.h"
#include "os_interface/os_context.h"
#include "os_interface/windows/wddm_allocation.h"
#include "os_interface/windows/windows_wrapper.h"
#include <d3dkmthk.h>
#include <map>
#include <mutex>
#include <vector>
namespace NEO {
class Gmm;
class OsContextWin;
class Wddm;
class WddmMemoryManager : public MemoryManager {
public:
using MemoryManager::allocateGraphicsMemoryWithProperties;
~WddmMemoryManager() override;
WddmMemoryManager(ExecutionEnvironment &executionEnvironment);
WddmMemoryManager(const WddmMemoryManager &) = delete;
WddmMemoryManager &operator=(const WddmMemoryManager &) = delete;
void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override;
void handleFenceCompletion(GraphicsAllocation *allocation) override;
GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) override;
GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) override;
void addAllocationToHostPtrManager(GraphicsAllocation *memory) override;
void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) override;
AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
void cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
void obtainGpuAddressFromFragments(WddmAllocation *allocation, OsHandleStorage &handleStorage);
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override;
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex) override;
bool tryDeferDeletions(const D3DKMT_HANDLE *handles, uint32_t allocationCount, D3DKMT_HANDLE resourceHandle, uint32_t rootDeviceIndex);
bool isMemoryBudgetExhausted() const override;
AlignedMallocRestrictions *getAlignedMallocRestrictions() override;
bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, const void *memoryToCopy, size_t sizeToCopy) override;
void *reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) override;
void releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex) override;
protected:
GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override;
GraphicsAllocation *allocateShareableMemory(const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override;
void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
void freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) override;
GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
GraphicsAllocation *createAllocationFromHandle(osHandle handle, bool requireSpecificBitness, bool ntHandle, GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex);
static bool validateAllocation(WddmAllocation *alloc);
bool createWddmAllocation(WddmAllocation *allocation, void *requiredGpuPtr);
bool mapGpuVirtualAddress(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr);
bool mapGpuVaForOneHandleAllocation(WddmAllocation *graphicsAllocation, const void *requiredGpuPtr);
bool createGpuAllocationsWithRetry(WddmAllocation *graphicsAllocation);
AlignedMallocRestrictions mallocRestrictions;
Wddm &getWddm(uint32_t rootDeviceIndex) const;
};
} // namespace NEO

View File

@@ -0,0 +1,25 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "helpers/basic_math.h"
#include "os_interface/windows/wddm_memory_manager.h"
namespace NEO {
GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) {
status = AllocationStatus::RetryInNonDevicePool;
return nullptr;
}
bool WddmMemoryManager::copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, const void *memoryToCopy, size_t sizeToCopy) {
return MemoryManager::copyMemoryToAllocation(graphicsAllocation, memoryToCopy, sizeToCopy);
}
bool WddmMemoryManager::mapGpuVirtualAddress(WddmAllocation *allocation, const void *requiredPtr) {
return mapGpuVaForOneHandleAllocation(allocation, requiredPtr);
}
uint64_t WddmMemoryManager::getLocalMemorySize(uint32_t rootDeviceIndex) {
return 0 * GB;
}
} // namespace NEO

View File

@@ -0,0 +1,79 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm_memory_operations_handler.h"
#include "memory_manager/host_ptr_defines.h"
#include "os_interface/windows/wddm_allocation.h"
#include "os_interface/windows/wddm_residency_allocations_container.h"
#include "utilities/stackvec.h"
namespace NEO {
WddmMemoryOperationsHandler::WddmMemoryOperationsHandler(Wddm *wddm) : wddm(wddm) {
residentAllocations = std::make_unique<WddmResidentAllocationsContainer>(wddm);
}
MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRef<GraphicsAllocation *> gfxAllocations) {
uint32_t totalHandlesCount = 0;
constexpr uint32_t stackAllocations = 64;
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount * stackAllocations;
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForResidency;
for (const auto &allocation : gfxAllocations) {
WddmAllocation *wddmAllocation = reinterpret_cast<WddmAllocation *>(allocation);
if (wddmAllocation->fragmentsStorage.fragmentCount > 0) {
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
handlesForResidency[totalHandlesCount++] = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
}
} else {
memcpy_s(&handlesForResidency[totalHandlesCount],
wddmAllocation->getNumHandles() * sizeof(D3DKMT_HANDLE),
wddmAllocation->getHandles().data(),
wddmAllocation->getNumHandles() * sizeof(D3DKMT_HANDLE));
totalHandlesCount += wddmAllocation->getNumHandles();
}
}
return residentAllocations->makeResidentResources(handlesForResidency.begin(), totalHandlesCount);
}
MemoryOperationsStatus WddmMemoryOperationsHandler::evict(GraphicsAllocation &gfxAllocation) {
constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount;
StackVec<D3DKMT_HANDLE, stackHandlesCount> handlesForEviction;
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
uint32_t totalHandleCount = 0;
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {
OsHandleStorage &fragmentStorage = wddmAllocation.fragmentsStorage;
for (uint32_t allocId = 0; allocId < fragmentStorage.fragmentCount; allocId++) {
handlesForEviction.push_back(fragmentStorage.fragmentStorageData[allocId].osHandleStorage->handle);
totalHandleCount++;
}
} else {
const D3DKMT_HANDLE *handlePtr = wddmAllocation.getHandles().data();
size_t handleCount = wddmAllocation.getNumHandles();
for (uint32_t i = 0; i < handleCount; i++, totalHandleCount++) {
handlesForEviction.push_back(*handlePtr);
handlePtr++;
}
}
return residentAllocations->evictResources(handlesForEviction.begin(), totalHandleCount);
}
MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(GraphicsAllocation &gfxAllocation) {
WddmAllocation &wddmAllocation = reinterpret_cast<WddmAllocation &>(gfxAllocation);
D3DKMT_HANDLE defaultHandle = 0u;
if (wddmAllocation.fragmentsStorage.fragmentCount > 0) {
defaultHandle = wddmAllocation.fragmentsStorage.fragmentStorageData[0].osHandleStorage->handle;
} else {
defaultHandle = wddmAllocation.getDefaultHandle();
}
return residentAllocations->isAllocationResident(defaultHandle);
}
} // namespace NEO

View File

@@ -0,0 +1,31 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "memory_manager/memory_operations_handler.h"
#include <memory>
namespace NEO {
class Wddm;
class WddmResidentAllocationsContainer;
class WddmMemoryOperationsHandler : public MemoryOperationsHandler {
public:
WddmMemoryOperationsHandler(Wddm *wddm);
~WddmMemoryOperationsHandler() override = default;
MemoryOperationsStatus makeResident(ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override;
protected:
Wddm *wddm;
std::unique_ptr<WddmResidentAllocationsContainer> residentAllocations;
};
} // namespace NEO

View File

@@ -0,0 +1,95 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm_residency_allocations_container.h"
#include "os_interface/windows/wddm/wddm.h"
#include "os_interface/windows/wddm_allocation.h"
#include <algorithm>
namespace NEO {
WddmResidentAllocationsContainer::~WddmResidentAllocationsContainer() {
evictAllResources();
}
MemoryOperationsStatus WddmResidentAllocationsContainer::isAllocationResident(const D3DKMT_HANDLE &handle) {
auto lock = acquireLock(resourcesLock);
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handle);
return position != resourceHandles.end() ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::MEMORY_NOT_FOUND;
}
MemoryOperationsStatus WddmResidentAllocationsContainer::evictAllResources() {
decltype(resourceHandles) resourcesToEvict;
auto lock = acquireLock(resourcesLock);
resourceHandles.swap(resourcesToEvict);
if (resourcesToEvict.empty()) {
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
}
uint64_t sizeToTrim = 0;
uint32_t evictedResources = static_cast<uint32_t>(resourcesToEvict.size());
bool success = wddm->evict(resourcesToEvict.data(), evictedResources, sizeToTrim);
return success ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED;
}
MemoryOperationsStatus WddmResidentAllocationsContainer::evictResource(const D3DKMT_HANDLE &handle) {
return evictResources(&handle, 1u);
}
MemoryOperationsStatus WddmResidentAllocationsContainer::evictResources(const D3DKMT_HANDLE *handles, const uint32_t count) {
auto lock = acquireLock(resourcesLock);
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handles[0]);
if (position == resourceHandles.end()) {
return MemoryOperationsStatus::MEMORY_NOT_FOUND;
}
auto distance = static_cast<size_t>(std::distance(resourceHandles.begin(), position));
UNRECOVERABLE_IF(distance + count > resourceHandles.size());
resourceHandles.erase(position, position + count);
uint64_t sizeToTrim = 0;
if (!wddm->evict(handles, count, sizeToTrim)) {
return MemoryOperationsStatus::FAILED;
}
return MemoryOperationsStatus::SUCCESS;
}
MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResource(const D3DKMT_HANDLE &handle) {
return makeResidentResources(&handle, 1u);
}
MemoryOperationsStatus WddmResidentAllocationsContainer::makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count) {
bool madeResident = false;
while (!(madeResident = wddm->makeResident(handles, count, false, nullptr))) {
if (evictAllResources() == MemoryOperationsStatus::SUCCESS) {
continue;
}
if (!wddm->makeResident(handles, count, false, nullptr)) {
DEBUG_BREAK_IF(true);
return MemoryOperationsStatus::OUT_OF_MEMORY;
};
break;
}
DEBUG_BREAK_IF(!madeResident);
auto lock = acquireLock(resourcesLock);
for (uint32_t i = 0; i < count; i++) {
resourceHandles.push_back(handles[i]);
}
lock.unlock();
wddm->waitOnPagingFenceFromCpu();
return madeResident ? MemoryOperationsStatus::SUCCESS : MemoryOperationsStatus::FAILED;
}
void WddmResidentAllocationsContainer::removeResource(const D3DKMT_HANDLE &handle) {
auto lock = acquireLock(resourcesLock);
auto position = std::find(resourceHandles.begin(), resourceHandles.end(), handle);
if (position == resourceHandles.end()) {
return;
}
*position = resourceHandles.back();
resourceHandles.pop_back();
}
} // namespace NEO

View File

@@ -0,0 +1,42 @@
/*
* Copyright (C) 2019-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "memory_manager/memory_operations_status.h"
#include "os_interface/windows/windows_defs.h"
#include "utilities/spinlock.h"
#include <mutex>
#include <vector>
namespace NEO {
class Wddm;
class WddmResidentAllocationsContainer {
public:
WddmResidentAllocationsContainer(Wddm *wddm) : wddm(wddm) {}
virtual ~WddmResidentAllocationsContainer();
MemoryOperationsStatus isAllocationResident(const D3DKMT_HANDLE &handle);
MOCKABLE_VIRTUAL MemoryOperationsStatus evictAllResources();
MOCKABLE_VIRTUAL MemoryOperationsStatus evictResource(const D3DKMT_HANDLE &handle);
MemoryOperationsStatus evictResources(const D3DKMT_HANDLE *handles, const uint32_t count);
MOCKABLE_VIRTUAL MemoryOperationsStatus makeResidentResource(const D3DKMT_HANDLE &handle);
MemoryOperationsStatus makeResidentResources(const D3DKMT_HANDLE *handles, const uint32_t count);
MOCKABLE_VIRTUAL void removeResource(const D3DKMT_HANDLE &handle);
protected:
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock(SpinLock &lock) {
return std::unique_lock<SpinLock>{lock};
}
Wddm *wddm;
std::vector<D3DKMT_HANDLE> resourceHandles;
SpinLock resourcesLock;
};
} // namespace NEO

View File

@@ -0,0 +1,392 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "os_interface/windows/wddm_residency_controller.h"
#include "debug_settings/debug_settings_manager.h"
#include "os_interface/windows/wddm/wddm.h"
#include "os_interface/windows/wddm_allocation.h"
#include "os_interface/windows/wddm_residency_allocations_container.h"
#include "utilities/spinlock.h"
namespace NEO {
WddmResidencyController::WddmResidencyController(Wddm &wddm, uint32_t osContextId) : wddm(wddm), osContextId(osContextId) {
}
void WddmResidencyController::registerCallback() {
this->trimCallbackHandle = wddm.registerTrimCallback(WddmResidencyController::trimCallback, *this);
}
WddmResidencyController::~WddmResidencyController() {
auto lock = this->acquireTrimCallbackLock();
wddm.unregisterTrimCallback(WddmResidencyController::trimCallback, this->trimCallbackHandle);
lock.unlock();
// Wait for lock to ensure trimCallback ended
lock.lock();
}
void APIENTRY WddmResidencyController::trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification) {
auto residencyController = static_cast<WddmResidencyController *>(trimNotification->Context);
DEBUG_BREAK_IF(residencyController == nullptr);
auto lock = residencyController->acquireTrimCallbackLock();
residencyController->trimResidency(trimNotification->Flags, trimNotification->NumBytesToTrim);
}
std::unique_lock<SpinLock> WddmResidencyController::acquireLock() {
return std::unique_lock<SpinLock>{this->lock};
}
std::unique_lock<SpinLock> WddmResidencyController::acquireTrimCallbackLock() {
return std::unique_lock<SpinLock>{this->trimCallbackLock};
}
WddmAllocation *WddmResidencyController::getTrimCandidateHead() {
uint32_t i = 0;
const size_t size = trimCandidateList.size();
if (size == 0) {
return nullptr;
}
while ((trimCandidateList[i] == nullptr) && (i < size))
i++;
return static_cast<WddmAllocation *>(trimCandidateList[i]);
}
void WddmResidencyController::addToTrimCandidateList(GraphicsAllocation *allocation) {
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
size_t position = trimCandidateList.size();
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
if (wddmAllocation->getTrimCandidateListPosition(this->osContextId) == trimListUnusedPosition) {
trimCandidatesCount++;
trimCandidateList.push_back(allocation);
wddmAllocation->setTrimCandidateListPosition(this->osContextId, position);
}
checkTrimCandidateCount();
}
void WddmResidencyController::removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList) {
WddmAllocation *wddmAllocation = (WddmAllocation *)allocation;
size_t position = wddmAllocation->getTrimCandidateListPosition(this->osContextId);
DEBUG_BREAK_IF(!(trimCandidatesCount > (trimCandidatesCount - 1)));
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
trimCandidatesCount--;
trimCandidateList[position] = nullptr;
checkTrimCandidateCount();
if (position == trimCandidateList.size() - 1) {
size_t erasePosition = position;
if (position == 0) {
trimCandidateList.resize(0);
} else {
while (trimCandidateList[erasePosition] == nullptr && erasePosition > 0) {
erasePosition--;
}
size_t sizeRemaining = erasePosition + 1;
if (erasePosition == 0 && trimCandidateList[erasePosition] == nullptr) {
sizeRemaining = 0;
}
trimCandidateList.resize(sizeRemaining);
}
}
wddmAllocation->setTrimCandidateListPosition(this->osContextId, trimListUnusedPosition);
if (compactList && checkTrimCandidateListCompaction()) {
compactTrimCandidateList();
}
checkTrimCandidateCount();
}
void WddmResidencyController::removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList) {
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
this->removeFromTrimCandidateList(allocation, true);
}
}
void WddmResidencyController::checkTrimCandidateCount() {
if (DebugManager.flags.ResidencyDebugEnable.get()) {
uint32_t sum = 0;
for (auto trimCandidate : trimCandidateList) {
if (trimCandidate != nullptr) {
sum++;
}
}
DEBUG_BREAK_IF(sum != trimCandidatesCount);
}
}
bool WddmResidencyController::checkTrimCandidateListCompaction() {
return 2 * trimCandidatesCount <= trimCandidateList.size();
}
void WddmResidencyController::compactTrimCandidateList() {
size_t size = trimCandidateList.size();
size_t freePosition = 0;
if (size == 0 || size == trimCandidatesCount) {
return;
}
DEBUG_BREAK_IF(!(trimCandidateList[size - 1] != nullptr));
uint32_t previousCount = trimCandidatesCount;
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
while (freePosition < trimCandidatesCount && trimCandidateList[freePosition] != nullptr)
freePosition++;
for (uint32_t i = 1; i < size; i++) {
if (trimCandidateList[i] != nullptr && freePosition < i) {
trimCandidateList[freePosition] = trimCandidateList[i];
trimCandidateList[i] = nullptr;
static_cast<WddmAllocation *>(trimCandidateList[freePosition])->setTrimCandidateListPosition(this->osContextId, freePosition);
freePosition++;
// Last element was moved, erase elements from freePosition
if (i == size - 1) {
trimCandidateList.resize(freePosition);
}
}
}
DEBUG_BREAK_IF(trimCandidatesCount > trimCandidateList.size());
DEBUG_BREAK_IF(trimCandidatesCount != previousCount);
checkTrimCandidateCount();
}
void WddmResidencyController::resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress) {
monitoredFence.lastSubmittedFence = 0;
monitoredFence.currentFenceValue = 1;
monitoredFence.fenceHandle = handle;
monitoredFence.cpuAddress = cpuAddress;
monitoredFence.gpuAddress = gpuAddress;
}
void WddmResidencyController::trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes) {
if (flags.PeriodicTrim) {
D3DKMT_HANDLE fragmentEvictHandles[3] = {0};
uint64_t sizeToTrim = 0;
auto lock = this->acquireLock();
WddmAllocation *wddmAllocation = nullptr;
while ((wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "lastPeriodicTrimFenceValue = ", lastTrimFenceValue);
if (wasAllocationUsedSinceLastTrim(wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId))) {
break;
}
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId));
uint32_t fragmentsToEvict = 0;
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict allocation: default handle =", wddmAllocation->getDefaultHandle(), "lastFence =", (wddmAllocation)->getResidencyData().getFenceValueForContextId(osContextId));
this->wddm.evict(wddmAllocation->getHandles().data(), wddmAllocation->getNumHandles(), sizeToTrim);
}
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
AllocationStorageData &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId];
if (!wasAllocationUsedSinceLastTrim(fragmentStorageData.residency->getFenceValueForContextId(osContextId))) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "Evict fragment: handle =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, "lastFence =", wddmAllocation->fragmentsStorage.fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId));
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData.osHandleStorage->handle;
fragmentStorageData.residency->resident[osContextId] = false;
}
}
if (fragmentsToEvict != 0) {
this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
}
wddmAllocation->getResidencyData().resident[osContextId] = false;
this->removeFromTrimCandidateList(wddmAllocation, false);
}
if (this->checkTrimCandidateListCompaction()) {
this->compactTrimCandidateList();
}
}
if (flags.TrimToBudget) {
auto lock = this->acquireLock();
trimResidencyToBudget(bytes);
}
if (flags.PeriodicTrim || flags.RestartPeriodicTrim) {
this->updateLastTrimFenceValue();
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "updated lastPeriodicTrimFenceValue =", lastTrimFenceValue);
}
}
bool WddmResidencyController::trimResidencyToBudget(uint64_t bytes) {
D3DKMT_HANDLE fragmentEvictHandles[maxFragmentsCount] = {0};
uint64_t numberOfBytesToTrim = bytes;
WddmAllocation *wddmAllocation = nullptr;
while (numberOfBytesToTrim > 0 && (wddmAllocation = this->getTrimCandidateHead()) != nullptr) {
uint64_t lastFence = wddmAllocation->getResidencyData().getFenceValueForContextId(osContextId);
auto &monitoredFence = this->getMonitoredFence();
if (lastFence > monitoredFence.lastSubmittedFence) {
break;
}
uint32_t fragmentsToEvict = 0;
uint64_t sizeEvicted = 0;
uint64_t sizeToTrim = 0;
if (lastFence > *monitoredFence.cpuAddress) {
this->wddm.waitFromCpu(lastFence, this->getMonitoredFence());
}
if (wddmAllocation->fragmentsStorage.fragmentCount == 0) {
this->wddm.evict(wddmAllocation->getHandles().data(), wddmAllocation->getNumHandles(), sizeToTrim);
sizeEvicted = wddmAllocation->getAlignedSize();
} else {
auto &fragmentStorageData = wddmAllocation->fragmentsStorage.fragmentStorageData;
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) {
fragmentEvictHandles[fragmentsToEvict++] = fragmentStorageData[allocationId].osHandleStorage->handle;
}
}
if (fragmentsToEvict != 0) {
this->wddm.evict((D3DKMT_HANDLE *)fragmentEvictHandles, fragmentsToEvict, sizeToTrim);
for (uint32_t allocationId = 0; allocationId < wddmAllocation->fragmentsStorage.fragmentCount; allocationId++) {
if (fragmentStorageData[allocationId].residency->getFenceValueForContextId(osContextId) <= monitoredFence.lastSubmittedFence) {
fragmentStorageData[allocationId].residency->resident[osContextId] = false;
sizeEvicted += fragmentStorageData[allocationId].fragmentSize;
}
}
}
}
if (sizeEvicted >= numberOfBytesToTrim) {
numberOfBytesToTrim = 0;
} else {
numberOfBytesToTrim -= sizeEvicted;
}
wddmAllocation->getResidencyData().resident[osContextId] = false;
this->removeFromTrimCandidateList(wddmAllocation, false);
}
if (bytes > numberOfBytesToTrim && this->checkTrimCandidateListCompaction()) {
this->compactTrimCandidateList();
}
return numberOfBytesToTrim == 0;
}
bool WddmResidencyController::makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency) {
const size_t residencyCount = allocationsForResidency.size();
std::unique_ptr<D3DKMT_HANDLE[]> handlesForResidency(new D3DKMT_HANDLE[residencyCount * maxFragmentsCount * EngineLimits::maxHandleCount]);
uint32_t totalHandlesCount = 0;
auto lock = this->acquireLock();
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "currentFenceValue =", this->getMonitoredFence().currentFenceValue);
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
ResidencyData &residencyData = allocation->getResidencyData();
bool fragmentResidency[3] = {false, false, false};
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, residencyData.resident[osContextId] ? "resident" : "not resident");
if (allocation->getTrimCandidateListPosition(this->osContextId) != trimListUnusedPosition) {
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "allocation =", allocation, "on trimCandidateList");
this->removeFromTrimCandidateList(allocation, false);
} else {
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
fragmentResidency[allocationId] = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency->resident[osContextId];
DBG_LOG(ResidencyDebugEnable, "Residency:", __FUNCTION__, "fragment handle =", allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle, fragmentResidency[allocationId] ? "resident" : "not resident");
}
}
if (allocation->fragmentsStorage.fragmentCount > 0) {
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
if (!fragmentResidency[allocationId])
handlesForResidency[totalHandlesCount++] = allocation->fragmentsStorage.fragmentStorageData[allocationId].osHandleStorage->handle;
}
} else if (!residencyData.resident[osContextId]) {
memcpy_s(&handlesForResidency[totalHandlesCount], allocation->getNumHandles() * sizeof(D3DKMT_HANDLE), allocation->getHandles().data(), allocation->getNumHandles() * sizeof(D3DKMT_HANDLE));
totalHandlesCount += allocation->getNumHandles();
}
}
bool result = true;
if (totalHandlesCount) {
uint64_t bytesToTrim = 0;
while ((result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, false, &bytesToTrim)) == false) {
this->setMemoryBudgetExhausted();
const bool trimmingDone = this->trimResidencyToBudget(bytesToTrim);
if (!trimmingDone) {
auto evictionStatus = wddm.getTemporaryResourcesContainer()->evictAllResources();
if (evictionStatus == MemoryOperationsStatus::SUCCESS) {
continue;
}
DEBUG_BREAK_IF(evictionStatus != MemoryOperationsStatus::MEMORY_NOT_FOUND);
result = wddm.makeResident(handlesForResidency.get(), totalHandlesCount, true, &bytesToTrim);
break;
}
}
}
if (result == true) {
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(allocationsForResidency[i]);
// Update fence value not to early destroy / evict allocation
const auto currentFence = this->getMonitoredFence().currentFenceValue;
allocation->getResidencyData().updateCompletionData(currentFence, this->osContextId);
allocation->getResidencyData().resident[osContextId] = true;
for (uint32_t allocationId = 0; allocationId < allocation->fragmentsStorage.fragmentCount; allocationId++) {
auto residencyData = allocation->fragmentsStorage.fragmentStorageData[allocationId].residency;
// Update fence value not to remove the fragment referenced by different GA in trimming callback
residencyData->updateCompletionData(currentFence, this->osContextId);
residencyData->resident[osContextId] = true;
}
}
}
return result;
}
void WddmResidencyController::makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations) {
auto lock = this->acquireLock();
const size_t residencyCount = evictionAllocations.size();
for (uint32_t i = 0; i < residencyCount; i++) {
WddmAllocation *allocation = static_cast<WddmAllocation *>(evictionAllocations[i]);
this->addToTrimCandidateList(allocation);
}
}
bool WddmResidencyController::isInitialized() const {
if (!DebugManager.flags.DoNotRegisterTrimCallback.get()) {
return trimCallbackHandle != nullptr;
}
return true;
}
} // namespace NEO

View File

@@ -0,0 +1,79 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "memory_manager/residency_container.h"
#include "os_interface/windows/windows_defs.h"
#include "os_interface/windows/windows_wrapper.h"
#include "utilities/spinlock.h"
#include <atomic>
#include <mutex>
namespace NEO {
class GraphicsAllocation;
class WddmAllocation;
class Wddm;
class WddmResidencyController {
public:
WddmResidencyController(Wddm &wddm, uint32_t osContextId);
MOCKABLE_VIRTUAL ~WddmResidencyController();
static void APIENTRY trimCallback(_Inout_ D3DKMT_TRIMNOTIFICATION *trimNotification);
MOCKABLE_VIRTUAL std::unique_lock<SpinLock> acquireLock();
std::unique_lock<SpinLock> acquireTrimCallbackLock();
WddmAllocation *getTrimCandidateHead();
void addToTrimCandidateList(GraphicsAllocation *allocation);
void removeFromTrimCandidateList(GraphicsAllocation *allocation, bool compactList);
void removeFromTrimCandidateListIfUsed(WddmAllocation *allocation, bool compactList);
void checkTrimCandidateCount();
bool checkTrimCandidateListCompaction();
void compactTrimCandidateList();
bool wasAllocationUsedSinceLastTrim(uint64_t fenceValue) { return fenceValue > lastTrimFenceValue; }
void updateLastTrimFenceValue() { lastTrimFenceValue = *this->getMonitoredFence().cpuAddress; }
const ResidencyContainer &peekTrimCandidateList() const { return trimCandidateList; }
uint32_t peekTrimCandidatesCount() const { return trimCandidatesCount; }
MonitoredFence &getMonitoredFence() { return monitoredFence; }
void resetMonitoredFenceParams(D3DKMT_HANDLE &handle, uint64_t *cpuAddress, D3DGPU_VIRTUAL_ADDRESS &gpuAddress);
void registerCallback();
void trimResidency(D3DDDI_TRIMRESIDENCYSET_FLAGS flags, uint64_t bytes);
bool trimResidencyToBudget(uint64_t bytes);
bool isMemoryBudgetExhausted() const { return memoryBudgetExhausted; }
void setMemoryBudgetExhausted() { memoryBudgetExhausted = true; }
bool makeResidentResidencyAllocations(const ResidencyContainer &allocationsForResidency);
void makeNonResidentEvictionAllocations(const ResidencyContainer &evictionAllocations);
bool isInitialized() const;
protected:
Wddm &wddm;
uint32_t osContextId;
MonitoredFence monitoredFence = {};
SpinLock lock;
SpinLock trimCallbackLock;
bool memoryBudgetExhausted = false;
uint64_t lastTrimFenceValue = 0u;
ResidencyContainer trimCandidateList;
uint32_t trimCandidatesCount = 0;
VOID *trimCallbackHandle = nullptr;
};
} // namespace NEO

View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2018-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "Windows.h"
#include <d3dkmthk.h>
#include <cstdint>
namespace NEO {
constexpr uintptr_t windowsMinAddress = 0x200000;
struct MonitoredFence {
D3DKMT_HANDLE fenceHandle = 0;
D3DGPU_VIRTUAL_ADDRESS gpuAddress = 0;
volatile uint64_t *cpuAddress = nullptr;
volatile uint64_t currentFenceValue = 0;
uint64_t lastSubmittedFence = 0;
};
} // namespace NEO

View File

@@ -0,0 +1,10 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
namespace Os {
const char *fileSeparator = "/";
}

View File

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include <Windows.h>
#pragma warning(push)
#pragma warning(disable : 4005)
#include <ntstatus.h>
#pragma warning(pop)
// There is a conflict with max/min defined as macro in windows headers with std::max/std::min
#undef min
#undef max
#undef RegOpenKeyExA
#undef RegQueryValueExA
#pragma warning(disable : 4273)
LSTATUS APIENTRY RegOpenKeyExA(
HKEY hKey,
LPCSTR lpSubKey,
DWORD ulOptions,
REGSAM samDesired,
PHKEY phkResult);
LSTATUS APIENTRY RegQueryValueExA(
HKEY hKey,
LPCSTR lpValueName,
LPDWORD lpReserved,
LPDWORD lpType,
LPBYTE lpData,
LPDWORD lpcbData);