intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-21 01:04:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: unblock xekmd recoverable pagefaults vmbind

Browse Source 
Related-To: HSD-13011898606
Signed-off-by: Naklicki, Mateusz <mateusz.naklicki@intel.com>
This commit is contained in:
Naklicki, Mateusz
2024-06-19 13:09:10 +00:00
committed by Compute-Runtime-Automation
parent d19f58019f
commit 5e144dae16
16 changed files with 261 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@@ -2398,7 +2398,7 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const
auto ioctlHelper = drm.getIoctlHelper();
const auto vmAdviseAttribute = ioctlHelper->getVmAdviseAtomicAttribute();
if (vmAdviseAttribute == 0) {
if (vmAdviseAttribute.has_value() && vmAdviseAttribute.value() == 0) {
return nullptr;
}
@@ -2481,7 +2481,7 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const
std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new BufferObject(allocationData.rootDeviceIndex, &drm, patIndex, handle, currentSize, maxOsContextCount));
if (!ioctlHelper->setVmBoAdvise(bo->peekHandle(), vmAdviseAttribute, nullptr)) {
if (vmAdviseAttribute.has_value() && !ioctlHelper->setVmBoAdvise(bo->peekHandle(), vmAdviseAttribute.value(), nullptr)) {
this->munmapFunction(cpuBasePointer, totalSizeToAlloc);
releaseGpuRange(reinterpret_cast<void *>(preferredAddress), totalSizeToAlloc, allocationData.rootDeviceIndex);
return nullptr;

6
shared/source/os_interface/linux/drm_neo.cpp
View File

@@ -1037,11 +1037,7 @@ void Drm::queryPageFaultSupport() {
return;
}
if (const auto paramId = ioctlHelper->getHasPageFaultParamId(); paramId) {
int support = 0;
const auto ret = getParamIoctl(*paramId, &support);
pageFaultSupported = (0 == ret) && (support > 0);
}
pageFaultSupported = this->ioctlHelper->isPageFaultSupported();
}
bool Drm::hasPageFaultSupport() const {

12
shared/source/os_interface/linux/ioctl_helper.h
View File

@@ -123,7 +123,7 @@ class IoctlHelper {
virtual uint16_t getWaitUserFenceSoftFlag() = 0;
virtual int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) = 0;
virtual bool completionFenceExtensionSupported(const bool isVmBindAvailable) = 0;
virtual std::optional<DrmParam> getHasPageFaultParamId() = 0;
virtual bool isPageFaultSupported() = 0;
virtual std::unique_ptr<uint8_t[]> createVmControlExtRegion(const std::optional<MemoryClassInstance> &regionInstanceClass) = 0;
virtual uint32_t getFlagsForVmCreate(bool disableScratch, bool enablePageFault, bool useVmBind) = 0;
virtual uint32_t createContextWithAccessCounters(GemContextCreateExt &gcc) = 0;
@@ -133,7 +133,7 @@ class IoctlHelper {
virtual void setVmBindUserFence(VmBindParams &vmBind, VmBindExtUserFenceT vmBindUserFence) = 0;
virtual std::optional<uint64_t> getCopyClassSaturatePCIECapability() = 0;
virtual std::optional<uint64_t> getCopyClassSaturateLinkCapability() = 0;
virtual uint32_t getVmAdviseAtomicAttribute() = 0;
virtual std::optional<uint32_t> getVmAdviseAtomicAttribute() = 0;
virtual int vmBind(const VmBindParams &vmBindParams) = 0;
virtual int vmUnbind(const VmBindParams &vmBindParams) = 0;
virtual int getResetStats(ResetStats &resetStats, uint32_t *status, ResetStatsFault *resetStatsFault) = 0;
@@ -274,7 +274,7 @@ class IoctlHelperUpstream : public IoctlHelperI915 {
uint16_t getWaitUserFenceSoftFlag() override;
int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) override;
bool completionFenceExtensionSupported(const bool isVmBindAvailable) override;
std::optional<DrmParam> getHasPageFaultParamId() override;
bool isPageFaultSupported() override;
std::unique_ptr<uint8_t[]> createVmControlExtRegion(const std::optional<MemoryClassInstance> &regionInstanceClass) override;
uint32_t getFlagsForVmCreate(bool disableScratch, bool enablePageFault, bool useVmBind) override;
uint32_t createContextWithAccessCounters(GemContextCreateExt &gcc) override;
@@ -284,7 +284,7 @@ class IoctlHelperUpstream : public IoctlHelperI915 {
void setVmBindUserFence(VmBindParams &vmBind, VmBindExtUserFenceT vmBindUserFence) override;
std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
uint32_t getVmAdviseAtomicAttribute() override;
std::optional<uint32_t> getVmAdviseAtomicAttribute() override;
int vmBind(const VmBindParams &vmBindParams) override;
int vmUnbind(const VmBindParams &vmBindParams) override;
int getResetStats(ResetStats &resetStats, uint32_t *status, ResetStatsFault *resetStatsFault) override;
@@ -352,7 +352,7 @@ class IoctlHelperPrelim20 : public IoctlHelperI915 {
uint16_t getWaitUserFenceSoftFlag() override;
int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) override;
bool completionFenceExtensionSupported(const bool isVmBindAvailable) override;
std::optional<DrmParam> getHasPageFaultParamId() override;
bool isPageFaultSupported() override;
std::unique_ptr<uint8_t[]> createVmControlExtRegion(const std::optional<MemoryClassInstance> &regionInstanceClass) override;
uint32_t getFlagsForVmCreate(bool disableScratch, bool enablePageFault, bool useVmBind) override;
uint32_t createContextWithAccessCounters(GemContextCreateExt &gcc) override;
@@ -362,7 +362,7 @@ class IoctlHelperPrelim20 : public IoctlHelperI915 {
void setVmBindUserFence(VmBindParams &vmBind, VmBindExtUserFenceT vmBindUserFence) override;
std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
uint32_t getVmAdviseAtomicAttribute() override;
std::optional<uint32_t> getVmAdviseAtomicAttribute() override;
int vmBind(const VmBindParams &vmBindParams) override;
int vmUnbind(const VmBindParams &vmBindParams) override;
int getResetStats(ResetStats &resetStats, uint32_t *status, ResetStatsFault *resetStatsFault) override;

17
shared/source/os_interface/linux/ioctl_helper_prelim.cpp
View File

@@ -557,8 +557,19 @@ int IoctlHelperPrelim20::queryDistances(std::vector<QueryItem> &queryItems, std:
return ret;
}
std::optional<DrmParam> IoctlHelperPrelim20::getHasPageFaultParamId() {
return DrmParam::paramHasPageFault;
bool IoctlHelperPrelim20::isPageFaultSupported() {
int pagefaultSupport{};
GetParam getParam{};
getParam.param = PRELIM_I915_PARAM_HAS_PAGE_FAULT;
getParam.value = &pagefaultSupport;
int retVal = ioctl(DrmIoctl::getparam, &getParam);
if (debugManager.flags.PrintIoctlEntries.get()) {
printf("DRM_IOCTL_I915_GETPARAM: param: PRELIM_I915_PARAM_HAS_PAGE_FAULT, output value: %d, retCode:% d\n",
*getParam.value,
retVal);
}
return (retVal == 0) && (pagefaultSupport > 0);
};
bool IoctlHelperPrelim20::getEuStallProperties(std::array<uint64_t, 12u> &properties, uint64_t dssBufferSize, uint64_t samplingRate,
@@ -731,7 +742,7 @@ std::optional<uint64_t> IoctlHelperPrelim20::getCopyClassSaturateLinkCapability(
return PRELIM_I915_COPY_CLASS_CAP_SATURATE_LINK;
}
uint32_t IoctlHelperPrelim20::getVmAdviseAtomicAttribute() {
std::optional<uint32_t> IoctlHelperPrelim20::getVmAdviseAtomicAttribute() {
switch (NEO::debugManager.flags.SetVmAdviseAtomicAttribute.get()) {
case 0:
return PRELIM_I915_VM_ADVISE_ATOMIC_NONE;

6
shared/source/os_interface/linux/ioctl_helper_upstream.cpp
View File

@@ -191,8 +191,8 @@ bool IoctlHelperUpstream::completionFenceExtensionSupported(const bool isVmBindA
return false;
}
std::optional<DrmParam> IoctlHelperUpstream::getHasPageFaultParamId() {
return std::nullopt;
bool IoctlHelperUpstream::isPageFaultSupported() {
return false;
};
bool IoctlHelperUpstream::getEuStallProperties(std::array<uint64_t, 12u> &properties, uint64_t dssBufferSize,
@@ -243,7 +243,7 @@ std::optional<uint64_t> IoctlHelperUpstream::getCopyClassSaturateLinkCapability(
return std::nullopt;
}
uint32_t IoctlHelperUpstream::getVmAdviseAtomicAttribute() {
std::optional<uint32_t> IoctlHelperUpstream::getVmAdviseAtomicAttribute() {
return 0;
}

2
shared/source/os_interface/linux/xe/CMakeLists.txt
View File

@@ -11,7 +11,7 @@ set(NEO_CORE_OS_INTERFACE_LINUX_XE
${CMAKE_CURRENT_SOURCE_DIR}/ioctl_helper_xe.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ioctl_helper_xe.h
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/ioctl_helper_xe_query_hw_ip_version.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/ioctl_helper_xe_vm_bind_flags.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/ioctl_helper_xe_query_features.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/ioctl_helper_xe_perf.cpp
)

72
shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp
View File

@@ -642,6 +642,7 @@ void IoctlHelperXe::setupXeWaitUserFenceStruct(void *arg, uint32_t ctxId, uint16
}
int IoctlHelperXe::xeWaitUserFence(uint32_t ctxId, uint16_t op, uint64_t addr, uint64_t value, int64_t timeout, bool userInterrupt, uint32_t externalInterruptId, GraphicsAllocation *allocForInterruptWait) {
UNRECOVERABLE_IF(addr == 0x0)
drm_xe_wait_user_fence waitUserFence = {};
setupXeWaitUserFenceStruct(&waitUserFence, ctxId, op, addr, value, timeout);
@@ -684,11 +685,14 @@ std::optional<MemoryClassInstance> IoctlHelperXe::getPreferredLocationRegion(Pre
bool IoctlHelperXe::setVmBoAdvise(int32_t handle, uint32_t attribute, void *region) {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return false;
// There is no vmAdvise attribute in Xe, so return success
return true;
}
bool IoctlHelperXe::setVmBoAdviseForChunking(int32_t handle, uint64_t start, uint64_t length, uint32_t attribute, void *region) {
return false;
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
// There is no vmAdvise attribute in Xe, so return success
return true;
}
bool IoctlHelperXe::setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) {
@@ -796,9 +800,10 @@ int IoctlHelperXe::queryDistances(std::vector<QueryItem> &queryItems, std::vecto
return 0;
}
std::optional<DrmParam> IoctlHelperXe::getHasPageFaultParamId() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return {};
bool IoctlHelperXe::isPageFaultSupported() {
xeLog(" -> IoctlHelperXe::%s %d\n", __FUNCTION__, supportedFeatures.flags.pageFault == true);
return supportedFeatures.flags.pageFault;
};
uint32_t IoctlHelperXe::getEuStallFdParameter() {
@@ -859,9 +864,10 @@ std::optional<uint64_t> IoctlHelperXe::getCopyClassSaturateLinkCapability() {
return {};
}
uint32_t IoctlHelperXe::getVmAdviseAtomicAttribute() {
std::optional<uint32_t> IoctlHelperXe::getVmAdviseAtomicAttribute() {
xeLog(" -> IoctlHelperXe::%s\n", __FUNCTION__);
return 0;
// There is no vmAdvise attribute in Xe
return {};
}
int IoctlHelperXe::vmBind(const VmBindParams &vmBindParams) {
@@ -1240,21 +1246,11 @@ int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool isBind) {
}
if (index != invalidIndex) {
drm_xe_sync sync[1] = {};
sync[0].type = DRM_XE_SYNC_TYPE_USER_FENCE;
sync[0].flags = DRM_XE_SYNC_FLAG_SIGNAL;
auto xeBindExtUserFence = reinterpret_cast<UserFenceExtension *>(vmBindParams.userFence);
UNRECOVERABLE_IF(!xeBindExtUserFence);
UNRECOVERABLE_IF(xeBindExtUserFence->tag != UserFenceExtension::tagValue);
sync[0].addr = xeBindExtUserFence->addr;
sync[0].timeline_value = xeBindExtUserFence->value;
drm_xe_vm_bind bind = {};
bind.vm_id = vmBindParams.vmId;
bind.num_syncs = 0;
bind.num_binds = 1;
bind.num_syncs = 1;
bind.syncs = reinterpret_cast<uintptr_t>(&sync);
bind.bind.range = vmBindParams.length;
bind.bind.addr = gmmHelper->decanonize(vmBindParams.start);
bind.bind.obj_offset = vmBindParams.offset;
@@ -1262,6 +1258,20 @@ int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool isBind) {
bind.bind.extensions = vmBindParams.extensions;
bind.bind.flags = static_cast<uint32_t>(vmBindParams.flags);
drm_xe_sync sync[1] = {};
bool residencyRequired = vmBindParams.userFence == 0x0 ? false : true;
if (residencyRequired) {
auto xeBindExtUserFence = reinterpret_cast<UserFenceExtension *>(vmBindParams.userFence);
UNRECOVERABLE_IF(xeBindExtUserFence->tag != UserFenceExtension::tagValue);
sync[0].type = DRM_XE_SYNC_TYPE_USER_FENCE;
sync[0].flags = DRM_XE_SYNC_FLAG_SIGNAL;
sync[0].addr = xeBindExtUserFence->addr;
sync[0].timeline_value = xeBindExtUserFence->value;
bind.syncs = reinterpret_cast<uintptr_t>(&sync);
bind.num_syncs = 1;
}
if (isBind) {
bind.bind.op = DRM_XE_VM_BIND_OP_MAP;
bind.bind.obj = vmBindParams.handle;
@@ -1301,17 +1311,21 @@ int IoctlHelperXe::xeVmBind(const VmBindParams &vmBindParams, bool isBind) {
return ret;
}
constexpr auto oneSecTimeout = 1000000000ll;
constexpr auto infiniteTimeout = -1;
bool debuggingEnabled = drm.getRootDeviceEnvironment().executionEnvironment.isDebuggingEnabled();
uint64_t timeout = debuggingEnabled ? infiniteTimeout : oneSecTimeout;
if (debugManager.flags.VmBindWaitUserFenceTimeout.get() != -1) {
timeout = debugManager.flags.VmBindWaitUserFenceTimeout.get();
if (residencyRequired) {
constexpr auto oneSecTimeout = 1000000000ll;
constexpr auto infiniteTimeout = -1;
bool debuggingEnabled = drm.getRootDeviceEnvironment().executionEnvironment.isDebuggingEnabled();
uint64_t timeout = debuggingEnabled ? infiniteTimeout : oneSecTimeout;
if (debugManager.flags.VmBindWaitUserFenceTimeout.get() != -1) {
timeout = debugManager.flags.VmBindWaitUserFenceTimeout.get();
}
return xeWaitUserFence(bind.exec_queue_id, DRM_XE_UFENCE_WAIT_OP_EQ,
sync[0].addr,
sync[0].timeline_value, timeout,
false, NEO::InterruptId::notUsed, nullptr);
}
return xeWaitUserFence(bind.exec_queue_id, DRM_XE_UFENCE_WAIT_OP_EQ,
sync[0].addr,
sync[0].timeline_value, timeout,
false, NEO::InterruptId::notUsed, nullptr);
return ret;
}
xeLog("error: -> IoctlHelperXe::%s %s index=%d vmid=0x%x h=0x%x s=0x%llx o=0x%llx l=0x%llx f=0x%llx pat=%hu r=%d\n",

7
shared/source/os_interface/linux/xe/ioctl_helper_xe.h
View File

@@ -66,7 +66,7 @@ class IoctlHelperXe : public IoctlHelper {
uint16_t getWaitUserFenceSoftFlag() override;
int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) override;
bool completionFenceExtensionSupported(const bool isVmBindAvailable) override;
std::optional<DrmParam> getHasPageFaultParamId() override;
bool isPageFaultSupported() override;
std::unique_ptr<uint8_t[]> createVmControlExtRegion(const std::optional<MemoryClassInstance> &regionInstanceClass) override;
uint32_t getFlagsForVmCreate(bool disableScratch, bool enablePageFault, bool useVmBind) override;
uint32_t createContextWithAccessCounters(GemContextCreateExt &gcc) override;
@@ -76,7 +76,7 @@ class IoctlHelperXe : public IoctlHelper {
void setVmBindUserFence(VmBindParams &vmBind, VmBindExtUserFenceT vmBindUserFence) override;
std::optional<uint64_t> getCopyClassSaturatePCIECapability() override;
std::optional<uint64_t> getCopyClassSaturateLinkCapability() override;
uint32_t getVmAdviseAtomicAttribute() override;
std::optional<uint32_t> getVmAdviseAtomicAttribute() override;
int vmBind(const VmBindParams &vmBindParams) override;
int vmUnbind(const VmBindParams &vmBindParams) override;
int getResetStats(ResetStats &resetStats, uint32_t *status, ResetStatsFault *resetStatsFault) override;
@@ -215,7 +215,8 @@ class IoctlHelperXe : public IoctlHelper {
struct {
uint32_t vmBindReadOnly : 1;
uint32_t vmBindImmediate : 1;
uint32_t reserved : 30;
uint32_t pageFault : 1;
uint32_t reserved : 29;
} flags;
uint32_t allFlags = 0;
};

34
shared/source/os_interface/linux/xe/ioctl_helper_xe_query_features.cpp Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/xe/ioctl_helper_xe.h"
#include "xe_drm.h"
namespace NEO {
void IoctlHelperXe::querySupportedFeatures() {
auto checkVmCreateFlagsSupport = [&](uint32_t flags) -> bool {
struct drm_xe_vm_create vmCreate = {};
vmCreate.flags = flags;
auto ret = IoctlHelper::ioctl(DrmIoctl::gemVmCreate, &vmCreate);
if (ret == 0) {
struct drm_xe_vm_destroy vmDestroy = {};
vmDestroy.vm_id = vmCreate.vm_id;
ret = IoctlHelper::ioctl(DrmIoctl::gemVmDestroy, &vmDestroy);
DEBUG_BREAK_IF(ret != 0);
return true;
}
return false;
};
supportedFeatures.flags.pageFault = checkVmCreateFlagsSupport(DRM_XE_VM_CREATE_FLAG_LR_MODE | DRM_XE_VM_CREATE_FLAG_FAULT_MODE);
};
uint64_t IoctlHelperXe::getAdditionalFlagsForVmBind(bool bindImmediate, bool readOnlyResource) {
return 0;
}
} // namespace NEO

16
shared/source/os_interface/linux/xe/ioctl_helper_xe_vm_bind_flags.cpp
View File

@@ -1,16 +0,0 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/xe/ioctl_helper_xe.h"
namespace NEO {
void IoctlHelperXe::querySupportedFeatures(){};
uint64_t IoctlHelperXe::getAdditionalFlagsForVmBind(bool bindImmediate, bool readOnlyResource) {
return 0;
}
} // namespace NEO