intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-24 21:18:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature: support explicit memory locking

Browse Source 
Added lockMemory in context to explicitly locking memory,
Added a boolean flag in graphics_allocation to indicate the allocation
is locked, and modified memory_operations_handler to add lock().

Related-To: NEO-8277
Signed-off-by: Young Jin Yoon <young.jin.yoon@intel.com>
This commit is contained in:
Young Jin Yoon
2024-03-28 08:06:58 +00:00
committed by Compute-Runtime-Automation
parent 15420aa094
commit d6a14d4ed5
49 changed files with 615 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
shared/source/os_interface/linux/drm_buffer_object.h
View File

@@ -217,6 +217,9 @@ class BufferObject {
std::vector<uint64_t> &getColourAddresses() {
return this->bindAddresses;
}
void requireExplicitLockedMemory(bool locked) { requiresLocked = locked; }
bool isExplicitLockedMemoryRequired() { return requiresLocked; }
uint64_t peekPatIndex() const { return patIndex; }
void setPatIndex(uint64_t newPatIndex) { this->patIndex = newPatIndex; }
BOType peekBOType() const { return boType; }
@@ -266,6 +269,7 @@ class BufferObject {
bool allowCapture = false;
bool requiresImmediateBinding = false;
bool requiresExplicitResidency = false;
bool requiresLocked = false;
bool chunked = false;
bool isReused = false;
bool readOnlyGpuResource = false;

15
shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp
View File

@@ -39,6 +39,13 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResident(Device *devi
return result;
}
MemoryOperationsStatus DrmMemoryOperationsHandlerBind::lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) {
for (auto gfxAllocation = gfxAllocations.begin(); gfxAllocation != gfxAllocations.end(); gfxAllocation++) {
(*gfxAllocation)->setLockedMemory(true);
}
return makeResident(device, gfxAllocations);
}
MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable) {
auto deviceBitfield = osContext->getDeviceBitfield();
@@ -59,12 +66,12 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsConte
}
if (!bo->getBindInfo()[bo->getOsContextId(osContext)][drmIterator]) {
bo->requireExplicitLockedMemory(drmAllocation->isLockedMemory());
int result = drmAllocation->makeBOsResident(osContext, drmIterator, nullptr, true);
if (result) {
return MemoryOperationsStatus::outOfMemory;
}
}
if (!evictable) {
drmAllocation->updateResidencyTaskCount(GraphicsAllocation::objectAlwaysResident, osContext->getContextId());
}
@@ -77,6 +84,7 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResidentWithinOsConte
MemoryOperationsStatus DrmMemoryOperationsHandlerBind::evict(Device *device, GraphicsAllocation &gfxAllocation) {
auto &engines = device->getAllEngines();
auto retVal = MemoryOperationsStatus::success;
gfxAllocation.setLockedMemory(false);
for (const auto &engine : engines) {
retVal = this->evictWithinOsContext(engine.osContext, gfxAllocation);
if (retVal != MemoryOperationsStatus::success) {
@@ -185,7 +193,10 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerBind::evictUnusedAllocationsImp
evict = false;
break;
}
if (allocation->isLockedMemory()) {
evict = false;
break;
}
if (waitForCompletion) {
const auto waitStatus = engine.commandStreamReceiver->waitForCompletionWithTimeout(WaitParams{false, false, 0}, engine.commandStreamReceiver->peekLatestFlushedTaskCount());
if (waitStatus == WaitStatus::gpuHang) {

1
shared/source/os_interface/linux/drm_memory_operations_handler_bind.h
View File

@@ -18,6 +18,7 @@ class DrmMemoryOperationsHandlerBind : public DrmMemoryOperationsHandler {
MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable) override;
MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override;

24
shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp
View File

@@ -8,6 +8,8 @@
#include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/os_interface/linux/drm_allocation.h"
#include "shared/source/os_interface/linux/drm_buffer_object.h"
#include <algorithm>
@@ -29,6 +31,18 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResident(Device *d
return this->makeResidentWithinOsContext(osContext, gfxAllocations, false);
}
MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) {
OsContext *osContext = nullptr;
for (auto gfxAllocation = gfxAllocations.begin(); gfxAllocation != gfxAllocations.end(); gfxAllocation++) {
auto drmAllocation = static_cast<DrmAllocation *>(*gfxAllocation);
drmAllocation->setLockedMemory(true);
for (auto bo : drmAllocation->getBOs()) {
bo->requireExplicitLockedMemory(true);
}
}
return this->makeResidentWithinOsContext(osContext, gfxAllocations, false);
}
MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) {
std::lock_guard<std::mutex> lock(mutex);
this->residency.erase(&gfxAllocation);
@@ -37,6 +51,16 @@ MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::evictWithinOsContext(O
MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::evict(Device *device, GraphicsAllocation &gfxAllocation) {
OsContext *osContext = nullptr;
auto drmAllocation = static_cast<DrmAllocation *>(&gfxAllocation);
drmAllocation->setLockedMemory(false);
if (drmAllocation->storageInfo.isChunked || drmAllocation->storageInfo.getNumBanks() == 1) {
auto bo = drmAllocation->getBO();
bo->requireExplicitLockedMemory(false);
} else {
for (auto bo : drmAllocation->getBOs()) {
bo->requireExplicitLockedMemory(false);
}
}
return this->evictWithinOsContext(osContext, gfxAllocation);
}

1
shared/source/os_interface/linux/drm_memory_operations_handler_default.h
View File

@@ -21,6 +21,7 @@ class DrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler {
MemoryOperationsStatus makeResidentWithinOsContext(OsContext *osContext, ArrayRef<GraphicsAllocation *> gfxAllocations, bool evictable) override;
MemoryOperationsStatus makeResident(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override;
MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override;
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override;

5
shared/source/os_interface/linux/drm_memory_operations_handler_with_aub_dump.h
View File

@@ -41,6 +41,11 @@ class DrmMemoryOperationsHandlerWithAubDump : public BaseOperationsHandler {
return BaseOperationsHandler::makeResident(device, gfxAllocations);
}
MemoryOperationsStatus lock(Device *device, ArrayRef<GraphicsAllocation *> gfxAllocations) override {
aubMemoryOperationsHandler->makeResident(device, gfxAllocations);
return BaseOperationsHandler::lock(device, gfxAllocations);
}
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override {
aubMemoryOperationsHandler->evict(device, gfxAllocation);
return BaseOperationsHandler::evict(device, gfxAllocation);

3
shared/source/os_interface/linux/drm_neo.cpp
View File

@@ -1315,7 +1315,8 @@ int changeBufferObjectBinding(Drm *drm, OsContext *osContext, uint32_t vmHandleI
bindMakeResident = bo->isExplicitResidencyRequired();
bindImmediate = true;
}
flags |= ioctlHelper->getFlagsForVmBind(bindCapture, bindImmediate, bindMakeResident, readOnlyResource);
bool bindLock = bo->isExplicitLockedMemoryRequired();
flags |= ioctlHelper->getFlagsForVmBind(bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource);
}
auto &bindAddresses = bo->getColourAddresses();

6
shared/source/os_interface/linux/ioctl_helper.h
View File

@@ -116,7 +116,7 @@ class IoctlHelper {
virtual bool getGemTiling(void *setTiling) = 0;
virtual uint32_t getDirectSubmissionFlag() = 0;
virtual std::unique_ptr<uint8_t[]> prepareVmBindExt(const StackVec<uint32_t, 2> &bindExtHandles) = 0;
virtual uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) = 0;
virtual uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLockedMemory, bool readOnlyResource) = 0;
virtual int queryDistances(std::vector<QueryItem> &queryItems, std::vector<DistanceInfo> &distanceInfos) = 0;
virtual uint16_t getWaitUserFenceSoftFlag() = 0;
virtual int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) = 0;
@@ -259,7 +259,7 @@ class IoctlHelperUpstream : public IoctlHelperI915 {
bool setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) override;
uint32_t getDirectSubmissionFlag() override;
std::unique_ptr<uint8_t[]> prepareVmBindExt(const StackVec<uint32_t, 2> &bindExtHandles) override;
uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) override;
uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLockedMemory, bool readOnlyResource) override;
int queryDistances(std::vector<QueryItem> &queryItems, std::vector<DistanceInfo> &distanceInfos) override;
uint16_t getWaitUserFenceSoftFlag() override;
int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) override;
@@ -336,7 +336,7 @@ class IoctlHelperPrelim20 : public IoctlHelperI915 {
bool setVmPrefetch(uint64_t start, uint64_t length, uint32_t region, uint32_t vmId) override;
uint32_t getDirectSubmissionFlag() override;
std::unique_ptr<uint8_t[]> prepareVmBindExt(const StackVec<uint32_t, 2> &bindExtHandles) override;
uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) override;
uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLockedMemory, bool readOnlyResource) override;
int queryDistances(std::vector<QueryItem> &queryItems, std::vector<DistanceInfo> &distanceInfos) override;
uint16_t getWaitUserFenceSoftFlag() override;
int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) override;

4
shared/source/os_interface/linux/ioctl_helper_prelim.cpp
View File

@@ -507,7 +507,7 @@ std::unique_ptr<uint8_t[]> IoctlHelperPrelim20::prepareVmBindExt(const StackVec<
return extensionsBuffer;
}
uint64_t IoctlHelperPrelim20::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) {
uint64_t IoctlHelperPrelim20::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLockedMemory, bool readOnlyResource) {
uint64_t flags = 0u;
if (bindCapture) {
flags |= PRELIM_I915_GEM_VM_BIND_CAPTURE;
@@ -515,7 +515,7 @@ uint64_t IoctlHelperPrelim20::getFlagsForVmBind(bool bindCapture, bool bindImmed
if (bindImmediate) {
flags |= PRELIM_I915_GEM_VM_BIND_IMMEDIATE;
}
if (bindMakeResident) {
if (bindMakeResident || bindLockedMemory) { // lockedMemory is equal to residency in i915_prelim
flags |= PRELIM_I915_GEM_VM_BIND_MAKE_RESIDENT;
}
if (readOnlyResource) {

2
shared/source/os_interface/linux/ioctl_helper_upstream.cpp
View File

@@ -168,7 +168,7 @@ std::unique_ptr<uint8_t[]> IoctlHelperUpstream::prepareVmBindExt(const StackVec<
return {};
}
uint64_t IoctlHelperUpstream::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) {
uint64_t IoctlHelperUpstream::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLockedMemory, bool readOnlyResource) {
return 0u;
}

1
shared/source/os_interface/linux/xe/CMakeLists.txt
View File

@@ -12,6 +12,7 @@ set(NEO_CORE_OS_INTERFACE_LINUX_XE
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}ioctl_helper_xe_vm_export.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}ioctl_helper_xe_context.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}ioctl_helper_xe_perf.cpp
${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}ioctl_helper_xe_vm_bind_flag.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ioctl_helper_xe.h
)

5
shared/source/os_interface/linux/xe/ioctl_helper_xe.cpp
View File

@@ -730,12 +730,13 @@ bool IoctlHelperXe::completionFenceExtensionSupported(const bool isVmBindAvailab
return isVmBindAvailable;
}
uint64_t IoctlHelperXe::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) {
uint64_t IoctlHelperXe::getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLock, bool readOnlyResource) {
uint64_t ret = 0;
xeLog(" -> IoctlHelperXe::%s %d %d %d %d\n", __FUNCTION__, bindCapture, bindImmediate, bindMakeResident, readOnlyResource);
xeLog(" -> IoctlHelperXe::%s %d %d %d %d %d\n", __FUNCTION__, bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource);
if (bindCapture) {
ret |= DRM_XE_VM_BIND_FLAG_DUMPABLE;
}
ret |= getExtraFlagsForVmBind(bindCapture, bindImmediate, bindMakeResident, bindLock, readOnlyResource);
return ret;
}

3
shared/source/os_interface/linux/xe/ioctl_helper_xe.h
View File

@@ -58,7 +58,7 @@ class IoctlHelperXe : public IoctlHelper {
bool getGemTiling(void *setTiling) override;
uint32_t getDirectSubmissionFlag() override;
std::unique_ptr<uint8_t[]> prepareVmBindExt(const StackVec<uint32_t, 2> &bindExtHandles) override;
uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool readOnlyResource) override;
uint64_t getFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLock, bool readOnlyResource) override;
int queryDistances(std::vector<QueryItem> &queryItems, std::vector<DistanceInfo> &distanceInfos) override;
uint16_t getWaitUserFenceSoftFlag() override;
int execBuffer(ExecBuffer *execBuffer, uint64_t completionGpuAddress, TaskCountType counterValue) override;
@@ -134,6 +134,7 @@ class IoctlHelperXe : public IoctlHelper {
const char *xeGetBindOperationName(int bindOperation);
const char *xeGetBindFlagsName(int bindFlags);
uint64_t getExtraFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLock, bool readOnlyResource);
const char *xeGetengineClassName(uint32_t engineClass);
template <typename DataType>
std::vector<DataType> queryData(uint32_t queryId);

16
shared/source/os_interface/linux/xe/ioctl_helper_xe_vm_bind_flag.cpp Normal file
View File

@@ -0,0 +1,16 @@
/*
* Copyright (C) 2024 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "shared/source/os_interface/linux/xe/ioctl_helper_xe.h"
namespace NEO {
uint64_t IoctlHelperXe::getExtraFlagsForVmBind(bool bindCapture, bool bindImmediate, bool bindMakeResident, bool bindLock, bool readOnlyResource) {
return 0;
}
} // namespace NEO