intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2025-12-19 16:24:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: debug flag to override PAT index for given memory type

Browse Source 
Signed-off-by: Dunajski, Bartosz <bartosz.dunajski@intel.com>
This commit is contained in:
Dunajski, Bartosz
2023-10-12 14:06:59 +00:00
committed by Compute-Runtime-Automation
parent 19aba581d4
commit 06a02552ce
14 changed files with 149 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
shared/source/debug_settings/debug_variables_base.inl
View File

@@ -210,6 +210,8 @@ DECLARE_DEBUG_VARIABLE(int32_t, CreateContextWithAccessCounters, -1, "-1: defaul
DECLARE_DEBUG_VARIABLE(int32_t, AccessCountersTrigger, -1, "-1: default - disabled, 0: disabled, >= 0: triggering thresholds")
DECLARE_DEBUG_VARIABLE(int32_t, AccessCountersGranularity, -1, "-1: default - ACG_2MB, >= 0: granularites - 0: ACG_128K, 1: ACG_2M, 2: ACG_16M, 3: ACG_16M")
DECLARE_DEBUG_VARIABLE(int32_t, OverridePatIndex, -1, "-1: default, >=0: PatIndex to override")
DECLARE_DEBUG_VARIABLE(int32_t, OverridePatIndexForSystemMemory, -1, "-1: default, >=0: PatIndex to override. Applicable only for System memory.")
DECLARE_DEBUG_VARIABLE(int32_t, OverridePatIndexForDeviceMemory, -1, "-1: default, >=0: PatIndex to override. Applicable only for Device memory.")
DECLARE_DEBUG_VARIABLE(int32_t, UseTileMemoryBankInVirtualMemoryCreation, -1, "-1: default - on, 0: do not assign tile memory bank to virtual memory space, 1: assign tile memory bank to virtual memory space")
DECLARE_DEBUG_VARIABLE(int32_t, OverrideTimestampEvents, -1, "-1: default (based on user settings), 0: Force disable timestamp events (no timestamps will be reported), 1: Force enable timestamp events")
DECLARE_DEBUG_VARIABLE(int32_t, ForcePreParserEnabledForMiArbCheck, -1, "-1: default , 0: PreParser disabled, 1: PreParser enabled")

6
shared/source/os_interface/linux/drm_allocation.cpp
View File

@@ -154,15 +154,15 @@ bool DrmAllocation::setCacheRegion(Drm *drm, CacheRegion regionIndex) {
return false;
}
return setCacheAdvice(drm, regionSize, regionIndex);
return setCacheAdvice(drm, regionSize, regionIndex, !isAllocatedInLocalMemoryPool());
}
bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex) {
bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex, bool isSystemMemoryPool) {
if (!drm->getCacheInfo()->getCacheRegion(regionSize, regionIndex)) {
return false;
}
auto patIndex = drm->getPatIndex(getDefaultGmm(), allocationType, regionIndex, CachePolicy::WriteBack, true);
auto patIndex = drm->getPatIndex(getDefaultGmm(), allocationType, regionIndex, CachePolicy::WriteBack, true, isSystemMemoryPool);
if (fragmentsStorage.fragmentCount > 0) {
for (uint32_t i = 0; i < fragmentsStorage.fragmentCount; i++) {

2
shared/source/os_interface/linux/drm_allocation.h
View File

@@ -104,7 +104,7 @@ class DrmAllocation : public GraphicsAllocation {
int peekInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) override;
bool setCacheRegion(Drm *drm, CacheRegion regionIndex);
bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex);
bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex, bool isSystemMemoryPool);
void setCachePolicy(CachePolicy memType);
bool setPreferredLocation(Drm *drm, PreferredLocation memoryLocation);

69
shared/source/os_interface/linux/drm_memory_manager.cpp
View File

@@ -293,7 +293,7 @@ NEO::BufferObject *DrmMemoryManager::allocUserptr(uintptr_t address, size_t size
PRINT_DEBUG_STRING(DebugManager.flags.PrintBOCreateDestroyResult.get(), stdout, "Created new BO with GEM_USERPTR, handle: BO-%d\n", userptr.handle);
auto patIndex = drm.getPatIndex(nullptr, AllocationType::EXTERNAL_HOST_PTR, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(nullptr, AllocationType::EXTERNAL_HOST_PTR, CacheRegion::Default, CachePolicy::WriteBack, false, true);
auto res = new (std::nothrow) BufferObject(rootDeviceIndex, &drm, patIndex, userptr.handle, size, maxOsContextCount);
if (!res) {
@@ -590,7 +590,7 @@ bool DrmMemoryManager::mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAl
}
GraphicsAllocation *DrmMemoryManager::allocatePhysicalDeviceMemory(const AllocationData &allocationData, AllocationStatus &status) {
const auto memoryPool = MemoryPool::SystemCpuInaccessible;
StorageInfo systemMemoryStorageInfo = {};
auto &productHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHelper<ProductHelper>();
@@ -603,11 +603,11 @@ GraphicsAllocation *DrmMemoryManager::allocatePhysicalDeviceMemory(const Allocat
uint32_t handle = ioctlHelper->createGem(bufferSize, static_cast<uint32_t>(allocationData.storageInfo.memoryBanks.to_ulong()));
auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new BufferObject(allocationData.rootDeviceIndex, &drm, patIndex, handle, bufferSize, maxOsContextCount));
auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, 0u, bufferSize, MemoryPool::SystemCpuInaccessible);
auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, 0u, bufferSize, memoryPool);
allocation->setDefaultGmm(gmm.release());
bo.release();
@@ -616,6 +616,7 @@ GraphicsAllocation *DrmMemoryManager::allocatePhysicalDeviceMemory(const Allocat
}
GraphicsAllocation *DrmMemoryManager::allocateMemoryByKMD(const AllocationData &allocationData) {
const auto memoryPool = MemoryPool::SystemCpuInaccessible;
auto &productHelper = executionEnvironment.rootDeviceEnvironments[allocationData.rootDeviceIndex]->getHelper<ProductHelper>();
StorageInfo systemMemoryStorageInfo = {};
@@ -629,12 +630,12 @@ GraphicsAllocation *DrmMemoryManager::allocateMemoryByKMD(const AllocationData &
uint32_t handle = ioctlHelper->createGem(bufferSize, static_cast<uint32_t>(allocationData.storageInfo.memoryBanks.to_ulong()));
auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new BufferObject(allocationData.rootDeviceIndex, &drm, patIndex, handle, bufferSize, maxOsContextCount));
bo->setAddress(gpuRange);
auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, gpuRange, bufferSize, MemoryPool::SystemCpuInaccessible);
auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, gpuRange, bufferSize, memoryPool);
if (!allocation) {
return nullptr;
}
@@ -656,13 +657,15 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryForImageImpl(const A
return alloc;
}
const auto memoryPool = MemoryPool::SystemCpuInaccessible;
uint64_t gpuRange = acquireGpuRange(allocationData.imgInfo->size, allocationData.rootDeviceIndex, HeapIndex::HEAP_STANDARD);
auto &drm = this->getDrm(allocationData.rootDeviceIndex);
auto ioctlHelper = drm.getIoctlHelper();
uint32_t handle = ioctlHelper->createGem(allocationData.imgInfo->size, static_cast<uint32_t>(allocationData.storageInfo.memoryBanks.to_ulong()));
auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(gmm.get(), allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
std::unique_ptr<BufferObject, BufferObject::Deleter> bo(new (std::nothrow) BufferObject(allocationData.rootDeviceIndex, &drm, patIndex, handle, allocationData.imgInfo->size, maxOsContextCount));
if (!bo) {
@@ -673,7 +676,7 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryForImageImpl(const A
[[maybe_unused]] auto ret2 = bo->setTiling(ioctlHelper->getDrmParamValue(DrmParam::TilingY), static_cast<uint32_t>(allocationData.imgInfo->rowPitch));
DEBUG_BREAK_IF(ret2 != true);
auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, gpuRange, allocationData.imgInfo->size, MemoryPool::SystemCpuInaccessible);
auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo.get(), nullptr, gpuRange, allocationData.imgInfo->size, memoryPool);
allocation->setDefaultGmm(gmm.release());
allocation->setReservedAddressRange(reinterpret_cast<void *>(gpuRange), allocationData.imgInfo->size);
@@ -794,6 +797,8 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromMultipleShared
bool areBosSharedObjects = true;
auto ioctlHelper = drm.getIoctlHelper();
const auto memoryPool = MemoryPool::LocalMemory;
for (auto handle : handles) {
PrimeHandle openFd = {0, 0, 0};
openFd.fileDescriptor = handle;
@@ -820,7 +825,7 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromMultipleShared
UNRECOVERABLE_IF(size == std::numeric_limits<size_t>::max());
totalSize += size;
auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
auto boHandleWrapper = reuseSharedAllocation ? BufferObjectHandleWrapper{boHandle} : tryToGetBoHandleWrapperWithSharedOwnership(boHandle);
bo = new (std::nothrow) BufferObject(properties.rootDeviceIndex, &drm, patIndex, boHandle, size, maxOsContextCount);
@@ -854,7 +859,7 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromMultipleShared
nullptr,
gpuRange,
totalSize,
MemoryPool::LocalMemory);
memoryPool);
drmAllocation->storageInfo = allocationData.storageInfo;
auto gmmHelper = executionEnvironment.rootDeviceEnvironments[properties.rootDeviceIndex]->getGmmHelper();
auto &productHelper = gmmHelper->getRootDeviceEnvironment().getHelper<ProductHelper>();
@@ -964,11 +969,13 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromSharedHandle(o
bo = findAndReferenceSharedBufferObject(boHandle, properties.rootDeviceIndex);
}
const auto memoryPool = MemoryPool::SystemCpuInaccessible;
if (bo == nullptr) {
size_t size = SysCalls::lseek(handle, 0, SEEK_END);
UNRECOVERABLE_IF(size == std::numeric_limits<size_t>::max());
auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
auto boHandleWrapper = reuseSharedAllocation ? BufferObjectHandleWrapper{boHandle} : tryToGetBoHandleWrapperWithSharedOwnership(boHandle);
bo = new (std::nothrow) BufferObject(properties.rootDeviceIndex, &drm, patIndex, std::move(boHandleWrapper), size, maxOsContextCount);
@@ -1025,7 +1032,7 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromSharedHandle(o
auto gmmHelper = getGmmHelper(properties.rootDeviceIndex);
auto canonizedGpuAddress = gmmHelper->canonize(castToUint64(reinterpret_cast<void *>(bo->peekAddress())));
auto drmAllocation = new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible, canonizedGpuAddress);
handle, memoryPool, canonizedGpuAddress);
if (requireSpecificBitness && this->force32bitAllocations) {
drmAllocation->set32BitAllocation(true);
@@ -1050,7 +1057,7 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromSharedHandle(o
gmm->updateImgInfoAndDesc(*properties.imgInfo, 0, NEO::ImagePlane::NO_PLANE);
drmAllocation->setDefaultGmm(gmm);
bo->setPatIndex(drm.getPatIndex(gmm, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false));
bo->setPatIndex(drm.getPatIndex(gmm, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool)));
}
if (!reuseSharedAllocation) {
@@ -1808,7 +1815,7 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const A
}
BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(uint32_t rootDeviceIndex, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress,
size_t size, uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle) {
size_t size, uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, bool isSystemMemoryPool) {
auto drm = &getDrm(rootDeviceIndex);
auto memoryInfo = drm->getMemoryInfo();
if (!memoryInfo) {
@@ -1818,7 +1825,7 @@ BufferObject *DrmMemoryManager::createBufferObjectInMemoryRegion(uint32_t rootDe
uint32_t handle = 0;
int ret = 0;
auto patIndex = drm->getPatIndex(gmm, allocationType, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm->getPatIndex(gmm, allocationType, CacheRegion::Default, CachePolicy::WriteBack, false, isSystemMemoryPool);
auto banks = std::bitset<4>(memoryBanks);
if (banks.count() > 1) {
@@ -1849,7 +1856,7 @@ bool DrmMemoryManager::createDrmChunkedAllocation(Drm *drm, DrmAllocation *alloc
uint32_t numOfChunks = DebugManager.flags.NumberOfBOChunks.get();
auto gmm = allocation->getGmm(0u);
auto patIndex = drm->getPatIndex(gmm, allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm->getPatIndex(gmm, allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false, !allocation->isAllocatedInLocalMemoryPool());
int ret = memoryInfo->createGemExtWithMultipleRegions(memoryBanks, boSize, handle, patIndex, -1, true, numOfChunks);
if (ret != 0) {
return false;
@@ -1935,7 +1942,8 @@ bool DrmMemoryManager::createDrmAllocation(Drm *drm, DrmAllocation *allocation,
}
auto gmm = allocation->getGmm(handleId);
auto boSize = alignUp(gmm->gmmResourceInfo->getSizeAllocation(), MemoryConstants::pageSize64k);
bos[handleId] = createBufferObjectInMemoryRegion(allocation->getRootDeviceIndex(), gmm, allocation->getAllocationType(), boAddress, boSize, memoryBanks, maxOsContextCount, pairHandle);
bos[handleId] = createBufferObjectInMemoryRegion(allocation->getRootDeviceIndex(), gmm, allocation->getAllocationType(), boAddress, boSize, memoryBanks, maxOsContextCount, pairHandle,
!allocation->isAllocatedInLocalMemoryPool());
if (nullptr == bos[handleId]) {
return false;
}
@@ -2052,6 +2060,8 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &
}
if (useBooMmap) {
const auto memoryPool = MemoryPool::System4KBPages;
auto totalSizeToAlloc = alignedSize + alignment;
uint64_t preferredAddress = 0;
auto gfxPartition = getGfxPartition(allocationData.rootDeviceIndex);
@@ -2072,7 +2082,8 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &
auto pointerDiff = ptrDiff(cpuPointer, cpuBasePointer);
std::unique_ptr<BufferObject, BufferObject::Deleter> bo(this->createBufferObjectInMemoryRegion(allocationData.rootDeviceIndex, nullptr, allocationData.type,
reinterpret_cast<uintptr_t>(cpuPointer), alignedSize, 0u, maxOsContextCount, -1));
reinterpret_cast<uintptr_t>(cpuPointer), alignedSize, 0u, maxOsContextCount, -1,
MemoryPoolHelper::isSystemMemoryPool(memoryPool)));
if (!bo) {
releaseGpuRange(reinterpret_cast<void *>(preferredAddress), totalSizeToAlloc, allocationData.rootDeviceIndex);
@@ -2097,7 +2108,7 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignment(const AllocationData &
auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex);
auto canonizedGpuAddress = gmmHelper->canonize(bo->peekAddress());
auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, canonizedGpuAddress, alignedSize, MemoryPool::System4KBPages);
auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bo.get(), cpuPointer, canonizedGpuAddress, alignedSize, memoryPool);
allocation->setMmapPtr(cpuPointer);
allocation->setMmapSize(alignedSize);
if (pointerDiff != 0) {
@@ -2248,6 +2259,8 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const
useChunking = true;
}
const auto memoryPool = MemoryPool::LocalMemory;
for (auto handleId = 0u; handleId < numHandles; handleId++) {
uint32_t handle = 0;
@@ -2260,7 +2273,7 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const
auto memoryBanks = (DebugManager.flags.KMDSupportForCrossTileMigrationPolicy.get() > 0 || useChunking) ? allocationData.storageInfo.memoryBanks : DeviceBitfield(1 << memoryInstance);
auto memRegions = createMemoryRegionsForSharedAllocation(*pHwInfo, *memoryInfo, allocationData, memoryBanks);
auto patIndex = drm.getPatIndex(nullptr, allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(nullptr, allocationData.type, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
int ret = memoryInfo->createGemExt(memRegions, currentSize, handle, patIndex, {}, -1, useChunking, numOfChunks);
@@ -2301,7 +2314,7 @@ GraphicsAllocation *DrmMemoryManager::createSharedUnifiedMemoryAllocation(const
auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex);
auto canonizedGpuAddress = gmmHelper->canonize(reinterpret_cast<uintptr_t>(cpuPointer));
auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bos, cpuPointer, canonizedGpuAddress, size, MemoryPool::LocalMemory);
auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, allocationData.type, bos, cpuPointer, canonizedGpuAddress, size, memoryPool);
allocation->setMmapPtr(cpuBasePointer);
allocation->setMmapSize(totalSizeToAlloc);
allocation->setReservedAddressRange(reinterpret_cast<void *>(preferredAddress), totalSizeToAlloc);
@@ -2349,8 +2362,10 @@ DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandl
PrimeHandle openFd{};
openFd.fileDescriptor = handle;
auto memoryPool = MemoryPool::SystemCpuInaccessible;
auto &drm = this->getDrm(properties.rootDeviceIndex);
auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
auto ioctlHelper = drm.getIoctlHelper();
auto ret = ioctlHelper->ioctl(DrmIoctl::PrimeFdToHandle, &openFd);
@@ -2368,7 +2383,7 @@ DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandl
auto gmmHelper = getGmmHelper(properties.rootDeviceIndex);
auto canonizedGpuAddress = gmmHelper->canonize(castToUint64(reinterpret_cast<void *>(bo->peekAddress())));
return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible, canonizedGpuAddress);
handle, memoryPool, canonizedGpuAddress);
}
const bool useBooMmap = drm.getMemoryInfo() && properties.useMmapObject;
@@ -2379,7 +2394,7 @@ DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandl
auto gmmHelper = getGmmHelper(properties.rootDeviceIndex);
auto canonizedGpuAddress = gmmHelper->canonize(castToUint64(reinterpret_cast<void *>(bo->peekAddress())));
return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible, canonizedGpuAddress);
handle, memoryPool, canonizedGpuAddress);
}
auto boHandle = openFd.handle;
@@ -2393,6 +2408,8 @@ DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandl
size_t size = SysCalls::lseek(handle, 0, SEEK_END);
UNRECOVERABLE_IF(size == std::numeric_limits<size_t>::max());
memoryPool = MemoryPool::System4KBPages;
patIndex = drm.getPatIndex(nullptr, properties.allocationType, CacheRegion::Default, CachePolicy::WriteBack, false, MemoryPoolHelper::isSystemMemoryPool(memoryPool));
bo = new BufferObject(properties.rootDeviceIndex, &drm, patIndex, boHandle, size, maxOsContextCount);
if (properties.allocationType == AllocationType::GPU_TIMESTAMP_DEVICE_BUFFER) {
@@ -2442,7 +2459,7 @@ DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandl
pushSharedBufferObject(bo);
auto drmAllocation = std::make_unique<DrmAllocation>(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), MemoryPool::System4KBPages);
auto drmAllocation = std::make_unique<DrmAllocation>(properties.rootDeviceIndex, properties.allocationType, bo, cpuPointer, bo->peekAddress(), bo->peekSize(), memoryPool);
drmAllocation->setMmapPtr(cpuPointer);
drmAllocation->setMmapSize(size);
drmAllocation->setReservedAddressRange(reinterpret_cast<void *>(cpuPointer), size);
@@ -2458,7 +2475,7 @@ DrmAllocation *DrmMemoryManager::createUSMHostAllocationFromSharedHandle(osHandl
auto gmmHelper = getGmmHelper(properties.rootDeviceIndex);
auto canonizedGpuAddress = gmmHelper->canonize(castToUint64(reinterpret_cast<void *>(bo->peekAddress())));
return new DrmAllocation(properties.rootDeviceIndex, properties.allocationType, bo, reinterpret_cast<void *>(bo->peekAddress()), bo->peekSize(),
handle, MemoryPool::SystemCpuInaccessible, canonizedGpuAddress);
handle, memoryPool, canonizedGpuAddress);
}
bool DrmMemoryManager::allowIndirectAllocationsAsPack(uint32_t rootDeviceIndex) {
return this->getDrm(rootDeviceIndex).isVmBindAvailable();

2
shared/source/os_interface/linux/drm_memory_manager.h
View File

@@ -71,7 +71,7 @@ class DrmMemoryManager : public MemoryManager {
AddressRange reserveGpuAddress(const uint64_t requiredStartAddress, size_t size, RootDeviceIndicesContainer rootDeviceIndices, uint32_t *reservedOnRootDeviceIndex) override;
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
MOCKABLE_VIRTUAL BufferObject *createBufferObjectInMemoryRegion(uint32_t rootDeviceIndex, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size,
uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle);
uint32_t memoryBanks, size_t maxOsContextCount, int32_t pairHandle, bool isSystemMemoryPool);
bool hasPageFaultsEnabled(const Device &neoDevice) override;
bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override;

10
shared/source/os_interface/linux/drm_neo.cpp
View File

@@ -1193,7 +1193,15 @@ bool Drm::isVmBindAvailable() {
return bindAvailable;
}
uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled) const {
uint64_t Drm::getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled, bool isSystemMemory) const {
if ((DebugManager.flags.OverridePatIndexForSystemMemory.get() != -1) && isSystemMemory) {
return static_cast<uint64_t>(DebugManager.flags.OverridePatIndexForSystemMemory.get());
}
if ((DebugManager.flags.OverridePatIndexForDeviceMemory.get() != -1) && !isSystemMemory) {
return static_cast<uint64_t>(DebugManager.flags.OverridePatIndexForDeviceMemory.get());
}
if (DebugManager.flags.OverridePatIndex.get() != -1) {
return static_cast<uint64_t>(DebugManager.flags.OverridePatIndex.get());
}

2
shared/source/os_interface/linux/drm_neo.h
View File

@@ -245,7 +245,7 @@ class Drm : public DriverModel {
MOCKABLE_VIRTUAL uint32_t notifyFirstCommandQueueCreated(const void *data, size_t size);
MOCKABLE_VIRTUAL void notifyLastCommandQueueDestroyed(uint32_t handle);
uint64_t getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled) const;
uint64_t getPatIndex(Gmm *gmm, AllocationType allocationType, CacheRegion cacheRegion, CachePolicy cachePolicy, bool closEnabled, bool isSystemMemory) const;
bool isVmBindPatIndexProgrammingSupported() const { return vmBindPatIndexProgrammingSupported; }
MOCKABLE_VIRTUAL bool getDeviceMemoryMaxClockRateInMhz(uint32_t tileId, uint32_t &clkRate);
MOCKABLE_VIRTUAL bool getDeviceMemoryPhysicalSizeInBytes(uint32_t tileId, uint64_t &physicalSize);

1
shared/test/common/libult/linux/drm_mock.h
View File

@@ -54,6 +54,7 @@ class DrmMock : public Drm {
using Drm::sliceCountChangeSupported;
using Drm::systemInfo;
using Drm::virtualMemoryIds;
using Drm::vmBindPatIndexProgrammingSupported;
DrmMock(int fd, RootDeviceEnvironment &rootDeviceEnvironment);
DrmMock(RootDeviceEnvironment &rootDeviceEnvironment) : DrmMock(mockFd, rootDeviceEnvironment) {}

2
shared/test/common/mocks/linux/mock_drm_memory_manager.cpp
View File

@@ -53,7 +53,7 @@ TestedDrmMemoryManager::TestedDrmMemoryManager(bool enableLocalMemory,
BufferObject *TestedDrmMemoryManager::findAndReferenceSharedBufferObject(int boHandle, uint32_t rootDeviceIndex) {
if (failOnfindAndReferenceSharedBufferObject) {
DrmMockCustom drmMock(*executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]);
auto patIndex = drmMock.getPatIndex(nullptr, AllocationType::BUFFER, CacheRegion::Default, CachePolicy::WriteBack, false);
auto patIndex = drmMock.getPatIndex(nullptr, AllocationType::BUFFER, CacheRegion::Default, CachePolicy::WriteBack, false, false);
return new (std::nothrow) BufferObject(rootDeviceIndex, &drmMock, patIndex, boHandle, 4096u, 2u);
}
return MemoryManagerCreate<DrmMemoryManager>::findAndReferenceSharedBufferObject(boHandle, rootDeviceIndex);

2
shared/test/common/test_files/igdrcl.config
View File

@@ -550,4 +550,6 @@ DoNotValidateDriverPath = 0
EnableInOrderRegularCmdListPatching = -1
ForceInOrderEvents = -1
EnableInOrderRelaxedOrderingForEventsChaining = -1
OverridePatIndexForSystemMemory = -1
OverridePatIndexForDeviceMemory = -1
# Please don't edit below this line

6
shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_prelim_tests.cpp
View File

@@ -61,7 +61,8 @@ TEST_F(DrmMemoryManagerLocalMemoryPrelimTest, givenDrmMemoryManagerWithPrelimSup
size,
(1 << (MemoryBanks::getBankForLocalMemory(0) - 1)),
1,
-1));
-1,
false));
ASSERT_NE(nullptr, bo);
EXPECT_EQ(1u, mock->ioctlCallsCount);
@@ -1980,7 +1981,8 @@ TEST_F(DrmMemoryManagerLocalMemoryPrelimTest, givenPrintBOCreateDestroyResultFla
size,
(1 << (MemoryBanks::getBankForLocalMemory(0) - 1)),
1,
-1));
-1,
false));
EXPECT_NE(nullptr, bo);
std::string output = testing::internal::GetCapturedStdout();

8
shared/test/unit_test/os_interface/linux/drm_memory_manager_localmem_upstream_tests.cpp
View File

@@ -54,7 +54,7 @@ class DrmMemoryManagerFixtureImpl : public DrmMemoryManagerFixture {
std::unique_ptr<VariableBackup<UltHwConfig>> backup;
};
BufferObject *createBufferObjectInMemoryRegion(Drm *drm, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount);
BufferObject *createBufferObjectInMemoryRegion(Drm *drm, Gmm *gmm, AllocationType allocationType, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount, bool isSystemMemoryPool);
class DrmMemoryManagerLocalMemoryTest : public ::testing::Test {
public:
@@ -126,7 +126,8 @@ HWTEST2_F(DrmMemoryManagerLocalMemoryTest, givenDrmMemoryManagerWhenCreateBuffer
size,
(1 << (MemoryBanks::getBankForLocalMemory(0) - 1)),
1,
-1));
-1,
false));
ASSERT_NE(nullptr, bo);
EXPECT_EQ(1u, mock->ioctlCallsCount);
EXPECT_EQ(1u, mock->createExt.handle);
@@ -447,7 +448,8 @@ class DrmMemoryManagerLocalMemoryMemoryBankMock : public TestedDrmMemoryManager
size_t size,
uint32_t memoryBanks,
size_t maxOsContextCount,
int32_t pairHandle) override {
int32_t pairHandle,
bool isSystemMemoryPool) override {
memoryBankIsOne = (memoryBanks == 1) ? true : false;
return nullptr;
}

22
shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp
View File

@@ -4287,7 +4287,7 @@ TEST_F(DrmAllocationTests, givenDrmAllocationWhenCacheRegionIsNotSetThenReturnFa
MockDrmAllocation allocation(rootDeviceIndex, AllocationType::BUFFER, MemoryPool::LocalMemory);
EXPECT_FALSE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::None));
EXPECT_FALSE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::None, false));
}
TEST_F(DrmAllocationTests, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenReturnTrue) {
@@ -4302,9 +4302,9 @@ TEST_F(DrmAllocationTests, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThe
if ((gfxCoreHelper.getNumCacheRegions() == 0) &&
productHelper.isVmBindPatIndexProgrammingSupported()) {
EXPECT_ANY_THROW(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1));
EXPECT_ANY_THROW(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1, false));
} else {
EXPECT_TRUE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1));
EXPECT_TRUE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1, false));
}
}
@@ -4323,9 +4323,9 @@ TEST_F(DrmAllocationTests, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThe
if ((gfxCoreHelper.getNumCacheRegions() == 0) &&
productHelper.isVmBindPatIndexProgrammingSupported()) {
EXPECT_ANY_THROW(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1));
EXPECT_ANY_THROW(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1, false));
} else {
EXPECT_TRUE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1));
EXPECT_TRUE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1, false));
for (auto bo : allocation.bufferObjects) {
if (bo != nullptr) {
@@ -4429,7 +4429,7 @@ TEST_F(DrmMemoryManagerTest, givenDrmAllocationWithHostPtrWhenItIsCreatedWithCac
nullptr, ptr, castToUint64(ptr), size, MemoryPool::System4KBPages);
allocation->fragmentsStorage = storage;
allocation->setCacheAdvice(drm, 1024, CacheRegion::Region1);
allocation->setCacheAdvice(drm, 1024, CacheRegion::Region1, false);
for (uint32_t i = 0; i < storage.fragmentCount; i++) {
auto bo = static_cast<OsHandleLinux *>(allocation->fragmentsStorage.fragmentStorageData[i].osHandleStorage)->bo;
@@ -5178,7 +5178,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenDrmMemor
auto gpuAddress = 0x1234u;
auto size = MemoryConstants::pageSize;
auto bo = std::unique_ptr<BufferObject>(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::BUFFER, gpuAddress, size, MemoryBanks::MainBank, 1, -1));
auto bo = std::unique_ptr<BufferObject>(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::BUFFER, gpuAddress, size, MemoryBanks::MainBank, 1, -1, false));
EXPECT_EQ(nullptr, bo);
}
@@ -5186,7 +5186,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenDrmMemor
auto gpuAddress = 0x1234u;
auto size = 0u;
auto bo = std::unique_ptr<BufferObject>(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::BUFFER, gpuAddress, size, MemoryBanks::MainBank, 1, -1));
auto bo = std::unique_ptr<BufferObject>(memoryManager->createBufferObjectInMemoryRegion(rootDeviceIndex, nullptr, AllocationType::BUFFER, gpuAddress, size, MemoryBanks::MainBank, 1, -1, false));
EXPECT_EQ(nullptr, bo);
}
@@ -6194,7 +6194,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenPatIndex
EXPECT_EQ(isVmBindPatIndexProgrammingSupported, mock->isVmBindPatIndexProgrammingSupported());
if (isVmBindPatIndexProgrammingSupported) {
auto expectedIndex = mock->getPatIndex(allocation->getDefaultGmm(), allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false);
auto expectedIndex = mock->getPatIndex(allocation->getDefaultGmm(), allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false, false);
EXPECT_NE(CommonConstants::unsupportedPatIndex, drmAllocation->getBO()->peekPatIndex());
EXPECT_EQ(expectedIndex, drmAllocation->getBO()->peekPatIndex());
@@ -6230,7 +6230,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenCompress
gmm->isCompressionEnabled = true;
gmm->gmmResourceInfo->getResourceFlags()->Info.Cacheable = 1;
mock->getPatIndex(allocation->getDefaultGmm(), allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false);
mock->getPatIndex(allocation->getDefaultGmm(), allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false, false);
EXPECT_TRUE(mockClientContext->passedCachableSettingForGetPatIndexQuery);
EXPECT_TRUE(mockClientContext->passedCompressedSettingForGetPatIndexQuery);
@@ -6240,7 +6240,7 @@ TEST_F(DrmMemoryManagerWithLocalMemoryAndExplicitExpectationsTest, givenCompress
gmm->isCompressionEnabled = false;
gmm->gmmResourceInfo->getResourceFlags()->Info.Cacheable = 0;
mock->getPatIndex(allocation->getDefaultGmm(), allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false);
mock->getPatIndex(allocation->getDefaultGmm(), allocation->getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false, false);
EXPECT_FALSE(mockClientContext->passedCachableSettingForGetPatIndexQuery);
EXPECT_FALSE(mockClientContext->passedCompressedSettingForGetPatIndexQuery);

68
shared/test/unit_test/os_interface/linux/drm_residency_handler_prelim_tests.cpp
View File

@@ -1112,7 +1112,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenPatIndexProgrammingEnabledWhen
mock->context.receivedVmBindPatIndex.reset();
mock->context.receivedVmUnbindPatIndex.reset();
bo.setPatIndex(mock->getPatIndex(allocation.getDefaultGmm(), allocation.getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, (debugFlag == 1 && closSupported)));
bo.setPatIndex(mock->getPatIndex(allocation.getDefaultGmm(), allocation.getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, (debugFlag == 1 && closSupported), true));
operationHandler->makeResident(device, ArrayRef<GraphicsAllocation *>(&allocationPtr, 1));
@@ -1163,7 +1163,7 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenPatIndexErrorAndUncachedDebugF
BufferObject bo(0, mock, static_cast<uint64_t>(MockGmmClientContextBase::MockPatIndex::cached), 0, 1, 1);
DrmAllocation allocation(0, 1, AllocationType::BUFFER, &bo, nullptr, gpuAddress, size, MemoryPool::System4KBPages);
EXPECT_ANY_THROW(mock->getPatIndex(allocation.getDefaultGmm(), allocation.getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false));
EXPECT_ANY_THROW(mock->getPatIndex(allocation.getDefaultGmm(), allocation.getAllocationType(), CacheRegion::Default, CachePolicy::WriteBack, false, false));
}
HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenUncachedDebugFlagSetWhenVmBindCalledThenSetCorrectPatIndexExtension) {
@@ -1224,6 +1224,68 @@ HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenDebugFlagSetWhenVmBindCalledTh
EXPECT_EQ(1u, mock->context.receivedVmUnbindPatIndex.value());
}
HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenDebugFlagSetWhenVmBindCalledThenOverridePatIndexForDeviceMem) {
DebugManager.flags.UseVmBind.set(1);
DebugManager.flags.ClosEnabled.set(1);
DebugManager.flags.OverridePatIndex.set(1);
DebugManager.flags.OverridePatIndexForDeviceMemory.set(2);
DebugManager.flags.OverridePatIndexForSystemMemory.set(3);
mock->bindAvailable = true;
mock->vmBindPatIndexProgrammingSupported = true;
auto csr = std::make_unique<UltCommandStreamReceiver<FamilyType>>(*executionEnvironment, 0, DeviceBitfield(1));
auto osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor());
csr->setupContext(*osContext);
auto patIndex = mock->getPatIndex(nullptr, AllocationType::BUFFER, CacheRegion::Default, CachePolicy::WriteBack, false, false);
EXPECT_EQ(2u, patIndex);
MockBufferObject bo(0, mock, patIndex, 0, 0, 1);
DrmAllocation allocation(0, AllocationType::BUFFER, &bo, nullptr, 0x1234000, 1, MemoryPool::LocalMemory);
GraphicsAllocation *allocPtr = &allocation;
operationHandler->makeResident(device, ArrayRef<GraphicsAllocation *>(&allocPtr, 1));
EXPECT_EQ(2u, mock->context.receivedVmBindPatIndex.value());
operationHandler->evict(device, allocation);
EXPECT_EQ(2u, mock->context.receivedVmUnbindPatIndex.value());
}
HWTEST_F(DrmMemoryOperationsHandlerBindTest, givenDebugFlagSetWhenVmBindCalledThenOverridePatIndexForSystemMem) {
DebugManager.flags.UseVmBind.set(1);
DebugManager.flags.ClosEnabled.set(1);
DebugManager.flags.OverridePatIndex.set(1);
DebugManager.flags.OverridePatIndexForDeviceMemory.set(2);
DebugManager.flags.OverridePatIndexForSystemMemory.set(3);
mock->bindAvailable = true;
mock->vmBindPatIndexProgrammingSupported = true;
auto csr = std::make_unique<UltCommandStreamReceiver<FamilyType>>(*executionEnvironment, 0, DeviceBitfield(1));
auto osContext = memoryManager->createAndRegisterOsContext(csr.get(), EngineDescriptorHelper::getDefaultDescriptor());
csr->setupContext(*osContext);
auto patIndex = mock->getPatIndex(nullptr, AllocationType::BUFFER, CacheRegion::Default, CachePolicy::WriteBack, false, true);
EXPECT_EQ(3u, patIndex);
MockBufferObject bo(0, mock, patIndex, 0, 0, 1);
DrmAllocation allocation(0, AllocationType::BUFFER, &bo, nullptr, 0x1234000, 1, MemoryPool::System4KBPages);
GraphicsAllocation *allocPtr = &allocation;
operationHandler->makeResident(device, ArrayRef<GraphicsAllocation *>(&allocPtr, 1));
EXPECT_EQ(3u, mock->context.receivedVmBindPatIndex.value());
operationHandler->evict(device, allocation);
EXPECT_EQ(3u, mock->context.receivedVmUnbindPatIndex.value());
}
TEST_F(DrmMemoryOperationsHandlerBindTest, givenClosEnabledAndAllocationToBeCachedInCacheRegionWhenVmBindIsCalledThenSetPatIndexCorrespondingToRequestedRegion) {
DebugManager.flags.UseVmBind.set(1);
DebugManager.flags.ClosEnabled.set(1);
@@ -1244,7 +1306,7 @@ TEST_F(DrmMemoryOperationsHandlerBindTest, givenClosEnabledAndAllocationToBeCach
auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{device->getRootDeviceIndex(), MemoryConstants::pageSize});
for (auto cacheRegion : {CacheRegion::Default, CacheRegion::Region1, CacheRegion::Region2}) {
EXPECT_TRUE(static_cast<DrmAllocation *>(allocation)->setCacheAdvice(mock, 32 * MemoryConstants::kiloByte, cacheRegion));
EXPECT_TRUE(static_cast<DrmAllocation *>(allocation)->setCacheAdvice(mock, 32 * MemoryConstants::kiloByte, cacheRegion, false));
mock->context.receivedVmBindPatIndex.reset();
operationHandler->makeResident(device, ArrayRef<GraphicsAllocation *>(&allocation, 1));