intel/compute-runtime
1
0
Fork 0
mirror of https://github.com/intel/compute-runtime.git synced 2026-02-05 14:52:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add new SVM types

Browse Source 
Related-To: NEO-2917

Change-Id: Ica127129799c1e617a326a110348c2f70160b15c
Signed-off-by: Zbigniew Zdanowicz <zbigniew.zdanowicz@intel.com>
This commit is contained in:
Zbigniew Zdanowicz
2019-03-06 16:35:21 +01:00
committed by sys_ocldev
parent 653986aea1
commit 971cbd55f3
34 changed files with 1019 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
runtime/api/api.cpp
View File

@@ -3628,11 +3628,12 @@ cl_int CL_API_CALL clSetKernelArgSVMPointer(cl_kernel kernel,
GraphicsAllocation *pSvmAlloc = nullptr;
if (argValue != nullptr) {
pSvmAlloc = pKernel->getContext().getSVMAllocsManager()->getSVMAlloc(argValue);
if (pSvmAlloc == nullptr) {
auto svmData = pKernel->getContext().getSVMAllocsManager()->getSVMAlloc(argValue);
if (svmData == nullptr) {
retVal = CL_INVALID_ARG_VALUE;
return retVal;
}
pSvmAlloc = svmData->gpuAllocation;
}
retVal = pKernel->setArgSvmAlloc(argIndex, const_cast<void *>(argValue), pSvmAlloc);
return retVal;
@@ -3667,13 +3668,13 @@ cl_int CL_API_CALL clSetKernelExecInfo(cl_kernel kernel,
pKernel->clearKernelExecInfo();
for (uint32_t i = 0; i < numPointers; i++) {
NEO::GraphicsAllocation *pSvmAlloc =
pKernel->getContext().getSVMAllocsManager()->getSVMAlloc((const void *)pSvmPtrList[i]);
if (pSvmAlloc == nullptr) {
auto svmData = pKernel->getContext().getSVMAllocsManager()->getSVMAlloc((const void *)pSvmPtrList[i]);
if (svmData == nullptr) {
retVal = CL_INVALID_VALUE;
return retVal;
}
pKernel->setKernelExecInfo(pSvmAlloc);
GraphicsAllocation *svmAlloc = svmData->gpuAllocation;
pKernel->setKernelExecInfo(svmAlloc);
}
break;
}
@@ -4145,14 +4146,14 @@ cl_int CL_API_CALL clEnqueueSVMMigrateMem(cl_command_queue commandQueue,
for (uint32_t i = 0; i < numSvmPointers; i++) {
SVMAllocsManager *pSvmAllocMgr = pCommandQueue->getContext().getSVMAllocsManager();
GraphicsAllocation *pSvmAlloc = pSvmAllocMgr->getSVMAlloc(svmPointers[i]);
if (pSvmAlloc == nullptr) {
auto svmData = pSvmAllocMgr->getSVMAlloc(svmPointers[i]);
if (svmData == nullptr) {
retVal = CL_INVALID_VALUE;
return retVal;
}
if (sizes != nullptr && sizes[i] != 0) {
pSvmAlloc = pSvmAllocMgr->getSVMAlloc(reinterpret_cast<void *>((size_t)svmPointers[i] + sizes[i] - 1));
if (pSvmAlloc == nullptr) {
svmData = pSvmAllocMgr->getSVMAlloc(reinterpret_cast<void *>((size_t)svmPointers[i] + sizes[i] - 1));
if (svmData == nullptr) {
retVal = CL_INVALID_VALUE;
return retVal;
}

188
runtime/command_queue/enqueue_svm.h
View File

@@ -6,9 +6,11 @@
*/
#pragma once
#include "runtime/built_ins/built_ins.h"
#include "runtime/command_queue/command_queue_hw.h"
#include "runtime/command_queue/enqueue_common.h"
#include "runtime/event/event.h"
#include "runtime/memory_manager/surface.h"
#include "runtime/memory_manager/svm_memory_manager.h"
#include <new>
@@ -67,25 +69,78 @@ cl_int CommandQueueHw<GfxFamily>::enqueueSVMMap(cl_bool blockingMap,
const cl_event *eventWaitList,
cl_event *event) {
NEO::GraphicsAllocation *svmAllocation = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
if (svmAllocation == nullptr) {
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
if (svmData == nullptr) {
return CL_INVALID_VALUE;
}
bool blocking = blockingMap == CL_TRUE;
NullSurface s;
Surface *surfaces[] = {&s};
if (context->isProvidingPerformanceHints()) {
context->providePerformanceHint(CL_CONTEXT_DIAGNOSTICS_LEVEL_GOOD_INTEL, CL_ENQUEUE_SVM_MAP_DOESNT_REQUIRE_COPY_DATA, svmPtr);
if (svmData->gpuAllocation->getAllocationType() == GraphicsAllocation::AllocationType::SVM_ZERO_COPY) {
NullSurface s;
Surface *surfaces[] = {&s};
if (context->isProvidingPerformanceHints()) {
context->providePerformanceHint(CL_CONTEXT_DIAGNOSTICS_LEVEL_GOOD_INTEL, CL_ENQUEUE_SVM_MAP_DOESNT_REQUIRE_COPY_DATA, svmPtr);
}
enqueueHandler<CL_COMMAND_SVM_MAP>(surfaces,
blocking,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
return CL_SUCCESS;
} else {
auto svmOperation = context->getSVMAllocsManager()->getSvmMapOperation(svmPtr);
if (svmOperation) {
NullSurface s;
Surface *surfaces[] = {&s};
enqueueHandler<CL_COMMAND_SVM_MAP>(surfaces,
blocking,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
return CL_SUCCESS;
}
MultiDispatchInfo dispatchInfo;
auto &builder = getDevice().getExecutionEnvironment()->getBuiltIns()->getBuiltinDispatchInfoBuilder(EBuiltInOps::CopyBufferToBuffer,
this->getContext(), this->getDevice());
BuiltInOwnershipWrapper builtInLock(builder, this->context);
GeneralSurface dstSurface(svmData->cpuAllocation);
GeneralSurface srcSurface(svmData->gpuAllocation);
Surface *surfaces[] = {&dstSurface, &srcSurface};
void *svmBasePtr = svmData->cpuAllocation->getUnderlyingBuffer();
size_t svmOffset = ptrDiff(svmPtr, svmBasePtr);
BuiltinDispatchInfoBuilder::BuiltinOpParams dc;
dc.dstPtr = reinterpret_cast<void *>(svmData->cpuAllocation->getGpuAddressToPatch());
dc.dstSvmAlloc = svmData->cpuAllocation;
dc.dstOffset = {svmOffset, 0, 0};
dc.srcPtr = reinterpret_cast<void *>(svmData->gpuAllocation->getGpuAddressToPatch());
dc.srcSvmAlloc = svmData->gpuAllocation;
dc.srcOffset = {svmOffset, 0, 0};
dc.size = {size, 0, 0};
builder.buildDispatchInfos(dispatchInfo, dc);
enqueueHandler<CL_COMMAND_READ_BUFFER>(
surfaces,
blocking,
dispatchInfo,
numEventsInWaitList,
eventWaitList,
event);
if (event) {
castToObjectOrAbort<Event>(*event)->setCmdType(CL_COMMAND_SVM_MAP);
}
bool readOnlyMap = SVMAllocsManager::mapFlagIsReadOnly(mapFlags);
context->getSVMAllocsManager()->insertSvmMapOperation(svmPtr, size, svmBasePtr, svmOffset, readOnlyMap);
return CL_SUCCESS;
}
enqueueHandler<CL_COMMAND_SVM_MAP>(surfaces,
blockingMap ? true : false,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
return CL_SUCCESS;
}
template <typename GfxFamily>
@@ -94,21 +149,82 @@ cl_int CommandQueueHw<GfxFamily>::enqueueSVMUnmap(void *svmPtr,
const cl_event *eventWaitList,
cl_event *event) {
NEO::GraphicsAllocation *svmAllocation = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
if (svmAllocation == nullptr) {
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
if (svmData == nullptr) {
return CL_INVALID_VALUE;
}
NullSurface s;
Surface *surfaces[] = {&s};
enqueueHandler<CL_COMMAND_SVM_UNMAP>(surfaces,
false,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
if (svmData->gpuAllocation->getAllocationType() == GraphicsAllocation::AllocationType::SVM_ZERO_COPY) {
NullSurface s;
Surface *surfaces[] = {&s};
enqueueHandler<CL_COMMAND_SVM_UNMAP>(surfaces,
false,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
return CL_SUCCESS;
return CL_SUCCESS;
} else {
auto svmOperation = context->getSVMAllocsManager()->getSvmMapOperation(svmPtr);
if (!svmOperation) {
NullSurface s;
Surface *surfaces[] = {&s};
enqueueHandler<CL_COMMAND_SVM_UNMAP>(surfaces,
false,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
return CL_SUCCESS;
}
if (svmOperation->readOnlyMap) {
NullSurface s;
Surface *surfaces[] = {&s};
enqueueHandler<CL_COMMAND_SVM_UNMAP>(surfaces,
false,
MultiDispatchInfo(),
numEventsInWaitList,
eventWaitList,
event);
context->getSVMAllocsManager()->removeSvmMapOperation(svmPtr);
return CL_SUCCESS;
}
MultiDispatchInfo dispatchInfo;
auto &builder = getDevice().getExecutionEnvironment()->getBuiltIns()->getBuiltinDispatchInfoBuilder(EBuiltInOps::CopyBufferToBuffer,
this->getContext(), this->getDevice());
BuiltInOwnershipWrapper builtInLock(builder, this->context);
GeneralSurface dstSurface(svmData->gpuAllocation);
GeneralSurface srcSurface(svmData->cpuAllocation);
Surface *surfaces[] = {&dstSurface, &srcSurface};
BuiltinDispatchInfoBuilder::BuiltinOpParams dc;
dc.dstPtr = reinterpret_cast<void *>(svmData->gpuAllocation->getGpuAddressToPatch());
dc.dstSvmAlloc = svmData->gpuAllocation;
dc.dstOffset = {svmOperation->offset, 0, 0};
dc.srcPtr = reinterpret_cast<void *>(svmData->cpuAllocation->getGpuAddressToPatch());
dc.srcSvmAlloc = svmData->cpuAllocation;
dc.srcOffset = {svmOperation->offset, 0, 0};
dc.size = {svmOperation->regionSize, 0, 0};
builder.buildDispatchInfos(dispatchInfo, dc);
enqueueHandler<CL_COMMAND_READ_BUFFER>(
surfaces,
false,
dispatchInfo,
numEventsInWaitList,
eventWaitList,
event);
if (event) {
castToObjectOrAbort<Event>(*event)->setCmdType(CL_COMMAND_SVM_UNMAP);
}
context->getSVMAllocsManager()->removeSvmMapOperation(svmPtr);
return CL_SUCCESS;
}
}
template <typename GfxFamily>
@@ -157,9 +273,9 @@ cl_int CommandQueueHw<GfxFamily>::enqueueSVMMemcpy(cl_bool blockingCopy,
const cl_event *eventWaitList,
cl_event *event) {
GraphicsAllocation *pDstSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(dstPtr);
GraphicsAllocation *pSrcSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(srcPtr);
if ((pDstSvmAlloc == nullptr) || (pSrcSvmAlloc == nullptr)) {
auto dstSvmData = context->getSVMAllocsManager()->getSVMAlloc(dstPtr);
auto srcSvmData = context->getSVMAllocsManager()->getSVMAlloc(srcPtr);
if ((dstSvmData == nullptr) || (srcSvmData == nullptr)) {
return CL_INVALID_VALUE;
}
@@ -173,14 +289,14 @@ cl_int CommandQueueHw<GfxFamily>::enqueueSVMMemcpy(cl_bool blockingCopy,
BuiltinDispatchInfoBuilder::BuiltinOpParams operationParams;
operationParams.srcPtr = const_cast<void *>(srcPtr);
operationParams.dstPtr = dstPtr;
operationParams.srcSvmAlloc = pSrcSvmAlloc;
operationParams.dstSvmAlloc = pDstSvmAlloc;
operationParams.srcSvmAlloc = srcSvmData->gpuAllocation;
operationParams.dstSvmAlloc = dstSvmData->gpuAllocation;
operationParams.srcOffset = {0, 0, 0};
operationParams.dstOffset = {0, 0, 0};
operationParams.size = {size, 0, 0};
builder.buildDispatchInfos(dispatchInfo, operationParams);
GeneralSurface s1(pSrcSvmAlloc), s2(pDstSvmAlloc);
GeneralSurface s1(srcSvmData->gpuAllocation), s2(dstSvmData->gpuAllocation);
Surface *surfaces[] = {&s1, &s2};
enqueueHandler<CL_COMMAND_SVM_MEMCPY>(
@@ -203,8 +319,8 @@ cl_int CommandQueueHw<GfxFamily>::enqueueSVMMemFill(void *svmPtr,
const cl_event *eventWaitList,
cl_event *event) {
NEO::GraphicsAllocation *pSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
if (pSvmAlloc == nullptr) {
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
if (svmData == nullptr) {
return CL_INVALID_VALUE;
}
@@ -243,12 +359,12 @@ cl_int CommandQueueHw<GfxFamily>::enqueueSVMMemFill(void *svmPtr,
patternAllocation->getUnderlyingBuffer(), patternAllocation, false, false, true);
operationParams.srcMemObj = &patternMemObj;
operationParams.dstPtr = svmPtr;
operationParams.dstSvmAlloc = pSvmAlloc;
operationParams.dstSvmAlloc = svmData->gpuAllocation;
operationParams.dstOffset = {0, 0, 0};
operationParams.size = {size, 0, 0};
builder.buildDispatchInfos(dispatchInfo, operationParams);
GeneralSurface s1(pSvmAlloc);
GeneralSurface s1(svmData->gpuAllocation);
GeneralSurface s2(patternAllocation);
Surface *surfaces[] = {&s1, &s2};

16
runtime/mem_obj/buffer.cpp
View File

@@ -127,6 +127,7 @@ Buffer *Buffer::create(Context *context,
errcodeRet = CL_SUCCESS;
GraphicsAllocation *memory = nullptr;
GraphicsAllocation *mapAllocation = nullptr;
bool zeroCopyAllowed = true;
bool isHostPtrSVM = false;
@@ -173,13 +174,15 @@ Buffer *Buffer::create(Context *context,
allocateMemory = true;
}
memory = context->getSVMAllocsManager()->getSVMAlloc(hostPtr);
if (memory) {
allocationType = GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY;
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(hostPtr);
if (svmData) {
memory = svmData->gpuAllocation;
allocationType = memory->getAllocationType();
isHostPtrSVM = true;
zeroCopyAllowed = true;
zeroCopyAllowed = memory->getAllocationType() == GraphicsAllocation::AllocationType::SVM_ZERO_COPY;
copyMemoryFromHostPtr = false;
allocateMemory = false;
mapAllocation = svmData->cpuAllocation;
}
}
@@ -231,7 +234,9 @@ Buffer *Buffer::create(Context *context,
if (!MemoryPool::isSystemMemoryPool(memory->getMemoryPool())) {
zeroCopyAllowed = false;
if (hostPtr) {
copyMemoryFromHostPtr = true;
if (!isHostPtrSVM) {
copyMemoryFromHostPtr = true;
}
}
} else if (allocationType == GraphicsAllocation::AllocationType::BUFFER) {
allocationType = GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY;
@@ -259,6 +264,7 @@ Buffer *Buffer::create(Context *context,
}
pBuffer->setHostPtrMinSize(size);
pBuffer->mapAllocation = mapAllocation;
if (copyMemoryFromHostPtr) {
auto gmm = memory->getDefaultGmm();

18
runtime/mem_obj/mem_obj.cpp
View File

@@ -287,7 +287,7 @@ void MemObj::releaseAllocatedMapPtr() {
}
void MemObj::releaseMapAllocation() {
if (mapAllocation) {
if (mapAllocation && !isHostPtrSVM) {
destroyGraphicsAllocation(mapAllocation, false);
}
}
@@ -312,20 +312,20 @@ bool MemObj::checkIfMemoryTransferIsRequired(size_t offsetInMemObjest, size_t of
void *MemObj::getBasePtrForMap() {
if (associatedMemObject) {
TakeOwnershipWrapper<MemObj> memObjOwnership(*this);
return associatedMemObject->getBasePtrForMap();
}
if (getMapAllocation()) {
return getMapAllocation()->getUnderlyingBuffer();
}
if (getFlags() & CL_MEM_USE_HOST_PTR) {
return getHostPtr();
} else {
TakeOwnershipWrapper<MemObj> memObjOwnership(*this);
if (!getMapAllocation()) {
auto memory = memoryManager->allocateSystemMemory(getSize(), MemoryConstants::pageSize);
setAllocatedMapPtr(memory);
AllocationProperties properties{false, getSize(), GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR};
auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(properties, memory);
setMapAllocation(allocation);
}
auto memory = memoryManager->allocateSystemMemory(getSize(), MemoryConstants::pageSize);
setAllocatedMapPtr(memory);
AllocationProperties properties{false, getSize(), GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR};
auto allocation = memoryManager->allocateGraphicsMemoryWithProperties(properties, memory);
setMapAllocation(allocation);
return getAllocatedMapPtr();
}
}

4
runtime/memory_manager/graphics_allocation.h
View File

@@ -63,7 +63,9 @@ class GraphicsAllocation : public IDNode<GraphicsAllocation> {
SHARED_BUFFER,
SHARED_RESOURCE_COPY,
SURFACE_STATE_HEAP,
SVM,
SVM_CPU,
SVM_GPU,
SVM_ZERO_COPY,
TAG_BUFFER,
TIMESTAMP_PACKET_TAG_BUFFER,
UNDECIDED,

12
runtime/memory_manager/memory_manager.cpp
View File

@@ -232,7 +232,8 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo
}
switch (properties.allocationType) {
case GraphicsAllocation::AllocationType::SVM:
case GraphicsAllocation::AllocationType::SVM_ZERO_COPY:
case GraphicsAllocation::AllocationType::SVM_GPU:
allow64KbPages = true;
default:
break;
@@ -253,8 +254,9 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo
case GraphicsAllocation::AllocationType::PRINTF_SURFACE:
case GraphicsAllocation::AllocationType::CONSTANT_SURFACE:
case GraphicsAllocation::AllocationType::GLOBAL_SURFACE:
case GraphicsAllocation::AllocationType::SVM:
case GraphicsAllocation::AllocationType::SVM_ZERO_COPY:
case GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR:
case GraphicsAllocation::AllocationType::SVM_CPU:
mustBeZeroCopy = true;
default:
break;
@@ -271,7 +273,9 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo
case GraphicsAllocation::AllocationType::SHARED_IMAGE:
case GraphicsAllocation::AllocationType::SHARED_BUFFER:
case GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY:
case GraphicsAllocation::AllocationType::SVM:
case GraphicsAllocation::AllocationType::SVM_ZERO_COPY:
case GraphicsAllocation::AllocationType::SVM_GPU:
case GraphicsAllocation::AllocationType::SVM_CPU:
case GraphicsAllocation::AllocationType::UNDECIDED:
mayRequireL3Flush = true;
default:
@@ -283,6 +287,8 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo
case GraphicsAllocation::AllocationType::FILL_PATTERN:
case GraphicsAllocation::AllocationType::PROFILING_TAG_BUFFER:
case GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR:
case GraphicsAllocation::AllocationType::SVM_ZERO_COPY:
case GraphicsAllocation::AllocationType::SVM_CPU:
allocationData.flags.useSystemMemory = true;
default:
break;

18
runtime/memory_manager/memory_manager.h
View File

@@ -212,7 +212,23 @@ class MemoryManager {
break;
default:
if (!allocationData.flags.useSystemMemory && !(allocationData.flags.allow32Bit && this->force32bitAllocations)) {
auto allocation = allocateGraphicsMemory(allocationData);
GraphicsAllocation *allocation = nullptr;
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_GPU) {
void *cpuAllocation = allocateSystemMemory(allocationData.size, allocationData.alignment);
if (!cpuAllocation) {
return nullptr;
}
uint64_t gpuAddress = reinterpret_cast<uint64_t>(allocationData.hostPtr);
allocation = new GraphicsAllocation(allocationData.type,
cpuAllocation,
gpuAddress,
gpuAddress,
allocationData.size, MemoryPool::LocalMemory, false);
allocation->setDriverAllocatedCpuPtr(cpuAllocation);
} else {
allocation = allocateGraphicsMemory(allocationData);
}
if (allocation) {
allocation->storageInfo = allocationData.storageInfo;
allocation->setFlushL3Required(allocationData.flags.flushL3);

14
runtime/memory_manager/os_agnostic_memory_manager.cpp
View File

@@ -30,7 +30,6 @@ struct OsHandle {
};
GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) {
auto sizeAligned = alignUp(allocationData.size, MemoryConstants::pageSize);
MemoryAllocation *memoryAllocation = nullptr;
@@ -51,6 +50,19 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryWithAlignment
alignedFreeWrapper(ptr);
return nullptr;
}
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_CPU) {
//add 2MB padding in case mapPtr is not 2MB aligned
size_t reserveSize = sizeAligned + allocationData.alignment;
void *gpuPtr = reserveCpuAddressRange(reserveSize);
if (!gpuPtr) {
delete memoryAllocation;
alignedFreeWrapper(ptr);
return nullptr;
}
memoryAllocation->setReservedAddressRange(gpuPtr, reserveSize);
gpuPtr = alignUp(gpuPtr, allocationData.alignment);
memoryAllocation->setCpuPtrAndGpuAddress(ptr, reinterpret_cast<uint64_t>(gpuPtr));
}
}
counter++;
return memoryAllocation;

162
runtime/memory_manager/svm_memory_manager.cpp
View File

@@ -14,40 +14,60 @@
namespace NEO {
void SVMAllocsManager::MapBasedAllocationTracker::insert(GraphicsAllocation &ga) {
allocs.insert(std::make_pair(ga.getUnderlyingBuffer(), &ga));
void SVMAllocsManager::MapBasedAllocationTracker::insert(SvmAllocationData allocationsPair) {
allocations.insert(std::make_pair(reinterpret_cast<void *>(allocationsPair.gpuAllocation->getGpuAddress()), allocationsPair));
}
void SVMAllocsManager::MapBasedAllocationTracker::remove(GraphicsAllocation &ga) {
std::map<const void *, GraphicsAllocation *>::iterator iter;
iter = allocs.find(ga.getUnderlyingBuffer());
allocs.erase(iter);
void SVMAllocsManager::MapBasedAllocationTracker::remove(SvmAllocationData allocationsPair) {
SvmAllocationContainer::iterator iter;
iter = allocations.find(reinterpret_cast<void *>(allocationsPair.gpuAllocation->getGpuAddress()));
allocations.erase(iter);
}
GraphicsAllocation *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) {
std::map<const void *, GraphicsAllocation *>::iterator Iter, End;
GraphicsAllocation *GA;
SvmAllocationData *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) {
SvmAllocationContainer::iterator Iter, End;
SvmAllocationData *svmAllocData;
if (ptr == nullptr)
return nullptr;
End = allocs.end();
Iter = allocs.lower_bound(ptr);
End = allocations.end();
Iter = allocations.lower_bound(ptr);
if (((Iter != End) && (Iter->first != ptr)) ||
(Iter == End)) {
if (Iter == allocs.begin()) {
if (Iter == allocations.begin()) {
Iter = End;
} else {
Iter--;
}
}
if (Iter != End) {
GA = Iter->second;
if (ptr < ((char *)GA->getUnderlyingBuffer() + GA->getUnderlyingBufferSize())) {
return GA;
svmAllocData = &Iter->second;
char *charPtr = reinterpret_cast<char *>(svmAllocData->gpuAllocation->getGpuAddress());
if (ptr < (charPtr + svmAllocData->size)) {
return svmAllocData;
}
}
return nullptr;
}
void SVMAllocsManager::MapOperationsTracker::insert(SvmMapOperation mapOperation) {
operations.insert(std::make_pair(mapOperation.regionSvmPtr, mapOperation));
}
void SVMAllocsManager::MapOperationsTracker::remove(const void *regionPtr) {
SvmMapOperationsContainer::iterator iter;
iter = operations.find(regionPtr);
operations.erase(iter);
}
SvmMapOperation *SVMAllocsManager::MapOperationsTracker::get(const void *regionPtr) {
SvmMapOperationsContainer::iterator iter;
iter = operations.find(regionPtr);
if (iter == operations.end()) {
return nullptr;
}
return &iter->second;
}
SVMAllocsManager::SVMAllocsManager(MemoryManager *memoryManager) : memoryManager(memoryManager) {
}
@@ -56,7 +76,32 @@ void *SVMAllocsManager::createSVMAlloc(size_t size, cl_mem_flags flags) {
return nullptr;
std::unique_lock<std::mutex> lock(mtx);
AllocationProperties properties{true, size, GraphicsAllocation::AllocationType::SVM};
if (!memoryManager->isLocalMemorySupported()) {
return createZeroCopySvmAllocation(size, flags);
} else {
return createSvmAllocationWithDeviceStorage(size, flags);
}
}
SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) {
std::unique_lock<std::mutex> lock(mtx);
return SVMAllocs.get(ptr);
}
void SVMAllocsManager::freeSVMAlloc(void *ptr) {
SvmAllocationData *svmData = getSVMAlloc(ptr);
if (svmData) {
std::unique_lock<std::mutex> lock(mtx);
if (svmData->gpuAllocation->getAllocationType() == GraphicsAllocation::AllocationType::SVM_ZERO_COPY) {
freeZeroCopySvmAllocation(svmData);
} else {
freeSvmAllocationWithDeviceStorage(svmData);
}
}
}
void *SVMAllocsManager::createZeroCopySvmAllocation(size_t size, cl_mem_flags flags) {
AllocationProperties properties{true, size, GraphicsAllocation::AllocationType::SVM_ZERO_COPY};
MemObjHelper::fillCachePolicyInProperties(properties, flags);
GraphicsAllocation *allocation = memoryManager->allocateGraphicsMemoryWithProperties(properties);
if (!allocation) {
@@ -64,26 +109,83 @@ void *SVMAllocsManager::createSVMAlloc(size_t size, cl_mem_flags flags) {
}
allocation->setMemObjectsAllocationWithWritableFlags(!SVMAllocsManager::memFlagIsReadOnly(flags));
allocation->setCoherent(isValueSet(flags, CL_MEM_SVM_FINE_GRAIN_BUFFER));
this->SVMAllocs.insert(*allocation);
SvmAllocationData allocData;
allocData.gpuAllocation = allocation;
allocData.size = size;
this->SVMAllocs.insert(allocData);
return allocation->getUnderlyingBuffer();
}
GraphicsAllocation *SVMAllocsManager::getSVMAlloc(const void *ptr) {
std::unique_lock<std::mutex> lock(mtx);
return SVMAllocs.get(ptr);
}
void SVMAllocsManager::freeSVMAlloc(void *ptr) {
GraphicsAllocation *GA = getSVMAlloc(ptr);
if (GA) {
std::unique_lock<std::mutex> lock(mtx);
SVMAllocs.remove(*GA);
memoryManager->freeGraphicsMemory(GA);
void *SVMAllocsManager::createSvmAllocationWithDeviceStorage(size_t size, cl_mem_flags flags) {
size_t alignedSize = alignUp<size_t>(size, 2 * MemoryConstants::megaByte);
AllocationProperties cpuProperties{true, alignedSize, GraphicsAllocation::AllocationType::SVM_CPU};
cpuProperties.alignment = 2 * MemoryConstants::megaByte;
MemObjHelper::fillCachePolicyInProperties(cpuProperties, flags);
GraphicsAllocation *allocationCpu = memoryManager->allocateGraphicsMemoryWithProperties(cpuProperties);
if (!allocationCpu) {
return nullptr;
}
allocationCpu->setMemObjectsAllocationWithWritableFlags(!SVMAllocsManager::memFlagIsReadOnly(flags));
allocationCpu->setCoherent(isValueSet(flags, CL_MEM_SVM_FINE_GRAIN_BUFFER));
void *svmPtr = allocationCpu->getUnderlyingBuffer();
AllocationProperties gpuProperties{false, alignedSize, GraphicsAllocation::AllocationType::SVM_GPU};
gpuProperties.alignment = 2 * MemoryConstants::megaByte;
MemObjHelper::fillCachePolicyInProperties(gpuProperties, flags);
GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties, svmPtr);
if (!allocationGpu) {
memoryManager->freeGraphicsMemory(allocationCpu);
return nullptr;
}
allocationGpu->setMemObjectsAllocationWithWritableFlags(!SVMAllocsManager::memFlagIsReadOnly(flags));
allocationGpu->setCoherent(isValueSet(flags, CL_MEM_SVM_FINE_GRAIN_BUFFER));
SvmAllocationData allocData;
allocData.gpuAllocation = allocationGpu;
allocData.cpuAllocation = allocationCpu;
allocData.size = size;
this->SVMAllocs.insert(allocData);
return svmPtr;
}
bool SVMAllocsManager::memFlagIsReadOnly(cl_svm_mem_flags flags) {
return (flags & (CL_MEM_READ_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS)) != 0;
void SVMAllocsManager::freeZeroCopySvmAllocation(SvmAllocationData *svmData) {
GraphicsAllocation *gpuAllocation = svmData->gpuAllocation;
SVMAllocs.remove(*svmData);
memoryManager->freeGraphicsMemory(gpuAllocation);
}
void SVMAllocsManager::freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData) {
GraphicsAllocation *gpuAllocation = svmData->gpuAllocation;
GraphicsAllocation *cpuAllocation = svmData->cpuAllocation;
SVMAllocs.remove(*svmData);
memoryManager->freeGraphicsMemory(gpuAllocation);
memoryManager->freeGraphicsMemory(cpuAllocation);
}
SvmMapOperation *SVMAllocsManager::getSvmMapOperation(const void *ptr) {
std::unique_lock<std::mutex> lock(mtx);
return svmMapOperations.get(ptr);
}
void SVMAllocsManager::insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap) {
SvmMapOperation svmMapOperation;
svmMapOperation.regionSvmPtr = regionSvmPtr;
svmMapOperation.baseSvmPtr = baseSvmPtr;
svmMapOperation.offset = offset;
svmMapOperation.regionSize = regionSize;
svmMapOperation.readOnlyMap = readOnlyMap;
std::unique_lock<std::mutex> lock(mtx);
svmMapOperations.insert(svmMapOperation);
}
void SVMAllocsManager::removeSvmMapOperation(const void *regionSvmPtr) {
std::unique_lock<std::mutex> lock(mtx);
svmMapOperations.remove(regionSvmPtr);
}
} // namespace NEO

57
runtime/memory_manager/svm_memory_manager.h
View File

@@ -18,28 +18,71 @@ class GraphicsAllocation;
class CommandStreamReceiver;
class MemoryManager;
struct SvmAllocationData {
GraphicsAllocation *cpuAllocation = nullptr;
GraphicsAllocation *gpuAllocation = nullptr;
size_t size = 0;
};
struct SvmMapOperation {
void *regionSvmPtr = nullptr;
size_t regionSize = 0;
void *baseSvmPtr = nullptr;
size_t offset = 0;
bool readOnlyMap = false;
};
class SVMAllocsManager {
public:
class MapBasedAllocationTracker {
public:
void insert(GraphicsAllocation &);
void remove(GraphicsAllocation &);
GraphicsAllocation *get(const void *);
size_t getNumAllocs() const { return allocs.size(); };
using SvmAllocationContainer = std::map<const void *, SvmAllocationData>;
void insert(SvmAllocationData);
void remove(SvmAllocationData);
SvmAllocationData *get(const void *);
size_t getNumAllocs() const { return allocations.size(); };
protected:
std::map<const void *, GraphicsAllocation *> allocs;
SvmAllocationContainer allocations;
};
struct MapOperationsTracker {
using SvmMapOperationsContainer = std::map<const void *, SvmMapOperation>;
void insert(SvmMapOperation);
void remove(const void *);
SvmMapOperation *get(const void *);
size_t getNumMapOperations() const { return operations.size(); };
protected:
SvmMapOperationsContainer operations;
};
SVMAllocsManager(MemoryManager *memoryManager);
void *createSVMAlloc(size_t size, cl_mem_flags flags);
GraphicsAllocation *getSVMAlloc(const void *ptr);
SvmAllocationData *getSVMAlloc(const void *ptr);
void freeSVMAlloc(void *ptr);
size_t getNumAllocs() const { return SVMAllocs.getNumAllocs(); }
static bool memFlagIsReadOnly(cl_svm_mem_flags flags);
static bool memFlagIsReadOnly(cl_svm_mem_flags flags) {
return (flags & (CL_MEM_READ_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS)) != 0;
}
static bool mapFlagIsReadOnly(cl_map_flags mapFlags) {
return !((mapFlags & (CL_MAP_WRITE | CL_MAP_WRITE_INVALIDATE_REGION)) != 0);
}
void insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap);
void removeSvmMapOperation(const void *regionSvmPtr);
SvmMapOperation *getSvmMapOperation(const void *regionPtr);
protected:
void *createZeroCopySvmAllocation(size_t size, cl_mem_flags flags);
void *createSvmAllocationWithDeviceStorage(size_t size, cl_mem_flags flags);
void freeZeroCopySvmAllocation(SvmAllocationData *svmData);
void freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData);
MapBasedAllocationTracker SVMAllocs;
MapOperationsTracker svmMapOperations;
MemoryManager *memoryManager;
std::mutex mtx;
};

8
runtime/os_interface/debug_settings_manager.cpp
View File

@@ -349,8 +349,12 @@ const char *DebugSettingsManager<DebugLevel>::getAllocationTypeString(GraphicsAl
return "SURFACE_STATE_HEAP";
case GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY:
return "SHARED_RESOURCE_COPY";
case GraphicsAllocation::AllocationType::SVM:
return "SVM";
case GraphicsAllocation::AllocationType::SVM_ZERO_COPY:
return "SVM_ZERO_COPY";
case GraphicsAllocation::AllocationType::SVM_CPU:
return "SVM_CPU";
case GraphicsAllocation::AllocationType::SVM_GPU:
return "SVM_GPU";
case GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR:
return "EXTERNAL_HOST_PTR";
case GraphicsAllocation::AllocationType::UNDECIDED:

15
runtime/os_interface/windows/wddm_memory_manager.cpp
View File

@@ -103,8 +103,21 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryWithAlignment(const
gmm = new Gmm(pSysMem, sizeAligned, allocationData.flags.uncacheable);
wddmAllocation->setDefaultGmm(gmm);
void *mapPtr = wddmAllocation->getAlignedCpuPtr();
if (allocationData.type == GraphicsAllocation::AllocationType::SVM_CPU) {
//add 2MB padding in case mapPtr is not 2MB aligned
size_t reserveSizeAligned = sizeAligned + allocationData.alignment;
bool ret = wddm->reserveValidAddressRange(reserveSizeAligned, mapPtr);
if (!ret) {
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;
}
wddmAllocation->setReservedAddressRange(mapPtr, reserveSizeAligned);
mapPtr = alignUp(mapPtr, newAlignment);
}
if (!createWddmAllocation(wddmAllocation.get(), wddmAllocation->getAlignedCpuPtr())) {
if (!createWddmAllocation(wddmAllocation.get(), mapPtr)) {
delete gmm;
freeSystemMemory(pSysMem);
return nullptr;

4
unit_tests/api/cl_enqueue_svm_migrate_mem_tests.cpp
View File

@@ -97,7 +97,9 @@ TEST_F(clEnqueueSVMMigrateMemTests, GivenNonZeroSizeIsNotContainedWithinAllocati
void *ptrSvm = clSVMAlloc(pContext, CL_MEM_READ_WRITE, 256, 4);
ASSERT_NE(nullptr, ptrSvm);
auto svmAlloc = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSvm);
auto svmData = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSvm);
ASSERT_NE(nullptr, svmData);
auto svmAlloc = svmData->gpuAllocation;
EXPECT_NE(nullptr, svmAlloc);
size_t allocSize = svmAlloc->getUnderlyingBufferSize();

4
unit_tests/api/cl_set_kernel_arg_svm_pointer_tests.inl
View File

@@ -179,7 +179,9 @@ TEST_F(clSetKernelArgSVMPointer_, SetKernelArgSVMPointerWithOffset_invalidArgVal
const DeviceInfo &devInfo = pDevice->getDeviceInfo();
if (devInfo.svmCapabilities != 0) {
void *ptrSvm = clSVMAlloc(pContext, CL_MEM_READ_WRITE, 256, 4);
auto svmAlloc = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSvm);
auto svmData = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSvm);
ASSERT_NE(nullptr, svmData);
auto svmAlloc = svmData->gpuAllocation;
EXPECT_NE(nullptr, svmAlloc);
size_t offset = svmAlloc->getUnderlyingBufferSize() + 1;

8
unit_tests/command_queue/enqueue_svm_mem_copy_tests.cpp
View File

@@ -29,9 +29,13 @@ struct EnqueueSvmMemCopyTest : public DeviceFixture,
ASSERT_NE(nullptr, srcSvmPtr);
dstSvmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, 0);
ASSERT_NE(nullptr, dstSvmPtr);
srcSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(srcSvmPtr);
auto srcSvmData = context->getSVMAllocsManager()->getSVMAlloc(srcSvmPtr);
ASSERT_NE(nullptr, srcSvmData);
srcSvmAlloc = srcSvmData->gpuAllocation;
ASSERT_NE(nullptr, srcSvmAlloc);
dstSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(dstSvmPtr);
auto dstSvmData = context->getSVMAllocsManager()->getSVMAlloc(dstSvmPtr);
ASSERT_NE(nullptr, dstSvmData);
dstSvmAlloc = dstSvmData->gpuAllocation;
ASSERT_NE(nullptr, dstSvmAlloc);
}

4
unit_tests/command_queue/enqueue_svm_mem_fill_tests.cpp
View File

@@ -29,7 +29,9 @@ struct EnqueueSvmMemFillTest : public DeviceFixture,
ASSERT_TRUE((0 < patternSize) && (patternSize <= 128));
svmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, CL_MEM_SVM_FINE_GRAIN_BUFFER);
ASSERT_NE(nullptr, svmPtr);
svmAlloc = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(svmPtr);
ASSERT_NE(nullptr, svmData);
svmAlloc = svmData->gpuAllocation;
ASSERT_NE(nullptr, svmAlloc);
}

283
unit_tests/command_queue/enqueue_svm_tests.cpp
View File

@@ -11,14 +11,18 @@
#include "runtime/memory_manager/allocations_list.h"
#include "runtime/memory_manager/surface.h"
#include "runtime/memory_manager/svm_memory_manager.h"
#include "runtime/os_interface/debug_settings_manager.h"
#include "test.h"
#include "unit_tests/command_queue/command_queue_fixture.h"
#include "unit_tests/command_queue/enqueue_map_buffer_fixture.h"
#include "unit_tests/fixtures/buffer_fixture.h"
#include "unit_tests/fixtures/device_fixture.h"
#include "unit_tests/helpers/debug_manager_state_restore.h"
#include "unit_tests/helpers/hw_parse.h"
#include "unit_tests/mocks/mock_command_queue.h"
#include "unit_tests/mocks/mock_context.h"
#include "unit_tests/mocks/mock_kernel.h"
#include "unit_tests/mocks/mock_svm_manager.h"
#include "unit_tests/utilities/base_object_utils.h"
using namespace NEO;
@@ -454,7 +458,9 @@ TEST_F(EnqueueSvmTest, givenEnqueueSVMMemFillWhenPatternAllocationIsObtainedThen
}
TEST_F(EnqueueSvmTest, enqueueTaskWithKernelExecInfo_success) {
GraphicsAllocation *pSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *pSvmAlloc = svmData->gpuAllocation;
EXPECT_NE(nullptr, ptrSVM);
std::unique_ptr<Program> program(Program::create("FillBufferBytes", context, *pDevice, true, &retVal));
@@ -481,7 +487,9 @@ TEST_F(EnqueueSvmTest, enqueueTaskWithKernelExecInfo_success) {
}
TEST_F(EnqueueSvmTest, givenEnqueueTaskBlockedOnUserEventWhenItIsEnqueuedThenSurfacesAreMadeResident) {
GraphicsAllocation *pSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *pSvmAlloc = svmData->gpuAllocation;
EXPECT_NE(nullptr, ptrSVM);
auto program = clUniquePtr(Program::create("FillBufferBytes", context, *pDevice, true, &retVal));
@@ -528,7 +536,9 @@ TEST_F(EnqueueSvmTest, concurentMapAccess) {
auto allocSvm = [&](uint32_t from, uint32_t to) {
for (uint32_t i = from; i <= to; i++) {
svmPtrs[i] = context->getSVMAllocsManager()->createSVMAlloc(1, 0);
auto ga = context->getSVMAllocsManager()->getSVMAlloc(svmPtrs[i]);
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(svmPtrs[i]);
ASSERT_NE(nullptr, svmData);
auto ga = svmData->gpuAllocation;
EXPECT_NE(nullptr, ga);
EXPECT_EQ(ga->getUnderlyingBuffer(), svmPtrs[i]);
}
@@ -582,3 +592,270 @@ TEST_F(EnqueueSvmTest, enqueueSVMMigrateMem_Success) {
);
EXPECT_EQ(CL_SUCCESS, retVal);
}
struct EnqueueSvmTestLocalMemory : public DeviceFixture,
public ::testing::Test {
void SetUp() override {
dbgRestore = std::make_unique<DebugManagerStateRestore>();
DebugManager.flags.EnableLocalMemory.set(1);
DeviceFixture::SetUp();
context = std::make_unique<MockContext>(pDevice, true);
size = 256;
svmPtr = context->getSVMAllocsManager()->createSVMAlloc(size, 0);
ASSERT_NE(nullptr, svmPtr);
mockSvmManager = reinterpret_cast<MockSVMAllocsManager *>(context->getSVMAllocsManager());
}
void TearDown() override {
context->getSVMAllocsManager()->freeSVMAlloc(svmPtr);
context.reset(nullptr);
DeviceFixture::TearDown();
}
cl_int retVal = CL_SUCCESS;
void *svmPtr = nullptr;
size_t size;
MockSVMAllocsManager *mockSvmManager;
std::unique_ptr<DebugManagerStateRestore> dbgRestore;
std::unique_ptr<MockContext> context;
HardwareParse hwParse;
};
HWTEST_F(EnqueueSvmTestLocalMemory, givenEnabledLocalMemoryWhenEnqeueMapValidSvmPtrThenExpectSingleWalker) {
using WALKER_TYPE = typename FamilyType::WALKER_TYPE;
MockCommandQueueHw<FamilyType> queue(context.get(), pDevice, nullptr);
LinearStream &stream = queue.getCS(0x1000);
cl_event event = nullptr;
uintptr_t offset = 64;
void *regionSvmPtr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(svmPtr) + offset);
size_t regionSize = 64;
retVal = queue.enqueueSVMMap(
CL_FALSE,
CL_MAP_READ,
regionSvmPtr,
regionSize,
0,
nullptr,
&event);
EXPECT_EQ(CL_SUCCESS, retVal);
auto svmMap = mockSvmManager->svmMapOperations.get(regionSvmPtr);
ASSERT_NE(nullptr, svmMap);
EXPECT_EQ(regionSvmPtr, svmMap->regionSvmPtr);
EXPECT_EQ(svmPtr, svmMap->baseSvmPtr);
EXPECT_EQ(regionSize, svmMap->regionSize);
EXPECT_EQ(offset, svmMap->offset);
EXPECT_TRUE(svmMap->readOnlyMap);
queue.flush();
hwParse.parseCommands<FamilyType>(stream);
auto walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(1u, walkerCount);
constexpr cl_command_type expectedCmd = CL_COMMAND_SVM_MAP;
cl_command_type actualCmd = castToObjectOrAbort<Event>(event)->getCommandType();
EXPECT_EQ(expectedCmd, actualCmd);
clReleaseEvent(event);
}
HWTEST_F(EnqueueSvmTestLocalMemory, givenEnabledLocalMemoryWhenEnqeueMapSvmPtrTwiceThenExpectSingleWalker) {
using WALKER_TYPE = typename FamilyType::WALKER_TYPE;
MockCommandQueueHw<FamilyType> queue(context.get(), pDevice, nullptr);
LinearStream &stream = queue.getCS(0x1000);
uintptr_t offset = 64;
void *regionSvmPtr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(svmPtr) + offset);
size_t regionSize = 64;
retVal = queue.enqueueSVMMap(
CL_FALSE,
CL_MAP_WRITE,
regionSvmPtr,
regionSize,
0,
nullptr,
nullptr);
EXPECT_EQ(CL_SUCCESS, retVal);
auto svmMap = mockSvmManager->svmMapOperations.get(regionSvmPtr);
ASSERT_NE(nullptr, svmMap);
EXPECT_EQ(regionSvmPtr, svmMap->regionSvmPtr);
EXPECT_EQ(svmPtr, svmMap->baseSvmPtr);
EXPECT_EQ(regionSize, svmMap->regionSize);
EXPECT_EQ(offset, svmMap->offset);
EXPECT_FALSE(svmMap->readOnlyMap);
EXPECT_EQ(1u, mockSvmManager->svmMapOperations.getNumMapOperations());
cl_event event = nullptr;
retVal = queue.enqueueSVMMap(
CL_FALSE,
CL_MAP_WRITE,
regionSvmPtr,
regionSize,
0,
nullptr,
&event);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(1u, mockSvmManager->svmMapOperations.getNumMapOperations());
queue.flush();
hwParse.parseCommands<FamilyType>(stream);
auto walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(1u, walkerCount);
constexpr cl_command_type expectedCmd = CL_COMMAND_SVM_MAP;
cl_command_type actualCmd = castToObjectOrAbort<Event>(event)->getCommandType();
EXPECT_EQ(expectedCmd, actualCmd);
clReleaseEvent(event);
}
HWTEST_F(EnqueueSvmTestLocalMemory, givenEnabledLocalMemoryWhenNoMappedSvmPtrThenExpectNoUnmapCopyKernel) {
using WALKER_TYPE = typename FamilyType::WALKER_TYPE;
MockCommandQueueHw<FamilyType> queue(context.get(), pDevice, nullptr);
LinearStream &stream = queue.getCS(0x1000);
cl_event event = nullptr;
retVal = queue.enqueueSVMUnmap(
svmPtr,
0,
nullptr,
&event);
EXPECT_EQ(CL_SUCCESS, retVal);
queue.flush();
hwParse.parseCommands<FamilyType>(stream);
auto walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(0u, walkerCount);
constexpr cl_command_type expectedCmd = CL_COMMAND_SVM_UNMAP;
cl_command_type actualCmd = castToObjectOrAbort<Event>(event)->getCommandType();
EXPECT_EQ(expectedCmd, actualCmd);
clReleaseEvent(event);
}
HWTEST_F(EnqueueSvmTestLocalMemory, givenEnabledLocalMemoryWhenMappedSvmRegionIsReadOnlyThenExpectNoUnmapCopyKernel) {
using WALKER_TYPE = typename FamilyType::WALKER_TYPE;
MockCommandQueueHw<FamilyType> queue(context.get(), pDevice, nullptr);
LinearStream &stream = queue.getCS(0x1000);
retVal = queue.enqueueSVMMap(
CL_FALSE,
CL_MAP_READ,
svmPtr,
size,
0,
nullptr,
nullptr);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(1u, mockSvmManager->svmMapOperations.getNumMapOperations());
auto svmMap = mockSvmManager->svmMapOperations.get(svmPtr);
ASSERT_NE(nullptr, svmMap);
queue.flush();
size_t offset = stream.getUsed();
hwParse.parseCommands<FamilyType>(stream);
auto walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(1u, walkerCount);
hwParse.TearDown();
cl_event event = nullptr;
retVal = queue.enqueueSVMUnmap(
svmPtr,
0,
nullptr,
&event);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(0u, mockSvmManager->svmMapOperations.getNumMapOperations());
queue.flush();
hwParse.parseCommands<FamilyType>(stream, offset);
walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(0u, walkerCount);
constexpr cl_command_type expectedCmd = CL_COMMAND_SVM_UNMAP;
cl_command_type actualCmd = castToObjectOrAbort<Event>(event)->getCommandType();
EXPECT_EQ(expectedCmd, actualCmd);
clReleaseEvent(event);
}
HWTEST_F(EnqueueSvmTestLocalMemory, givenEnabledLocalMemoryWhenMappedSvmRegionIsWritableThenExpectMapAndUnmapCopyKernel) {
using WALKER_TYPE = typename FamilyType::WALKER_TYPE;
MockCommandQueueHw<FamilyType> queue(context.get(), pDevice, nullptr);
LinearStream &stream = queue.getCS(0x1000);
cl_event eventMap = nullptr;
retVal = queue.enqueueSVMMap(
CL_FALSE,
CL_MAP_WRITE,
svmPtr,
size,
0,
nullptr,
&eventMap);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(1u, mockSvmManager->svmMapOperations.getNumMapOperations());
auto svmMap = mockSvmManager->svmMapOperations.get(svmPtr);
ASSERT_NE(nullptr, svmMap);
cl_event eventUnmap = nullptr;
retVal = queue.enqueueSVMUnmap(
svmPtr,
0,
nullptr,
&eventUnmap);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(0u, mockSvmManager->svmMapOperations.getNumMapOperations());
queue.flush();
hwParse.parseCommands<FamilyType>(stream);
auto walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(2u, walkerCount);
constexpr cl_command_type expectedMapCmd = CL_COMMAND_SVM_MAP;
cl_command_type actualMapCmd = castToObjectOrAbort<Event>(eventMap)->getCommandType();
EXPECT_EQ(expectedMapCmd, actualMapCmd);
constexpr cl_command_type expectedUnmapCmd = CL_COMMAND_SVM_UNMAP;
cl_command_type actualUnmapCmd = castToObjectOrAbort<Event>(eventUnmap)->getCommandType();
EXPECT_EQ(expectedUnmapCmd, actualUnmapCmd);
clReleaseEvent(eventMap);
clReleaseEvent(eventUnmap);
}
HWTEST_F(EnqueueSvmTestLocalMemory, givenEnabledLocalMemoryWhenMappedSvmRegionAndNoEventIsUsedIsWritableThenExpectMapAndUnmapCopyKernelAnNo) {
using WALKER_TYPE = typename FamilyType::WALKER_TYPE;
MockCommandQueueHw<FamilyType> queue(context.get(), pDevice, nullptr);
LinearStream &stream = queue.getCS(0x1000);
retVal = queue.enqueueSVMMap(
CL_FALSE,
CL_MAP_WRITE,
svmPtr,
size,
0,
nullptr,
nullptr);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(1u, mockSvmManager->svmMapOperations.getNumMapOperations());
auto svmMap = mockSvmManager->svmMapOperations.get(svmPtr);
ASSERT_NE(nullptr, svmMap);
retVal = queue.enqueueSVMUnmap(
svmPtr,
0,
nullptr,
nullptr);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(0u, mockSvmManager->svmMapOperations.getNumMapOperations());
queue.flush();
hwParse.parseCommands<FamilyType>(stream);
auto walkerCount = hwParse.getCommandCount<WALKER_TYPE>();
EXPECT_EQ(2u, walkerCount);
}

4
unit_tests/kernel/clone_kernel_tests.cpp
View File

@@ -512,7 +512,9 @@ TEST_F(CloneKernelTest, cloneKernelWithExecInfo) {
void *ptrSVM = pContext->getSVMAllocsManager()->createSVMAlloc(256, 0);
ASSERT_NE(nullptr, ptrSVM);
GraphicsAllocation *pSvmAlloc = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
auto svmData = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *pSvmAlloc = svmData->gpuAllocation;
ASSERT_NE(nullptr, pSvmAlloc);
pSourceKernel->setKernelExecInfo(pSvmAlloc);

2
unit_tests/kernel/kernel_arg_buffer_tests.cpp
View File

@@ -169,7 +169,7 @@ TEST_F(KernelArgBufferTest, given32BitDeviceWhenArgPassedIsNullThenOnly4BytesAre
EXPECT_NE(expValue, *pKernelArg64bit);
}
TEST_F(KernelArgBufferTest, givenWritebleBufferWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) {
TEST_F(KernelArgBufferTest, givenWritableBufferWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) {
auto buffer = std::make_unique<MockBuffer>();
buffer->mockGfxAllocation.setMemObjectsAllocationWithWritableFlags(true);
buffer->mockGfxAllocation.setFlushL3Required(false);

2
unit_tests/kernel/kernel_arg_svm_tests.cpp
View File

@@ -414,7 +414,7 @@ HWTEST_TYPED_TEST(KernelArgSvmTestTyped, GivenBufferKernelArgWhenBufferOffsetIsN
alignedFree(svmPtr);
}
TEST_F(KernelArgSvmTest, givenWritebleSvmAllocationWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) {
TEST_F(KernelArgSvmTest, givenWritableSvmAllocationWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) {
size_t svmSize = 4096;
void *svmPtr = alignedMalloc(svmSize, MemoryConstants::pageSize);
MockGraphicsAllocation svmAlloc(svmPtr, svmSize);

2
unit_tests/kernel/kernel_image_arg_tests.cpp
View File

@@ -261,7 +261,7 @@ TEST_F(KernelImageArgTest, givenKernelWithSharedImageWhenSetArgCalledThenUsingSh
EXPECT_TRUE(pKernel->isUsingSharedObjArgs());
}
TEST_F(KernelImageArgTest, givenWritebleImageWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) {
TEST_F(KernelImageArgTest, givenWritableImageWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) {
MockImageBase image;
image.graphicsAllocation->setMemObjectsAllocationWithWritableFlags(true);
image.graphicsAllocation->setFlushL3Required(false);

4
unit_tests/mem_obj/buffer_set_arg_tests.cpp
View File

@@ -264,7 +264,9 @@ TEST_F(BufferSetArgTest, clSetKernelArgSVMPointer) {
void *ptrSVM = pContext->getSVMAllocsManager()->createSVMAlloc(256, 0);
EXPECT_NE(nullptr, ptrSVM);
GraphicsAllocation *pSvmAlloc = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
auto svmData = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSVM);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *pSvmAlloc = svmData->gpuAllocation;
EXPECT_NE(nullptr, pSvmAlloc);
retVal = pKernel->setArgSvmAlloc(

14
unit_tests/mem_obj/buffer_tests.cpp
View File

@@ -562,12 +562,12 @@ TEST_F(RenderCompressedBuffersTests, givenDebugVariableSetWhenHwFlagIsNotSetThen
TEST_F(RenderCompressedBuffersTests, givenSvmAllocationWhenCreatingBufferThenForceDisableCompression) {
localHwInfo.capabilityTable.ftrRenderCompressedBuffers = true;
auto svmAlloc = context->getSVMAllocsManager()->createSVMAlloc(sizeof(uint32_t), 0);
auto svmPtr = context->getSVMAllocsManager()->createSVMAlloc(sizeof(uint32_t), 0);
auto expectedAllocationType = context->getSVMAllocsManager()->getSVMAlloc(svmPtr)->gpuAllocation->getAllocationType();
buffer.reset(Buffer::create(context.get(), CL_MEM_USE_HOST_PTR, sizeof(uint32_t), svmPtr, retVal));
EXPECT_EQ(expectedAllocationType, buffer->getGraphicsAllocation()->getAllocationType());
buffer.reset(Buffer::create(context.get(), CL_MEM_USE_HOST_PTR, sizeof(uint32_t), svmAlloc, retVal));
EXPECT_EQ(buffer->getGraphicsAllocation()->getAllocationType(), GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY);
context->getSVMAllocsManager()->freeSVMAlloc(svmAlloc);
context->getSVMAllocsManager()->freeSVMAlloc(svmPtr);
}
struct RenderCompressedBuffersCopyHostMemoryTests : public RenderCompressedBuffersTests {
@@ -961,7 +961,9 @@ TEST_P(ValidHostPtr, SvmHostPtr) {
auto bufferSvm = Buffer::create(context.get(), CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, 64, ptr, retVal);
EXPECT_NE(nullptr, bufferSvm);
EXPECT_TRUE(bufferSvm->isMemObjWithHostPtrSVM());
EXPECT_EQ(context->getSVMAllocsManager()->getSVMAlloc(ptr), bufferSvm->getGraphicsAllocation());
auto svmData = context->getSVMAllocsManager()->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
EXPECT_EQ(svmData->gpuAllocation, bufferSvm->getGraphicsAllocation());
EXPECT_EQ(CL_SUCCESS, retVal);
context->getSVMAllocsManager()->freeSVMAlloc(ptr);

62
unit_tests/mem_obj/get_mem_object_info_tests.cpp
View File

@@ -8,9 +8,11 @@
#include "runtime/helpers/aligned_memory.h"
#include "runtime/helpers/options.h"
#include "runtime/helpers/ptr_math.h"
#include "runtime/os_interface/debug_settings_manager.h"
#include "unit_tests/fixtures/buffer_fixture.h"
#include "unit_tests/fixtures/device_fixture.h"
#include "unit_tests/fixtures/platform_fixture.h"
#include "unit_tests/helpers/debug_manager_state_restore.h"
#include "unit_tests/mocks/mock_context.h"
#include "gtest/gtest.h"
@@ -32,8 +34,8 @@ class GetMemObjectInfo : public ::testing::Test, public PlatformFixture, public
void TearDown() override {
delete BufferDefaults::context;
PlatformFixture::TearDown();
DeviceFixture::TearDown();
PlatformFixture::TearDown();
}
};
@@ -308,3 +310,61 @@ TEST_F(GetMemObjectInfo, MEM_REFERENCE_COUNT) {
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(sizeof(refCount), sizeReturned);
}
class GetMemObjectInfoLocalMemory : public GetMemObjectInfo {
using GetMemObjectInfo::SetUp;
public:
void SetUp() override {
dbgRestore = std::make_unique<DebugManagerStateRestore>();
DebugManager.flags.EnableLocalMemory.set(1);
GetMemObjectInfo::SetUp();
delete BufferDefaults::context;
BufferDefaults::context = new MockContext(pDevice, true);
}
std::unique_ptr<DebugManagerStateRestore> dbgRestore;
};
TEST_F(GetMemObjectInfoLocalMemory, givenLocalMemoryEnabledWhenNoZeroCopySvmAllocationUsedThenBufferAllocationInheritsZeroCopyFlag) {
const DeviceInfo &devInfo = pDevice->getDeviceInfo();
if (devInfo.svmCapabilities != 0) {
auto hostPtr = clSVMAlloc(BufferDefaults::context, CL_MEM_READ_WRITE, BufferUseHostPtr<>::sizeInBytes, 64);
ASSERT_NE(nullptr, hostPtr);
cl_int retVal;
auto buffer = Buffer::create(
BufferDefaults::context,
CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR,
BufferUseHostPtr<>::sizeInBytes,
hostPtr,
retVal);
size_t sizeReturned = 0;
cl_bool usesSVMPointer = false;
retVal = buffer->getMemObjectInfo(
CL_MEM_USES_SVM_POINTER,
0,
nullptr,
&sizeReturned);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(sizeof(usesSVMPointer), sizeReturned);
retVal = buffer->getMemObjectInfo(
CL_MEM_USES_SVM_POINTER,
sizeof(usesSVMPointer),
&usesSVMPointer,
nullptr);
EXPECT_EQ(CL_SUCCESS, retVal);
EXPECT_EQ(static_cast<cl_bool>(CL_TRUE), usesSVMPointer);
EXPECT_TRUE(buffer->isMemObjWithHostPtrSVM());
EXPECT_FALSE(buffer->isMemObjZeroCopy());
delete buffer;
clSVMFree(BufferDefaults::context, hostPtr);
}
}

36
unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp
View File

@@ -44,3 +44,39 @@ TEST(MemoryManagerTest, givenImageOrSharedResourceCopyWhenGraphicsAllocationInDe
EXPECT_EQ(MemoryManager::AllocationStatus::RetryInNonDevicePool, status);
}
}
TEST(MemoryManagerTest, givenSvmGpuAllocationTypeWhenAllocationSystemMemoryFailsThenReturnNull) {
MockExecutionEnvironment executionEnvironment(*platformDevices);
MockMemoryManager memoryManager(false, false, executionEnvironment);
MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error;
AllocationData allocData;
allocData.allFlags = 0;
allocData.size = MemoryConstants::pageSize;
allocData.type = GraphicsAllocation::AllocationType::SVM_GPU;
allocData.hostPtr = reinterpret_cast<void *>(0x1000);
memoryManager.failAllocateSystemMemory = true;
auto allocation = memoryManager.allocateGraphicsMemoryInDevicePool(allocData, status);
EXPECT_EQ(nullptr, allocation);
EXPECT_EQ(MemoryManager::AllocationStatus::Error, status);
}
TEST(MemoryManagerTest, givenSvmGpuAllocationTypeWhenAllocationSucceedThenReturnGpuAddressAsHostPtr) {
MockExecutionEnvironment executionEnvironment(*platformDevices);
MockMemoryManager memoryManager(false, false, executionEnvironment);
MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Error;
AllocationData allocData;
allocData.size = MemoryConstants::pageSize;
allocData.type = GraphicsAllocation::AllocationType::SVM_GPU;
allocData.hostPtr = reinterpret_cast<void *>(0x1000);
auto allocation = memoryManager.allocateGraphicsMemoryInDevicePool(allocData, status);
ASSERT_NE(nullptr, allocation);
EXPECT_EQ(MemoryManager::AllocationStatus::Success, status);
EXPECT_EQ(reinterpret_cast<uint64_t>(allocData.hostPtr), allocation->getGpuAddress());
EXPECT_NE(reinterpret_cast<uint64_t>(allocation->getUnderlyingBuffer()), allocation->getGpuAddress());
memoryManager.freeGraphicsMemory(allocation);
}

4
unit_tests/memory_manager/memory_manager_allocate_in_preferred_pool_tests.inl
View File

@@ -347,14 +347,14 @@ TEST(MemoryManagerTest, givenMemoryManagerWhenBufferTypeIsPassedAndAllocateInDev
TEST(MemoryManagerTest, givenSvmAllocationTypeWhenGetAllocationDataIsCalledThenZeroCopyIsRequested) {
AllocationData allocData;
MockMemoryManager::getAllocationData(allocData, {1, GraphicsAllocation::AllocationType::SVM}, {}, nullptr);
MockMemoryManager::getAllocationData(allocData, {1, GraphicsAllocation::AllocationType::SVM_ZERO_COPY}, {}, nullptr);
EXPECT_TRUE(allocData.flags.mustBeZeroCopy);
EXPECT_TRUE(allocData.flags.allocateMemory);
}
TEST(MemoryManagerTest, givenSvmAllocationTypeWhenGetAllocationDataIsCalledThen64kbPagesAreAllowedAnd32BitAllocationIsDisallowed) {
AllocationData allocData;
MockMemoryManager::getAllocationData(allocData, {1, GraphicsAllocation::AllocationType::SVM}, {}, nullptr);
MockMemoryManager::getAllocationData(allocData, {1, GraphicsAllocation::AllocationType::SVM_ZERO_COPY}, {}, nullptr);
EXPECT_TRUE(allocData.flags.allow64kbPages);
EXPECT_FALSE(allocData.flags.allow32Bit);
}

8
unit_tests/memory_manager/memory_manager_tests.cpp
View File

@@ -321,6 +321,7 @@ TEST_F(MemoryAllocatorTest, GivenPointerAndSizeWhenAskedToCreateGrahicsAllocatio
allocationData.hostPtr = ptr;
auto allocation = std::unique_ptr<GraphicsAllocation>(memoryManager->createGraphicsAllocation(handleStorage, allocationData));
EXPECT_EQ(MemoryPool::System4KBPages, allocation->getMemoryPool());
EXPECT_EQ(ptr, allocation->getUnderlyingBuffer());
EXPECT_EQ(size, allocation->getUnderlyingBufferSize());
@@ -773,7 +774,7 @@ TEST(OsAgnosticMemoryManager, givenMemoryManagerWhenAllocate32BitGraphicsMemoryW
TEST(OsAgnosticMemoryManager, givenMemoryManagerWith64KBPagesEnabledWhenAllocateGraphicsMemoryThenMemoryPoolIsSystem64KBPages) {
MockExecutionEnvironment executionEnvironment(*platformDevices);
MemoryManagerCreate<OsAgnosticMemoryManager> memoryManager(true, false, executionEnvironment);
auto svmAllocation = memoryManager.allocateGraphicsMemoryWithProperties({MemoryConstants::pageSize, GraphicsAllocation::AllocationType::SVM});
auto svmAllocation = memoryManager.allocateGraphicsMemoryWithProperties({MemoryConstants::pageSize, GraphicsAllocation::AllocationType::SVM_ZERO_COPY});
EXPECT_NE(nullptr, svmAllocation);
EXPECT_EQ(MemoryPool::System64KBPages, svmAllocation->getMemoryPool());
memoryManager.freeGraphicsMemory(svmAllocation);
@@ -782,7 +783,7 @@ TEST(OsAgnosticMemoryManager, givenMemoryManagerWith64KBPagesEnabledWhenAllocate
TEST(OsAgnosticMemoryManager, givenMemoryManagerWith64KBPagesDisabledWhenAllocateGraphicsMemoryThen4KBGraphicsAllocationIsReturned) {
MockExecutionEnvironment executionEnvironment(*platformDevices);
MemoryManagerCreate<OsAgnosticMemoryManager> memoryManager(false, false, executionEnvironment);
auto svmAllocation = memoryManager.allocateGraphicsMemoryWithProperties({MemoryConstants::pageSize, GraphicsAllocation::AllocationType::SVM});
auto svmAllocation = memoryManager.allocateGraphicsMemoryWithProperties({MemoryConstants::pageSize, GraphicsAllocation::AllocationType::SVM_ZERO_COPY});
EXPECT_EQ(MemoryPool::System4KBPages, svmAllocation->getMemoryPool());
memoryManager.freeGraphicsMemory(svmAllocation);
}
@@ -1664,7 +1665,8 @@ TEST(MemoryManagerTest, givenAllocationTypesThatMayNeedL3FlushWhenCallingGetAllo
GraphicsAllocation::AllocationType::GLOBAL_SURFACE, GraphicsAllocation::AllocationType::IMAGE,
GraphicsAllocation::AllocationType::PIPE, GraphicsAllocation::AllocationType::SHARED_IMAGE,
GraphicsAllocation::AllocationType::SHARED_BUFFER, GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY,
GraphicsAllocation::AllocationType::SVM, GraphicsAllocation::AllocationType::UNDECIDED};
GraphicsAllocation::AllocationType::SVM_ZERO_COPY, GraphicsAllocation::AllocationType::UNDECIDED,
GraphicsAllocation::AllocationType::SVM_GPU, GraphicsAllocation::AllocationType::SVM_CPU};
for (auto allocationType : allocationTypesThatMayNeedL3Flush) {
properties.allocationType = allocationType;

179
unit_tests/memory_manager/svm_memory_manager_tests.cpp
View File

@@ -5,6 +5,7 @@
*
*/
#include "test.h"
#include "unit_tests/mocks/mock_execution_environment.h"
#include "unit_tests/mocks/mock_memory_manager.h"
#include "unit_tests/mocks/mock_svm_manager.h"
@@ -13,100 +14,124 @@
using namespace NEO;
struct SVMMemoryAllocatorTest : ::testing::Test {
SVMMemoryAllocatorTest() : executionEnvironment(*platformDevices), memoryManager(false, false, executionEnvironment), svmManager(&memoryManager) {}
template <bool enableLocalMemory>
struct SVMMemoryAllocatorFixture {
SVMMemoryAllocatorFixture() : executionEnvironment(*platformDevices) {}
virtual void SetUp() {
memoryManager = std::make_unique<MockMemoryManager>(false, enableLocalMemory, executionEnvironment);
svmManager = std::make_unique<MockSVMAllocsManager>(memoryManager.get());
}
virtual void TearDown() {
}
MockExecutionEnvironment executionEnvironment;
MockMemoryManager memoryManager;
MockSVMAllocsManager svmManager;
std::unique_ptr<MockMemoryManager> memoryManager;
std::unique_ptr<MockSVMAllocsManager> svmManager;
};
TEST_F(SVMMemoryAllocatorTest, whenCreateZeroSizedSVMAllocationThenReturnNullptr) {
auto ptr = svmManager.createSVMAlloc(0, 0);
using SVMMemoryAllocatorTest = Test<SVMMemoryAllocatorFixture<false>>;
EXPECT_EQ(0u, svmManager.SVMAllocs.getNumAllocs());
using SVMLocalMemoryAllocatorTest = Test<SVMMemoryAllocatorFixture<true>>;
TEST_F(SVMMemoryAllocatorTest, whenCreateZeroSizedSVMAllocationThenReturnNullptr) {
auto ptr = svmManager->createSVMAlloc(0, 0);
EXPECT_EQ(0u, svmManager->SVMAllocs.getNumAllocs());
EXPECT_EQ(ptr, nullptr);
}
TEST_F(SVMMemoryAllocatorTest, whenSVMAllocationIsFreedThenCannotBeGotAgain) {
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_NE(nullptr, ptr);
EXPECT_NE(nullptr, svmManager.getSVMAlloc(ptr));
EXPECT_NE(nullptr, svmManager.getSVMAlloc(ptr));
EXPECT_EQ(1u, svmManager.SVMAllocs.getNumAllocs());
EXPECT_FALSE(svmManager.getSVMAlloc(ptr)->isCoherent());
auto svmData = svmManager->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
EXPECT_NE(nullptr, svmData->gpuAllocation);
svmData = svmManager->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
EXPECT_NE(nullptr, svmData->gpuAllocation);
EXPECT_EQ(1u, svmManager->SVMAllocs.getNumAllocs());
auto svmAllocation = svmManager->getSVMAlloc(ptr)->gpuAllocation;
EXPECT_FALSE(svmAllocation->isCoherent());
svmManager.freeSVMAlloc(ptr);
EXPECT_EQ(nullptr, svmManager.getSVMAlloc(ptr));
EXPECT_EQ(0u, svmManager.SVMAllocs.getNumAllocs());
svmManager->freeSVMAlloc(ptr);
EXPECT_EQ(nullptr, svmManager->getSVMAlloc(ptr));
EXPECT_EQ(0u, svmManager->SVMAllocs.getNumAllocs());
}
TEST_F(SVMMemoryAllocatorTest, whenGetSVMAllocationFromReturnedPointerAreaThenReturnSameAllocation) {
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_NE(ptr, nullptr);
GraphicsAllocation *graphicsAllocation = svmManager.getSVMAlloc(ptr);
auto svmData = svmManager->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *graphicsAllocation = svmData->gpuAllocation;
EXPECT_NE(nullptr, graphicsAllocation);
auto ptrInRange = ptrOffset(ptr, MemoryConstants::pageSize - 4);
GraphicsAllocation *graphicsAllocationInRange = svmManager.getSVMAlloc(ptrInRange);
svmData = svmManager->getSVMAlloc(ptrInRange);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *graphicsAllocationInRange = svmData->gpuAllocation;
EXPECT_NE(nullptr, graphicsAllocationInRange);
EXPECT_EQ(graphicsAllocation, graphicsAllocationInRange);
svmManager.freeSVMAlloc(ptr);
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, whenGetSVMAllocationFromOutsideOfReturnedPointerAreaThenDontReturnThisAllocation) {
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_NE(ptr, nullptr);
GraphicsAllocation *graphicsAllocation = svmManager.getSVMAlloc(ptr);
auto svmData = svmManager->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *graphicsAllocation = svmData->gpuAllocation;
EXPECT_NE(nullptr, graphicsAllocation);
auto ptrBefore = ptrOffset(ptr, -4);
GraphicsAllocation *graphicsAllocationBefore = svmManager.getSVMAlloc(ptrBefore);
EXPECT_EQ(nullptr, graphicsAllocationBefore);
svmData = svmManager->getSVMAlloc(ptrBefore);
EXPECT_EQ(nullptr, svmData);
auto ptrAfter = ptrOffset(ptr, MemoryConstants::pageSize);
GraphicsAllocation *graphicsAllocationAfter = svmManager.getSVMAlloc(ptrAfter);
EXPECT_EQ(nullptr, graphicsAllocationAfter);
svmData = svmManager->getSVMAlloc(ptrAfter);
EXPECT_EQ(nullptr, svmData);
svmManager.freeSVMAlloc(ptr);
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, whenCouldNotAllocateInMemoryManagerThenReturnsNullAndDoesNotChangeAllocsMap) {
FailMemoryManager failMemoryManager(executionEnvironment);
svmManager.memoryManager = &failMemoryManager;
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
svmManager->memoryManager = &failMemoryManager;
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(nullptr, ptr);
EXPECT_EQ(0u, svmManager.SVMAllocs.getNumAllocs());
svmManager.freeSVMAlloc(ptr);
EXPECT_EQ(0u, svmManager->SVMAllocs.getNumAllocs());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, given64kbAllowedWhenAllocatingSvmMemoryThenDontPreferRenderCompression) {
MockMemoryManager memoryManager64Kb(true, false, executionEnvironment);
svmManager.memoryManager = &memoryManager64Kb;
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
svmManager->memoryManager = &memoryManager64Kb;
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_FALSE(memoryManager64Kb.preferRenderCompressedFlagPassed);
svmManager.freeSVMAlloc(ptr);
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, given64kbAllowedwhenAllocatingSvmMemoryThenAllocationIsIn64kbPagePool) {
MockMemoryManager memoryManager64Kb(true, false, executionEnvironment);
svmManager.memoryManager = &memoryManager64Kb;
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(MemoryPool::System64KBPages, svmManager.getSVMAlloc(ptr)->getMemoryPool());
svmManager.freeSVMAlloc(ptr);
svmManager->memoryManager = &memoryManager64Kb;
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(MemoryPool::System64KBPages, svmManager->getSVMAlloc(ptr)->gpuAllocation->getMemoryPool());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, given64kbDisallowedWhenAllocatingSvmMemoryThenAllocationIsIn4kbPagePool) {
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(MemoryPool::System4KBPages, svmManager.getSVMAlloc(ptr)->getMemoryPool());
svmManager.freeSVMAlloc(ptr);
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(MemoryPool::System4KBPages, svmManager->getSVMAlloc(ptr)->gpuAllocation->getMemoryPool());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, whenCoherentFlagIsPassedThenAllocationIsCoherent) {
auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, CL_MEM_SVM_FINE_GRAIN_BUFFER);
EXPECT_TRUE(svmManager.getSVMAlloc(ptr)->isCoherent());
svmManager.freeSVMAlloc(ptr);
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, CL_MEM_SVM_FINE_GRAIN_BUFFER);
EXPECT_TRUE(svmManager->getSVMAlloc(ptr)->gpuAllocation->isCoherent());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMMemoryAllocatorTest, whenReadOnlyFlagIsPresentThenReturnTrue) {
@@ -121,12 +146,74 @@ TEST_F(SVMMemoryAllocatorTest, whenNoReadOnlyFlagIsPresentThenReturnFalse) {
}
TEST_F(SVMMemoryAllocatorTest, whenReadOnlySvmAllocationCreatedThenGraphicsAllocationHasWriteableFlagFalse) {
void *svm = svmManager.createSVMAlloc(4096, CL_MEM_READ_ONLY);
void *svm = svmManager->createSVMAlloc(4096, CL_MEM_READ_ONLY);
EXPECT_NE(nullptr, svm);
GraphicsAllocation *svmAllocation = svmManager.getSVMAlloc(svm);
auto svmData = svmManager->getSVMAlloc(svm);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *svmAllocation = svmData->gpuAllocation;
EXPECT_NE(nullptr, svmAllocation);
EXPECT_FALSE(svmAllocation->isMemObjectsAllocationWithWritableFlags());
svmManager.freeSVMAlloc(svm);
svmManager->freeSVMAlloc(svm);
}
TEST_F(SVMLocalMemoryAllocatorTest, whenAllocatingSvmThenExpectCpuAllocationWithPointerAndGpuAllocationWithSameGpuAddress) {
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_NE(ptr, nullptr);
auto svmData = svmManager->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *cpuAllocation = svmData->cpuAllocation;
EXPECT_NE(nullptr, cpuAllocation);
EXPECT_EQ(ptr, cpuAllocation->getUnderlyingBuffer());
GraphicsAllocation *gpuAllocation = svmData->gpuAllocation;
EXPECT_NE(nullptr, gpuAllocation);
EXPECT_EQ(reinterpret_cast<uint64_t>(ptr), gpuAllocation->getGpuAddress());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMLocalMemoryAllocatorTest, whenGetSVMAllocationFromOutsideOfReturnedPointerAreaThenDontReturnThisAllocation) {
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_NE(ptr, nullptr);
auto svmData = svmManager->getSVMAlloc(ptr);
ASSERT_NE(nullptr, svmData);
GraphicsAllocation *graphicsAllocation = svmData->gpuAllocation;
EXPECT_NE(nullptr, graphicsAllocation);
auto ptrBefore = ptrOffset(ptr, -4);
svmData = svmManager->getSVMAlloc(ptrBefore);
EXPECT_EQ(nullptr, svmData);
auto ptrAfter = ptrOffset(ptr, MemoryConstants::pageSize);
svmData = svmManager->getSVMAlloc(ptrAfter);
EXPECT_EQ(nullptr, svmData);
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMLocalMemoryAllocatorTest, whenCouldNotAllocateCpuAllocationInMemoryManagerThenReturnsNullAndDoesNotChangeAllocsMap) {
FailMemoryManager failMemoryManager(false, true, executionEnvironment);
svmManager->memoryManager = &failMemoryManager;
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(nullptr, ptr);
EXPECT_EQ(0u, svmManager->SVMAllocs.getNumAllocs());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMLocalMemoryAllocatorTest, whenCouldNotAllocateGpuAllocationInMemoryManagerThenReturnsNullAndDoesNotChangeAllocsMap) {
FailMemoryManager failMemoryManager(1, executionEnvironment, true);
svmManager->memoryManager = &failMemoryManager;
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(nullptr, ptr);
EXPECT_EQ(0u, svmManager->SVMAllocs.getNumAllocs());
svmManager->freeSVMAlloc(ptr);
}
TEST_F(SVMLocalMemoryAllocatorTest, whenCouldNotReserveCpuAddressRangeInMemoryManagerThenReturnsNullAndDoesNotChangeAllocsMap) {
memoryManager->failReserveAddress = true;
auto ptr = svmManager->createSVMAlloc(MemoryConstants::pageSize, 0);
EXPECT_EQ(nullptr, ptr);
EXPECT_EQ(0u, svmManager->SVMAllocs.getNumAllocs());
}

8
unit_tests/mocks/mock_memory_manager.cpp
View File

@@ -29,6 +29,9 @@ void MockMemoryManager::overrideAsyncDeleterFlag(bool newValue) {
}
void *MockMemoryManager::allocateSystemMemory(size_t size, size_t alignment) {
if (failAllocateSystemMemory) {
return nullptr;
}
return OsAgnosticMemoryManager::allocateSystemMemory(redundancyRatio * size, alignment);
}
@@ -95,4 +98,9 @@ FailMemoryManager::FailMemoryManager(int32_t failedAllocationsCount, ExecutionEn
this->failedAllocationsCount = failedAllocationsCount;
}
FailMemoryManager::FailMemoryManager(int32_t failedAllocationsCount, ExecutionEnvironment &executionEnvironment, bool enableLocalMemory)
: MockMemoryManager(enableLocalMemory, executionEnvironment) {
this->failedAllocationsCount = failedAllocationsCount;
}
} // namespace NEO

10
unit_tests/mocks/mock_memory_manager.h
View File

@@ -85,6 +85,13 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
OsAgnosticMemoryManager::handleFenceCompletion(graphicsAllocation);
}
void *reserveCpuAddressRange(size_t size) override {
if (failReserveAddress) {
return nullptr;
}
return OsAgnosticMemoryManager::reserveCpuAddressRange(size);
}
GraphicsAllocation *allocate32BitGraphicsMemory(size_t size, const void *ptr, GraphicsAllocation::AllocationType allocationType);
uint32_t freeGraphicsMemoryCalled = 0u;
@@ -99,6 +106,8 @@ class MockMemoryManager : public MemoryManagerCreate<OsAgnosticMemoryManager> {
bool failInAllocateWithSizeAndAlignment = false;
bool preferRenderCompressedFlagPassed = false;
bool allocateForImageCalled = false;
bool failReserveAddress = false;
bool failAllocateSystemMemory = false;
std::unique_ptr<ExecutionEnvironment> mockExecutionEnvironment;
};
@@ -139,6 +148,7 @@ class FailMemoryManager : public MockMemoryManager {
using MemoryManager::allocateGraphicsMemoryWithProperties;
using MockMemoryManager::MockMemoryManager;
FailMemoryManager(int32_t failedAllocationsCount, ExecutionEnvironment &executionEnvironment);
FailMemoryManager(int32_t failedAllocationsCount, ExecutionEnvironment &executionEnvironment, bool localMemory);
GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override {
if (failedAllocationsCount <= 0) {

1
unit_tests/mocks/mock_svm_manager.h
View File

@@ -13,5 +13,6 @@ struct MockSVMAllocsManager : SVMAllocsManager {
using SVMAllocsManager::memoryManager;
using SVMAllocsManager::SVMAllocs;
using SVMAllocsManager::SVMAllocsManager;
using SVMAllocsManager::svmMapOperations;
};
} // namespace NEO

4
unit_tests/os_interface/debug_settings_manager_tests.cpp
View File

@@ -913,7 +913,9 @@ AllocationTypeTestCase allocationTypeValues[] = {
{GraphicsAllocation::AllocationType::INDIRECT_OBJECT_HEAP, "INDIRECT_OBJECT_HEAP"},
{GraphicsAllocation::AllocationType::SURFACE_STATE_HEAP, "SURFACE_STATE_HEAP"},
{GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY, "SHARED_RESOURCE_COPY"},
{GraphicsAllocation::AllocationType::SVM, "SVM"},
{GraphicsAllocation::AllocationType::SVM_ZERO_COPY, "SVM_ZERO_COPY"},
{GraphicsAllocation::AllocationType::SVM_CPU, "SVM_CPU"},
{GraphicsAllocation::AllocationType::SVM_GPU, "SVM_GPU"},
{GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR, "EXTERNAL_HOST_PTR"},
{GraphicsAllocation::AllocationType::UNDECIDED, "UNDECIDED"}};

1
unit_tests/os_interface/windows/mock_wddm_memory_manager.h
View File

@@ -23,6 +23,7 @@ class MockWddmMemoryManager : public MemoryManagerCreate<WddmMemoryManager> {
using BaseClass::createGraphicsAllocation;
using BaseClass::createWddmAllocation;
using BaseClass::gfxPartition;
using BaseClass::localMemorySupported;
using MemoryManagerCreate<WddmMemoryManager>::MemoryManagerCreate;
MockWddmMemoryManager(ExecutionEnvironment &executionEnvironment) : MemoryManagerCreate(false, false, executionEnvironment) {

22
unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp
View File

@@ -185,7 +185,7 @@ TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWith64KBPagesDisabledWhenA
memoryManager.reset(new MockWddmMemoryManager(false, false, *executionEnvironment));
auto size = MemoryConstants::pageSize;
auto svmAllocation = memoryManager->allocateGraphicsMemoryWithProperties({size, GraphicsAllocation::AllocationType::SVM});
auto svmAllocation = memoryManager->allocateGraphicsMemoryWithProperties({size, GraphicsAllocation::AllocationType::SVM_ZERO_COPY});
EXPECT_NE(nullptr, svmAllocation);
EXPECT_EQ(MemoryPool::System4KBPages, svmAllocation->getMemoryPool());
EXPECT_TRUE(svmAllocation->getDefaultGmm()->useSystemMemoryPool);
@@ -196,7 +196,7 @@ TEST_F(WddmMemoryManagerSimpleTest, givenMemoryManagerWith64KBPagesEnabledWhenAl
memoryManager.reset(new MockWddmMemoryManager(true, false, *executionEnvironment));
auto size = MemoryConstants::pageSize;
auto svmAllocation = memoryManager->allocateGraphicsMemoryWithProperties({size, GraphicsAllocation::AllocationType::SVM});
auto svmAllocation = memoryManager->allocateGraphicsMemoryWithProperties({size, GraphicsAllocation::AllocationType::SVM_ZERO_COPY});
EXPECT_NE(nullptr, svmAllocation);
EXPECT_EQ(MemoryPool::System64KBPages, svmAllocation->getMemoryPool());
memoryManager->freeGraphicsMemory(svmAllocation);
@@ -1671,3 +1671,21 @@ TEST_F(WddmMemoryManagerSimpleTest, givenAllocationWithPreferredGpuAddressWhenMa
EXPECT_EQ(1u, wddm->freeGpuVirtualAddressResult.called);
EXPECT_EQ(gpuAddress, wddm->freeGpuVirtualAddressResult.uint64ParamPassed);
}
TEST_F(WddmMemoryManagerSimpleTest, givenSvmCpuAllocationWhenSizeAndAlignmentProvidedThenAllocateMemoryReserveGpuVa) {
size_t size = 2 * MemoryConstants::megaByte;
MockAllocationProperties properties{true, size, GraphicsAllocation::AllocationType::SVM_CPU};
properties.alignment = size;
auto allocation = static_cast<WddmAllocation *>(memoryManager->allocateGraphicsMemoryWithProperties(properties));
ASSERT_NE(nullptr, allocation);
EXPECT_EQ(size, allocation->getUnderlyingBufferSize());
EXPECT_NE(nullptr, allocation->getUnderlyingBuffer());
EXPECT_EQ(allocation->getUnderlyingBuffer(), allocation->getDriverAllocatedCpuPtr());
//limited platforms will not use heap HeapIndex::HEAP_SVM
if (executionEnvironment->isFullRangeSvm()) {
EXPECT_EQ(alignUp(allocation->getReservedAddressPtr(), size), reinterpret_cast<void *>(allocation->getGpuAddress()));
}
EXPECT_EQ((2 * size), allocation->getReservedAddressSize());
memoryManager->freeGraphicsMemory(allocation);
}