From b2e16b78975b5f437bed3f11b12bd65f0972b92f Mon Sep 17 00:00:00 2001 From: Filip Hazubski Date: Wed, 3 Apr 2019 11:22:04 +0200 Subject: [PATCH] Update allocationForCacheFlush method Related-To: NEO-2535 Change-Id: Ia24556814188263e2ebb54b6419feddd5d8ed707 Signed-off-by: Filip Hazubski --- runtime/api/api.cpp | 3 +- .../command_stream_receiver.cpp | 2 +- runtime/kernel/kernel.cpp | 2 +- runtime/mem_obj/buffer.cpp | 4 +- runtime/mem_obj/mem_obj_helper.cpp | 5 +- runtime/mem_obj/mem_obj_helper.h | 2 +- runtime/mem_obj/pipe.cpp | 3 +- runtime/memory_manager/memory_manager.cpp | 23 ++++++++- runtime/memory_manager/memory_manager.h | 51 ++++++++++++++----- .../os_agnostic_memory_manager.cpp | 40 ++++++++------- .../os_agnostic_memory_manager.h | 10 ++-- runtime/memory_manager/svm_memory_manager.cpp | 9 ++-- runtime/memory_manager/svm_memory_manager.h | 2 +- .../api/cl_set_kernel_exec_info_tests.inl | 2 +- .../enqueue_svm_mem_copy_tests.cpp | 4 +- .../enqueue_svm_mem_fill_tests.cpp | 2 +- .../command_queue/enqueue_svm_tests.cpp | 16 +++--- .../command_queue/zero_size_enqueue_tests.cpp | 12 ++--- .../driver_diagnostics_enqueue_tests.cpp | 2 +- unit_tests/helpers/kernel_commands_tests.cpp | 33 +++++++++++- unit_tests/kernel/clone_kernel_tests.cpp | 2 +- unit_tests/kernel/kernel_arg_buffer_tests.cpp | 4 +- unit_tests/kernel/kernel_arg_svm_tests.cpp | 8 +-- unit_tests/kernel/kernel_image_arg_tests.cpp | 4 +- unit_tests/kernel/kernel_tests.cpp | 4 +- unit_tests/mem_obj/buffer_set_arg_tests.cpp | 2 +- unit_tests/mem_obj/buffer_tests.cpp | 4 +- ..._manager_allocate_in_device_pool_tests.inl | 16 +++++- .../memory_manager/memory_manager_tests.cpp | 36 ++++++++++++- .../svm_memory_manager_tests.cpp | 20 ++++---- 30 files changed, 229 insertions(+), 98 deletions(-) diff --git a/runtime/api/api.cpp b/runtime/api/api.cpp index 3a55616f24..c2227cc3dd 100644 --- a/runtime/api/api.cpp +++ b/runtime/api/api.cpp @@ -3352,8 +3352,7 @@ void *CL_API_CALL clSVMAlloc(cl_context context, return pAlloc; } - pAlloc = pContext->getSVMAllocsManager()->createSVMAlloc(size, !!(flags & CL_MEM_SVM_FINE_GRAIN_BUFFER), - SVMAllocsManager::memFlagIsReadOnly(flags)); + pAlloc = pContext->getSVMAllocsManager()->createSVMAlloc(size, flags); if (pContext->isProvidingPerformanceHints()) { pContext->providePerformanceHint(CL_CONTEXT_DIAGNOSTICS_LEVEL_GOOD_INTEL, CL_SVM_ALLOC_MEETS_ALIGNMENT_RESTRICTIONS, pAlloc, size); diff --git a/runtime/command_stream/command_stream_receiver.cpp b/runtime/command_stream/command_stream_receiver.cpp index b455df68f9..d8dcd43a42 100644 --- a/runtime/command_stream/command_stream_receiver.cpp +++ b/runtime/command_stream/command_stream_receiver.cpp @@ -126,7 +126,7 @@ void CommandStreamReceiver::ensureCommandBufferAllocation(LinearStream &commandS constexpr static auto allocationType = GraphicsAllocation::AllocationType::COMMAND_BUFFER; auto allocation = this->getInternalAllocationStorage()->obtainReusableAllocation(allocationSize, allocationType).release(); if (allocation == nullptr) { - const AllocationProperties commandStreamAllocationProperties{true, allocationSize, allocationType, this->isMultiOsContextCapable(), this->deviceIndex}; + const AllocationProperties commandStreamAllocationProperties{true, allocationSize, allocationType, this->isMultiOsContextCapable(), this->deviceIndex, 0}; allocation = this->getMemoryManager()->allocateGraphicsMemoryWithProperties(commandStreamAllocationProperties); } DEBUG_BREAK_IF(allocation == nullptr); diff --git a/runtime/kernel/kernel.cpp b/runtime/kernel/kernel.cpp index 5aa27ba0a5..7ae17b10d3 100644 --- a/runtime/kernel/kernel.cpp +++ b/runtime/kernel/kernel.cpp @@ -2181,7 +2181,7 @@ void Kernel::getAllocationsForCacheFlush(CacheFlushAllocationsVec &out) const { } bool Kernel::allocationForCacheFlush(GraphicsAllocation *argAllocation) const { - return argAllocation->isFlushL3Required() || argAllocation->isMemObjectsAllocationWithWritableFlags(); + return argAllocation->isFlushL3Required(); } void Kernel::addAllocationToCacheFlushVector(uint32_t argIndex, GraphicsAllocation *argAllocation) { diff --git a/runtime/mem_obj/buffer.cpp b/runtime/mem_obj/buffer.cpp index 34982f1665..858f87eb95 100644 --- a/runtime/mem_obj/buffer.cpp +++ b/runtime/mem_obj/buffer.cpp @@ -206,7 +206,7 @@ Buffer *Buffer::create(Context *context, } if (!memory) { - AllocationProperties allocProperties = MemObjHelper::getAllocationProperties(properties.flags_intel, allocateMemory, size, allocationType); + AllocationProperties allocProperties = MemObjHelper::getAllocationProperties(properties, allocateMemory, size, allocationType); StorageInfo storageInfo = MemObjHelper::getStorageInfo(properties); memory = memoryManager->allocateGraphicsMemoryInPreferredPool(allocProperties, storageInfo, hostPtr); } @@ -220,7 +220,7 @@ Buffer *Buffer::create(Context *context, allocationType = GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY; zeroCopyAllowed = false; copyMemoryFromHostPtr = true; - AllocationProperties allocProperties = MemObjHelper::getAllocationProperties(properties.flags_intel, true, size, allocationType); + AllocationProperties allocProperties = MemObjHelper::getAllocationProperties(properties, true, size, allocationType); StorageInfo storageInfo = MemObjHelper::getStorageInfo(properties); memory = memoryManager->allocateGraphicsMemoryInPreferredPool(allocProperties, storageInfo, nullptr); } diff --git a/runtime/mem_obj/mem_obj_helper.cpp b/runtime/mem_obj/mem_obj_helper.cpp index b3f2d5532c..ee0aab614b 100644 --- a/runtime/mem_obj/mem_obj_helper.cpp +++ b/runtime/mem_obj/mem_obj_helper.cpp @@ -31,10 +31,9 @@ bool MemObjHelper::parseMemoryProperties(const cl_mem_properties_intel *properti return true; } -AllocationProperties MemObjHelper::getAllocationProperties(cl_mem_flags_intel flags, bool allocateMemory, +AllocationProperties MemObjHelper::getAllocationProperties(MemoryProperties properties, bool allocateMemory, size_t size, GraphicsAllocation::AllocationType type) { - AllocationProperties allocationProperties(allocateMemory, size, type); - allocationProperties.flags.uncacheable = isValueSet(flags, CL_MEM_LOCALLY_UNCACHED_RESOURCE); + AllocationProperties allocationProperties(allocateMemory, size, type, properties); return allocationProperties; } diff --git a/runtime/mem_obj/mem_obj_helper.h b/runtime/mem_obj/mem_obj_helper.h index 4eb973d8a4..ed6e9d9bbe 100644 --- a/runtime/mem_obj/mem_obj_helper.h +++ b/runtime/mem_obj/mem_obj_helper.h @@ -87,7 +87,7 @@ class MemObjHelper { return validateExtraMemoryProperties(properties); } - static AllocationProperties getAllocationProperties(cl_mem_flags_intel flags, bool allocateMemory, + static AllocationProperties getAllocationProperties(MemoryProperties properties, bool allocateMemory, size_t size, GraphicsAllocation::AllocationType type); static AllocationProperties getAllocationProperties(ImageInfo *imgInfo, bool allocateMemory); diff --git a/runtime/mem_obj/pipe.cpp b/runtime/mem_obj/pipe.cpp index e8a0e86197..ecb4c564f8 100644 --- a/runtime/mem_obj/pipe.cpp +++ b/runtime/mem_obj/pipe.cpp @@ -52,7 +52,8 @@ Pipe *Pipe::create(Context *context, memoryProperties.flags = flags; while (true) { auto size = static_cast(packetSize * (maxPackets + 1) + intelPipeHeaderReservedSpace); - AllocationProperties allocProperties = MemObjHelper::getAllocationProperties(flags, true, size, GraphicsAllocation::AllocationType::PIPE); + AllocationProperties allocProperties = + MemObjHelper::getAllocationProperties(memoryProperties, true, size, GraphicsAllocation::AllocationType::PIPE); StorageInfo storageInfo = MemObjHelper::getStorageInfo(memoryProperties); GraphicsAllocation *memory = memoryManager->allocateGraphicsMemoryInPreferredPool(allocProperties, storageInfo, nullptr); if (!memory) { diff --git a/runtime/memory_manager/memory_manager.cpp b/runtime/memory_manager/memory_manager.cpp index fa17ad7845..ea2d81fd1d 100644 --- a/runtime/memory_manager/memory_manager.cpp +++ b/runtime/memory_manager/memory_manager.cpp @@ -214,6 +214,7 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo bool forcePin = properties.flags.forcePin; bool uncacheable = properties.flags.uncacheable; bool mustBeZeroCopy = false; + bool mayRequireL3Flush = false; bool multiOsContextCapable = properties.flags.multiOsContextCapable; switch (properties.allocationType) { @@ -265,6 +266,25 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo break; } + switch (properties.allocationType) { + case GraphicsAllocation::AllocationType::BUFFER: + case GraphicsAllocation::AllocationType::BUFFER_COMPRESSED: + case GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY: + case GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR: + case GraphicsAllocation::AllocationType::GLOBAL_SURFACE: + case GraphicsAllocation::AllocationType::IMAGE: + case GraphicsAllocation::AllocationType::PIPE: + case GraphicsAllocation::AllocationType::SHARED_IMAGE: + case GraphicsAllocation::AllocationType::SHARED_BUFFER: + case GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY: + case GraphicsAllocation::AllocationType::SVM: + case GraphicsAllocation::AllocationType::UNDECIDED: + mayRequireL3Flush = true; + break; + default: + break; + } + switch (properties.allocationType) { case GraphicsAllocation::AllocationType::UNDECIDED: case GraphicsAllocation::AllocationType::FILL_PATTERN: @@ -283,7 +303,8 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo allocationData.flags.allow64kbPages = allow64KbPages; allocationData.flags.forcePin = forcePin; allocationData.flags.uncacheable = uncacheable; - allocationData.flags.flushL3 = properties.flags.flushL3RequiredForRead | properties.flags.flushL3RequiredForWrite; + allocationData.flags.flushL3 = + (mayRequireL3Flush ? properties.flags.flushL3RequiredForRead | properties.flags.flushL3RequiredForWrite : 0u); allocationData.flags.preferRenderCompressed = GraphicsAllocation::AllocationType::BUFFER_COMPRESSED == properties.allocationType; allocationData.flags.multiOsContextCapable = multiOsContextCapable; diff --git a/runtime/memory_manager/memory_manager.h b/runtime/memory_manager/memory_manager.h index 19f98d915c..6c3b14753c 100644 --- a/runtime/memory_manager/memory_manager.h +++ b/runtime/memory_manager/memory_manager.h @@ -6,6 +6,7 @@ */ #pragma once +#include "common/helpers/bit_helpers.h" #include "public/cl_ext_private.h" #include "runtime/command_stream/preemption_mode.h" #include "runtime/helpers/aligned_memory.h" @@ -16,6 +17,7 @@ #include "runtime/os_interface/32bit_memory.h" #include "engine_node.h" +#include "mem_obj_types.h" #include #include @@ -67,26 +69,51 @@ struct AllocationProperties { ImageInfo *imgInfo = nullptr; uint32_t deviceIndex = AllocationProperties::noDeviceSpecified; - AllocationProperties(size_t size, GraphicsAllocation::AllocationType allocationType) - : AllocationProperties(true, size, allocationType) {} + AllocationProperties(size_t size, + GraphicsAllocation::AllocationType allocationType) + : AllocationProperties(true, size, allocationType, 0) {} - AllocationProperties(size_t size, GraphicsAllocation::AllocationType allocationType, uint32_t deviceIndex) - : AllocationProperties(true, size, allocationType, false, deviceIndex) {} - AllocationProperties(bool allocateMemory, size_t size, GraphicsAllocation::AllocationType allocationType) - : AllocationProperties(allocateMemory, size, allocationType, false, AllocationProperties::noDeviceSpecified) {} - AllocationProperties(bool allocateMemory, size_t size, GraphicsAllocation::AllocationType allocationType, - bool multiOsContextCapable, uint32_t deviceIndex) + AllocationProperties(size_t size, + GraphicsAllocation::AllocationType allocationType, + uint32_t deviceIndex) + : AllocationProperties(true, size, allocationType, false, deviceIndex, 0) {} + + AllocationProperties(bool allocateMemory, + size_t size, + GraphicsAllocation::AllocationType allocationType) + : AllocationProperties(allocateMemory, size, allocationType, 0) {} + + AllocationProperties(bool allocateMemory, + size_t size, + GraphicsAllocation::AllocationType allocationType, + MemoryProperties memoryProperties) + : AllocationProperties(allocateMemory, size, allocationType, false, AllocationProperties::noDeviceSpecified, + memoryProperties) {} + + AllocationProperties(bool allocateMemory, + size_t size, + GraphicsAllocation::AllocationType allocationType, + bool multiOsContextCapable, + uint32_t deviceIndex, + MemoryProperties memoryProperties) : size(size), allocationType(allocationType), deviceIndex(deviceIndex) { allFlags = 0; - flags.flushL3RequiredForRead = 1; - flags.flushL3RequiredForWrite = 1; + flags.uncacheable = isValueSet(memoryProperties.flags_intel, CL_MEM_LOCALLY_UNCACHED_RESOURCE); + auto cacheFlushRequired = !flags.uncacheable && !isValueSet(memoryProperties.flags, CL_MEM_READ_ONLY); + flags.flushL3RequiredForRead = cacheFlushRequired; + flags.flushL3RequiredForWrite = cacheFlushRequired; flags.allocateMemory = allocateMemory; flags.multiOsContextCapable = multiOsContextCapable; } - AllocationProperties(ImageInfo *imgInfo, bool allocateMemory) : AllocationProperties(allocateMemory, 0, GraphicsAllocation::AllocationType::IMAGE) { + AllocationProperties(ImageInfo *imgInfo, + bool allocateMemory) + : AllocationProperties(allocateMemory, 0, GraphicsAllocation::AllocationType::IMAGE, 0) { this->imgInfo = imgInfo; } - AllocationProperties(ImageInfo *imgInfo, bool allocateMemory, GraphicsAllocation::AllocationType allocationType) : AllocationProperties(allocateMemory, 0, allocationType) { + AllocationProperties(ImageInfo *imgInfo, + bool allocateMemory, + GraphicsAllocation::AllocationType allocationType) + : AllocationProperties(allocateMemory, 0, allocationType, 0) { this->imgInfo = imgInfo; } }; diff --git a/runtime/memory_manager/os_agnostic_memory_manager.cpp b/runtime/memory_manager/os_agnostic_memory_manager.cpp index 7b050c98ef..f2dee8c796 100644 --- a/runtime/memory_manager/os_agnostic_memory_manager.cpp +++ b/runtime/memory_manager/os_agnostic_memory_manager.cpp @@ -35,21 +35,22 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryWithAlignment MemoryAllocation *memoryAllocation = nullptr; if (fakeBigAllocations && allocationData.size > bigAllocation) { - memoryAllocation = new MemoryAllocation(allocationData.type, nullptr, (void *)dummyAddress, static_cast(dummyAddress), - allocationData.size, counter, MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable); + memoryAllocation = new MemoryAllocation( + allocationData.type, nullptr, (void *)dummyAddress, static_cast(dummyAddress), allocationData.size, counter, + MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable, allocationData.flags.uncacheable, + allocationData.flags.flushL3); counter++; - memoryAllocation->uncacheable = allocationData.flags.uncacheable; return memoryAllocation; } auto ptr = allocateSystemMemory(sizeAligned, allocationData.alignment ? alignUp(allocationData.alignment, MemoryConstants::pageSize) : MemoryConstants::pageSize); if (ptr != nullptr) { - memoryAllocation = new MemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast(ptr), - allocationData.size, counter, MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable); + memoryAllocation = new MemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast(ptr), allocationData.size, + counter, MemoryPool::System4KBPages, allocationData.flags.multiOsContextCapable, + allocationData.flags.uncacheable, allocationData.flags.flushL3); if (!memoryAllocation) { alignedFreeWrapper(ptr); return nullptr; } - memoryAllocation->uncacheable = allocationData.flags.uncacheable; } counter++; return memoryAllocation; @@ -59,12 +60,11 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForNonSvmHost auto alignedPtr = alignDown(allocationData.hostPtr, MemoryConstants::pageSize); auto offsetInPage = ptrDiff(allocationData.hostPtr, alignedPtr); - auto memoryAllocation = new MemoryAllocation(allocationData.type, nullptr, const_cast(allocationData.hostPtr), reinterpret_cast(alignedPtr), - allocationData.size, counter, MemoryPool::System4KBPages, false); + auto memoryAllocation = new MemoryAllocation(allocationData.type, nullptr, const_cast(allocationData.hostPtr), + reinterpret_cast(alignedPtr), allocationData.size, counter, + MemoryPool::System4KBPages, false, false, allocationData.flags.flushL3); memoryAllocation->setAllocationOffset(offsetInPage); - memoryAllocation->uncacheable = false; - counter++; return memoryAllocation; } @@ -88,8 +88,9 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(con return nullptr; } uint64_t offset = static_cast(reinterpret_cast(allocationData.hostPtr) & MemoryConstants::pageMask); - MemoryAllocation *memAlloc = new MemoryAllocation(allocationData.type, nullptr, const_cast(allocationData.hostPtr), GmmHelper::canonize(gpuVirtualAddress + offset), - allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false); + MemoryAllocation *memAlloc = new MemoryAllocation( + allocationData.type, nullptr, const_cast(allocationData.hostPtr), GmmHelper::canonize(gpuVirtualAddress + offset), + allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false, false, false); memAlloc->set32BitAllocation(true); memAlloc->setGpuBaseAddress(GmmHelper::canonize(allocator32Bit->getBase())); memAlloc->sizeToFree = allocationSize; @@ -109,7 +110,8 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(con MemoryAllocation *memoryAllocation = nullptr; if (ptrAlloc != nullptr) { memoryAllocation = new MemoryAllocation(allocationData.type, ptrAlloc, ptrAlloc, GmmHelper::canonize(gpuAddress), - allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false); + allocationData.size, counter, MemoryPool::System4KBPagesWith32BitGpuAddressing, false, + false, false); memoryAllocation->set32BitAllocation(true); memoryAllocation->setGpuBaseAddress(GmmHelper::canonize(allocator32Bit->getBase())); memoryAllocation->sizeToFree = allocationSize; @@ -120,7 +122,8 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocate32BitGraphicsMemoryImpl(con GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) { auto graphicsAllocation = new MemoryAllocation(GraphicsAllocation::AllocationType::UNDECIDED, nullptr, reinterpret_cast(1), 1, - 4096u, static_cast(handle), MemoryPool::SystemCpuInaccessible, false); + 4096u, static_cast(handle), MemoryPool::SystemCpuInaccessible, false, + false, false); graphicsAllocation->setSharedHandle(handle); graphicsAllocation->set32BitAllocation(requireSpecificBitness); @@ -196,8 +199,9 @@ uint64_t OsAgnosticMemoryManager::getInternalHeapBaseAddress() { } GraphicsAllocation *OsAgnosticMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) { - auto allocation = new MemoryAllocation(allocationData.type, nullptr, const_cast(allocationData.hostPtr), reinterpret_cast(allocationData.hostPtr), - allocationData.size, counter++, MemoryPool::System4KBPages, false); + auto allocation = new MemoryAllocation(allocationData.type, nullptr, const_cast(allocationData.hostPtr), + reinterpret_cast(allocationData.hostPtr), allocationData.size, counter++, + MemoryPool::System4KBPages, false, false, false); allocation->fragmentsStorage = handleStorage; return allocation; } @@ -241,8 +245,8 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryForImageImpl( auto ptr = allocateSystemMemory(alignUp(allocationData.imgInfo->size, MemoryConstants::pageSize), MemoryConstants::pageSize); if (ptr != nullptr) { - alloc = new MemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast(ptr), - allocationData.imgInfo->size, counter, MemoryPool::SystemCpuInaccessible, false); + alloc = new MemoryAllocation(allocationData.type, ptr, ptr, reinterpret_cast(ptr), allocationData.imgInfo->size, + counter, MemoryPool::SystemCpuInaccessible, false, false, false); counter++; } diff --git a/runtime/memory_manager/os_agnostic_memory_manager.h b/runtime/memory_manager/os_agnostic_memory_manager.h index 38dca211a6..a6ef78a627 100644 --- a/runtime/memory_manager/os_agnostic_memory_manager.h +++ b/runtime/memory_manager/os_agnostic_memory_manager.h @@ -16,18 +16,20 @@ constexpr size_t bigAllocation = 1 * MB; constexpr uintptr_t dummyAddress = 0xFFFFF000u; class MemoryAllocation : public GraphicsAllocation { public: - unsigned long long id; + const unsigned long long id; size_t sizeToFree = 0; - bool uncacheable = false; + const bool uncacheable; void setSharedHandle(osHandle handle) { sharingInfo.sharedHandle = handle; } - MemoryAllocation(AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize, uint64_t count, MemoryPool::Type pool, bool multiOsContextCapable) + MemoryAllocation(AllocationType allocationType, void *driverAllocatedCpuPointer, void *pMem, uint64_t gpuAddress, size_t memSize, + uint64_t count, MemoryPool::Type pool, bool multiOsContextCapable, bool uncacheable, bool flushL3Required) : GraphicsAllocation(allocationType, pMem, gpuAddress, 0u, memSize, pool, multiOsContextCapable), - id(count) { + id(count), uncacheable(uncacheable) { this->driverAllocatedCpuPointer = driverAllocatedCpuPointer; overrideMemoryPool(pool); + allocationInfo.flags.flushL3Required = flushL3Required; } void overrideMemoryPool(MemoryPool::Type pool); diff --git a/runtime/memory_manager/svm_memory_manager.cpp b/runtime/memory_manager/svm_memory_manager.cpp index a41718a78d..dbf3dc8ddc 100644 --- a/runtime/memory_manager/svm_memory_manager.cpp +++ b/runtime/memory_manager/svm_memory_manager.cpp @@ -50,17 +50,18 @@ GraphicsAllocation *SVMAllocsManager::MapBasedAllocationTracker::get(const void SVMAllocsManager::SVMAllocsManager(MemoryManager *memoryManager) : memoryManager(memoryManager) { } -void *SVMAllocsManager::createSVMAlloc(size_t size, bool coherent, bool readOnly) { +void *SVMAllocsManager::createSVMAlloc(size_t size, cl_mem_flags flags) { if (size == 0) return nullptr; std::unique_lock lock(mtx); - GraphicsAllocation *GA = memoryManager->allocateGraphicsMemoryWithProperties({size, GraphicsAllocation::AllocationType::SVM}); + GraphicsAllocation *GA = memoryManager->allocateGraphicsMemoryWithProperties( + {true, size, GraphicsAllocation::AllocationType::SVM, flags}); if (!GA) { return nullptr; } - GA->setMemObjectsAllocationWithWritableFlags(!readOnly); - GA->setCoherent(coherent); + GA->setMemObjectsAllocationWithWritableFlags(!SVMAllocsManager::memFlagIsReadOnly(flags)); + GA->setCoherent(isValueSet(flags, CL_MEM_SVM_FINE_GRAIN_BUFFER)); this->SVMAllocs.insert(*GA); return GA->getUnderlyingBuffer(); diff --git a/runtime/memory_manager/svm_memory_manager.h b/runtime/memory_manager/svm_memory_manager.h index 832e988641..0f2fe7650b 100644 --- a/runtime/memory_manager/svm_memory_manager.h +++ b/runtime/memory_manager/svm_memory_manager.h @@ -32,7 +32,7 @@ class SVMAllocsManager { }; SVMAllocsManager(MemoryManager *memoryManager); - void *createSVMAlloc(size_t size, bool coherent, bool readOnly); + void *createSVMAlloc(size_t size, cl_mem_flags flags); GraphicsAllocation *getSVMAlloc(const void *ptr); void freeSVMAlloc(void *ptr); size_t getNumAllocs() const { return SVMAllocs.getNumAllocs(); } diff --git a/unit_tests/api/cl_set_kernel_exec_info_tests.inl b/unit_tests/api/cl_set_kernel_exec_info_tests.inl index 6133eb4797..fa08bd228e 100644 --- a/unit_tests/api/cl_set_kernel_exec_info_tests.inl +++ b/unit_tests/api/cl_set_kernel_exec_info_tests.inl @@ -181,7 +181,7 @@ TEST_F(clSetKernelExecInfoTests, success_SvmPtrListWithMultiplePointers) { } } -TEST_F(clSetKernelExecInfoTests, givenReadOnlySvmPtrListWhenUsedAsKernelPointersThenNoCacheFlushRequire) { +TEST_F(clSetKernelExecInfoTests, givenReadOnlySvmPtrListWhenUsedAsKernelPointersThenCacheFlushIsNotRequired) { if (svmCapabilities != 0) { void *ptrSvm1 = clSVMAlloc(pContext, CL_MEM_READ_ONLY, 256, 4); EXPECT_NE(nullptr, ptrSvm1); diff --git a/unit_tests/command_queue/enqueue_svm_mem_copy_tests.cpp b/unit_tests/command_queue/enqueue_svm_mem_copy_tests.cpp index 3729090d16..9526832946 100644 --- a/unit_tests/command_queue/enqueue_svm_mem_copy_tests.cpp +++ b/unit_tests/command_queue/enqueue_svm_mem_copy_tests.cpp @@ -25,9 +25,9 @@ struct EnqueueSvmMemCopyTest : public DeviceFixture, void SetUp() override { DeviceFixture::SetUp(); CommandQueueFixture::SetUp(pDevice, 0); - srcSvmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + srcSvmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, 0); ASSERT_NE(nullptr, srcSvmPtr); - dstSvmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + dstSvmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, 0); ASSERT_NE(nullptr, dstSvmPtr); srcSvmAlloc = context->getSVMAllocsManager()->getSVMAlloc(srcSvmPtr); ASSERT_NE(nullptr, srcSvmAlloc); diff --git a/unit_tests/command_queue/enqueue_svm_mem_fill_tests.cpp b/unit_tests/command_queue/enqueue_svm_mem_fill_tests.cpp index 9def5ed82d..38ee239b78 100644 --- a/unit_tests/command_queue/enqueue_svm_mem_fill_tests.cpp +++ b/unit_tests/command_queue/enqueue_svm_mem_fill_tests.cpp @@ -27,7 +27,7 @@ struct EnqueueSvmMemFillTest : public DeviceFixture, CommandQueueFixture::SetUp(pDevice, 0); patternSize = (size_t)GetParam(); ASSERT_TRUE((0 < patternSize) && (patternSize <= 128)); - svmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, true, false); + svmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, CL_MEM_SVM_FINE_GRAIN_BUFFER); ASSERT_NE(nullptr, svmPtr); svmAlloc = context->getSVMAllocsManager()->getSVMAlloc(svmPtr); ASSERT_NE(nullptr, svmAlloc); diff --git a/unit_tests/command_queue/enqueue_svm_tests.cpp b/unit_tests/command_queue/enqueue_svm_tests.cpp index 1dc05f585f..5fd676504f 100644 --- a/unit_tests/command_queue/enqueue_svm_tests.cpp +++ b/unit_tests/command_queue/enqueue_svm_tests.cpp @@ -34,7 +34,7 @@ struct EnqueueSvmTest : public DeviceFixture, void SetUp() override { DeviceFixture::SetUp(); CommandQueueFixture::SetUp(pDevice, 0); - ptrSVM = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + ptrSVM = context->getSVMAllocsManager()->createSVMAlloc(256, 0); } void TearDown() override { @@ -239,7 +239,7 @@ TEST_F(EnqueueSvmTest, enqueueSVMMemcpy_InvalidValueDstPtrIsNull) { DebugManagerStateRestore dbgRestore; DebugManager.flags.EnableAsyncEventsHandler.set(false); void *pDstSVM = nullptr; - void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, 0); retVal = this->pCmdQ->enqueueSVMMemcpy( false, // cl_bool blocking_copy pDstSVM, // void *dst_ptr @@ -270,7 +270,7 @@ TEST_F(EnqueueSvmTest, enqueueSVMMemcpy_InvalidValueSrcPtrIsNull) { TEST_F(EnqueueSvmTest, enqueueSVMMemcpy_Success) { void *pDstSVM = ptrSVM; - void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, 0); retVal = this->pCmdQ->enqueueSVMMemcpy( false, // cl_bool blocking_copy pDstSVM, // void *dst_ptr @@ -286,7 +286,7 @@ TEST_F(EnqueueSvmTest, enqueueSVMMemcpy_Success) { TEST_F(EnqueueSvmTest, enqueueSVMMemcpyBlocking_Success) { void *pDstSVM = ptrSVM; - void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, 0); retVal = this->pCmdQ->enqueueSVMMemcpy( true, // cl_bool blocking_copy pDstSVM, // void *dst_ptr @@ -302,7 +302,7 @@ TEST_F(EnqueueSvmTest, enqueueSVMMemcpyBlocking_Success) { TEST_F(EnqueueSvmTest, enqueueSVMMemcpyBlockedOnEvent_Success) { void *pDstSVM = ptrSVM; - void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, 0); auto uEvent = make_releaseable(); cl_event eventWaitList[] = {uEvent.get()}; retVal = this->pCmdQ->enqueueSVMMemcpy( @@ -321,7 +321,7 @@ TEST_F(EnqueueSvmTest, enqueueSVMMemcpyBlockedOnEvent_Success) { TEST_F(EnqueueSvmTest, enqueueSVMMemcpyCoherent_Success) { void *pDstSVM = ptrSVM; - void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, true, false); + void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, CL_MEM_SVM_FINE_GRAIN_BUFFER); retVal = this->pCmdQ->enqueueSVMMemcpy( false, // cl_bool blocking_copy pDstSVM, // void *dst_ptr @@ -337,7 +337,7 @@ TEST_F(EnqueueSvmTest, enqueueSVMMemcpyCoherent_Success) { TEST_F(EnqueueSvmTest, enqueueSVMMemcpyCoherentBlockedOnEvent_Success) { void *pDstSVM = ptrSVM; - void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, true, false); + void *pSrcSVM = context->getSVMAllocsManager()->createSVMAlloc(256, CL_MEM_SVM_FINE_GRAIN_BUFFER); auto uEvent = make_releaseable(); cl_event eventWaitList[] = {uEvent.get()}; retVal = this->pCmdQ->enqueueSVMMemcpy( @@ -527,7 +527,7 @@ TEST_F(EnqueueSvmTest, concurentMapAccess) { auto allocSvm = [&](uint32_t from, uint32_t to) { for (uint32_t i = from; i <= to; i++) { - svmPtrs[i] = context->getSVMAllocsManager()->createSVMAlloc(1, false, false); + svmPtrs[i] = context->getSVMAllocsManager()->createSVMAlloc(1, 0); auto ga = context->getSVMAllocsManager()->getSVMAlloc(svmPtrs[i]); EXPECT_NE(nullptr, ga); EXPECT_EQ(ga->getUnderlyingBuffer(), svmPtrs[i]); diff --git a/unit_tests/command_queue/zero_size_enqueue_tests.cpp b/unit_tests/command_queue/zero_size_enqueue_tests.cpp index ad8d479486..0ef6710d96 100644 --- a/unit_tests/command_queue/zero_size_enqueue_tests.cpp +++ b/unit_tests/command_queue/zero_size_enqueue_tests.cpp @@ -756,8 +756,8 @@ HWTEST_F(ZeroSizeEnqueueHandlerTest, enqueueFillImageWhenZeroSizeEnqueueIsDetect HWTEST_F(ZeroSizeEnqueueHandlerTest, enqueueSVMMemcpyWhenZeroSizeEnqueueIsDetectedThenCommandMarkerShouldBeEnqueued) { auto mockCmdQ = std::unique_ptr>(new MockCommandQueueHw(&context, pDevice, 0)); - void *pSrcSVM = context.getSVMAllocsManager()->createSVMAlloc(256, false, false); - void *pDstSVM = context.getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSrcSVM = context.getSVMAllocsManager()->createSVMAlloc(256, 0); + void *pDstSVM = context.getSVMAllocsManager()->createSVMAlloc(256, 0); size_t zeroSize = 0; mockCmdQ->enqueueSVMMemcpy(false, pSrcSVM, pDstSVM, zeroSize, 0, nullptr, nullptr); EXPECT_EQ(static_cast(CL_COMMAND_MARKER), mockCmdQ->lastCommandType); @@ -770,8 +770,8 @@ HWTEST_F(ZeroSizeEnqueueHandlerTest, enqueueSVMMemcpyWhenZeroSizeEnqueueIsDetect auto mockCmdQ = std::unique_ptr>(new MockCommandQueueHw(&context, pDevice, 0)); cl_event event; - void *pSrcSVM = context.getSVMAllocsManager()->createSVMAlloc(256, false, false); - void *pDstSVM = context.getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSrcSVM = context.getSVMAllocsManager()->createSVMAlloc(256, 0); + void *pDstSVM = context.getSVMAllocsManager()->createSVMAlloc(256, 0); size_t zeroSize = 0; mockCmdQ->enqueueSVMMemcpy(false, pSrcSVM, pDstSVM, zeroSize, 0, nullptr, &event); EXPECT_EQ(static_cast(CL_COMMAND_MARKER), mockCmdQ->lastCommandType); @@ -792,7 +792,7 @@ HWTEST_F(ZeroSizeEnqueueHandlerTest, enqueueSVMMemcpyWhenZeroSizeEnqueueIsDetect HWTEST_F(ZeroSizeEnqueueHandlerTest, enqueueSVMMemFillWhenZeroSizeEnqueueIsDetectedThenCommandMarkerShouldBeEnqueued) { auto mockCmdQ = std::unique_ptr>(new MockCommandQueueHw(&context, pDevice, 0)); - void *pSVM = context.getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSVM = context.getSVMAllocsManager()->createSVMAlloc(256, 0); const float pattern[1] = {1.2345f}; size_t zeroSize = 0; mockCmdQ->enqueueSVMMemFill(pSVM, &pattern, sizeof(pattern), zeroSize, 0, nullptr, nullptr); @@ -805,7 +805,7 @@ HWTEST_F(ZeroSizeEnqueueHandlerTest, enqueueSVMMemFillWhenZeroSizeEnqueueIsDetec auto mockCmdQ = std::unique_ptr>(new MockCommandQueueHw(&context, pDevice, 0)); cl_event event; - void *pSVM = context.getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *pSVM = context.getSVMAllocsManager()->createSVMAlloc(256, 0); const float pattern[1] = {1.2345f}; size_t zeroSize = 0; mockCmdQ->enqueueSVMMemFill(pSVM, &pattern, sizeof(pattern), zeroSize, 0, nullptr, &event); diff --git a/unit_tests/context/driver_diagnostics_enqueue_tests.cpp b/unit_tests/context/driver_diagnostics_enqueue_tests.cpp index e8b898b8d2..4243fe5038 100644 --- a/unit_tests/context/driver_diagnostics_enqueue_tests.cpp +++ b/unit_tests/context/driver_diagnostics_enqueue_tests.cpp @@ -648,7 +648,7 @@ TEST_P(PerformanceHintEnqueueMapTest, GivenZeroCopyFlagWhenEnqueueUnmapIsCalling TEST_F(PerformanceHintEnqueueTest, GivenSVMPointerWhenEnqueueSVMMapIsCallingThenContextProvidesProperHint) { - void *svmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *svmPtr = context->getSVMAllocsManager()->createSVMAlloc(256, 0); pCmdQ->enqueueSVMMap(CL_FALSE, 0, svmPtr, 256, 0, nullptr, nullptr); diff --git a/unit_tests/helpers/kernel_commands_tests.cpp b/unit_tests/helpers/kernel_commands_tests.cpp index f1eb9f2945..19a026e63b 100644 --- a/unit_tests/helpers/kernel_commands_tests.cpp +++ b/unit_tests/helpers/kernel_commands_tests.cpp @@ -7,17 +7,20 @@ #include "unit_tests/helpers/kernel_commands_tests.h" +#include "runtime/api/api.h" #include "runtime/built_ins/builtins_dispatch_builder.h" #include "runtime/command_queue/command_queue_hw.h" #include "runtime/helpers/basic_math.h" #include "runtime/helpers/kernel_commands.h" #include "runtime/memory_manager/svm_memory_manager.h" #include "unit_tests/fixtures/execution_model_kernel_fixture.h" +#include "unit_tests/fixtures/hello_world_fixture.h" #include "unit_tests/fixtures/image_fixture.h" #include "unit_tests/helpers/debug_manager_state_restore.h" #include "unit_tests/helpers/hw_parse.h" #include "unit_tests/indirect_heap/indirect_heap_fixture.h" #include "unit_tests/mocks/mock_graphics_allocation.h" +#include "unit_tests/utilities/base_object_utils.h" #include "hw_cmds.h" @@ -1359,4 +1362,32 @@ TEST_F(KernelCommandsTest, givenCacheFlushAfterWalkerEnabledWhenPlatformNotSuppo StackVec allocationsForCacheFlush; mockKernelWithInternal->mockKernel->getAllocationsForCacheFlush(allocationsForCacheFlush); EXPECT_EQ(0U, allocationsForCacheFlush.size()); -} \ No newline at end of file +} + +using KernelCacheFlushTests = Test>; + +HWTEST_F(KernelCacheFlushTests, givenLocallyUncachedBufferWhenGettingAllocationsForFlushThenEmptyVectorIsReturned) { + DebugManagerStateRestore dbgRestore; + DebugManager.flags.EnableCacheFlushAfterWalker.set(-1); + + auto kernel = clUniquePtr(Kernel::create(pProgram, *pProgram->getKernelInfo("CopyBuffer"), &retVal)); + + cl_mem_properties_intel bufferPropertiesUncachedResource[] = {CL_MEM_FLAGS_INTEL, CL_MEM_LOCALLY_UNCACHED_RESOURCE, 0}; + auto bufferLocallyUncached = clCreateBufferWithPropertiesINTEL(context, bufferPropertiesUncachedResource, 1, nullptr, nullptr); + kernel->setArg(0, sizeof(bufferLocallyUncached), &bufferLocallyUncached); + + using CacheFlushAllocationsVec = StackVec; + CacheFlushAllocationsVec cacheFlushVec; + kernel->getAllocationsForCacheFlush(cacheFlushVec); + EXPECT_EQ(0u, cacheFlushVec.size()); + + auto bufferRegular = clCreateBufferWithPropertiesINTEL(context, nullptr, 1, nullptr, nullptr); + kernel->setArg(1, sizeof(bufferRegular), &bufferRegular); + + kernel->getAllocationsForCacheFlush(cacheFlushVec); + size_t expectedCacheFlushVecSize = (hwInfoHelper.capabilityTable.supportCacheFlushAfterWalker ? 1u : 0u); + EXPECT_EQ(expectedCacheFlushVecSize, cacheFlushVec.size()); + + clReleaseMemObject(bufferLocallyUncached); + clReleaseMemObject(bufferRegular); +} diff --git a/unit_tests/kernel/clone_kernel_tests.cpp b/unit_tests/kernel/clone_kernel_tests.cpp index 4929b50712..0ca0b65968 100644 --- a/unit_tests/kernel/clone_kernel_tests.cpp +++ b/unit_tests/kernel/clone_kernel_tests.cpp @@ -509,7 +509,7 @@ TEST_F(CloneKernelTest, cloneKernelWithArgImmediate) { } TEST_F(CloneKernelTest, cloneKernelWithExecInfo) { - void *ptrSVM = pContext->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *ptrSVM = pContext->getSVMAllocsManager()->createSVMAlloc(256, 0); ASSERT_NE(nullptr, ptrSVM); GraphicsAllocation *pSvmAlloc = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSVM); diff --git a/unit_tests/kernel/kernel_arg_buffer_tests.cpp b/unit_tests/kernel/kernel_arg_buffer_tests.cpp index e8524bf373..c25fddc32c 100644 --- a/unit_tests/kernel/kernel_arg_buffer_tests.cpp +++ b/unit_tests/kernel/kernel_arg_buffer_tests.cpp @@ -169,7 +169,7 @@ TEST_F(KernelArgBufferTest, given32BitDeviceWhenArgPassedIsNullThenOnly4BytesAre EXPECT_NE(expValue, *pKernelArg64bit); } -TEST_F(KernelArgBufferTest, givenWritebleBufferWhenSettingAsArgThenExpectAllocationInCacheFlushVector) { +TEST_F(KernelArgBufferTest, givenWritebleBufferWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) { auto buffer = std::make_unique(); buffer->mockGfxAllocation.setMemObjectsAllocationWithWritableFlags(true); buffer->mockGfxAllocation.setFlushL3Required(false); @@ -179,7 +179,7 @@ TEST_F(KernelArgBufferTest, givenWritebleBufferWhenSettingAsArgThenExpectAllocat auto retVal = pKernel->setArg(0, sizeof(cl_mem *), pVal); EXPECT_EQ(CL_SUCCESS, retVal); - EXPECT_EQ(&buffer->mockGfxAllocation, pKernel->kernelArgRequiresCacheFlush[0]); + EXPECT_EQ(nullptr, pKernel->kernelArgRequiresCacheFlush[0]); } TEST_F(KernelArgBufferTest, givenCacheFlushBufferWhenSettingAsArgThenExpectAllocationInCacheFlushVector) { diff --git a/unit_tests/kernel/kernel_arg_svm_tests.cpp b/unit_tests/kernel/kernel_arg_svm_tests.cpp index 545e1964f4..5817da6d1d 100644 --- a/unit_tests/kernel/kernel_arg_svm_tests.cpp +++ b/unit_tests/kernel/kernel_arg_svm_tests.cpp @@ -414,7 +414,7 @@ HWTEST_TYPED_TEST(KernelArgSvmTestTyped, GivenBufferKernelArgWhenBufferOffsetIsN alignedFree(svmPtr); } -TEST_F(KernelArgSvmTest, givenWritebleSvmAllocationWhenSettingAsArgThenExpectAllocationInCacheFlushVector) { +TEST_F(KernelArgSvmTest, givenWritebleSvmAllocationWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) { size_t svmSize = 4096; void *svmPtr = alignedMalloc(svmSize, MemoryConstants::pageSize); MockGraphicsAllocation svmAlloc(svmPtr, svmSize); @@ -424,7 +424,7 @@ TEST_F(KernelArgSvmTest, givenWritebleSvmAllocationWhenSettingAsArgThenExpectAll auto retVal = pKernel->setArgSvmAlloc(0, svmPtr, &svmAlloc); EXPECT_EQ(CL_SUCCESS, retVal); - EXPECT_EQ(&svmAlloc, pKernel->kernelArgRequiresCacheFlush[0]); + EXPECT_EQ(nullptr, pKernel->kernelArgRequiresCacheFlush[0]); alignedFree(svmPtr); } @@ -459,7 +459,7 @@ TEST_F(KernelArgSvmTest, givenNoCacheFlushSvmAllocationWhenSettingAsArgThenNotEx alignedFree(svmPtr); } -TEST_F(KernelArgSvmTest, givenWritableSvmAllocationWhenSettingKernelExecInfoThenExpectSvmFlushFlagTrue) { +TEST_F(KernelArgSvmTest, givenWritableSvmAllocationWhenSettingKernelExecInfoThenDoNotExpectSvmFlushFlagTrue) { size_t svmSize = 4096; void *svmPtr = alignedMalloc(svmSize, MemoryConstants::pageSize); MockGraphicsAllocation svmAlloc(svmPtr, svmSize); @@ -468,7 +468,7 @@ TEST_F(KernelArgSvmTest, givenWritableSvmAllocationWhenSettingKernelExecInfoThen svmAlloc.setFlushL3Required(false); pKernel->setKernelExecInfo(&svmAlloc); - EXPECT_TRUE(pKernel->svmAllocationsRequireCacheFlush); + EXPECT_FALSE(pKernel->svmAllocationsRequireCacheFlush); alignedFree(svmPtr); } diff --git a/unit_tests/kernel/kernel_image_arg_tests.cpp b/unit_tests/kernel/kernel_image_arg_tests.cpp index d81afa170d..c6ecf0f81c 100644 --- a/unit_tests/kernel/kernel_image_arg_tests.cpp +++ b/unit_tests/kernel/kernel_image_arg_tests.cpp @@ -261,7 +261,7 @@ TEST_F(KernelImageArgTest, givenKernelWithSharedImageWhenSetArgCalledThenUsingSh EXPECT_TRUE(pKernel->isUsingSharedObjArgs()); } -TEST_F(KernelImageArgTest, givenWritebleImageWhenSettingAsArgThenExpectAllocationInCacheFlushVector) { +TEST_F(KernelImageArgTest, givenWritebleImageWhenSettingAsArgThenDoNotExpectAllocationInCacheFlushVector) { MockImageBase image; image.graphicsAllocation->setMemObjectsAllocationWithWritableFlags(true); image.graphicsAllocation->setFlushL3Required(false); @@ -270,7 +270,7 @@ TEST_F(KernelImageArgTest, givenWritebleImageWhenSettingAsArgThenExpectAllocatio pKernel->setArg(0, sizeof(imageObj), &imageObj); EXPECT_EQ(CL_SUCCESS, retVal); - EXPECT_EQ(image.graphicsAllocation, pKernel->kernelArgRequiresCacheFlush[0]); + EXPECT_EQ(nullptr, pKernel->kernelArgRequiresCacheFlush[0]); } TEST_F(KernelImageArgTest, givenCacheFlushImageWhenSettingAsArgThenExpectAllocationInCacheFlushVector) { diff --git a/unit_tests/kernel/kernel_tests.cpp b/unit_tests/kernel/kernel_tests.cpp index 8d88b1aa5a..49efa3d6f4 100644 --- a/unit_tests/kernel/kernel_tests.cpp +++ b/unit_tests/kernel/kernel_tests.cpp @@ -2420,7 +2420,7 @@ TEST(KernelTest, whenCacheFlushEnabledForAllQueuesAndKernelRequireCacheFlushAfte EXPECT_TRUE(kernel.mockKernel->requiresCacheFlushCommand(queue)); } -TEST(KernelTest, whenAllocationWriteableThenAssignAllocationPointerToCacheFlushVector) { +TEST(KernelTest, whenAllocationWriteableThenDoNotAssignAllocationPointerToCacheFlushVector) { MockGraphicsAllocation mockAllocation; auto device = std::unique_ptr(MockDevice::createWithNewExecutionEnvironment(platformDevices[0])); MockKernelWithInternals kernel(*device); @@ -2430,7 +2430,7 @@ TEST(KernelTest, whenAllocationWriteableThenAssignAllocationPointerToCacheFlushV mockAllocation.setFlushL3Required(false); kernel.mockKernel->addAllocationToCacheFlushVector(0, &mockAllocation); - EXPECT_EQ(&mockAllocation, kernel.mockKernel->kernelArgRequiresCacheFlush[0]); + EXPECT_EQ(nullptr, kernel.mockKernel->kernelArgRequiresCacheFlush[0]); } TEST(KernelTest, whenAllocationReadOnlyNonFlushRequiredThenAssignNullPointerToCacheFlushVector) { diff --git a/unit_tests/mem_obj/buffer_set_arg_tests.cpp b/unit_tests/mem_obj/buffer_set_arg_tests.cpp index 9c273e7282..f22811395b 100644 --- a/unit_tests/mem_obj/buffer_set_arg_tests.cpp +++ b/unit_tests/mem_obj/buffer_set_arg_tests.cpp @@ -261,7 +261,7 @@ TEST_F(BufferSetArgTest, clSetKernelArgBuffer) { } TEST_F(BufferSetArgTest, clSetKernelArgSVMPointer) { - void *ptrSVM = pContext->getSVMAllocsManager()->createSVMAlloc(256, false, false); + void *ptrSVM = pContext->getSVMAllocsManager()->createSVMAlloc(256, 0); EXPECT_NE(nullptr, ptrSVM); GraphicsAllocation *pSvmAlloc = pContext->getSVMAllocsManager()->getSVMAlloc(ptrSVM); diff --git a/unit_tests/mem_obj/buffer_tests.cpp b/unit_tests/mem_obj/buffer_tests.cpp index ac937282d0..c011ee244a 100644 --- a/unit_tests/mem_obj/buffer_tests.cpp +++ b/unit_tests/mem_obj/buffer_tests.cpp @@ -562,7 +562,7 @@ TEST_F(RenderCompressedBuffersTests, givenDebugVariableSetWhenHwFlagIsNotSetThen TEST_F(RenderCompressedBuffersTests, givenSvmAllocationWhenCreatingBufferThenForceDisableCompression) { localHwInfo.capabilityTable.ftrRenderCompressedBuffers = true; - auto svmAlloc = context->getSVMAllocsManager()->createSVMAlloc(sizeof(uint32_t), false, false); + auto svmAlloc = context->getSVMAllocsManager()->createSVMAlloc(sizeof(uint32_t), 0); buffer.reset(Buffer::create(context.get(), CL_MEM_USE_HOST_PTR, sizeof(uint32_t), svmAlloc, retVal)); EXPECT_EQ(buffer->getGraphicsAllocation()->getAllocationType(), GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY); @@ -956,7 +956,7 @@ TEST_P(ValidHostPtr, failedAllocationInjection) { TEST_P(ValidHostPtr, SvmHostPtr) { const DeviceInfo &devInfo = pDevice->getDeviceInfo(); if (devInfo.svmCapabilities != 0) { - auto ptr = context->getSVMAllocsManager()->createSVMAlloc(64, false, false); + auto ptr = context->getSVMAllocsManager()->createSVMAlloc(64, 0); auto bufferSvm = Buffer::create(context.get(), CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, 64, ptr, retVal); EXPECT_NE(nullptr, bufferSvm); diff --git a/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl b/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl index ad71674227..12b2926b96 100644 --- a/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl +++ b/unit_tests/memory_manager/memory_manager_allocate_in_device_pool_tests.inl @@ -55,12 +55,26 @@ TEST(AllocationFlagsTest, givenAllocateMemoryFlagWhenGetAllocationFlagsIsCalledT } TEST(UncacheableFlagsTest, givenUncachedResourceFlagWhenGetAllocationFlagsIsCalledThenUncacheableFlagIsCorrectlySet) { - auto allocationFlags = MemObjHelper::getAllocationProperties(CL_MEM_LOCALLY_UNCACHED_RESOURCE, false, 0, GraphicsAllocation::AllocationType::BUFFER); + MemoryProperties memoryProperties; + memoryProperties.flags_intel = CL_MEM_LOCALLY_UNCACHED_RESOURCE; + auto allocationFlags = MemObjHelper::getAllocationProperties(memoryProperties, false, 0, GraphicsAllocation::AllocationType::BUFFER); EXPECT_TRUE(allocationFlags.flags.uncacheable); allocationFlags = MemObjHelper::getAllocationProperties(0, false, 0, GraphicsAllocation::AllocationType::BUFFER); EXPECT_FALSE(allocationFlags.flags.uncacheable); } + +TEST(AllocationFlagsTest, givenReadOnlyResourceFlagWhenGetAllocationFlagsIsCalledThenFlushL3FlagsAreCorrectlySet) { + auto allocationFlags = + MemObjHelper::getAllocationProperties(CL_MEM_READ_ONLY, true, 0, GraphicsAllocation::AllocationType::BUFFER); + EXPECT_FALSE(allocationFlags.flags.flushL3RequiredForRead); + EXPECT_FALSE(allocationFlags.flags.flushL3RequiredForWrite); + + allocationFlags = MemObjHelper::getAllocationProperties(0, true, 0, GraphicsAllocation::AllocationType::BUFFER); + EXPECT_TRUE(allocationFlags.flags.flushL3RequiredForRead); + EXPECT_TRUE(allocationFlags.flags.flushL3RequiredForWrite); +} + TEST(StorageInfoTest, whenStorageInfoIsCreatedWithDefaultConstructorThenReturnsOneHandle) { StorageInfo storageInfo; EXPECT_EQ(1u, storageInfo.getNumHandles()); diff --git a/unit_tests/memory_manager/memory_manager_tests.cpp b/unit_tests/memory_manager/memory_manager_tests.cpp index 8955795b71..e5cda43204 100644 --- a/unit_tests/memory_manager/memory_manager_tests.cpp +++ b/unit_tests/memory_manager/memory_manager_tests.cpp @@ -1644,6 +1644,36 @@ TEST(MemoryManagerTest, givenMemoryManagerWhenAllocationWasNotUnlockedThenItIsUn EXPECT_EQ(1u, memoryManager.unlockResourceCalled); } +TEST(MemoryManagerTest, givenAllocationTypesThatMayNeedL3FlushWhenCallingGetAllocationDataThenFlushL3FlagIsCorrectlySet) { + AllocationData allocData; + AllocationProperties properties(1, GraphicsAllocation::AllocationType::UNDECIDED); + properties.flags.flushL3RequiredForRead = 1; + properties.flags.flushL3RequiredForWrite = 1; + + GraphicsAllocation::AllocationType allocationTypesThatMayNeedL3Flush[] = { + GraphicsAllocation::AllocationType::BUFFER, GraphicsAllocation::AllocationType::BUFFER_COMPRESSED, + GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY, GraphicsAllocation::AllocationType::EXTERNAL_HOST_PTR, + GraphicsAllocation::AllocationType::GLOBAL_SURFACE, GraphicsAllocation::AllocationType::IMAGE, + GraphicsAllocation::AllocationType::PIPE, GraphicsAllocation::AllocationType::SHARED_IMAGE, + GraphicsAllocation::AllocationType::SHARED_BUFFER, GraphicsAllocation::AllocationType::SHARED_RESOURCE_COPY, + GraphicsAllocation::AllocationType::SVM, GraphicsAllocation::AllocationType::UNDECIDED}; + + for (auto allocationType : allocationTypesThatMayNeedL3Flush) { + properties.allocationType = allocationType; + MockMemoryManager::getAllocationData(allocData, properties, {}, nullptr); + EXPECT_TRUE(allocData.flags.flushL3); + } + + properties.flags.flushL3RequiredForRead = 0; + properties.flags.flushL3RequiredForWrite = 0; + + for (auto allocationType : allocationTypesThatMayNeedL3Flush) { + properties.allocationType = allocationType; + MockMemoryManager::getAllocationData(allocData, properties, {}, nullptr); + EXPECT_FALSE(allocData.flags.flushL3); + } +} + TEST(HeapSelectorTest, given32bitInternalAllocationWhenSelectingHeapThenInternalHeapIsUsed) { GraphicsAllocation allocation{GraphicsAllocation::AllocationType::KERNEL_ISA, nullptr, 0, 0, 0, MemoryPool::MemoryNull, false}; allocation.set32BitAllocation(true); @@ -1699,12 +1729,14 @@ TEST(HeapSelectorTest, givenLimitedAddressSpaceWhenSelectingHeapForNullAllocatio } TEST(MemoryAllocationTest, givenAllocationTypeWhenPassedToMemoryAllocationConstructorThenAllocationTypeIsStored) { - MemoryAllocation allocation{GraphicsAllocation::AllocationType::COMMAND_BUFFER, nullptr, nullptr, 0, 0, 0, MemoryPool::MemoryNull, false}; + MemoryAllocation allocation{GraphicsAllocation::AllocationType::COMMAND_BUFFER, nullptr, nullptr, 0, 0, 0, + MemoryPool::MemoryNull, false, false, false}; EXPECT_EQ(GraphicsAllocation::AllocationType::COMMAND_BUFFER, allocation.getAllocationType()); } TEST(MemoryAllocationTest, givenMemoryPoolWhenPassedToMemoryAllocationConstructorThenMemoryPoolIsStored) { - MemoryAllocation allocation{GraphicsAllocation::AllocationType::COMMAND_BUFFER, nullptr, nullptr, 0, 0, 0, MemoryPool::System64KBPages, false}; + MemoryAllocation allocation{GraphicsAllocation::AllocationType::COMMAND_BUFFER, nullptr, nullptr, 0, 0, 0, + MemoryPool::System64KBPages, false, false, false}; EXPECT_EQ(MemoryPool::System64KBPages, allocation.getMemoryPool()); } diff --git a/unit_tests/memory_manager/svm_memory_manager_tests.cpp b/unit_tests/memory_manager/svm_memory_manager_tests.cpp index 3fb43b4381..f29eddd047 100644 --- a/unit_tests/memory_manager/svm_memory_manager_tests.cpp +++ b/unit_tests/memory_manager/svm_memory_manager_tests.cpp @@ -21,14 +21,14 @@ struct SVMMemoryAllocatorTest : ::testing::Test { }; TEST_F(SVMMemoryAllocatorTest, whenCreateZeroSizedSVMAllocationThenReturnNullptr) { - auto ptr = svmManager.createSVMAlloc(0, false, false); + auto ptr = svmManager.createSVMAlloc(0, 0); EXPECT_EQ(0u, svmManager.SVMAllocs.getNumAllocs()); EXPECT_EQ(ptr, nullptr); } TEST_F(SVMMemoryAllocatorTest, whenSVMAllocationIsFreedThenCannotBeGotAgain) { - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_NE(nullptr, ptr); EXPECT_NE(nullptr, svmManager.getSVMAlloc(ptr)); EXPECT_NE(nullptr, svmManager.getSVMAlloc(ptr)); @@ -41,7 +41,7 @@ TEST_F(SVMMemoryAllocatorTest, whenSVMAllocationIsFreedThenCannotBeGotAgain) { } TEST_F(SVMMemoryAllocatorTest, whenGetSVMAllocationFromReturnedPointerAreaThenReturnSameAllocation) { - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_NE(ptr, nullptr); GraphicsAllocation *graphicsAllocation = svmManager.getSVMAlloc(ptr); EXPECT_NE(nullptr, graphicsAllocation); @@ -56,7 +56,7 @@ TEST_F(SVMMemoryAllocatorTest, whenGetSVMAllocationFromReturnedPointerAreaThenRe } TEST_F(SVMMemoryAllocatorTest, whenGetSVMAllocationFromOutsideOfReturnedPointerAreaThenDontReturnThisAllocation) { - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_NE(ptr, nullptr); GraphicsAllocation *graphicsAllocation = svmManager.getSVMAlloc(ptr); EXPECT_NE(nullptr, graphicsAllocation); @@ -75,7 +75,7 @@ TEST_F(SVMMemoryAllocatorTest, whenGetSVMAllocationFromOutsideOfReturnedPointerA TEST_F(SVMMemoryAllocatorTest, whenCouldNotAllocateInMemoryManagerThenReturnsNullAndDoesNotChangeAllocsMap) { FailMemoryManager failMemoryManager(executionEnvironment); svmManager.memoryManager = &failMemoryManager; - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_EQ(nullptr, ptr); EXPECT_EQ(0u, svmManager.SVMAllocs.getNumAllocs()); svmManager.freeSVMAlloc(ptr); @@ -84,7 +84,7 @@ TEST_F(SVMMemoryAllocatorTest, whenCouldNotAllocateInMemoryManagerThenReturnsNul TEST_F(SVMMemoryAllocatorTest, given64kbAllowedWhenAllocatingSvmMemoryThenDontPreferRenderCompression) { MockMemoryManager memoryManager64Kb(true, false, executionEnvironment); svmManager.memoryManager = &memoryManager64Kb; - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_FALSE(memoryManager64Kb.preferRenderCompressedFlagPassed); svmManager.freeSVMAlloc(ptr); } @@ -92,19 +92,19 @@ TEST_F(SVMMemoryAllocatorTest, given64kbAllowedWhenAllocatingSvmMemoryThenDontPr TEST_F(SVMMemoryAllocatorTest, given64kbAllowedwhenAllocatingSvmMemoryThenAllocationIsIn64kbPagePool) { MockMemoryManager memoryManager64Kb(true, false, executionEnvironment); svmManager.memoryManager = &memoryManager64Kb; - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_EQ(MemoryPool::System64KBPages, svmManager.getSVMAlloc(ptr)->getMemoryPool()); svmManager.freeSVMAlloc(ptr); } TEST_F(SVMMemoryAllocatorTest, given64kbDisallowedWhenAllocatingSvmMemoryThenAllocationIsIn4kbPagePool) { - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, false, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, 0); EXPECT_EQ(MemoryPool::System4KBPages, svmManager.getSVMAlloc(ptr)->getMemoryPool()); svmManager.freeSVMAlloc(ptr); } TEST_F(SVMMemoryAllocatorTest, whenCoherentFlagIsPassedThenAllocationIsCoherent) { - auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, true, false); + auto ptr = svmManager.createSVMAlloc(MemoryConstants::pageSize, CL_MEM_SVM_FINE_GRAIN_BUFFER); EXPECT_TRUE(svmManager.getSVMAlloc(ptr)->isCoherent()); svmManager.freeSVMAlloc(ptr); } @@ -121,7 +121,7 @@ TEST_F(SVMMemoryAllocatorTest, whenNoReadOnlyFlagIsPresentThenReturnFalse) { } TEST_F(SVMMemoryAllocatorTest, whenReadOnlySvmAllocationCreatedThenGraphicsAllocationHasWriteableFlagFalse) { - void *svm = svmManager.createSVMAlloc(4096, false, true); + void *svm = svmManager.createSVMAlloc(4096, CL_MEM_READ_ONLY); EXPECT_NE(nullptr, svm); GraphicsAllocation *svmAllocation = svmManager.getSVMAlloc(svm);