From 7dad49ccf45f8354a6a7cce464204fd213cb5045 Mon Sep 17 00:00:00 2001 From: Slawomir Milczarek Date: Tue, 9 Feb 2021 00:31:32 +0000 Subject: [PATCH] Add new setters and getters for cache policies Signed-off-by: Slawomir Milczarek --- .../helpers/memory_properties_helpers.cpp | 9 ++++-- .../helpers/memory_properties_helpers.h | 6 ++-- .../memory_properties_helpers_base.inl | 5 ++-- .../memory_properties_helpers_tests.cpp | 4 +-- .../linux/drm_memory_manager_tests.cpp | 29 +++++++++++++++++-- .../memory_manager/allocation_properties.h | 4 ++- .../memory_manager/unified_memory_manager.cpp | 12 +++++--- .../os_interface/linux/drm_allocation.cpp | 2 +- .../os_interface/linux/drm_allocation.h | 3 +- .../linux/drm_allocation_extended.cpp | 13 ++++++++- .../os_interface/linux/drm_memory_manager.cpp | 2 ++ 11 files changed, 70 insertions(+), 19 deletions(-) diff --git a/opencl/source/helpers/memory_properties_helpers.cpp b/opencl/source/helpers/memory_properties_helpers.cpp index 4c03d86f68..9d589c56cd 100644 --- a/opencl/source/helpers/memory_properties_helpers.cpp +++ b/opencl/source/helpers/memory_properties_helpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -63,7 +63,12 @@ void MemoryPropertiesHelper::fillPoliciesInProperties(AllocationProperties &allo fillCachePolicyInProperties(allocationProperties, memoryProperties.flags.locallyUncachedResource, memoryProperties.flags.readOnly, - false); + false, + 0); +} + +uint32_t MemoryPropertiesHelper::getCacheRegion(const MemoryProperties &memoryProperties) { + return 0; } } // namespace NEO diff --git a/opencl/source/helpers/memory_properties_helpers.h b/opencl/source/helpers/memory_properties_helpers.h index b168c12dac..998cbfda33 100644 --- a/opencl/source/helpers/memory_properties_helpers.h +++ b/opencl/source/helpers/memory_properties_helpers.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -46,6 +46,8 @@ class MemoryPropertiesHelper { static void fillPoliciesInProperties(AllocationProperties &allocationProperties, const MemoryProperties &memoryProperties, const HardwareInfo &hwInfo); static void fillCachePolicyInProperties(AllocationProperties &allocationProperties, bool uncached, bool readOnly, - bool deviceOnlyVisibilty); + bool deviceOnlyVisibilty, uint32_t cacheRegion); + + static uint32_t getCacheRegion(const MemoryProperties &memoryProperties); }; } // namespace NEO diff --git a/opencl/source/helpers/memory_properties_helpers_base.inl b/opencl/source/helpers/memory_properties_helpers_base.inl index d24282a194..a55c8494b1 100644 --- a/opencl/source/helpers/memory_properties_helpers_base.inl +++ b/opencl/source/helpers/memory_properties_helpers_base.inl @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -101,11 +101,12 @@ AllocationProperties MemoryPropertiesHelper::getAllocationProperties( } void MemoryPropertiesHelper::fillCachePolicyInProperties(AllocationProperties &allocationProperties, bool uncached, bool readOnly, - bool deviceOnlyVisibilty) { + bool deviceOnlyVisibilty, uint32_t cacheRegion) { allocationProperties.flags.uncacheable = uncached; auto cacheFlushRequired = !uncached && !readOnly && !deviceOnlyVisibilty; allocationProperties.flags.flushL3RequiredForRead = cacheFlushRequired; allocationProperties.flags.flushL3RequiredForWrite = cacheFlushRequired; + allocationProperties.cacheRegion = cacheRegion; } } // namespace NEO diff --git a/opencl/test/unit_test/helpers/memory_properties_helpers_tests.cpp b/opencl/test/unit_test/helpers/memory_properties_helpers_tests.cpp index 41f8bbad53..43d1ce64b6 100644 --- a/opencl/test/unit_test/helpers/memory_properties_helpers_tests.cpp +++ b/opencl/test/unit_test/helpers/memory_properties_helpers_tests.cpp @@ -294,13 +294,13 @@ TEST_F(MemoryPropertiesHelperTests, givenDifferentParametersWhenCallingFillCache if (uncached || readOnly || deviceOnlyVisibilty) { allocationProperties.flags.flushL3RequiredForRead = true; allocationProperties.flags.flushL3RequiredForWrite = true; - MemoryPropertiesHelper::fillCachePolicyInProperties(allocationProperties, uncached, readOnly, deviceOnlyVisibilty); + MemoryPropertiesHelper::fillCachePolicyInProperties(allocationProperties, uncached, readOnly, deviceOnlyVisibilty, 0); EXPECT_FALSE(allocationProperties.flags.flushL3RequiredForRead); EXPECT_FALSE(allocationProperties.flags.flushL3RequiredForWrite); } else { allocationProperties.flags.flushL3RequiredForRead = false; allocationProperties.flags.flushL3RequiredForWrite = false; - MemoryPropertiesHelper::fillCachePolicyInProperties(allocationProperties, uncached, readOnly, deviceOnlyVisibilty); + MemoryPropertiesHelper::fillCachePolicyInProperties(allocationProperties, uncached, readOnly, deviceOnlyVisibilty, 0); EXPECT_TRUE(allocationProperties.flags.flushL3RequiredForRead); EXPECT_TRUE(allocationProperties.flags.flushL3RequiredForWrite); } diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 48bf0bb337..804c9c6a7c 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -4185,6 +4185,29 @@ TEST(DrmAllocationTest, givenResourceRegistrationEnabledWhenIsaIsRegisteredThenC EXPECT_EQ(2u, drm.unregisterCalledCount); } +TEST(DrmAllocationTest, givenDrmAllocationWhenCacheInfoIsNotAvailableThenCacheRegionIsNotSet) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + + MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory); + + EXPECT_FALSE(allocation.setCacheRegion(&drm, CacheRegion::None)); +} + +TEST(DrmAllocationTest, givenDrmAllocationWhenCacheInfoIsAvailableThenCacheRegionIsNotSetForTheDefault) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + drm.setupCacheInfo(*defaultHwInfo.get()); + + MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory); + + EXPECT_FALSE(allocation.setCacheRegion(&drm, CacheRegion::Default)); +} + TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsNotSetThenReturnFalse) { auto executionEnvironment = std::make_unique(); executionEnvironment->prepareRootDeviceEnvironments(1); @@ -4194,7 +4217,7 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsNotSetThenReturnFalse MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory); - EXPECT_FALSE(allocation.setCacheRegion(&drm, 1024, CacheRegion::None)); + EXPECT_FALSE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::None)); } TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenReturnTrue) { @@ -4206,7 +4229,7 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenRe MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory); - EXPECT_TRUE(allocation.setCacheRegion(&drm, 1024, CacheRegion::Region1)); + EXPECT_TRUE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1)); } TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenSetRegionInBufferObject) { @@ -4220,7 +4243,7 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenSe MockDrmAllocation allocation(GraphicsAllocation::AllocationType::BUFFER, MemoryPool::LocalMemory); allocation.bufferObjects[0] = &bo; - EXPECT_TRUE(allocation.setCacheRegion(&drm, 1024, CacheRegion::Region1)); + EXPECT_TRUE(allocation.setCacheAdvice(&drm, 1024, CacheRegion::Region1)); for (auto bo : allocation.bufferObjects) { if (bo != nullptr) { diff --git a/shared/source/memory_manager/allocation_properties.h b/shared/source/memory_manager/allocation_properties.h index e8bfa7754b..faebd200e4 100644 --- a/shared/source/memory_manager/allocation_properties.h +++ b/shared/source/memory_manager/allocation_properties.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -41,6 +41,7 @@ struct AllocationProperties { uint64_t gpuAddress = 0; OsContext *osContext = nullptr; bool useMmapObject = true; + uint32_t cacheRegion = 0; AllocationProperties(uint32_t rootDeviceIndex, size_t size, GraphicsAllocation::AllocationType allocationType, DeviceBitfield subDevicesBitfieldParam) @@ -114,5 +115,6 @@ struct AllocationData { uint32_t rootDeviceIndex = 0; OsContext *osContext = nullptr; bool useMmapObject = true; + uint32_t cacheRegion = 0; }; } // namespace NEO diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index c43c4aff46..5b66d9567c 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -140,6 +140,7 @@ void *SVMAllocsManager::createHostUnifiedMemoryAllocation(size_t size, unifiedMemoryProperties.flags.shareable = memoryProperties.allocationFlags.flags.shareable; unifiedMemoryProperties.flags.isUSMHostAllocation = true; unifiedMemoryProperties.flags.isUSMDeviceAllocation = false; + unifiedMemoryProperties.cacheRegion = MemoryPropertiesHelper::getCacheRegion(memoryProperties.allocationFlags); auto maxRootDeviceIndex = *std::max_element(rootDeviceIndicesVector.begin(), rootDeviceIndicesVector.end(), std::less()); SvmAllocationData allocData(maxRootDeviceIndex); @@ -181,6 +182,7 @@ void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size, deviceBitfield}; unifiedMemoryProperties.flags.shareable = memoryProperties.allocationFlags.flags.shareable; unifiedMemoryProperties.flags.isUSMDeviceAllocation = true; + unifiedMemoryProperties.cacheRegion = MemoryPropertiesHelper::getCacheRegion(memoryProperties.allocationFlags); if (memoryProperties.memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY) { unifiedMemoryProperties.flags.isUSMHostAllocation = true; @@ -264,7 +266,8 @@ void *SVMAllocsManager::createUnifiedKmdMigratedAllocation(size_t size, const Sv deviceBitfield}; gpuProperties.alignment = 2 * MemoryConstants::megaByte; - MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false); + auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags); + MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false, cacheRegion); GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties); if (!allocationGpu) { return nullptr; @@ -344,7 +347,7 @@ void *SVMAllocsManager::createZeroCopySvmAllocation(size_t size, const SvmAlloca GraphicsAllocation::AllocationType::SVM_ZERO_COPY, false, // isMultiStorageAllocation deviceBitfield}; - MemoryPropertiesHelper::fillCachePolicyInProperties(properties, false, svmProperties.readOnly, false); + MemoryPropertiesHelper::fillCachePolicyInProperties(properties, false, svmProperties.readOnly, false, properties.cacheRegion); GraphicsAllocation *allocation = memoryManager->allocateGraphicsMemoryWithProperties(properties); if (!allocation) { return nullptr; @@ -372,7 +375,8 @@ void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(size_t size, co false, // isMultiStorageAllocation unifiedMemoryProperties.subdeviceBitfields.at(rootDeviceIndex)}; cpuProperties.alignment = 2 * MemoryConstants::megaByte; - MemoryPropertiesHelper::fillCachePolicyInProperties(cpuProperties, false, svmProperties.readOnly, false); + auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags); + MemoryPropertiesHelper::fillCachePolicyInProperties(cpuProperties, false, svmProperties.readOnly, false, cacheRegion); GraphicsAllocation *allocationCpu = memoryManager->allocateGraphicsMemoryWithProperties(cpuProperties); if (!allocationCpu) { return nullptr; @@ -389,7 +393,7 @@ void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(size_t size, co unifiedMemoryProperties.subdeviceBitfields.at(rootDeviceIndex)}; gpuProperties.alignment = 2 * MemoryConstants::megaByte; - MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false); + MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false, cacheRegion); GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties, svmPtr); if (!allocationGpu) { memoryManager->freeGraphicsMemory(allocationCpu); diff --git a/shared/source/os_interface/linux/drm_allocation.cpp b/shared/source/os_interface/linux/drm_allocation.cpp index a94062d075..d5d672636d 100644 --- a/shared/source/os_interface/linux/drm_allocation.cpp +++ b/shared/source/os_interface/linux/drm_allocation.cpp @@ -29,7 +29,7 @@ uint64_t DrmAllocation::peekInternalHandle(MemoryManager *memoryManager) { return static_cast((static_cast(memoryManager))->obtainFdFromHandle(getBO()->peekHandle(), this->rootDeviceIndex)); } -bool DrmAllocation::setCacheRegion(Drm *drm, size_t regionSize, CacheRegion regionIndex) { +bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex) { if (!drm->getCacheInfo()->getCacheRegion(regionSize, regionIndex)) { return false; } diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 18602f6c97..bf9f366d0a 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -64,7 +64,8 @@ class DrmAllocation : public GraphicsAllocation { uint64_t peekInternalHandle(MemoryManager *memoryManager) override; - bool setCacheRegion(Drm *drm, size_t regionSize, CacheRegion regionIndex); + bool setCacheRegion(Drm *drm, CacheRegion regionIndex); + bool setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex); void *getMmapPtr() { return this->mmapPtr; } void setMmapPtr(void *ptr) { this->mmapPtr = ptr; } diff --git a/shared/source/os_interface/linux/drm_allocation_extended.cpp b/shared/source/os_interface/linux/drm_allocation_extended.cpp index 3e2cf16a6a..f0fb84dc79 100644 --- a/shared/source/os_interface/linux/drm_allocation_extended.cpp +++ b/shared/source/os_interface/linux/drm_allocation_extended.cpp @@ -1,12 +1,14 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * */ +#include "shared/source/os_interface/linux/cache_info_impl.h" #include "shared/source/os_interface/linux/drm_allocation.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" +#include "shared/source/os_interface/linux/drm_neo.h" #include "shared/source/os_interface/os_context.h" namespace NEO { @@ -16,4 +18,13 @@ void DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vect bindBO(bo, osContext, vmHandleId, bufferObjects, bind); } +bool DrmAllocation::setCacheRegion(Drm *drm, CacheRegion regionIndex) { + auto cacheInfo = static_cast(drm->getCacheInfo()); + if (cacheInfo == nullptr) { + return false; + } + + return setCacheAdvice(drm, 0, regionIndex); +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 2a4ce6e4df..12f246a357 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -221,6 +221,7 @@ DrmAllocation *DrmMemoryManager::createGraphicsAllocation(OsHandleStorage &handl auto hostPtr = const_cast(allocationData.hostPtr); auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, nullptr, hostPtr, castToUint64(hostPtr), allocationData.size, MemoryPool::System4KBPages); allocation->fragmentsStorage = handleStorage; + allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion)); return allocation; } @@ -273,6 +274,7 @@ DrmAllocation *DrmMemoryManager::createAllocWithAlignmentFromUserptr(const Alloc auto allocation = new DrmAllocation(allocationData.rootDeviceIndex, allocationData.type, bo, res, bo->gpuAddress, size, MemoryPool::System4KBPages); allocation->setDriverAllocatedCpuPtr(res); allocation->setReservedAddressRange(reinterpret_cast(gpuAddress), alignedSVMSize); + allocation->setCacheRegion(&this->getDrm(allocationData.rootDeviceIndex), static_cast(allocationData.cacheRegion)); return allocation; }