feature: add support for cache reservation

Resolves: NEO-7849

Signed-off-by: Maciej Bielski <maciej.bielski@intel.com>

level_zero/core/source/cache/linux/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2021-2023 Intel Corporation
+# Copyright (C) 2021-2024 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -8,7 +8,19 @@ if(UNIX)
   target_sources(${L0_STATIC_LIB_NAME}
                  PRIVATE
                  ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
-                 ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}cache_reservation_impl.cpp
-                 ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}cache_reservation_impl.h
   )
+
+  if(NEO_ENABLE_i915_PRELIM_DETECTION)
+    target_sources(${L0_STATIC_LIB_NAME}
+                   PRIVATE
+                   ${CMAKE_CURRENT_SOURCE_DIR}/cache_reservation_impl_prelim.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/cache_reservation_impl_prelim.h
+    )
+  else()
+    target_sources(${L0_STATIC_LIB_NAME}
+                   PRIVATE
+                   ${CMAKE_CURRENT_SOURCE_DIR}/cache_reservation_impl_upstream.cpp
+                   ${CMAKE_CURRENT_SOURCE_DIR}/cache_reservation_impl_upstream.h
+    )
+  endif()
 endif()

level_zero/core/source/cache/linux/cache_reservation_impl_prelim.cpp

@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2021-2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/cache/linux/cache_reservation_impl_prelim.h"
+
+#include "shared/source/memory_manager/unified_memory_manager.h"
+#include "shared/source/os_interface/linux/cache_info.h"
+#include "shared/source/os_interface/linux/drm_allocation.h"
+#include "shared/source/os_interface/linux/drm_neo.h"
+
+#include "level_zero/core/source/device/device.h"
+#include "level_zero/core/source/driver/driver_handle.h"
+
+namespace L0 {
+
+std::unique_ptr<CacheReservation> CacheReservation::create(Device &device) {
+    return std::make_unique<CacheReservationImpl>(device);
+}
+
+bool CacheReservationImpl::reserveCache(size_t cacheLevel, size_t cacheReservationSize) {
+    auto drm = device.getOsInterface().getDriverModel()->as<NEO::Drm>();
+
+    auto cacheInfo = drm->getCacheInfo();
+
+    if (cacheReservationSize == 0) {
+        cacheInfo->freeCacheRegion(this->reservedCacheRegion);
+        this->reservedCacheRegion = NEO::CacheRegion::none;
+        this->reservedCacheSize = 0;
+        return true;
+    }
+
+    auto cacheRegion = cacheInfo->reserveCacheRegion(cacheReservationSize);
+    if (cacheRegion == NEO::CacheRegion::none) {
+        return false;
+    }
+
+    this->reservedCacheRegion = cacheRegion;
+    this->reservedCacheSize = cacheReservationSize;
+
+    return true;
+}
+
+bool CacheReservationImpl::setCacheAdvice(void *ptr, [[maybe_unused]] size_t regionSize, ze_cache_ext_region_t cacheRegion) {
+    auto cacheRegionIdx = NEO::CacheRegion::defaultRegion;
+    size_t cacheRegionSize = 0;
+
+    if (cacheRegion == ze_cache_ext_region_t::ZE_CACHE_EXT_REGION_ZE_CACHE_RESERVE_REGION) {
+        cacheRegionIdx = this->reservedCacheRegion;
+        cacheRegionSize = this->reservedCacheSize;
+    }
+
+    auto allocData = device.getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
+    if (allocData == nullptr) {
+        return false;
+    }
+
+    auto drm = device.getOsInterface().getDriverModel()->as<NEO::Drm>();
+
+    auto gpuAllocation = allocData->gpuAllocations.getGraphicsAllocation(device.getRootDeviceIndex());
+    auto drmAllocation = static_cast<NEO::DrmAllocation *>(gpuAllocation);
+
+    return drmAllocation->setCacheAdvice(drm, cacheRegionSize, cacheRegionIdx, !drmAllocation->isAllocatedInLocalMemoryPool());
+}
+
+size_t CacheReservationImpl::getMaxCacheReservationSize() {
+    auto drm = device.getOsInterface().getDriverModel()->as<NEO::Drm>();
+
+    auto cacheInfo = drm->getCacheInfo();
+    return cacheInfo->getMaxReservationCacheSize();
+}
+
+} // namespace L0

level_zero/core/source/cache/linux/cache_reservation_impl_prelim.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2021-2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "shared/source/helpers/common_types.h"
+#include "shared/source/os_interface/linux/cache_info.h"
+
+#include "level_zero/core/source/cache/cache_reservation.h"
+
+namespace L0 {
+
+class CacheReservationImpl : public CacheReservation {
+  public:
+    ~CacheReservationImpl() override = default;
+    CacheReservationImpl(Device &device) : device(device){};
+
+    bool reserveCache(size_t cacheLevel, size_t cacheReservationSize) override;
+    bool setCacheAdvice(void *ptr, size_t regionSize, ze_cache_ext_region_t cacheRegion) override;
+    size_t getMaxCacheReservationSize() override;
+
+  protected:
+    Device &device;
+    NEO::CacheRegion reservedCacheRegion = NEO::CacheRegion::none;
+    size_t reservedCacheSize = 0;
+};
+
+} // namespace L0

level_zero/core/source/cache/linux/cache_reservation_impl_upstream.cpp

@@ -1,11 +1,11 @@
 /*
- * Copyright (C) 2021 Intel Corporation
+ * Copyright (C) 2021-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
  */
 
-#include "level_zero/core/source/cache/linux/cache_reservation_impl.h"
+#include "level_zero/core/source/cache/linux/cache_reservation_impl_upstream.h"
 
 namespace L0 {
 

level_zero/core/source/cache/linux/cache_reservation_impl_upstream.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021 Intel Corporation
+ * Copyright (C) 2021-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -20,4 +20,4 @@ class CacheReservationImpl : public CacheReservation {
     size_t getMaxCacheReservationSize() override;
 };
 
-} // namespace L0
+} // namespace L0

level_zero/core/test/unit_tests/sources/cache/linux/CMakeLists.txt

@@ -1,12 +1,12 @@
 #
-# Copyright (C) 2021 Intel Corporation
+# Copyright (C) 2021-2024 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
 
-if(UNIX)
+if(UNIX AND NEO_ENABLE_i915_PRELIM_DETECTION)
   target_sources(${TARGET_NAME} PRIVATE
                  ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
-                 ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}test_cache_reservation_impl.cpp
+                 ${CMAKE_CURRENT_SOURCE_DIR}/test_cache_reservation_impl_prelim.cpp
   )
 endif()

level_zero/core/test/unit_tests/sources/cache/linux/test_cache_reservation_impl.cpp

@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2021-2022 Intel Corporation
- *
- * SPDX-License-Identifier: MIT
- *
- */
-
-#include "shared/test/common/test_macros/test.h"
-
-#include "level_zero/core/source/cache/linux/cache_reservation_impl.h"
-#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
-
-using namespace NEO;
-
-namespace L0 {
-namespace ult {
-
-class CacheReservationFixture : public DeviceFixture {
-  public:
-    void setUp() {
-        DeviceFixture::setUp();
-        auto deviceImp = static_cast<DeviceImp *>(device);
-        ASSERT_NE(nullptr, deviceImp->cacheReservation.get());
-
-        cache = deviceImp->cacheReservation.get();
-    }
-    void tearDown() {
-        DeviceFixture::tearDown();
-    }
-    CacheReservation *cache = nullptr;
-};
-
-using CacheReservationTest = Test<CacheReservationFixture>;
-
-TEST_F(CacheReservationTest, GivenCacheReservationCreatedWhenCallingReserveCacheThenReturnFalse) {
-    size_t cacheLevel = 3;
-    size_t cacheReservationSize = 1024;
-
-    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
-    EXPECT_FALSE(result);
-}
-
-TEST_F(CacheReservationTest, GivenCacheReservationCreatedWhenCallingSetCacheAdviceThenReturnFalse) {
-    void *ptr = reinterpret_cast<void *>(0x123456789);
-    size_t regionSize = 512;
-    ze_cache_ext_region_t cacheRegion = ze_cache_ext_region_t::ZE_CACHE_EXT_REGION_ZE_CACHE_REGION_DEFAULT;
-
-    auto result = cache->setCacheAdvice(ptr, regionSize, cacheRegion);
-    EXPECT_FALSE(result);
-}
-
-TEST_F(CacheReservationTest, GivenCacheReservationCreatedWhenCallingGetMaxCacheReservationSizeThenReturnZero) {
-    EXPECT_EQ(0u, cache->getMaxCacheReservationSize());
-}
-} // namespace ult
-} // namespace L0

level_zero/core/test/unit_tests/sources/cache/linux/test_cache_reservation_impl_prelim.cpp

@@ -0,0 +1,233 @@
+/*
+ * Copyright (C) 2021-2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/common/mocks/linux/mock_drm_allocation.h"
+#include "shared/test/common/mocks/mock_device.h"
+#include "shared/test/common/os_interface/linux/drm_mock_cache_info.h"
+#include "shared/test/common/os_interface/linux/drm_mock_extended.h"
+#include "shared/test/common/test_macros/hw_test.h"
+#include "shared/test/common/test_macros/test.h"
+
+#include "level_zero/core/source/cache/linux/cache_reservation_impl_prelim.h"
+#include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
+
+using namespace NEO;
+
+struct IsCacheReservationSupported {
+    template <PRODUCT_FAMILY productFamily>
+    static constexpr bool isMatched() {
+        if constexpr (IGFX_DG1 == productFamily) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+};
+
+namespace L0 {
+namespace ult {
+
+class MockCacheReservationImpl : public CacheReservationImpl {
+  public:
+    using CacheReservationImpl::reservedCacheRegion;
+    using CacheReservationImpl::reservedCacheSize;
+};
+
+class CacheReservationFixture : public DeviceFixture {
+  public:
+    void setUp() {
+        DeviceFixture::setUp();
+
+        auto &rootDeviceEnvironment{*neoDevice->executionEnvironment->rootDeviceEnvironments[0]};
+        const PRODUCT_FAMILY productFamily{rootDeviceEnvironment.getHardwareInfo()->platform.eProductFamily};
+
+        mockDrm = new DrmMockExtended(rootDeviceEnvironment);
+        mockDrm->ioctlHelper = IoctlHelper::getI915Helper(productFamily, "2.0", *mockDrm);
+        mockDrm->ioctlHelper->initialize();
+
+        mockDrm->cacheInfo.reset(new MockCacheInfo(*mockDrm, 1024, 1, 32));
+        rootDeviceEnvironment.osInterface.reset(new NEO::OSInterface);
+        rootDeviceEnvironment.osInterface->setDriverModel(std::unique_ptr<DriverModel>(mockDrm));
+        rootDeviceEnvironment.initGmm();
+
+        cache = static_cast<DeviceImp *>(device)->cacheReservation.get();
+        ASSERT_NE(nullptr, cache);
+    }
+    void tearDown() {
+        DeviceFixture::tearDown();
+    }
+
+    CacheReservation *cache = nullptr;
+    DrmMockExtended *mockDrm = nullptr;
+};
+
+using CacheReservationTest = Test<CacheReservationFixture>;
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingReserveCacheWithZeroSizeThenRemovePriorReservation, IsCacheReservationSupported) {
+    size_t cacheLevel = 3;
+    size_t cacheReservationSize = 0;
+
+    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
+    EXPECT_TRUE(result);
+
+    auto cacheImpl = static_cast<MockCacheReservationImpl *>(cache);
+    EXPECT_EQ(CacheRegion::none, cacheImpl->reservedCacheRegion);
+    EXPECT_EQ(0u, cacheImpl->reservedCacheSize);
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingReserveCacheWithInvalidSizeThenDontReserveCacheRegion, IsCacheReservationSupported) {
+    size_t cacheLevel = 3;
+    size_t cacheReservationSize = 2048;
+    ASSERT_GT(cacheReservationSize, mockDrm->getCacheInfo()->getMaxReservationCacheSize());
+
+    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
+    EXPECT_FALSE(result);
+
+    auto cacheImpl = static_cast<MockCacheReservationImpl *>(cache);
+    EXPECT_EQ(CacheRegion::none, cacheImpl->reservedCacheRegion);
+    EXPECT_EQ(0u, cacheImpl->reservedCacheSize);
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingReserveCacheWithValidSizeThenReserveCacheRegionWithGivenSize, IsCacheReservationSupported) {
+    size_t cacheLevel = 3;
+    size_t cacheReservationSize = 1024;
+    ASSERT_LE(cacheReservationSize, mockDrm->getCacheInfo()->getMaxReservationCacheSize());
+
+    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
+    EXPECT_TRUE(result);
+
+    auto cacheImpl = static_cast<MockCacheReservationImpl *>(cache);
+    EXPECT_NE(CacheRegion::none, cacheImpl->reservedCacheRegion);
+    EXPECT_EQ(1024u, cacheImpl->reservedCacheSize);
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingSetCacheAdviceWithInvalidPointerThenReturnFalse, IsCacheReservationSupported) {
+    size_t cacheLevel = 3;
+    size_t cacheReservationSize = 1024;
+    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
+    EXPECT_TRUE(result);
+
+    void *ptr = reinterpret_cast<void *>(0x123456789);
+    size_t size = 16384;
+    ze_cache_ext_region_t cacheRegion = ze_cache_ext_region_t::ZE_CACHE_EXT_REGION_ZE_CACHE_REGION_DEFAULT;
+
+    result = cache->setCacheAdvice(ptr, size, cacheRegion);
+    EXPECT_FALSE(result);
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingSetCacheAdviceOnReservedRegionWithoutPriorCacheReservationThenReturnFalse, IsCacheReservationSupported) {
+    auto cacheImpl = static_cast<MockCacheReservationImpl *>(cache);
+    EXPECT_EQ(CacheRegion::none, cacheImpl->reservedCacheRegion);
+    EXPECT_EQ(0u, cacheImpl->reservedCacheSize);
+
+    uint64_t gpuAddress = 0x1200;
+    void *ptr = reinterpret_cast<void *>(gpuAddress);
+    size_t size = 16384;
+
+    MockDrmAllocation mockAllocation(rootDeviceIndex, AllocationType::unifiedSharedMemory, MemoryPool::localMemory);
+    MockBufferObject bo(rootDeviceIndex, mockDrm, 3, 0, 0, 1);
+    mockAllocation.bufferObjects[0] = &bo;
+    mockAllocation.setCpuPtrAndGpuAddress(ptr, gpuAddress);
+
+    NEO::SvmAllocationData allocData(0);
+    allocData.size = size;
+    allocData.gpuAllocations.addAllocation(&mockAllocation);
+    device->getDriverHandle()->getSvmAllocsManager()->insertSVMAlloc(allocData);
+
+    ze_cache_ext_region_t cacheRegion = ze_cache_ext_region_t::ZE_CACHE_EXT_REGION_ZE_CACHE_RESERVE_REGION;
+
+    auto result = cache->setCacheAdvice(ptr, size, cacheRegion);
+    EXPECT_FALSE(result);
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingSetCacheAdviceOnReservedCacheRegionThenSetCacheRegionCorrectly, IsCacheReservationSupported) {
+    auto &gfxCoreHelper = neoDevice->getGfxCoreHelper();
+    if (gfxCoreHelper.getNumCacheRegions() == 0) {
+        GTEST_SKIP();
+    }
+    size_t cacheLevel = 3;
+    size_t cacheReservationSize = 1024;
+    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
+    EXPECT_TRUE(result);
+
+    uint64_t gpuAddress = 0x1200;
+    void *ptr = reinterpret_cast<void *>(gpuAddress);
+    size_t size = 16384;
+
+    MockDrmAllocation mockAllocation(rootDeviceIndex, AllocationType::unifiedSharedMemory, MemoryPool::localMemory);
+    MockBufferObject bo(rootDeviceIndex, mockDrm, 3, 0, 0, 1);
+    mockAllocation.bufferObjects[0] = &bo;
+    mockAllocation.setCpuPtrAndGpuAddress(ptr, gpuAddress);
+
+    NEO::SvmAllocationData allocData(0);
+    allocData.size = size;
+    allocData.gpuAllocations.addAllocation(&mockAllocation);
+    device->getDriverHandle()->getSvmAllocsManager()->insertSVMAlloc(allocData);
+
+    ze_cache_ext_region_t cacheRegion = ze_cache_ext_region_t::ZE_CACHE_EXT_REGION_ZE_CACHE_RESERVE_REGION;
+
+    result = cache->setCacheAdvice(ptr, size, cacheRegion);
+    EXPECT_TRUE(result);
+
+    auto svmData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
+    ASSERT_NE(nullptr, svmData);
+
+    auto drmAllocation = static_cast<DrmAllocation *>(svmData->gpuAllocations.getGraphicsAllocation(device->getRootDeviceIndex()));
+    for (auto bo : drmAllocation->getBOs()) {
+        if (bo != nullptr) {
+            EXPECT_NE(CacheRegion::defaultRegion, bo->peekCacheRegion());
+            EXPECT_EQ(static_cast<MockCacheReservationImpl *>(cache)->reservedCacheRegion, bo->peekCacheRegion());
+        }
+    }
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationSupportedWhenCallingSetCacheAdviceOnNonReservedCacheRegionThenSetCacheRegionCorrectly, IsCacheReservationSupported) {
+    auto &gfxCoreHelper = neoDevice->getGfxCoreHelper();
+    if (gfxCoreHelper.getNumCacheRegions() == 0) {
+        GTEST_SKIP();
+    }
+    size_t cacheLevel = 3;
+    size_t cacheReservationSize = 1024;
+    auto result = cache->reserveCache(cacheLevel, cacheReservationSize);
+    EXPECT_TRUE(result);
+
+    uint64_t gpuAddress = 0x1200;
+    void *ptr = reinterpret_cast<void *>(gpuAddress);
+    size_t size = 16384;
+
+    MockDrmAllocation mockAllocation(rootDeviceIndex, AllocationType::unifiedSharedMemory, MemoryPool::localMemory);
+    MockBufferObject bo(rootDeviceIndex, mockDrm, 3, 0, 0, 1);
+    mockAllocation.bufferObjects[0] = &bo;
+    mockAllocation.setCpuPtrAndGpuAddress(ptr, gpuAddress);
+
+    NEO::SvmAllocationData allocData(0);
+    allocData.size = size;
+    allocData.gpuAllocations.addAllocation(&mockAllocation);
+    device->getDriverHandle()->getSvmAllocsManager()->insertSVMAlloc(allocData);
+
+    ze_cache_ext_region_t cacheRegion = ze_cache_ext_region_t::ZE_CACHE_EXT_REGION_ZE_CACHE_NON_RESERVED_REGION;
+
+    result = cache->setCacheAdvice(ptr, size, cacheRegion);
+    EXPECT_TRUE(result);
+
+    auto svmData = device->getDriverHandle()->getSvmAllocsManager()->getSVMAlloc(ptr);
+    ASSERT_NE(nullptr, svmData);
+
+    auto drmAllocation = static_cast<DrmAllocation *>(svmData->gpuAllocations.getGraphicsAllocation(device->getRootDeviceIndex()));
+    for (auto bo : drmAllocation->getBOs()) {
+        if (bo != nullptr) {
+            EXPECT_EQ(CacheRegion::defaultRegion, bo->peekCacheRegion());
+        }
+    }
+}
+
+HWTEST2_F(CacheReservationTest, GivenCacheReservationCreatedWhenCallingGetMaxCacheReservationSizeThenReturnZero, IsCacheReservationSupported) {
+    EXPECT_EQ(mockDrm->getCacheInfo()->getMaxReservationCacheSize(), cache->getMaxCacheReservationSize());
+}
+
+} // namespace ult
+} // namespace L0