diff --git a/opencl/test/unit_test/os_interface/linux/CMakeLists.txt b/opencl/test/unit_test/os_interface/linux/CMakeLists.txt index f98d041fcc..39c1c3ea4c 100644 --- a/opencl/test/unit_test/os_interface/linux/CMakeLists.txt +++ b/opencl/test/unit_test/os_interface/linux/CMakeLists.txt @@ -14,7 +14,6 @@ set(IGDRCL_SRCS_tests_os_interface_linux ${CMAKE_CURRENT_SOURCE_DIR}/device_os_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/driver_info_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_buffer_object_tests.cpp - ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}drm_cache_info_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_command_stream_mm_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_command_stream_tests_1.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_command_stream_tests_2.cpp @@ -54,6 +53,7 @@ if(NEO_ENABLE_i915_PRELIM_DETECTION) list(APPEND IGDRCL_SRCS_tests_os_interface_linux ${CMAKE_CURRENT_SOURCE_DIR}/ioctl_helper_tests_prelim.cpp ${CMAKE_CURRENT_SOURCE_DIR}/prelim_helper_func.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/drm_cache_info_tests.cpp ) if(TESTS_DG1 AND "${BRANCH_TYPE}" STREQUAL "") diff --git a/opencl/test/unit_test/os_interface/linux/drm_cache_info_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_cache_info_tests.cpp index a99ab12e67..f88d98393e 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_cache_info_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_cache_info_tests.cpp @@ -1,18 +1,174 @@ /* - * Copyright (C) 2021 Intel Corporation + * Copyright (C) 2022 Intel Corporation * * SPDX-License-Identifier: MIT * */ +#include "shared/source/helpers/constants.h" #include "shared/source/os_interface/linux/cache_info_impl.h" +#include "shared/test/common/helpers/debug_manager_state_restore.h" +#include "shared/test/common/helpers/default_hw_info.h" +#include "shared/test/common/libult/linux/drm_query_mock.h" +#include "shared/test/common/os_interface/linux/drm_mock_cache_info.h" -#include "gtest/gtest.h" +#include "opencl/test/unit_test/helpers/gtest_helpers.h" + +#include "gmock/gmock.h" using namespace NEO; -TEST(DrmCacheInfoTest, givenCacheInfoCreatedWhenCallingGetCacheRegionThenReturnZero) { - CacheInfoImpl cacheInfo; +TEST(DrmCacheInfoTest, givenCacheRegionsExistsWhenCallingSetUpCacheInfoThenCacheInfoIsCreatedAndReturnsMaxReservationCacheLimits) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); - EXPECT_FALSE(cacheInfo.getCacheRegion(1024, CacheRegion::Default)); + auto &hwHelper = HwHelper::get(drm.context.hwInfo->platform.eRenderCoreFamily); + + drm.setupCacheInfo(*defaultHwInfo.get()); + EXPECT_NE(nullptr, drm.getCacheInfo()); + auto cacheInfo = static_cast(drm.getCacheInfo()); + + if (hwHelper.getNumCacheRegions() == 0) { + EXPECT_EQ(0u, cacheInfo->getMaxReservationCacheSize()); + EXPECT_EQ(0u, cacheInfo->getMaxReservationNumCacheRegions()); + EXPECT_EQ(0u, cacheInfo->getMaxReservationNumWays()); + } else { + const GT_SYSTEM_INFO *gtSysInfo = &defaultHwInfo->gtSystemInfo; + constexpr uint16_t maxNumWays = 32; + constexpr uint16_t globalReservationLimit = 16; + constexpr uint16_t clientReservationLimit = 8; + constexpr uint16_t maxReservationNumWays = std::min(globalReservationLimit, clientReservationLimit); + const size_t totalCacheSize = gtSysInfo->L3CacheSizeInKb * MemoryConstants::kiloByte; + const size_t maxReservationCacheSize = (totalCacheSize * maxReservationNumWays) / maxNumWays; + const size_t maxReservationNumCacheRegions = hwHelper.getNumCacheRegions() - 1; + + EXPECT_EQ(maxReservationCacheSize, cacheInfo->getMaxReservationCacheSize()); + EXPECT_EQ(maxReservationNumCacheRegions, cacheInfo->getMaxReservationNumCacheRegions()); + EXPECT_EQ(maxReservationNumWays, cacheInfo->getMaxReservationNumWays()); + } +} + +TEST(DrmCacheInfoTest, givenDebugFlagSetWhenCallingSetUpCacheInfoThenCacheInfoIsCreatedWithoutValues) { + DebugManagerStateRestore restorer; + DebugManager.flags.ClosEnabled.set(0); + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + + drm.setupCacheInfo(*defaultHwInfo.get()); + EXPECT_NE(nullptr, drm.getCacheInfo()); + auto cacheInfo = static_cast(drm.getCacheInfo()); + + EXPECT_EQ(0u, cacheInfo->getMaxReservationCacheSize()); + EXPECT_EQ(0u, cacheInfo->getMaxReservationNumCacheRegions()); + EXPECT_EQ(0u, cacheInfo->getMaxReservationNumWays()); +} + +TEST(DrmCacheInfoTest, givenCacheInfoCreatedWhenGetCacheRegionSucceedsToReserveCacheRegionThenReturnTrue) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + CacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, 32); + size_t cacheReservationSize = cacheInfo.getMaxReservationCacheSize(); + + EXPECT_TRUE(cacheInfo.getCacheRegion(cacheReservationSize, CacheRegion::Region1)); + + EXPECT_EQ(CacheRegion::Region1, cacheInfo.freeCacheRegion(CacheRegion::Region1)); +} + +TEST(DrmCacheInfoTest, givenCacheInfoCreatedWhenGetCacheRegionFailsToReserveCacheRegionThenReturnFalse) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + CacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, 32); + size_t cacheReservationSize = cacheInfo.getMaxReservationCacheSize(); + + drm.context.closIndex = 0xFFFF; + EXPECT_FALSE(cacheInfo.getCacheRegion(cacheReservationSize, CacheRegion::Region1)); + + EXPECT_EQ(CacheRegion::None, cacheInfo.freeCacheRegion(CacheRegion::Region1)); +} + +TEST(DrmCacheInfoTest, givenCacheInfoWithReservedCacheRegionWhenGetCacheRegionIsCalledForReservedCacheRegionThenReturnTrue) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + CacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, 32); + size_t cacheReservationSize = cacheInfo.getMaxReservationCacheSize(); + + EXPECT_EQ(CacheRegion::Region1, cacheInfo.reserveCacheRegion(cacheReservationSize)); + + EXPECT_TRUE(cacheInfo.getCacheRegion(cacheReservationSize, CacheRegion::Region1)); + + EXPECT_EQ(CacheRegion::Region1, cacheInfo.freeCacheRegion(CacheRegion::Region1)); +} + +TEST(DrmCacheInfoTest, givenCacheInfoCreatedWhenGetCacheRegionIsCalledForReservableRegionsWithRegionSizesInverselyProportionalToNumCacheRegionsThenReturnTrue) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + CacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, 32); + size_t regionSize = cacheInfo.getMaxReservationCacheSize() / cacheInfo.getMaxReservationNumCacheRegions(); + + EXPECT_TRUE(cacheInfo.getCacheRegion(regionSize, CacheRegion::Region1)); + EXPECT_TRUE(cacheInfo.getCacheRegion(regionSize, CacheRegion::Region2)); + + EXPECT_EQ(CacheRegion::Region2, cacheInfo.freeCacheRegion(CacheRegion::Region2)); + EXPECT_EQ(CacheRegion::Region1, cacheInfo.freeCacheRegion(CacheRegion::Region1)); +} + +TEST(DrmCacheInfoTest, givenCacheInfoWhenSpecificNumCacheWaysIsRequestedThenReserveAppropriateCacheSize) { + constexpr uint16_t maxNumCacheWays = 32; + + DebugManagerStateRestore restorer; + DebugManager.flags.ClosNumCacheWays.set(maxNumCacheWays / 2); + + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + MockCacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, maxNumCacheWays); + size_t maxReservationCacheSize = cacheInfo.getMaxReservationCacheSize(); + + EXPECT_EQ(CacheRegion::Region1, cacheInfo.reserveCacheRegion(maxReservationCacheSize)); + EXPECT_TRUE(cacheInfo.isRegionReserved(CacheRegion::Region1, maxReservationCacheSize)); + + auto cacheRegion = cacheInfo.cacheRegionsReserved.begin(); + EXPECT_EQ(CacheRegion::Region1, cacheRegion->first); + EXPECT_EQ(maxReservationCacheSize / 2, cacheRegion->second); + + EXPECT_EQ(CacheRegion::Region1, cacheInfo.freeCacheRegion(CacheRegion::Region1)); +} + +TEST(DrmCacheInfoTest, givenCacheInfoWhenNumCacheWaysIsExceededThenDontReserveCacheRegion) { + constexpr uint16_t maxNumCacheWays = 32; + + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + MockCacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, maxNumCacheWays); + size_t maxReservationCacheSize = cacheInfo.getMaxReservationCacheSize(); + + EXPECT_EQ(CacheRegion::Region1, cacheInfo.reserveCacheRegion(maxReservationCacheSize)); + EXPECT_TRUE(cacheInfo.isRegionReserved(CacheRegion::Region1, maxReservationCacheSize)); + + EXPECT_EQ(CacheRegion::None, cacheInfo.reserveCacheRegion(maxReservationCacheSize)); + EXPECT_FALSE(cacheInfo.isRegionReserved(CacheRegion::Region2, maxReservationCacheSize)); +} + +TEST(DrmCacheInfoTest, givenCacheInfoCreatedWhenFreeCacheRegionIsCalledForNonReservedRegionThenItFails) { + auto executionEnvironment = std::make_unique(); + executionEnvironment->prepareRootDeviceEnvironments(1); + + DrmQueryMock drm(*executionEnvironment->rootDeviceEnvironments[0]); + MockCacheInfoImpl cacheInfo(drm, 32 * MemoryConstants::kiloByte, 2, 32); + + cacheInfo.cacheRegionsReserved.insert({CacheRegion::Region1, MemoryConstants::kiloByte}); + EXPECT_EQ(CacheRegion::None, cacheInfo.freeCacheRegion(CacheRegion::Region1)); } diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 82f4c460af..a703eedab6 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -36,6 +36,7 @@ #include "shared/test/common/mocks/mock_allocation_properties.h" #include "shared/test/common/mocks/mock_gfx_partition.h" #include "shared/test/common/mocks/mock_gmm.h" +#include "shared/test/common/os_interface/linux/drm_mock_cache_info.h" #include "shared/test/common/test_macros/matchers.h" #include "shared/test/common/test_macros/test.h" @@ -46,7 +47,6 @@ #include "opencl/test/unit_test/mocks/mock_cl_device.h" #include "opencl/test/unit_test/mocks/mock_context.h" #include "opencl/test/unit_test/mocks/mock_platform.h" -#include "opencl/test/unit_test/os_interface/linux/drm_mock_cache_info.h" #include "drm/i915_drm.h" #include "gmock/gmock.h" @@ -4471,7 +4471,7 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsNotSetThenReturnFalse executionEnvironment->prepareRootDeviceEnvironments(1); DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]); - drm.cacheInfo.reset(new MockCacheInfo()); + drm.cacheInfo.reset(new MockCacheInfoImpl(drm, 32 * MemoryConstants::kiloByte, 2, 32)); MockDrmAllocation allocation(AllocationType::BUFFER, MemoryPool::LocalMemory); @@ -4483,7 +4483,7 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenRe executionEnvironment->prepareRootDeviceEnvironments(1); DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]); - drm.cacheInfo.reset(new MockCacheInfo()); + drm.cacheInfo.reset(new MockCacheInfoImpl(drm, 32 * MemoryConstants::kiloByte, 2, 32)); MockDrmAllocation allocation(AllocationType::BUFFER, MemoryPool::LocalMemory); @@ -4495,7 +4495,7 @@ TEST(DrmAllocationTest, givenDrmAllocationWhenCacheRegionIsSetSuccessfullyThenSe executionEnvironment->prepareRootDeviceEnvironments(1); DrmMock drm(*executionEnvironment->rootDeviceEnvironments[0]); - drm.cacheInfo.reset(new MockCacheInfo()); + drm.cacheInfo.reset(new MockCacheInfoImpl(drm, 32 * MemoryConstants::kiloByte, 2, 32)); MockBufferObject bo(&drm, 0, 0, 1); MockDrmAllocation allocation(AllocationType::BUFFER, MemoryPool::LocalMemory); @@ -4591,7 +4591,7 @@ TEST(DrmAllocationTest, givenBoWhenMarkingForCaptureThenBosAreMarked) { TEST_F(DrmMemoryManagerTest, givenDrmAllocationWithHostPtrWhenItIsCreatedWithCacheRegionThenSetRegionInBufferObject) { mock->ioctl_expected.total = -1; auto drm = static_cast(executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface->getDriverModel()->as()); - drm->cacheInfo.reset(new MockCacheInfo()); + drm->cacheInfo.reset(new MockCacheInfoImpl(*drm, 32 * MemoryConstants::kiloByte, 2, 32)); auto ptr = reinterpret_cast(0x1000); auto size = MemoryConstants::pageSize; diff --git a/opencl/test/unit_test/os_interface/linux/drm_mock_cache_info.h b/opencl/test/unit_test/os_interface/linux/drm_mock_cache_info.h deleted file mode 100644 index 4268f5a97d..0000000000 --- a/opencl/test/unit_test/os_interface/linux/drm_mock_cache_info.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2021 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#pragma once - -#include "shared/source/os_interface/linux/cache_info.h" - -namespace NEO { - -struct MockCacheInfo : public CacheInfo { - MockCacheInfo() {} - - ~MockCacheInfo() override = default; - - bool getCacheRegion(size_t regionSize, CacheRegion regionIndex) override { - return (regionIndex < CacheRegion::Count) ? true : false; - } -}; - -} // namespace NEO diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config index 785101cb46..2858e1c7c6 100644 --- a/opencl/test/unit_test/test_files/igdrcl.config +++ b/opencl/test/unit_test/test_files/igdrcl.config @@ -359,6 +359,7 @@ ProgramGlobalFenceAsKernelInstructionInEUKernel = -1 DoNotReportTile1BscWaActive = -1 ForceTile0PlacementForTile1ResourcesWaActive = -1 ClosEnabled = -1 +ClosNumCacheWays = -1 EngineUsageHint = -1 AddStatePrefetchCmdToMemoryPrefetchAPI = -1 UpdateCrossThreadDataSize = 0 diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 9bd035a398..68b03bf4ee 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -174,6 +174,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, ProgramGlobalFenceAsKernelInstructionInEUKernel, DECLARE_DEBUG_VARIABLE(int32_t, DoNotReportTile1BscWaActive, -1, "-1: default, 0: WA Disabled, 1: WA enabled") DECLARE_DEBUG_VARIABLE(int32_t, ForceTile0PlacementForTile1ResourcesWaActive, -1, "-1: default, 0: WA Disabled, 1: WA enabled") DECLARE_DEBUG_VARIABLE(int32_t, ClosEnabled, -1, "-1: default, 0: disabled, 1: enabled. Enable CLOS based cache reservation") +DECLARE_DEBUG_VARIABLE(int32_t, ClosNumCacheWays, -1, "-1: default, >= 0 Number of cache ways to allocate within previously reserved CLOS regions (0-32)") DECLARE_DEBUG_VARIABLE(int32_t, EngineUsageHint, -1, "-1: default, >=0: engine usage value to use when creating command queue on user selected engine") DECLARE_DEBUG_VARIABLE(int32_t, ForceBcsEngineIndex, -1, "-1: default, >=0 Copy Engine index") DECLARE_DEBUG_VARIABLE(int32_t, Force2dImageAsArray, -1, "-1: default, 0: WA Disabled, 1: Forces surface state of 2dImage to array") diff --git a/shared/source/os_interface/linux/CMakeLists.txt b/shared/source/os_interface/linux/CMakeLists.txt index 67d9f067fe..4e5ce79edf 100644 --- a/shared/source/os_interface/linux/CMakeLists.txt +++ b/shared/source/os_interface/linux/CMakeLists.txt @@ -8,7 +8,9 @@ set(NEO_CORE_OS_INTERFACE_LINUX ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/allocator_helper.h ${CMAKE_CURRENT_SOURCE_DIR}/cache_info.h - ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}cache_info_impl.h + ${CMAKE_CURRENT_SOURCE_DIR}/cache_info_impl.h + ${CMAKE_CURRENT_SOURCE_DIR}/clos_cache.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/clos_cache.h ${CMAKE_CURRENT_SOURCE_DIR}/device_command_stream.inl ${CMAKE_CURRENT_SOURCE_DIR}/device_time_drm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/device_time_drm.h diff --git a/shared/source/os_interface/linux/cache_info_impl.h b/shared/source/os_interface/linux/cache_info_impl.h index 220afed2a3..4d49238580 100644 --- a/shared/source/os_interface/linux/cache_info_impl.h +++ b/shared/source/os_interface/linux/cache_info_impl.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Intel Corporation + * Copyright (C) 2021-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,16 +7,121 @@ #pragma once +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/helpers/debug_helpers.h" #include "shared/source/os_interface/linux/cache_info.h" +#include "shared/source/os_interface/linux/clos_cache.h" +#include "shared/source/utilities/spinlock.h" + +#include namespace NEO { -struct CacheInfoImpl : public CacheInfo { - CacheInfoImpl() {} +class Drm; - ~CacheInfoImpl() override = default; +enum class CacheLevel : uint16_t { + Default = 0, + Level3 = 3 +}; - bool getCacheRegion(size_t regionSize, CacheRegion regionIndex) override { return false; } +class CacheInfoImpl : public CacheInfo { + public: + CacheInfoImpl(Drm &drm, size_t maxReservationCacheSize, uint32_t maxReservationNumCacheRegions, uint16_t maxReservationNumWays) + : maxReservationCacheSize(maxReservationCacheSize), maxReservationNumCacheRegions(maxReservationNumCacheRegions), maxReservationNumWays(maxReservationNumWays), cacheReserve(drm) { + } + + ~CacheInfoImpl() override { + for (auto const &cacheRegion : cacheRegionsReserved) { + cacheReserve.freeCache(CacheLevel::Level3, cacheRegion.first); + } + cacheRegionsReserved.clear(); + } + + size_t getMaxReservationCacheSize() const { + return maxReservationCacheSize; + } + + size_t getMaxReservationNumCacheRegions() const { + return maxReservationNumCacheRegions; + } + + size_t getMaxReservationNumWays() const { + return maxReservationNumWays; + } + + CacheRegion reserveCacheRegion(size_t cacheReservationSize) { + std::unique_lock lock{mtx}; + return reserveRegion(cacheReservationSize); + } + + CacheRegion freeCacheRegion(CacheRegion regionIndex) { + std::unique_lock lock{mtx}; + return freeRegion(regionIndex); + } + + bool getCacheRegion(size_t regionSize, CacheRegion regionIndex) override { + std::unique_lock lock{mtx}; + return getRegion(regionSize, regionIndex); + } + + protected: + CacheRegion reserveRegion(size_t cacheReservationSize) { + uint16_t numWays = (maxReservationNumWays * cacheReservationSize) / maxReservationCacheSize; + if (DebugManager.flags.ClosNumCacheWays.get() != -1) { + numWays = DebugManager.flags.ClosNumCacheWays.get(); + cacheReservationSize = (numWays * maxReservationCacheSize) / maxReservationNumWays; + } + auto regionIndex = cacheReserve.reserveCache(CacheLevel::Level3, numWays); + if (regionIndex == CacheRegion::None) { + return CacheRegion::None; + } + cacheRegionsReserved.insert({regionIndex, cacheReservationSize}); + + return regionIndex; + } + + CacheRegion freeRegion(CacheRegion regionIndex) { + auto search = cacheRegionsReserved.find(regionIndex); + if (search != cacheRegionsReserved.end()) { + cacheRegionsReserved.erase(search); + return cacheReserve.freeCache(CacheLevel::Level3, regionIndex); + } + return CacheRegion::None; + } + + bool isRegionReserved(CacheRegion regionIndex, [[maybe_unused]] size_t regionSize) const { + auto search = cacheRegionsReserved.find(regionIndex); + if (search != cacheRegionsReserved.end()) { + if (DebugManager.flags.ClosNumCacheWays.get() != -1) { + auto numWays = DebugManager.flags.ClosNumCacheWays.get(); + regionSize = (numWays * maxReservationCacheSize) / maxReservationNumWays; + } + DEBUG_BREAK_IF(search->second != regionSize); + return true; + } + return false; + } + + bool getRegion(size_t regionSize, CacheRegion regionIndex) { + if (regionIndex == CacheRegion::Default) { + return true; + } + if (!isRegionReserved(regionIndex, regionSize)) { + auto regionIdx = reserveRegion(regionSize); + if (regionIdx == CacheRegion::None) { + return false; + } + DEBUG_BREAK_IF(regionIdx != regionIndex); + } + return true; + } + + size_t maxReservationCacheSize; + uint32_t maxReservationNumCacheRegions; + uint16_t maxReservationNumWays; + ClosCacheReservation cacheReserve; + std::unordered_map cacheRegionsReserved; + SpinLock mtx; }; } // namespace NEO diff --git a/shared/source/os_interface/linux/clos_cache.cpp b/shared/source/os_interface/linux/clos_cache.cpp new file mode 100644 index 0000000000..6d66b7c13a --- /dev/null +++ b/shared/source/os_interface/linux/clos_cache.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2021-2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/linux/clos_cache.h" + +#include "shared/source/debug_settings/debug_settings_manager.h" +#include "shared/source/os_interface/linux/drm_neo.h" +#include "shared/source/os_interface/linux/ioctl_helper.h" + +#include +#include + +namespace NEO { + +CacheRegion ClosCacheReservation::reserveCache(CacheLevel cacheLevel, uint16_t numWays) { + auto closIndex = allocEntry(); + if (closIndex == CacheRegion::None) { + return CacheRegion::None; + } + + auto allocWays = allocCacheWay(closIndex, cacheLevel, numWays); + if (allocWays != numWays) { + freeEntry(closIndex); + return CacheRegion::None; + } + + return closIndex; +} + +CacheRegion ClosCacheReservation::freeCache(CacheLevel cacheLevel, CacheRegion closIndex) { + allocCacheWay(closIndex, cacheLevel, 0); + + return freeEntry(closIndex); +} + +CacheRegion ClosCacheReservation::allocEntry() { + return drm.getIoctlHelper()->closAlloc(&drm); +} + +CacheRegion ClosCacheReservation::freeEntry(CacheRegion closIndex) { + return drm.getIoctlHelper()->closFree(&drm, closIndex); +} + +uint16_t ClosCacheReservation::allocCacheWay(CacheRegion closIndex, CacheLevel cacheLevel, uint16_t numWays) { + return drm.getIoctlHelper()->closAllocWays(&drm, closIndex, static_cast(cacheLevel), numWays); +} + +} // namespace NEO diff --git a/shared/source/os_interface/linux/clos_cache.h b/shared/source/os_interface/linux/clos_cache.h new file mode 100644 index 0000000000..9f97012899 --- /dev/null +++ b/shared/source/os_interface/linux/clos_cache.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021-2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include +#include + +namespace NEO { + +class Drm; +enum class CacheLevel : uint16_t; +enum class CacheRegion : uint16_t; + +class ClosCacheReservation { + public: + ClosCacheReservation(Drm &drm) : drm(drm) {} + + CacheRegion reserveCache(CacheLevel cacheLevel, uint16_t numWays); + CacheRegion freeCache(CacheLevel cacheLevel, CacheRegion closIndex); + + protected: + CacheRegion allocEntry(); + CacheRegion freeEntry(CacheRegion closIndex); + uint16_t allocCacheWay(CacheRegion closIndex, CacheLevel cacheLevel, uint16_t numWays); + + Drm &drm; +}; + +} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_allocation.cpp b/shared/source/os_interface/linux/drm_allocation.cpp index be78a3f34c..6f0a4e327a 100644 --- a/shared/source/os_interface/linux/drm_allocation.cpp +++ b/shared/source/os_interface/linux/drm_allocation.cpp @@ -8,6 +8,7 @@ #include "shared/source/os_interface/linux/drm_allocation.h" #include "shared/source/memory_manager/residency.h" +#include "shared/source/os_interface/linux/cache_info_impl.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" #include "shared/source/os_interface/linux/drm_memory_manager.h" #include "shared/source/os_interface/linux/ioctl_helper.h" @@ -47,6 +48,24 @@ void DrmAllocation::setCachePolicy(CachePolicy memType) { } } +bool DrmAllocation::setCacheRegion(Drm *drm, CacheRegion regionIndex) { + if (regionIndex == CacheRegion::Default) { + return true; + } + + auto cacheInfo = static_cast(drm->getCacheInfo()); + if (cacheInfo == nullptr) { + return false; + } + + auto regionSize = (cacheInfo->getMaxReservationNumCacheRegions() > 0) ? cacheInfo->getMaxReservationCacheSize() / cacheInfo->getMaxReservationNumCacheRegions() : 0; + if (regionSize == 0) { + return false; + } + + return setCacheAdvice(drm, regionSize, regionIndex); +} + bool DrmAllocation::setCacheAdvice(Drm *drm, size_t regionSize, CacheRegion regionIndex) { if (!drm->getCacheInfo()->getCacheRegion(regionSize, regionIndex)) { return false; diff --git a/shared/source/os_interface/linux/drm_allocation_extended.cpp b/shared/source/os_interface/linux/drm_allocation_extended.cpp index 111d2046e6..42e5363847 100644 --- a/shared/source/os_interface/linux/drm_allocation_extended.cpp +++ b/shared/source/os_interface/linux/drm_allocation_extended.cpp @@ -5,7 +5,6 @@ * */ -#include "shared/source/os_interface/linux/cache_info_impl.h" #include "shared/source/os_interface/linux/drm_allocation.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" #include "shared/source/os_interface/linux/drm_neo.h" @@ -18,19 +17,6 @@ int DrmAllocation::bindBOs(OsContext *osContext, uint32_t vmHandleId, std::vecto return bindBO(bo, osContext, vmHandleId, bufferObjects, bind); } -bool DrmAllocation::setCacheRegion(Drm *drm, CacheRegion regionIndex) { - if (regionIndex == CacheRegion::Default) { - return true; - } - - auto cacheInfo = static_cast(drm->getCacheInfo()); - if (cacheInfo == nullptr) { - return false; - } - - return setCacheAdvice(drm, 0, regionIndex); -} - bool DrmAllocation::shouldAllocationPageFault(const Drm *drm) { return false; } diff --git a/shared/source/os_interface/linux/drm_neo.cpp b/shared/source/os_interface/linux/drm_neo.cpp index fb6125ed0c..815f45d630 100644 --- a/shared/source/os_interface/linux/drm_neo.cpp +++ b/shared/source/os_interface/linux/drm_neo.cpp @@ -16,6 +16,7 @@ #include "shared/source/helpers/hw_info.h" #include "shared/source/helpers/ptr_math.h" #include "shared/source/os_interface/driver_info.h" +#include "shared/source/os_interface/linux/cache_info_impl.h" #include "shared/source/os_interface/linux/drm_gem_close_worker.h" #include "shared/source/os_interface/linux/drm_memory_manager.h" #include "shared/source/os_interface/linux/hw_device_id.h" @@ -938,6 +939,27 @@ void Drm::setupSystemInfo(HardwareInfo *hwInfo, SystemInfo *sysInfo) { gtSysInfo->MaxDualSubSlicesSupported = sysInfo->getMaxDualSubSlicesSupported(); } +void Drm::setupCacheInfo(const HardwareInfo &hwInfo) { + auto &hwHelper = HwHelper::get(hwInfo.platform.eRenderCoreFamily); + + if (DebugManager.flags.ClosEnabled.get() == 0 || hwHelper.getNumCacheRegions() == 0) { + this->cacheInfo.reset(new CacheInfoImpl(*this, 0, 0, 0)); + return; + } + + const GT_SYSTEM_INFO *gtSysInfo = &hwInfo.gtSystemInfo; + + constexpr uint16_t maxNumWays = 32; + constexpr uint16_t globalReservationLimit = 16; + constexpr uint16_t clientReservationLimit = 8; + constexpr uint16_t maxReservationNumWays = std::min(globalReservationLimit, clientReservationLimit); + const size_t totalCacheSize = gtSysInfo->L3CacheSizeInKb * MemoryConstants::kiloByte; + const size_t maxReservationCacheSize = (totalCacheSize * maxReservationNumWays) / maxNumWays; + const uint32_t maxReservationNumCacheRegions = hwHelper.getNumCacheRegions() - 1; + + this->cacheInfo.reset(new CacheInfoImpl(*this, maxReservationCacheSize, maxReservationNumCacheRegions, maxReservationNumWays)); +} + void Drm::getPrelimVersion(std::string &prelimVersion) { std::string sysFsPciPath = getSysFsPciPath(); std::string prelimVersionPath = sysFsPciPath + "/prelim_uapi_version"; diff --git a/shared/source/os_interface/linux/drm_query.cpp b/shared/source/os_interface/linux/drm_query.cpp index 9b3628c7c4..989d278689 100644 --- a/shared/source/os_interface/linux/drm_query.cpp +++ b/shared/source/os_interface/linux/drm_query.cpp @@ -7,7 +7,6 @@ #include "shared/source/execution_environment/root_device_environment.h" #include "shared/source/helpers/string.h" -#include "shared/source/os_interface/linux/cache_info_impl.h" #include "shared/source/os_interface/linux/drm_engine_mapper.h" #include "shared/source/os_interface/linux/engine_info.h" #include "shared/source/os_interface/linux/ioctl_helper.h" @@ -53,10 +52,6 @@ bool Drm::isDebugAttachAvailable() { return false; } -void Drm::setupCacheInfo(const HardwareInfo &hwInfo) { - this->cacheInfo.reset(new CacheInfoImpl()); -} - int Drm::bindBufferObject(OsContext *osContext, uint32_t vmHandleId, BufferObject *bo) { return 0; } diff --git a/shared/test/common/libult/linux/CMakeLists.txt b/shared/test/common/libult/linux/CMakeLists.txt index 15ed91354f..434bad7762 100644 --- a/shared/test/common/libult/linux/CMakeLists.txt +++ b/shared/test/common/libult/linux/CMakeLists.txt @@ -16,10 +16,8 @@ if(UNIX) target_sources(neo_libult_common PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/drm_query_mock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_query_mock.h - ${CMAKE_CURRENT_SOURCE_DIR}/drm_query_mock_context.h - ${CMAKE_CURRENT_SOURCE_DIR}/drm_query_mock_prelim.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/drm_prelim_helper.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/drm_prelim_helper.h + ${CMAKE_CURRENT_SOURCE_DIR}/drm_mock_prelim_context.h + ${CMAKE_CURRENT_SOURCE_DIR}/drm_mock_prelim_context.cpp ) endif() diff --git a/shared/test/common/libult/linux/drm_query_mock_prelim.cpp b/shared/test/common/libult/linux/drm_mock_prelim_context.cpp similarity index 82% rename from shared/test/common/libult/linux/drm_query_mock_prelim.cpp rename to shared/test/common/libult/linux/drm_mock_prelim_context.cpp index d6d63db7ff..b1b6aef532 100644 --- a/shared/test/common/libult/linux/drm_query_mock_prelim.cpp +++ b/shared/test/common/libult/linux/drm_mock_prelim_context.cpp @@ -5,9 +5,11 @@ * */ +#include "shared/test/common/libult/linux/drm_mock_prelim_context.h" + #include "shared/source/helpers/basic_math.h" #include "shared/source/helpers/ptr_math.h" -#include "shared/test/common/libult/linux/drm_query_mock_context.h" +#include "shared/source/os_interface/linux/cache_info_impl.h" #include "third_party/uapi/prelim/drm/i915_drm.h" #include @@ -44,14 +46,51 @@ constexpr std::array copyEnginesCapsMap = {{ } // namespace -int handlePrelimDrmQueryRequests(DrmQueryMockContext &context, unsigned long request, void *arg) { - return -1; +int DrmMockPrelimContext::handlePrelimRequest(unsigned long request, void *arg) { + switch (request) { + case PRELIM_DRM_IOCTL_I915_GEM_CLOS_RESERVE: { + auto closReserveArg = static_cast(arg); + closIndex++; + if (closIndex == 0) { + return EINVAL; + } + closReserveArg->clos_index = closIndex; + } break; + case PRELIM_DRM_IOCTL_I915_GEM_CLOS_FREE: { + auto closFreeArg = static_cast(arg); + if (closFreeArg->clos_index > closIndex) { + return EINVAL; + } + closIndex--; + } break; + case PRELIM_DRM_IOCTL_I915_GEM_CACHE_RESERVE: { + auto cacheReserveArg = static_cast(arg); + if (cacheReserveArg->clos_index > closIndex) { + return EINVAL; + } + auto cacheInfoImpl = static_cast(cacheInfo); + auto maxReservationNumWays = cacheInfoImpl ? cacheInfoImpl->getMaxReservationNumWays() : maxNumWays; + if (cacheReserveArg->num_ways > maxReservationNumWays) { + return EINVAL; + } + auto freeNumWays = maxReservationNumWays - allocNumWays; + if (cacheReserveArg->num_ways > freeNumWays) { + return EINVAL; + } + allocNumWays += cacheReserveArg->num_ways; + } break; + + default: + return -1; + } + + return 0; } -bool handlePrelimDrmQueryItem(DrmQueryMockContext &context, void *arg) { +bool DrmMockPrelimContext::handlePrelimQueryItem(void *arg) { auto queryItem = reinterpret_cast(arg); - auto >SystemInfo = context.hwInfo->gtSystemInfo; + auto >SystemInfo = hwInfo->gtSystemInfo; auto numberOfCCS = gtSystemInfo.CCSInfo.IsValid ? gtSystemInfo.CCSInfo.NumberOfCCSEnabled : 0u; switch (queryItem->query_id) { @@ -133,7 +172,7 @@ bool handlePrelimDrmQueryItem(DrmQueryMockContext &context, void *arg) { } break; case PRELIM_DRM_I915_QUERY_COMPUTE_SLICES: { - auto >SystemInfo = context.rootDeviceEnvironment.getHardwareInfo()->gtSystemInfo; + auto >SystemInfo = rootDeviceEnvironment.getHardwareInfo()->gtSystemInfo; auto maxEuPerSubslice = gtSystemInfo.MaxEuPerSubSlice; auto maxSlices = gtSystemInfo.MaxSlicesSupported; auto maxSubslices = gtSystemInfo.MaxSubSlicesSupported / maxSlices; @@ -148,7 +187,7 @@ bool handlePrelimDrmQueryItem(DrmQueryMockContext &context, void *arg) { break; } else { auto topologyArg = reinterpret_cast(queryItem->data_ptr); - if (context.failRetTopology) { + if (failRetTopology) { return false; } topologyArg->max_slices = maxSlices; @@ -195,3 +234,7 @@ bool handlePrelimDrmQueryItem(DrmQueryMockContext &context, void *arg) { } return true; } + +uint32_t getQueryComputeSlicesIoctl() { + return PRELIM_DRM_I915_QUERY_COMPUTE_SLICES; +} diff --git a/shared/test/common/libult/linux/drm_mock_prelim_context.h b/shared/test/common/libult/linux/drm_mock_prelim_context.h new file mode 100644 index 0000000000..00a7323032 --- /dev/null +++ b/shared/test/common/libult/linux/drm_mock_prelim_context.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/execution_environment/root_device_environment.h" +#include "shared/source/helpers/hw_info.h" +#include "shared/source/os_interface/linux/cache_info.h" + +using namespace NEO; + +struct DrmMockPrelimContext { + const HardwareInfo *hwInfo; + const RootDeviceEnvironment &rootDeviceEnvironment; + const CacheInfo *cacheInfo; + const bool &failRetTopology; + uint16_t closIndex{0}; + uint16_t maxNumWays{32}; + uint32_t allocNumWays{0}; + + int handlePrelimRequest(unsigned long request, void *arg); + bool handlePrelimQueryItem(void *arg); +}; + +uint32_t getQueryComputeSlicesIoctl(); diff --git a/shared/test/common/libult/linux/drm_prelim_helper.cpp b/shared/test/common/libult/linux/drm_prelim_helper.cpp deleted file mode 100644 index 0dd163c510..0000000000 --- a/shared/test/common/libult/linux/drm_prelim_helper.cpp +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2022 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/test/common/libult/linux/drm_prelim_helper.h" - -#include "third_party/uapi/prelim/drm/i915_drm.h" - -uint32_t getQueryComputeSlicesIoctl() { - return PRELIM_DRM_I915_QUERY_COMPUTE_SLICES; -} diff --git a/shared/test/common/libult/linux/drm_prelim_helper.h b/shared/test/common/libult/linux/drm_prelim_helper.h deleted file mode 100644 index 29e3325204..0000000000 --- a/shared/test/common/libult/linux/drm_prelim_helper.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright (C) 2022 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#pragma once - -#include - -uint32_t getQueryComputeSlicesIoctl(); diff --git a/shared/test/common/libult/linux/drm_query_mock.cpp b/shared/test/common/libult/linux/drm_query_mock.cpp index 828be7c98d..9bc7c9f9f5 100644 --- a/shared/test/common/libult/linux/drm_query_mock.cpp +++ b/shared/test/common/libult/linux/drm_query_mock.cpp @@ -7,9 +7,6 @@ #include "shared/test/common/libult/linux/drm_query_mock.h" -extern int handlePrelimDrmQueryRequests(DrmQueryMockContext &context, unsigned long request, void *arg); -extern bool handlePrelimDrmQueryItem(DrmQueryMockContext &context, void *arg); - int DrmQueryMock::handleRemainingRequests(unsigned long request, void *arg) { if (request == DRM_IOCTL_I915_QUERY && arg) { auto query = static_cast(arg); @@ -27,9 +24,9 @@ int DrmQueryMock::handleRemainingRequests(unsigned long request, void *arg) { return 0; } - return handlePrelimDrmQueryRequests(context, request, arg); + return context.handlePrelimRequest(request, arg); } bool DrmQueryMock::handleQueryItem(void *arg) { - return handlePrelimDrmQueryItem(context, arg); + return context.handlePrelimQueryItem(arg); } diff --git a/shared/test/common/libult/linux/drm_query_mock.h b/shared/test/common/libult/linux/drm_query_mock.h index 0a34e7d7ad..c4aa92d5e7 100644 --- a/shared/test/common/libult/linux/drm_query_mock.h +++ b/shared/test/common/libult/linux/drm_query_mock.h @@ -11,7 +11,7 @@ #include "shared/source/os_interface/linux/ioctl_helper.h" #include "shared/test/common/helpers/default_hw_info.h" #include "shared/test/common/libult/linux/drm_mock.h" -#include "shared/test/common/libult/linux/drm_query_mock_context.h" +#include "shared/test/common/libult/linux/drm_mock_prelim_context.h" using namespace NEO; @@ -29,9 +29,10 @@ class DrmQueryMock : public DrmMock { prelimVersion = "2.0"; } - DrmQueryMockContext context{ + DrmMockPrelimContext context{ .hwInfo = nullptr, .rootDeviceEnvironment = rootDeviceEnvironment, + .cacheInfo = getCacheInfo(), .failRetTopology = failRetTopology, }; diff --git a/shared/test/common/libult/linux/drm_query_mock_context.h b/shared/test/common/libult/linux/drm_query_mock_context.h deleted file mode 100644 index c1f36a136a..0000000000 --- a/shared/test/common/libult/linux/drm_query_mock_context.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (C) 2022 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/execution_environment/root_device_environment.h" -#include "shared/source/helpers/hw_info.h" - -using namespace NEO; - -struct DrmQueryMockContext { - const HardwareInfo *hwInfo; - const RootDeviceEnvironment &rootDeviceEnvironment; - const bool &failRetTopology; -}; diff --git a/shared/test/common/os_interface/linux/CMakeLists.txt b/shared/test/common/os_interface/linux/CMakeLists.txt index c28ec07c36..b5b3dd287f 100644 --- a/shared/test/common/os_interface/linux/CMakeLists.txt +++ b/shared/test/common/os_interface/linux/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2021-2022 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -8,6 +8,7 @@ if(UNIX) target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_manager_tests.h + ${CMAKE_CURRENT_SOURCE_DIR}/drm_mock_cache_info.h ${CMAKE_CURRENT_SOURCE_DIR}/drm_mock_device_blob.h ) add_subdirectories() diff --git a/shared/test/common/os_interface/linux/drm_mock_cache_info.h b/shared/test/common/os_interface/linux/drm_mock_cache_info.h new file mode 100644 index 0000000000..21ebcef69b --- /dev/null +++ b/shared/test/common/os_interface/linux/drm_mock_cache_info.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021-2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include "shared/source/os_interface/linux/cache_info_impl.h" + +namespace NEO { + +struct MockCacheInfoImpl : public CacheInfoImpl { + using CacheInfoImpl::cacheRegionsReserved; + using CacheInfoImpl::isRegionReserved; + + MockCacheInfoImpl(Drm &drm, size_t maxReservationCacheSize, uint32_t maxReservationNumCacheRegions, uint16_t maxReservationNumWays) + : CacheInfoImpl(drm, maxReservationCacheSize, maxReservationNumCacheRegions, maxReservationNumWays) {} + + ~MockCacheInfoImpl() override = default; + + bool getCacheRegion(size_t regionSize, CacheRegion regionIndex) override { + if (regionIndex >= CacheRegion::Count) { + return false; + } + if (regionSize > (maxReservationCacheSize / maxReservationNumCacheRegions)) { + return false; + } + return true; + } +}; + +} // namespace NEO diff --git a/shared/test/unit_test/os_interface/linux/drm_query_topology_prelim_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_query_topology_prelim_tests.cpp index 98262aaa20..b7c6607f40 100644 --- a/shared/test/unit_test/os_interface/linux/drm_query_topology_prelim_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_query_topology_prelim_tests.cpp @@ -10,7 +10,7 @@ #include "shared/source/os_interface/os_interface.h" #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/helpers/default_hw_info.h" -#include "shared/test/common/libult/linux/drm_prelim_helper.h" +#include "shared/test/common/libult/linux/drm_mock_prelim_context.h" #include "shared/test/common/libult/linux/drm_query_mock.h" #include "shared/test/common/test_macros/test.h"