From 7b40b01f545de7a4a5b38dd8959afa76c09ef63a Mon Sep 17 00:00:00 2001 From: Mateusz Jablonski Date: Mon, 15 Jan 2024 14:47:26 +0000 Subject: [PATCH] feature: add debug key for toggling bit in 57bit GPU VA for specific allocations Related-To: NEO-9419 Signed-off-by: Mateusz Jablonski --- .../debug_settings/debug_variables_base.inl | 1 + .../source/memory_manager/memory_manager.cpp | 27 +++++++ shared/source/memory_manager/memory_manager.h | 3 +- .../os_agnostic_memory_manager.cpp | 5 +- .../os_interface/linux/drm_memory_manager.cpp | 3 + .../mocks/linux/mock_drm_memory_manager.h | 3 +- shared/test/common/test_files/igdrcl.config | 1 + ..._manager_allocate_in_device_pool_tests.cpp | 71 ++++++++++++++++- .../linux/drm_memory_manager_tests.cpp | 79 +++++++++++++++++++ 9 files changed, 189 insertions(+), 4 deletions(-) diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl index 1bb861f6b2..9f33ab28b2 100644 --- a/shared/source/debug_settings/debug_variables_base.inl +++ b/shared/source/debug_settings/debug_variables_base.inl @@ -95,6 +95,7 @@ DECLARE_DEBUG_VARIABLE(std::string, InjectApiBuildOptions, std::string("unk"), " DECLARE_DEBUG_VARIABLE(std::string, OverrideDeviceName, std::string("unk"), "Override device name to provided string; ignored when unk") DECLARE_DEBUG_VARIABLE(std::string, OverridePlatformName, std::string("unk"), "Override platform name to provided string; ignored when unk") DECLARE_DEBUG_VARIABLE(std::string, WddmResidencyLoggerOutputDirectory, std::string("unk"), "Selects non-default output directory for Wddm Residency logger file") +DECLARE_DEBUG_VARIABLE(std::string, ToggleBitIn57GpuVa, std::string("unk"), "Toggles specific bit in GPU VA for given allocation type from heap extended. Format :,:") DECLARE_DEBUG_VARIABLE(int64_t, OverrideMultiStoragePlacement, -1, "Place memory only in selected tiles indicated by bit mask; ignore when -1") DECLARE_DEBUG_VARIABLE(int64_t, ForceCompressionDisabledForCompressedBlitCopies, -1, "If compression is required, set AUX_CCS_E, but force CompressionEnable filed; 0 should result in uncompressed read/write; values = -1: default, 0: disabled, 1: enabled") DECLARE_DEBUG_VARIABLE(int64_t, WddmPagingFenceCpuWaitDelayTime, 0, "Amount of microseconds after waitng for paging fence on CPU") diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index 07ed3a60c5..5ee16069cf 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -19,11 +19,13 @@ #include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/bindless_heaps_helper.h" +#include "shared/source/helpers/bit_helpers.h" #include "shared/source/helpers/blit_helper.h" #include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/hw_info.h" #include "shared/source/helpers/memory_properties_helpers.h" #include "shared/source/helpers/string.h" +#include "shared/source/helpers/string_helpers.h" #include "shared/source/helpers/surface_format_info.h" #include "shared/source/memory_manager/allocation_properties.h" #include "shared/source/memory_manager/compression_selector.h" @@ -1079,4 +1081,29 @@ bool MemoryTransferHelper::transferMemoryToAllocationBanks(const Device &device, } return true; } + +uint64_t MemoryManager::adjustToggleBitFlagForGpuVa(AllocationType inputAllocationType, uint64_t gpuAddress) { + if (debugManager.flags.ToggleBitIn57GpuVa.get() != "unk") { + auto toggleBitIn57GpuVaEntries = StringHelpers::split(debugManager.flags.ToggleBitIn57GpuVa.get(), ","); + + for (const auto &entry : toggleBitIn57GpuVaEntries) { + auto subEntries = StringHelpers::split(entry, ":"); + UNRECOVERABLE_IF(subEntries.size() < 2u); + uint32_t allocationType = StringHelpers::toUint32t(subEntries[0]); + uint32_t bitNumber = StringHelpers::toUint32t(subEntries[1]); + + UNRECOVERABLE_IF(allocationType >= static_cast(AllocationType::count)); + UNRECOVERABLE_IF(bitNumber >= 56); + if (allocationType == static_cast(inputAllocationType)) { + if (isBitSet(gpuAddress, bitNumber)) { + gpuAddress &= ~(1ull << bitNumber); + } else { + gpuAddress |= 1ull << bitNumber; + } + } + } + } + return gpuAddress; +} + } // namespace NEO diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 7638a9408b..d5aa99740d 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -290,6 +290,7 @@ class MemoryManager { virtual bool mapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize) = 0; virtual void unMapPhysicalToVirtualMemory(GraphicsAllocation *physicalAllocation, uint64_t gpuRange, size_t bufferSize, OsContext *osContext, uint32_t rootDeviceIndex) = 0; bool allocateBindlessSlot(GraphicsAllocation *allocation); + static uint64_t adjustToggleBitFlagForGpuVa(AllocationType inputAllocationType, uint64_t gpuAddress); protected: bool getAllocationData(AllocationData &allocationData, const AllocationProperties &properties, const void *hostPtr, const StorageInfo &storageInfo); diff --git a/shared/source/memory_manager/os_agnostic_memory_manager.cpp b/shared/source/memory_manager/os_agnostic_memory_manager.cpp index 501e7f76dd..9e8ccd111e 100644 --- a/shared/source/memory_manager/os_agnostic_memory_manager.cpp +++ b/shared/source/memory_manager/os_agnostic_memory_manager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -654,6 +654,9 @@ GraphicsAllocation *OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool( auto sizeOfHeapChunk = sizeAligned64k; auto gmmHelper = getGmmHelper(allocationData.rootDeviceIndex); auto canonizedGpuAddress = gmmHelper->canonize(gfxPartition->heapAllocate(heapIndex, sizeOfHeapChunk)); + if (heapIndex == HeapIndex::heapExtended) { + canonizedGpuAddress = MemoryManager::adjustToggleBitFlagForGpuVa(allocationData.type, canonizedGpuAddress); + } allocation = new MemoryAllocation(allocationData.rootDeviceIndex, numHandles, allocationData.type, systemMemory, systemMemory, canonizedGpuAddress, sizeAligned64k, counter, MemoryPool::localMemory, false, allocationData.flags.flushL3, maxOsContextCount); diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 48379945cc..e5e6c6cd11 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -1671,6 +1671,9 @@ AllocationStatus getGpuAddress(const AlignmentSelector &alignmentSelector, HeapA alignment.alignment = allocationData.alignment; } gpuAddress = gmmHelper->canonize(gfxPartition->heapAllocateWithCustomAlignment(alignment.heap, sizeAllocated, alignment.alignment)); + if (alignment.heap == HeapIndex::heapExtended) { + gpuAddress = MemoryManager::adjustToggleBitFlagForGpuVa(allocationData.type, gpuAddress); + } break; } diff --git a/shared/test/common/mocks/linux/mock_drm_memory_manager.h b/shared/test/common/mocks/linux/mock_drm_memory_manager.h index 681972830f..e7b5abaf1f 100644 --- a/shared/test/common/mocks/linux/mock_drm_memory_manager.h +++ b/shared/test/common/mocks/linux/mock_drm_memory_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -81,6 +81,7 @@ class TestedDrmMemoryManager : public MemoryManagerCreate { using MemoryManager::allocateGraphicsMemoryInDevicePool; using MemoryManager::allRegisteredEngines; using MemoryManager::heapAssigners; + using MemoryManager::localMemorySupported; TestedDrmMemoryManager(ExecutionEnvironment &executionEnvironment); TestedDrmMemoryManager(bool enableLocalMemory, diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config index 9da064a784..2ed48dfbd7 100644 --- a/shared/test/common/test_files/igdrcl.config +++ b/shared/test/common/test_files/igdrcl.config @@ -480,6 +480,7 @@ ForceUncachedGmmUsageType = 0 OverrideDeviceName = unk OverridePlatformName = unk WddmResidencyLoggerOutputDirectory = unk +ToggleBitIn57GpuVa = unk EnablePrivateBO = 0 ExperimentalEnableDeviceAllocationCache = -1 OverrideL1CachePolicyInSurfaceStateAndStateless = -1 diff --git a/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp index 683494b1aa..bbda7ac60e 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_allocate_in_device_pool_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2023 Intel Corporation + * Copyright (C) 2018-2024 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -8,6 +8,7 @@ #include "shared/source/aub/aub_helper.h" #include "shared/source/execution_environment/execution_environment.h" #include "shared/source/gmm_helper/gmm_helper.h" +#include "shared/source/helpers/bit_helpers.h" #include "shared/source/helpers/gfx_core_helper.h" #include "shared/source/helpers/memory_properties_helpers.h" #include "shared/source/memory_manager/gfx_partition.h" @@ -15,6 +16,7 @@ #include "shared/test/common/helpers/debug_manager_state_restore.h" #include "shared/test/common/mocks/mock_device.h" #include "shared/test/common/mocks/mock_execution_environment.h" +#include "shared/test/common/mocks/mock_gfx_partition.h" #include "shared/test/common/mocks/mock_gmm.h" #include "shared/test/common/mocks/mock_memory_manager.h" #include "shared/test/common/mocks/ult_device_factory.h" @@ -872,3 +874,70 @@ HWTEST2_F(MemoryManagerDirectSubmissionImplicitScalingTest, givenDirectSubmissio } } } + +TEST(MemoryManagerTest, givenDebugVariableToToggleGpuVaBitsWhenAllocatingResourceInHeapExtendedThenSpecificBitIsToggled) { + if (defaultHwInfo->capabilityTable.gpuAddressSpace < maxNBitValue(57)) { + GTEST_SKIP(); + } + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + MockMemoryManager memoryManager(true, true, executionEnvironment); + + auto mockGfxPartition = std::make_unique(); + mockGfxPartition->initHeap(HeapIndex::heapExtended, maxNBitValue(56) + 1, MemoryConstants::teraByte, MemoryConstants::pageSize64k); + memoryManager.gfxPartitions[0] = std::move(mockGfxPartition); + + DebugManagerStateRestore restorer; + EXPECT_EQ(4u, static_cast(AllocationType::constantSurface)); + EXPECT_EQ(7u, static_cast(AllocationType::globalSurface)); + debugManager.flags.ToggleBitIn57GpuVa.set("4:55,7:32"); + + auto status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.size = static_cast(MemoryConstants::kiloByte); + allocData.type = AllocationType::buffer; + allocData.rootDeviceIndex = mockRootDeviceIndex; + + { + allocData.type = AllocationType::buffer; + auto allocation = memoryManager.allocateGraphicsMemoryInDevicePool(allocData, status); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + ASSERT_NE(nullptr, allocation); + + auto gpuVA = allocation->getGpuAddress(); + + EXPECT_TRUE(isBitSet(gpuVA, 56)); + EXPECT_FALSE(isBitSet(gpuVA, 55)); + EXPECT_TRUE(isBitSet(gpuVA, 32)); + + memoryManager.freeGraphicsMemory(allocation); + } + { + allocData.type = AllocationType::constantSurface; + auto allocation = memoryManager.allocateGraphicsMemoryInDevicePool(allocData, status); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + ASSERT_NE(nullptr, allocation); + + auto gpuVA = allocation->getGpuAddress(); + + EXPECT_TRUE(isBitSet(gpuVA, 56)); + EXPECT_TRUE(isBitSet(gpuVA, 55)); + EXPECT_TRUE(isBitSet(gpuVA, 32)); + + memoryManager.freeGraphicsMemory(allocation); + } + + { + allocData.type = AllocationType::globalSurface; + auto allocation = memoryManager.allocateGraphicsMemoryInDevicePool(allocData, status); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + ASSERT_NE(nullptr, allocation); + + auto gpuVA = allocation->getGpuAddress(); + + EXPECT_TRUE(isBitSet(gpuVA, 56)); + EXPECT_FALSE(isBitSet(gpuVA, 55)); + EXPECT_FALSE(isBitSet(gpuVA, 32)); + + memoryManager.freeGraphicsMemory(allocation); + } +} \ No newline at end of file diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index e3a16f51d7..af5f010ece 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -9,6 +9,7 @@ #include "shared/source/command_stream/tag_allocation_layout.h" #include "shared/source/gmm_helper/gmm_helper.h" #include "shared/source/helpers/basic_math.h" +#include "shared/source/helpers/bit_helpers.h" #include "shared/source/helpers/common_types.h" #include "shared/source/helpers/surface_format_info.h" #include "shared/source/indirect_heap/indirect_heap.h" @@ -7391,3 +7392,81 @@ TEST_F(DrmMemoryManagerTest, given57bAddressSpaceCpuAndGpuWhenAllocating48bResou EXPECT_EQ(sharedUSM->getReservedAddressPtr(), nullptr); memoryManager->freeGraphicsMemory(sharedUSM); } + +TEST_F(DrmMemoryManagerTest, givenDebugVariableToToggleGpuVaBitsWhenAllocatingResourceInHeapExtendedThenSpecificBitIsToggled) { + if (defaultHwInfo->capabilityTable.gpuAddressSpace < maxNBitValue(57)) { + GTEST_SKIP(); + } + auto mockGfxPartition = std::make_unique(); + mockGfxPartition->initHeap(HeapIndex::heapExtended, maxNBitValue(56) + 1, MemoryConstants::teraByte, MemoryConstants::pageSize64k); + memoryManager->gfxPartitions[1] = std::move(mockGfxPartition); + + std::vector regionInfo(1); + regionInfo[0].region = {drm_i915_gem_memory_class::I915_MEMORY_CLASS_SYSTEM, 0}; + + auto &drm = static_cast(memoryManager->getDrm(rootDeviceIndex)); + drm.memoryInfo.reset(new MemoryInfo(regionInfo, drm)); + drm.ioctlHelper = std::make_unique(drm); + memoryManager->localMemorySupported[rootDeviceIndex] = true; + + mock->ioctlExpected.gemCreateExt = 3; + mock->ioctlExpected.gemWait = 3; + mock->ioctlExpected.gemClose = 3; + + DebugManagerStateRestore restorer; + EXPECT_EQ(4u, static_cast(AllocationType::constantSurface)); + EXPECT_EQ(7u, static_cast(AllocationType::globalSurface)); + debugManager.flags.ToggleBitIn57GpuVa.set("4:55,7:32"); + + auto size = MemoryConstants::kiloByte; + + auto status = MemoryManager::AllocationStatus::Error; + AllocationData allocData; + allocData.size = size; + allocData.type = AllocationType::buffer; + allocData.rootDeviceIndex = rootDeviceIndex; + + { + allocData.type = AllocationType::buffer; + auto allocation = memoryManager->allocateGraphicsMemoryInDevicePool(allocData, status); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + ASSERT_NE(nullptr, allocation); + + auto gpuVA = allocation->getGpuAddress(); + + EXPECT_TRUE(isBitSet(gpuVA, 56)); + EXPECT_FALSE(isBitSet(gpuVA, 55)); + EXPECT_TRUE(isBitSet(gpuVA, 32)); + + memoryManager->freeGraphicsMemory(allocation); + } + { + allocData.type = AllocationType::constantSurface; + auto allocation = memoryManager->allocateGraphicsMemoryInDevicePool(allocData, status); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + ASSERT_NE(nullptr, allocation); + + auto gpuVA = allocation->getGpuAddress(); + + EXPECT_TRUE(isBitSet(gpuVA, 56)); + EXPECT_TRUE(isBitSet(gpuVA, 55)); + EXPECT_TRUE(isBitSet(gpuVA, 32)); + + memoryManager->freeGraphicsMemory(allocation); + } + + { + allocData.type = AllocationType::globalSurface; + auto allocation = memoryManager->allocateGraphicsMemoryInDevicePool(allocData, status); + EXPECT_EQ(MemoryManager::AllocationStatus::Success, status); + ASSERT_NE(nullptr, allocation); + + auto gpuVA = allocation->getGpuAddress(); + + EXPECT_TRUE(isBitSet(gpuVA, 56)); + EXPECT_FALSE(isBitSet(gpuVA, 55)); + EXPECT_FALSE(isBitSet(gpuVA, 32)); + + memoryManager->freeGraphicsMemory(allocation); + } +} \ No newline at end of file