diff --git a/runtime/os_interface/linux/drm_memory_manager.cpp b/runtime/os_interface/linux/drm_memory_manager.cpp index 195fb45ebd..252a722a7c 100644 --- a/runtime/os_interface/linux/drm_memory_manager.cpp +++ b/runtime/os_interface/linux/drm_memory_manager.cpp @@ -58,7 +58,9 @@ DrmMemoryManager::DrmMemoryManager(Drm *drm, gemCloseWorkerMode mode, bool force DrmMemoryManager::~DrmMemoryManager() { if (this->limitedGpuAddressRangeAllocator) { - this->limitedGpuAddressRangeAllocator->free(this->internal32bitAllocator->getBase(), (4 * MemoryConstants::gigaByte)); + // internal 32bit allocator size is reserved 1 page (offset to 0x0) when the allocator created. + uint64_t size = 4 * MemoryConstants::gigaByte - MemoryConstants::pageSize; + this->limitedGpuAddressRangeAllocator->free(this->internal32bitAllocator->getBase(), size); } applyCommonCleanup(); if (gemCloseWorker) { @@ -75,10 +77,12 @@ void DrmMemoryManager::initInternalRangeAllocator(size_t gpuRange) { // set the allocator with the whole reduced address space range this->limitedGpuAddressRangeAllocator.reset(new AllocatorLimitedRange(0, gpuRange)); - // when reduced address space is available, set the internal32bitAllocator inside this range. + // reserve 4G range for internal 32bit allocator, leave 1 page (offset to 0x0) to avoid mistakenly + // treating 0x0 as null pointer. uint64_t size = 4 * MemoryConstants::gigaByte; - uint64_t alloc4GRange = this->limitedGpuAddressRangeAllocator->allocate(size); - internal32bitAllocator.reset(new Allocator32bit(alloc4GRange, size)); + uint64_t internal32bitRange = size - MemoryConstants::pageSize; + uint64_t internal32bitBase = this->limitedGpuAddressRangeAllocator->allocate(size) - internal32bitRange; + internal32bitAllocator.reset(new Allocator32bit(internal32bitBase, internal32bitRange)); } else { // when in full range space, set the internal32bitAllocator using 32bit addressing allocator. internal32bitAllocator.reset(new Allocator32bit); @@ -371,9 +375,25 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemory(size_t size, const return nullptr; } - BufferObject *bo = allocUserptr(res, alignedAllocationSize, 0, true); + auto limitedRangeAllocation = false; + void *ptrAlloc = nullptr; + + if (limitedGpuAddressRangeAllocator.get() && allocatorType == BIT32_ALLOCATOR_INTERNAL) { + limitedRangeAllocation = true; + ptrAlloc = alignedMallocWrapper(alignedAllocationSize, MemoryConstants::allocationAlignment); + + if (!ptrAlloc) { + allocatorToUse->free(res, allocationSize); + return nullptr; + } + } + + BufferObject *bo = allocUserptr(limitedRangeAllocation ? reinterpret_cast(ptrAlloc) : res, alignedAllocationSize, 0, true); if (!bo) { + if (ptrAlloc != nullptr) { + alignedFreeWrapper(ptrAlloc); + } allocatorToUse->free(res, allocationSize); return nullptr; } @@ -383,10 +403,18 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemory(size_t size, const bo->setAllocationType(allocatorType); - auto drmAllocation = new DrmAllocation(bo, reinterpret_cast(res), alignedAllocationSize, - MemoryPool::System4KBPagesWith32BitGpuAddressing, getOsContextCount(), false); + DrmAllocation *drmAllocation = nullptr; + if (limitedRangeAllocation) { + drmAllocation = new DrmAllocation(bo, ptrAlloc, res, alignedAllocationSize, + MemoryPool::System4KBPagesWith32BitGpuAddressing, getOsContextCount(), false); + } else { + drmAllocation = new DrmAllocation(bo, reinterpret_cast(res), alignedAllocationSize, + MemoryPool::System4KBPagesWith32BitGpuAddressing, getOsContextCount(), false); + } + drmAllocation->is32BitAllocation = true; drmAllocation->gpuBaseAddress = allocatorToUse->getBase(); + drmAllocation->driverAllocatedCpuPointer = ptrAlloc; return drmAllocation; } @@ -523,6 +551,8 @@ void DrmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) if (input->gmm) delete input->gmm; + alignedFreeWrapper(gfxAllocation->driverAllocatedCpuPointer); + if (gfxAllocation->fragmentsStorage.fragmentCount) { cleanGraphicsMemoryCreatedFromHostPtr(gfxAllocation); delete gfxAllocation; diff --git a/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp b/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp index 6ee08a242d..f8ff02c34e 100644 --- a/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/linux/drm_memory_manager_tests.cpp @@ -983,6 +983,20 @@ TEST_F(DrmMemoryManagerTest, givenMemoryManagerWhenAskedFor32BitAllocationAndAll mock->ioctl_res_ext = &mock->NONE; } +TEST_F(DrmMemoryManagerTest, givenLimitedRangeAllocatorWhenAskedForInternal32BitAllocationAndAllocUserptrFailsThenFails) { + mock->ioctl_expected.gemUserptr = 1; + + this->ioctlResExt = {mock->ioctl_cnt.total, -1}; + mock->ioctl_res_ext = &ioctlResExt; + + auto size = 10u; + memoryManager->forceLimitedRangeAllocator(0xFFFFFFFFF); + auto allocation = memoryManager->allocate32BitGraphicsMemory(size, nullptr, AllocationOrigin::INTERNAL_ALLOCATION); + + EXPECT_EQ(nullptr, allocation); + mock->ioctl_res_ext = &mock->NONE; +} + TEST_F(DrmMemoryManagerTest, GivenSizeAbove2GBWhenUseHostPtrAndAllocHostPtrAreCreatedThenFirstSucceedsAndSecondFails) { DebugManagerStateRestore dbgRestorer; mock->ioctl_expected.total = -1; @@ -2062,6 +2076,74 @@ TEST_F(DrmMemoryManagerTest, givenMemoryManagerWhenAskedForInternalAllocationWit memoryManager->freeGraphicsMemory(drmAllocation); } +TEST_F(DrmMemoryManagerTest, givenLimitedRangeAllocatorWhenAskedForInternalAllocationWithNoPointerThenAllocationFromInternalHeapIsReturned) { + mock->ioctl_expected.gemUserptr = 1; + mock->ioctl_expected.gemWait = 1; + mock->ioctl_expected.gemClose = 1; + + memoryManager->forceLimitedRangeAllocator(0xFFFFFFFFF); + + auto bufferSize = MemoryConstants::pageSize; + void *ptr = nullptr; + auto drmAllocation = static_cast(memoryManager->allocate32BitGraphicsMemory(bufferSize, ptr, AllocationOrigin::INTERNAL_ALLOCATION)); + ASSERT_NE(nullptr, drmAllocation); + + auto internalAllocator = memoryManager->getDrmInternal32BitAllocator(); + + EXPECT_NE(nullptr, drmAllocation->getUnderlyingBuffer()); + EXPECT_EQ(bufferSize, drmAllocation->getUnderlyingBufferSize()); + + ASSERT_NE(nullptr, drmAllocation->driverAllocatedCpuPointer); + EXPECT_EQ(drmAllocation->driverAllocatedCpuPointer, drmAllocation->getUnderlyingBuffer()); + + EXPECT_TRUE(drmAllocation->is32BitAllocation); + + auto gpuPtr = drmAllocation->getGpuAddress(); + + auto heapBase = internalAllocator->getBase(); + auto heapSize = 4 * GB; + + EXPECT_GE(gpuPtr, heapBase); + EXPECT_LE(gpuPtr, heapBase + heapSize); + + EXPECT_EQ(drmAllocation->gpuBaseAddress, heapBase); + + auto bo = drmAllocation->getBO(); + EXPECT_TRUE(bo->peekIsAllocated()); + EXPECT_EQ(bo->peekAllocationType(), StorageAllocatorType::BIT32_ALLOCATOR_INTERNAL); + EXPECT_EQ(bo->peekUnmapSize(), bufferSize); + + memoryManager->freeGraphicsMemory(drmAllocation); +} + +TEST_F(DrmMemoryManagerTest, givenLimitedRangeAllocatorWhenAskedForExternalAllocationWithNoPointerThenAllocationFromInternalHeapIsReturned) { + mock->ioctl_expected.gemUserptr = 1; + mock->ioctl_expected.gemWait = 1; + mock->ioctl_expected.gemClose = 1; + + memoryManager->setForce32BitAllocations(true); + memoryManager->forceLimitedRangeAllocator(0xFFFFFFFFF); + + auto bufferSize = MemoryConstants::pageSize; + void *ptr = nullptr; + auto drmAllocation = static_cast(memoryManager->allocate32BitGraphicsMemory(bufferSize, ptr, AllocationOrigin::EXTERNAL_ALLOCATION)); + ASSERT_NE(nullptr, drmAllocation); + + EXPECT_NE(nullptr, drmAllocation->getUnderlyingBuffer()); + EXPECT_TRUE(drmAllocation->is32BitAllocation); + + memoryManager->freeGraphicsMemory(drmAllocation); +} + +TEST_F(DrmMemoryManagerTest, givenLimitedRangeAllocatorWhenAskedForInternalAllocationWithNoPointerandHugeBufferSizeThenAllocationFromInternalHeapFailed) { + memoryManager->forceLimitedRangeAllocator(0xFFFFFFFFF); + + auto bufferSize = 128 * MemoryConstants::megaByte + 4 * MemoryConstants::pageSize; + void *ptr = nullptr; + auto drmAllocation = static_cast(memoryManager->allocate32BitGraphicsMemory(bufferSize, ptr, AllocationOrigin::INTERNAL_ALLOCATION)); + ASSERT_EQ(nullptr, drmAllocation); +} + TEST_F(DrmMemoryManagerTest, givenMemoryManagerWhenAskedForInternalAllocationWithPointerThenAllocationFromInternalHeapIsReturned) { mock->ioctl_expected.gemUserptr = 1; mock->ioctl_expected.gemWait = 1;