diff --git a/runtime/os_interface/windows/wddm.cpp b/runtime/os_interface/windows/wddm.cpp index 0406486ee8..ff6b24eb57 100644 --- a/runtime/os_interface/windows/wddm.cpp +++ b/runtime/os_interface/windows/wddm.cpp @@ -34,6 +34,7 @@ #include "runtime/helpers/wddm_helper.h" #include "runtime/command_stream/linear_stream.h" #include "runtime/sku_info/operations/sku_info_receiver.h" +#include "runtime/utilities/stackvec.h" #include #include #include "CL/cl.h" @@ -341,7 +342,8 @@ bool Wddm::evict(OsHandleStorage &osHandles) { } bool Wddm::mapGpuVirtualAddress(WddmAllocation *allocation, void *cpuPtr, uint64_t size, bool allocation32bit, bool use64kbPages) { - return mapGpuVirtualAddressImpl(allocation->gmm, allocation->handle, cpuPtr, size, allocation->gpuPtr, allocation32bit, use64kbPages); + void *mapPtr = allocation->getReservedAddress() != nullptr ? allocation->getReservedAddress() : cpuPtr; + return mapGpuVirtualAddressImpl(allocation->gmm, allocation->handle, mapPtr, size, allocation->gpuPtr, allocation32bit, use64kbPages); } bool Wddm::mapGpuVirtualAddress(AllocationStorageData *allocationStorageData, bool allocation32bit, bool use64kbPages) { @@ -371,7 +373,7 @@ bool Wddm::mapGpuVirtualAddressImpl(Gmm *gmm, D3DKMT_HANDLE handle, void *cpuPtr } else { MapGPUVA.BaseAddress = reinterpret_cast(cpuPtr); MapGPUVA.MinimumAddress = static_cast(0x0); - MapGPUVA.MaximumAddress = static_cast((sizeof(size_t) == 8) ? 0x7ffffffffff : (D3DGPU_VIRTUAL_ADDRESS)0xffffffff); + MapGPUVA.MaximumAddress = static_cast((sizeof(size_t) == 8) ? 0x7fffffffffff : (D3DGPU_VIRTUAL_ADDRESS)0xffffffff); if (!cpuPtr) { MapGPUVA.MinimumAddress = adapterInfo->GfxPartition.Standard.Base; @@ -870,10 +872,10 @@ void Wddm::registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, Wdd } } -void Wddm::releaseGpuPtr(void *gpuPtr) { - if (gpuPtr) { - auto status = VirtualFree(gpuPtr, 0, MEM_RELEASE); - DEBUG_BREAK_IF(status != 1); +void Wddm::releaseReservedAddress(void *reservedAddress) { + if (reservedAddress) { + auto status = virtualFreeWrapper(reservedAddress, 0, MEM_RELEASE); + DEBUG_BREAK_IF(!status); } } @@ -902,4 +904,30 @@ void Wddm::resetPageTableManager(GmmPageTableMngr *newPageTableManager) { pageTableManager.reset(newPageTableManager); } +bool Wddm::reserveValidAddressRange(size_t size, void *&reservedMem) { + reservedMem = virtualAllocWrapper(nullptr, size, MEM_RESERVE, PAGE_READWRITE); + if (reservedMem == nullptr) { + return false; + } else if (minAddress > reinterpret_cast(reservedMem)) { + StackVec invalidAddrVector; + invalidAddrVector.push_back(reservedMem); + do { + reservedMem = virtualAllocWrapper(nullptr, size, MEM_RESERVE | MEM_TOP_DOWN, PAGE_READWRITE); + if (minAddress > reinterpret_cast(reservedMem) && reservedMem != nullptr) { + invalidAddrVector.push_back(reservedMem); + } else { + break; + } + } while (1); + for (auto &it : invalidAddrVector) { + auto status = virtualFreeWrapper(it, 0, MEM_RELEASE); + DEBUG_BREAK_IF(!status); + } + if (reservedMem == nullptr) { + return false; + } + } + return true; +} + } // namespace OCLRT diff --git a/runtime/os_interface/windows/wddm.h b/runtime/os_interface/windows/wddm.h index 7c75e02a00..57b483bfcc 100644 --- a/runtime/os_interface/windows/wddm.h +++ b/runtime/os_interface/windows/wddm.h @@ -96,7 +96,14 @@ class Wddm { NTSTATUS escape(D3DKMT_ESCAPE &escapeCommand); void registerTrimCallback(PFND3DKMT_TRIMNOTIFICATIONCALLBACK callback, WddmMemoryManager *memoryManager); - MOCKABLE_VIRTUAL void releaseGpuPtr(void *gpuPtr); + MOCKABLE_VIRTUAL void releaseReservedAddress(void *reservedAddress); + MOCKABLE_VIRTUAL bool reserveValidAddressRange(size_t size, void *&reservedMem); + MOCKABLE_VIRTUAL bool virtualFreeWrapper(void *ptr, size_t size, uint32_t flags) { + return VirtualFree(ptr, size, flags) != 0 ? true : false; + } + MOCKABLE_VIRTUAL void *virtualAllocWrapper(void *inPtr, size_t size, uint32_t flags, uint32_t type) { + return reinterpret_cast(VirtualAlloc(inPtr, size, flags, type)); + } template bool configureDeviceAddressSpace(); diff --git a/runtime/os_interface/windows/wddm_allocation.h b/runtime/os_interface/windows/wddm_allocation.h index 539ceea88d..17defa9d5c 100644 --- a/runtime/os_interface/windows/wddm_allocation.h +++ b/runtime/os_interface/windows/wddm_allocation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2017 - 2018, Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -48,7 +48,7 @@ class WddmAllocation : public GraphicsAllocation { D3DKMT_HANDLE resourceHandle = 0u; // used by shared resources D3DGPU_VIRTUAL_ADDRESS gpuPtr; // set by mapGpuVA - WddmAllocation(void *cpuPtrIn, size_t sizeIn, void *alignedCpuPtr, size_t alignedSize) + WddmAllocation(void *cpuPtrIn, size_t sizeIn, void *alignedCpuPtr, size_t alignedSize, void *reservedAddr) : GraphicsAllocation(cpuPtrIn, sizeIn), cpuPtrAllocated(false), handle(0), @@ -56,6 +56,7 @@ class WddmAllocation : public GraphicsAllocation { alignedCpuPtr(alignedCpuPtr), alignedSize(alignedSize) { trimListPosition = trimListUnusedPosition; + reservedAddressSpace = reservedAddr; } WddmAllocation(void *cpuPtrIn, size_t sizeIn, osHandle sharedHandle) : GraphicsAllocation(cpuPtrIn, sizeIn, sharedHandle), @@ -65,13 +66,14 @@ class WddmAllocation : public GraphicsAllocation { alignedCpuPtr(nullptr), alignedSize(sizeIn) { trimListPosition = trimListUnusedPosition; + reservedAddressSpace = nullptr; } - WddmAllocation(void *alignedCpuPtr, size_t sizeIn) - : WddmAllocation(alignedCpuPtr, sizeIn, alignedCpuPtr, sizeIn) { + WddmAllocation(void *alignedCpuPtr, size_t sizeIn, void *reservedAddress) + : WddmAllocation(alignedCpuPtr, sizeIn, alignedCpuPtr, sizeIn, reservedAddress) { } - WddmAllocation() : WddmAllocation(nullptr, 0, nullptr, 0) { + WddmAllocation() : WddmAllocation(nullptr, 0, nullptr, 0, nullptr) { } void *getAlignedCpuPtr() const { @@ -99,10 +101,19 @@ class WddmAllocation : public GraphicsAllocation { return trimListPosition; } + void *getReservedAddress() const { + return this->reservedAddressSpace; + } + + void setReservedAddress(void *reserveMem) { + this->reservedAddressSpace = reserveMem; + } + protected: void *alignedCpuPtr; size_t alignedSize; ResidencyData residency; size_t trimListPosition; + void *reservedAddressSpace; }; } // namespace OCLRT diff --git a/runtime/os_interface/windows/wddm_memory_manager.cpp b/runtime/os_interface/windows/wddm_memory_manager.cpp index 38320d594c..679cf7c360 100644 --- a/runtime/os_interface/windows/wddm_memory_manager.cpp +++ b/runtime/os_interface/windows/wddm_memory_manager.cpp @@ -71,7 +71,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryForImage(ImageInfo delete gmm; return allocateGraphicsMemory(imgInfo.size, MemoryConstants::preferredAlignment); } - WddmAllocation allocation(nullptr, imgInfo.size); + WddmAllocation allocation(nullptr, imgInfo.size, nullptr); allocation.gmm = gmm; auto status = wddm->createAllocation(&allocation); if (status == STATUS_GRAPHICS_NO_VIDEO_MEMORY && deferredDeleter) { @@ -91,7 +91,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory64kb(size_t size, s bool success = true; Gmm *gmm = nullptr; - WddmAllocation allocation(nullptr, sizeAligned, nullptr, sizeAligned); + WddmAllocation allocation(nullptr, sizeAligned, nullptr, sizeAligned, nullptr); gmm = Gmm::create(nullptr, sizeAligned, false); @@ -129,7 +129,7 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory(size_t size, size_ return nullptr; } - WddmAllocation allocation(pSysMem, sizeAligned, pSysMem, sizeAligned); + WddmAllocation allocation(pSysMem, sizeAligned, pSysMem, sizeAligned, nullptr); allocation.cpuPtrAllocated = true; gmm = Gmm::create(pSysMem, sizeAligned, uncacheable); @@ -157,6 +157,30 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemory(size_t size, const DEBUG_BREAK_IF(true); return nullptr; } + + if (mallocRestrictions.minAddress > reinterpret_cast(ptrArg)) { + void *reserve = nullptr; + void *ptrAligned = alignDown(ptr, MemoryConstants::allocationAlignment); + size_t sizeAligned = alignSizeWholePage(ptr, size); + size_t offset = ptrDiff(ptr, ptrAligned); + + if (!wddm->reserveValidAddressRange(sizeAligned, reserve)) { + return nullptr; + } + + auto allocation = new WddmAllocation(ptr, size, ptrAligned, sizeAligned, reserve); + allocation->allocationOffset = offset; + + Gmm *gmm = Gmm::create(ptrAligned, sizeAligned, false); + allocation->gmm = gmm; + if (wddm->createAllocation(allocation) == STATUS_SUCCESS) { + allocation->setGpuAddress(allocation->gpuPtr); + return allocation; + } + freeGraphicsMemory(allocation); + return nullptr; + } + return MemoryManager::allocateGraphicsMemory(size, ptr); } @@ -184,7 +208,7 @@ GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemory(size_t size, cpuPtrAllocated = true; } - WddmAllocation allocation((void *)ptrAligned, sizeAligned, (void *)ptrAligned, sizeAligned); + WddmAllocation allocation((void *)ptrAligned, sizeAligned, (void *)ptrAligned, sizeAligned, nullptr); allocation.cpuPtrAllocated = cpuPtrAllocated; allocation.is32BitAllocation = true; @@ -232,7 +256,10 @@ GraphicsAllocation *WddmMemoryManager::createAllocationFromHandle(osHandle handl void *ptr = nullptr; if (is32bit) { - ptr = (void *)VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); + if (!wddm->reserveValidAddressRange(size, ptr)) { + return nullptr; + } + allocation.setReservedAddress(ptr); } else if (requireSpecificBitness && this->force32bitAllocations) { is32BitAllocation = true; allocation.is32BitAllocation = true; @@ -291,12 +318,8 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation uint32_t allocationCount = 0; D3DKMT_HANDLE resourceHandle = 0; void *cpuPtr = nullptr; - void *gpuPtr = nullptr; if (input->peekSharedHandle()) { resourceHandle = input->resourceHandle; - if (is32bit) { - gpuPtr = (void *)input->gpuPtr; - } } else { allocationHandles = &input->handle; allocationCount = 1; @@ -311,8 +334,8 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation auto status = tryDeferDeletions(allocationHandles, allocationCount, input->getResidencyData().lastFence, resourceHandle); DEBUG_BREAK_IF(!status); alignedFreeWrapper(cpuPtr); - wddm->releaseGpuPtr(gpuPtr); } + wddm->releaseReservedAddress(input->getReservedAddress()); delete gfxAllocation; } @@ -381,7 +404,7 @@ void WddmMemoryManager::cleanOsHandles(OsHandleStorage &handleStorage) { } GraphicsAllocation *WddmMemoryManager::createGraphicsAllocation(OsHandleStorage &handleStorage, size_t hostPtrSize, const void *hostPtr) { - auto allocation = new WddmAllocation(const_cast(hostPtr), hostPtrSize, const_cast(hostPtr), hostPtrSize); + auto allocation = new WddmAllocation(const_cast(hostPtr), hostPtrSize, const_cast(hostPtr), hostPtrSize, nullptr); allocation->fragmentsStorage = handleStorage; return allocation; } diff --git a/unit_tests/mock_gdi/mock_gdi.cpp b/unit_tests/mock_gdi/mock_gdi.cpp index a4e9eff22b..a4c66c22d6 100644 --- a/unit_tests/mock_gdi/mock_gdi.cpp +++ b/unit_tests/mock_gdi/mock_gdi.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2017 - 2018, Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -221,6 +221,7 @@ NTSTATUS __stdcall D3DKMTDestroyAllocation2(IN CONST D3DKMT_DESTROYALLOCATION2 * } NTSTATUS __stdcall D3DKMTMapGpuVirtualAddress(IN OUT D3DDDI_MAPGPUVIRTUALADDRESS *mapGpuVA) { + uint64_t maxSvmAddress = sizeof(size_t) == 8 ? 0x7fffffffffff : 0xffffffff; if (mapGpuVA == nullptr || mapGpuVA->hPagingQueue != PAGINGQUEUE_HANDLE) { return STATUS_INVALID_PARAMETER; } @@ -240,6 +241,9 @@ NTSTATUS __stdcall D3DKMTMapGpuVirtualAddress(IN OUT D3DDDI_MAPGPUVIRTUALADDRESS if (mapGpuVA->BaseAddress == 0) { mapGpuVA->VirtualAddress = mapGpuVA->MinimumAddress; } else { + if (maxSvmAddress != mapGpuVA->MaximumAddress) { + return STATUS_INVALID_PARAMETER; + } mapGpuVA->VirtualAddress = mapGpuVA->BaseAddress; } diff --git a/unit_tests/os_interface/windows/device_command_stream_tests.cpp b/unit_tests/os_interface/windows/device_command_stream_tests.cpp index 578e8d2f97..e09877082b 100644 --- a/unit_tests/os_interface/windows/device_command_stream_tests.cpp +++ b/unit_tests/os_interface/windows/device_command_stream_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2017 - 2018, Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -56,7 +56,7 @@ using namespace ::testing; class WddmCommandStreamFixture : public WddmFixtureMock { public: DeviceCommandStreamReceiver *csr = nullptr; - MemoryManager *mm = nullptr; + MemoryManager *memManager = nullptr; MockDevice *device = nullptr; MockWddmMemoryManager *mockWddmMM = nullptr; DebugManagerStateRestore stateRestore; @@ -71,21 +71,21 @@ class WddmCommandStreamFixture : public WddmFixtureMock { ASSERT_NE(nullptr, csr); mockWddmMM = new MockWddmMemoryManager(wddm); - mm = mockWddmMM; - csr->setMemoryManager(mm); + memManager = mockWddmMM; + csr->setMemoryManager(memManager); device = MockDevice::create(platformDevices[0]); ASSERT_NE(nullptr, device); - mm->device = device; + memManager->device = device; - ASSERT_NE(nullptr, mm); + ASSERT_NE(nullptr, memManager); } void TearDown() { mockWddmMM = nullptr; delete csr->getTagAddress(); delete csr; - delete mm; + delete memManager; delete device; WddmFixtureMock::TearDown(); } @@ -94,7 +94,7 @@ class WddmCommandStreamFixture : public WddmFixtureMock { class WddmCommandStreamWithMockGdiFixture : public WddmFixture { public: DeviceCommandStreamReceiver *csr = nullptr; - MemoryManager *mm = nullptr; + MemoryManager *memManager = nullptr; MockDevice *device = nullptr; MockGdi gdi; DebugManagerStateRestore stateRestore; @@ -107,23 +107,23 @@ class WddmCommandStreamWithMockGdiFixture : public WddmFixture { csr = new WddmCommandStreamReceiver(*platformDevices[0], wddm); ASSERT_NE(nullptr, csr); - mm = csr->createMemoryManager(false); - ASSERT_NE(nullptr, mm); + memManager = csr->createMemoryManager(false); + ASSERT_NE(nullptr, memManager); device = MockDevice::create(platformDevices[0]); ASSERT_NE(nullptr, device); - mm->device = device; + memManager->device = device; - tagAllocation = mm->allocateGraphicsMemory(1024, 4096); + tagAllocation = memManager->allocateGraphicsMemory(1024, 4096); auto tagBuffer = (uint32_t *)tagAllocation->getUnderlyingBuffer(); tagBuffer[0] = initialHardwareTag; } void TearDown() { - mm->freeGraphicsMemory(tagAllocation); + memManager->freeGraphicsMemory(tagAllocation); delete csr->getTagAddress(); delete csr; - delete mm; + delete memManager; wddm = nullptr; delete device; WddmFixture::TearDown(); @@ -159,7 +159,7 @@ TEST_F(WddmCommandStreamTest, givenFlushStampWhenWaitCalledThenWaitForSpecifiedM } TEST_F(WddmCommandStreamTest, Flush) { - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, false, false, cs.getUsed(), &cs}; @@ -169,12 +169,12 @@ TEST_F(WddmCommandStreamTest, Flush) { EXPECT_TRUE(wddm->submitResult.success); EXPECT_EQ(flushStamp, wddm->getMonitoredFence().lastSubmittedFence); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, FlushWithOffset) { auto offset = 128u; - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); @@ -184,11 +184,11 @@ TEST_F(WddmCommandStreamTest, FlushWithOffset) { EXPECT_TRUE(wddm->submitResult.success); EXPECT_EQ(wddm->submitResult.commandBufferSubmitted, (char *)commandBuffer->getUnderlyingBuffer() + offset); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, givenWdmmWhenSubmitIsCalledThenCoherencyRequiredFlagIsSetToFalse) { - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); @@ -200,13 +200,13 @@ TEST_F(WddmCommandStreamTest, givenWdmmWhenSubmitIsCalledThenCoherencyRequiredFl EXPECT_FALSE(pHeader->RequiresCoherency); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, givenWdmmWhenSubmitIsCalledAndPreemptionIsDisabledThenSetHeaderFieldToFalse) { device->setPreemptionMode(PreemptionMode::Disabled); - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); @@ -218,13 +218,13 @@ TEST_F(WddmCommandStreamTest, givenWdmmWhenSubmitIsCalledAndPreemptionIsDisabled EXPECT_FALSE(pHeader->NeedsMidBatchPreEmptionSupport); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, givenWdmmWhenSubmitIsCalledAndPreemptionIsEnabledThenSetHeaderFieldToTrue) { device->setPreemptionMode(PreemptionMode::ThreadGroup); - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); @@ -236,29 +236,29 @@ TEST_F(WddmCommandStreamTest, givenWdmmWhenSubmitIsCalledAndPreemptionIsEnabledT EXPECT_TRUE(pHeader->NeedsMidBatchPreEmptionSupport); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, makeResident) { - WddmMemoryManager *wddmMM = reinterpret_cast(mm); + WddmMemoryManager *wddmMM = reinterpret_cast(memManager); - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); csr->makeResident(*commandBuffer); EXPECT_EQ(0u, wddm->makeResidentResult.called); - EXPECT_EQ(1u, mm->getResidencyAllocations().size()); - EXPECT_EQ(commandBuffer, mm->getResidencyAllocations()[0]); + EXPECT_EQ(1u, memManager->getResidencyAllocations().size()); + EXPECT_EQ(commandBuffer, memManager->getResidencyAllocations()[0]); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, makeNonResidentPutsAllocationInEvictionAllocations) { - WddmMemoryManager *wddmMM = reinterpret_cast(mm); + WddmMemoryManager *wddmMM = reinterpret_cast(memManager); - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); @@ -266,78 +266,92 @@ TEST_F(WddmCommandStreamTest, makeNonResidentPutsAllocationInEvictionAllocations csr->makeNonResident(*commandBuffer); - EXPECT_EQ(1u, mm->getEvictionAllocations().size()); + EXPECT_EQ(1u, memManager->getEvictionAllocations().size()); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamTest, processEvictionPlacesAllAllocationsOnTrimCandidateList) { - WddmMemoryManager *wddmMM = reinterpret_cast(mm); + WddmMemoryManager *wddmMM = reinterpret_cast(memManager); - GraphicsAllocation *allocation = mm->allocateGraphicsMemory(4096, 4096); - GraphicsAllocation *allocation2 = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *allocation = memManager->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *allocation2 = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, allocation); ASSERT_NE(nullptr, allocation2); - mm->pushAllocationForEviction(allocation); - mm->pushAllocationForEviction(allocation2); + memManager->pushAllocationForEviction(allocation); + memManager->pushAllocationForEviction(allocation2); - EXPECT_EQ(2u, mm->getEvictionAllocations().size()); + EXPECT_EQ(2u, memManager->getEvictionAllocations().size()); csr->processEviction(); EXPECT_EQ(2u, mockWddmMM->trimCandidateList.size()); - mm->freeGraphicsMemory(allocation); - mm->freeGraphicsMemory(allocation2); + memManager->freeGraphicsMemory(allocation); + memManager->freeGraphicsMemory(allocation2); } TEST_F(WddmCommandStreamTest, processEvictionClearsEvictionAllocations) { - WddmMemoryManager *wddmMM = reinterpret_cast(mm); + WddmMemoryManager *wddmMM = reinterpret_cast(memManager); - GraphicsAllocation *allocation = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *allocation = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, allocation); - mm->pushAllocationForEviction(allocation); + memManager->pushAllocationForEviction(allocation); - EXPECT_EQ(1u, mm->getEvictionAllocations().size()); + EXPECT_EQ(1u, memManager->getEvictionAllocations().size()); csr->processEviction(); - EXPECT_EQ(0u, mm->getEvictionAllocations().size()); + EXPECT_EQ(0u, memManager->getEvictionAllocations().size()); - mm->freeGraphicsMemory(allocation); + memManager->freeGraphicsMemory(allocation); } TEST_F(WddmCommandStreamTest, makeResidentNonResidentMemObj) { - GraphicsAllocation *gfxAllocation = mm->allocateGraphicsMemory(256); + GraphicsAllocation *gfxAllocation = memManager->allocateGraphicsMemory(256); Buffer *buffer = new AlignedBuffer(gfxAllocation); - WddmMemoryManager *wddmMM = reinterpret_cast(mm); + WddmMemoryManager *wddmMM = reinterpret_cast(memManager); csr->makeResident(*buffer->getGraphicsAllocation()); EXPECT_EQ(0u, wddm->makeResidentResult.called); - EXPECT_EQ(1u, mm->getResidencyAllocations().size()); - EXPECT_EQ(gfxAllocation, mm->getResidencyAllocations()[0]); + EXPECT_EQ(1u, memManager->getResidencyAllocations().size()); + EXPECT_EQ(gfxAllocation, memManager->getResidencyAllocations()[0]); csr->makeNonResident(*buffer->getGraphicsAllocation()); - EXPECT_EQ(gfxAllocation, mm->getEvictionAllocations()[0]); + EXPECT_EQ(gfxAllocation, memManager->getEvictionAllocations()[0]); delete buffer; - mm->freeGraphicsMemory(gfxAllocation); + memManager->freeGraphicsMemory(gfxAllocation); } TEST_F(WddmCommandStreamTest, createAllocationAndMakeResident) { - - void *host_ptr = (void *)0x1234; + void *hostPtr = reinterpret_cast(wddm->virtualAllocAddress + 0x1234); auto size = 1234u; - WddmAllocation *gfxAllocation = (WddmAllocation *)csr->createAllocationAndHandleResidency(host_ptr, size); + WddmAllocation *gfxAllocation = static_cast(csr->createAllocationAndHandleResidency(hostPtr, size)); ASSERT_NE(nullptr, gfxAllocation); - EXPECT_EQ(1u, mm->getResidencyAllocations().size()); + EXPECT_EQ(1u, memManager->getResidencyAllocations().size()); - EXPECT_EQ(host_ptr, gfxAllocation->getUnderlyingBuffer()); + EXPECT_EQ(hostPtr, gfxAllocation->getUnderlyingBuffer()); +} + +TEST_F(WddmCommandStreamTest, givenHostPtrWhenPtrBelowRestrictionThenCreateAllocationAndMakeResident) { + void *hostPtr = reinterpret_cast(wddm->virtualAllocAddress - 0x3000); + auto size = 0x2000u; + void *expectedReserve = reinterpret_cast(wddm->virtualAllocAddress); + WddmAllocation *gfxAllocation = static_cast(csr->createAllocationAndHandleResidency(hostPtr, size)); + + ASSERT_NE(nullptr, gfxAllocation); + + EXPECT_EQ(1u, memManager->getResidencyAllocations().size()); + + EXPECT_EQ(hostPtr, gfxAllocation->getUnderlyingBuffer()); + EXPECT_EQ(expectedReserve, gfxAllocation->getReservedAddress()); + EXPECT_EQ(reinterpret_cast(expectedReserve), gfxAllocation->getGpuAddress()); } TEST_F(WddmCommandStreamTest, killAllTemporaryAllocation) { @@ -385,13 +399,13 @@ TEST_F(WddmCommandStreamTest, killCompletedAllocations) { } TEST_F(WddmCommandStreamMockGdiTest, FlushCallsWddmMakeResidentForResidencyAllocations) { - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); csr->makeResident(*commandBuffer); - EXPECT_EQ(1u, mm->getResidencyAllocations().size()); + EXPECT_EQ(1u, memManager->getResidencyAllocations().size()); gdi.getMakeResidentArg().NumAllocations = 0; @@ -400,18 +414,18 @@ TEST_F(WddmCommandStreamMockGdiTest, FlushCallsWddmMakeResidentForResidencyAlloc EXPECT_NE(0u, gdi.getMakeResidentArg().NumAllocations); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } TEST_F(WddmCommandStreamMockGdiTest, makeResidentClearsResidencyAllocations) { - GraphicsAllocation *commandBuffer = mm->allocateGraphicsMemory(4096, 4096); + GraphicsAllocation *commandBuffer = memManager->allocateGraphicsMemory(4096, 4096); ASSERT_NE(nullptr, commandBuffer); LinearStream cs(commandBuffer); csr->makeResident(*commandBuffer); - EXPECT_EQ(1u, mm->getResidencyAllocations().size()); - EXPECT_EQ(0u, mm->getEvictionAllocations().size()); + EXPECT_EQ(1u, memManager->getResidencyAllocations().size()); + EXPECT_EQ(0u, memManager->getEvictionAllocations().size()); EXPECT_EQ(trimListUnusedPosition, ((WddmAllocation *)commandBuffer)->getTrimCandidateListPosition()); @@ -419,12 +433,12 @@ TEST_F(WddmCommandStreamMockGdiTest, makeResidentClearsResidencyAllocations) { csr->makeSurfacePackNonResident(nullptr); - EXPECT_EQ(0u, mm->getResidencyAllocations().size()); - EXPECT_EQ(0u, mm->getEvictionAllocations().size()); + EXPECT_EQ(0u, memManager->getResidencyAllocations().size()); + EXPECT_EQ(0u, memManager->getEvictionAllocations().size()); EXPECT_EQ(0u, ((WddmAllocation *)commandBuffer)->getTrimCandidateListPosition()); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } template @@ -452,13 +466,13 @@ struct MockWddmCsr : public WddmCommandStreamReceiver { HWTEST_F(WddmCommandStreamMockGdiTest, givenRecordedCommandBufferWhenItIsSubmittedThenFlushTaskIsProperlyCalled) { std::unique_ptr> mockCsr(new MockWddmCsr(*platformDevices[0], this->wddm)); - mockCsr->setMemoryManager(mm); + mockCsr->setMemoryManager(memManager); mockCsr->overrideDispatchPolicy(CommandStreamReceiver::DispatchMode::BatchedDispatch); auto mockedSubmissionsAggregator = new mockSubmissionsAggregator(); mockCsr->overrideSubmissionAggregator(mockedSubmissionsAggregator); - auto commandBuffer = mm->allocateGraphicsMemory(1024, 4096); + auto commandBuffer = memManager->allocateGraphicsMemory(1024, 4096); mockCsr->setTagAllocation(tagAllocation); @@ -504,7 +518,7 @@ HWTEST_F(WddmCommandStreamMockGdiTest, givenRecordedCommandBufferWhenItIsSubmitt EXPECT_NE(trimListUnusedPosition, ((WddmAllocation *)commandBuffer)->getTrimCandidateListPosition()); EXPECT_NE(trimListUnusedPosition, ((WddmAllocation *)csrCommandStream)->getTrimCandidateListPosition()); - mm->freeGraphicsMemory(commandBuffer); + memManager->freeGraphicsMemory(commandBuffer); } HWTEST_F(WddmDefaultTest, givenDefaultWddmCsrWhenItIsCreatedThenBatchingIsTurnedOn) { @@ -585,12 +599,12 @@ HWTEST_F(WddmCsrCompressionTests, givenEnabledCompressionWhenFlushingThenInitTra mockWddmCsr.overrideDispatchPolicy(CommandStreamReceiver::DispatchMode::BatchedDispatch); auto mockMngr = reinterpret_cast(myMockWddm->getPageTableManager()); - mockWddmCsr.setMemoryManager(mm); + mockWddmCsr.setMemoryManager(memManager); mockWddmCsr.setTagAllocation(tagAllocation); auto &csrCS = mockWddmCsr.getCS(); - auto graphicsAllocation = mm->allocateGraphicsMemory(1024, 4096); + auto graphicsAllocation = memManager->allocateGraphicsMemory(1024, 4096); LinearStream cs(graphicsAllocation); EXPECT_FALSE(myMockWddm->peekIsPageTableManagerInitialized()); @@ -606,7 +620,7 @@ HWTEST_F(WddmCsrCompressionTests, givenEnabledCompressionWhenFlushingThenInitTra // flush again to check if PT manager was initialized once mockWddmCsr.flushTask(cs, 0u, cs, cs, cs, cs, 0u, dispatchFlags); - mm->freeGraphicsMemory(graphicsAllocation); + memManager->freeGraphicsMemory(graphicsAllocation); } HWTEST_F(WddmCsrCompressionTests, givenDisabledCompressionWhenFlushingThenDontInitTranslationTable) { @@ -617,10 +631,10 @@ HWTEST_F(WddmCsrCompressionTests, givenDisabledCompressionWhenFlushingThenDontIn EXPECT_EQ(nullptr, myMockWddm->getPageTableManager()); - mockWddmCsr.setMemoryManager(mm); + mockWddmCsr.setMemoryManager(memManager); mockWddmCsr.setTagAllocation(tagAllocation); - auto graphicsAllocation = mm->allocateGraphicsMemory(1024, 4096); + auto graphicsAllocation = memManager->allocateGraphicsMemory(1024, 4096); LinearStream cs(graphicsAllocation); EXPECT_FALSE(myMockWddm->peekIsPageTableManagerInitialized()); @@ -630,5 +644,5 @@ HWTEST_F(WddmCsrCompressionTests, givenDisabledCompressionWhenFlushingThenDontIn EXPECT_FALSE(myMockWddm->peekIsPageTableManagerInitialized()); - mm->freeGraphicsMemory(graphicsAllocation); + memManager->freeGraphicsMemory(graphicsAllocation); } diff --git a/unit_tests/os_interface/windows/wddm_fixture.h b/unit_tests/os_interface/windows/wddm_fixture.h index 385ba60d65..2a2bb5728e 100644 --- a/unit_tests/os_interface/windows/wddm_fixture.h +++ b/unit_tests/os_interface/windows/wddm_fixture.h @@ -39,12 +39,13 @@ #include #pragma warning(pop) +#include + using namespace OCLRT; OsLibrary *setAdapterInfo(const void *platform, const void *gtSystemInfo); class WddmMock : public Wddm { - struct CallResult { uint32_t called = 0; uint64_t uint64ParamPassed = -1; @@ -80,9 +81,33 @@ class WddmMock : public Wddm { createContextResult(), lockResult(), unlockResult(), - waitFromCpuResult() {} + waitFromCpuResult(), + releaseReservedAddressResult(), + reserveValidAddressRangeResult() { + reservedAddresses.clear(); + virtualAllocAddress = OCLRT::windowsMinAddress; + } - WddmMock(Gdi *gdi) : Wddm(gdi) { + WddmMock(Gdi *gdi) : Wddm(gdi), + makeResidentResult(), + makeNonResidentResult(), + mapGpuVirtualAddressResult(), + freeGpuVirtualAddresResult(), + createAllocationResult(), + destroyAllocationResult(), + destroyContextResult(), + queryAdapterInfoResult(), + submitResult(), + waitOnGPUResult(), + configureDeviceAddressSpaceResult(), + createContextResult(), + lockResult(), + unlockResult(), + waitFromCpuResult(), + releaseReservedAddressResult(), + reserveValidAddressRangeResult() { + reservedAddresses.clear(); + virtualAllocAddress = OCLRT::windowsMinAddress; } bool makeResident(D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) override { @@ -134,12 +159,9 @@ class WddmMock : public Wddm { uint32_t allocationCount = 0; D3DKMT_HANDLE resourceHandle = 0; void *cpuPtr = nullptr; - void *gpuPtr = nullptr; + void *reserveAddress = alloc->getReservedAddress(); if (alloc->peekSharedHandle()) { resourceHandle = alloc->resourceHandle; - if (is32bit) { - gpuPtr = (void *)alloc->gpuPtr; - } } else { allocationHandles = &alloc->handle; allocationCount = 1; @@ -149,7 +171,7 @@ class WddmMock : public Wddm { } auto success = destroyAllocations(allocationHandles, allocationCount, alloc->getResidencyData().lastFence, resourceHandle); ::alignedFree(cpuPtr); - releaseGpuPtr(gpuPtr); + releaseReservedAddress(reserveAddress); return success; } @@ -228,9 +250,39 @@ class WddmMock : public Wddm { return waitFromCpuResult.success = Wddm::waitFromCpu(lastFenceValue); } - void releaseGpuPtr(void *gpuPtr) override { - releaseGpuPtrResult.called++; - Wddm::releaseGpuPtr(gpuPtr); + bool virtualFreeWrapper(void *ptr, size_t size, uint32_t flags) { + return true; + } + + void *virtualAllocWrapper(void *inPtr, size_t size, uint32_t flags, uint32_t type) { + void *tmp = reinterpret_cast(virtualAllocAddress); + size += MemoryConstants::pageSize; + size -= size % MemoryConstants::pageSize; + virtualAllocAddress += size; + return tmp; + } + + void releaseReservedAddress(void *reservedAddress) override { + releaseReservedAddressResult.called++; + if (reservedAddress != nullptr) { + std::set::iterator it; + it = reservedAddresses.find(reservedAddress); + EXPECT_NE(reservedAddresses.end(), it); + reservedAddresses.erase(it); + } + Wddm::releaseReservedAddress(reservedAddress); + } + + bool reserveValidAddressRange(size_t size, void *&reservedMem) { + reserveValidAddressRangeResult.called++; + bool ret = Wddm::reserveValidAddressRange(size, reservedMem); + if (reservedMem != nullptr) { + std::set::iterator it; + it = reservedAddresses.find(reservedMem); + EXPECT_EQ(reservedAddresses.end(), it); + reservedAddresses.insert(reservedMem); + } + return ret; } GmmMemory *getGmmMemory() { @@ -255,10 +307,60 @@ class WddmMock : public Wddm { CallResult lockResult; CallResult unlockResult; CallResult waitFromCpuResult; - CallResult releaseGpuPtrResult; + CallResult releaseReservedAddressResult; + CallResult reserveValidAddressRangeResult; NTSTATUS createAllocationStatus; bool callBaseDestroyAllocations = true; bool failOpenSharedHandle = false; + std::set reservedAddresses; + uintptr_t virtualAllocAddress; +}; + +class WddmMockReserveAddress : public WddmMock { + public: + WddmMockReserveAddress() : WddmMock() { + returnGood = 0; + returnInvalidCount = 0; + returnInvalidIter = 0; + returnNullCount = 0; + returnNullIter = 0; + } + WddmMockReserveAddress(Gdi *gdi) : WddmMock(gdi) { + returnGood = 0; + returnInvalidCount = 0; + returnInvalidIter = 0; + returnNullCount = 0; + returnNullIter = 0; + } + + void *virtualAllocWrapper(void *inPtr, size_t size, uint32_t flags, uint32_t type) { + if (returnGood != 0) { + return WddmMock::virtualAllocWrapper(inPtr, size, flags, type); + } + + if (returnInvalidCount != 0) { + returnInvalidIter++; + if (returnInvalidIter > returnInvalidCount) { + return WddmMock::virtualAllocWrapper(inPtr, size, flags, type); + } + if (returnNullCount != 0) { + returnNullIter++; + if (returnNullIter > returnNullCount) { + return nullptr; + } + return reinterpret_cast(0x1000); + } + return reinterpret_cast(0x1000); + } + + return nullptr; + } + + uint32_t returnGood; + uint32_t returnInvalidCount; + uint32_t returnInvalidIter; + uint32_t returnNullCount; + uint32_t returnNullIter; }; class WddmFixture { @@ -306,10 +408,13 @@ class WddmFixture { } virtual void TearDown() { - if (wddm != nullptr) + if (wddm != nullptr) { + EXPECT_EQ(0, mockWddm->reservedAddresses.size()); delete wddm; - if (mockGdiDll != nullptr) + } + if (mockGdiDll != nullptr) { delete mockGdiDll; + } } }; diff --git a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp index b781ca21db..a7cf0e47e7 100644 --- a/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_memory_manager_tests.cpp @@ -557,6 +557,20 @@ HWTEST_F(WddmMemoryManagerTest, givenDefaultWddmMemoryManagerWhenAskedForAligned EXPECT_EQ(OCLRT::windowsMinAddress, mallocRestrictions->minAddress); } +HWTEST_F(WddmMemoryManagerTest, givenWddmMemoryManagerWhenCpuMemNotMeetRestrictionsThenReserveMemRangeForMap) { + SetUpMm(); + WddmMock *mockWddm = static_cast(wddm); + void *cpuPtr = reinterpret_cast(mockWddm->virtualAllocAddress - 0x3000); + size_t size = 0x1000; + void *expectReserve = reinterpret_cast(mockWddm->virtualAllocAddress); + + WddmAllocation *allocation = static_cast(mm->allocateGraphicsMemory(size, cpuPtr)); + ASSERT_NE(nullptr, allocation); + EXPECT_EQ(expectReserve, allocation->getReservedAddress()); + EXPECT_EQ(expectReserve, reinterpret_cast(allocation->gpuPtr)); + mm->freeGraphicsMemory(allocation); +} + HWTEST_F(WddmMemoryManagerResidencyTest, addToTrimCandidateListPlacesAllocationInContainerAndAssignsPosition) { SetUpMm(); WddmAllocation allocation; @@ -780,9 +794,11 @@ HWTEST_F(WddmMemoryManagerResidencyTest, makeResidentResidencyAllocationsUpdates HWTEST_F(WddmMemoryManagerResidencyTest, makeResidentResidencyAllocationsMarksTripleAllocationsResident) { SetUpMm(); + WddmMock *mockWddm = static_cast(wddm); WddmAllocation allocation1, allocation2; + void *ptr = reinterpret_cast(mockWddm->virtualAllocAddress + 0x1500); - WddmAllocation *allocationTriple = (WddmAllocation *)mm->allocateGraphicsMemory(8196, (void *)0x1500); + WddmAllocation *allocationTriple = (WddmAllocation *)mm->allocateGraphicsMemory(8196, ptr); mm->pushAllocationForResidency(&allocation1); mm->pushAllocationForResidency(allocationTriple); @@ -901,12 +917,13 @@ HWTEST_F(WddmMemoryManagerResidencyTest, givenOneUsedAllocationFromPreviousPerio HWTEST_F(WddmMemoryManagerResidencyTest, givenTripleAllocationWithUsedAndUnusedFragmentsSincePreviousTrimWhenTrimResidencyPeriodicTrimIsCalledThenProperFragmentsAreEvictedAndMarked) { SetUpMm(); + WddmMock *mockWddm = static_cast(wddm); D3DKMT_TRIMNOTIFICATION trimNotification = {0}; trimNotification.Flags.PeriodicTrim = 1; trimNotification.NumBytesToTrim = 0; - + void *ptr = reinterpret_cast(mockWddm->virtualAllocAddress + 0x1500); // 3-fragment Allocation - WddmAllocation *allocationTriple = (WddmAllocation *)mm->allocateGraphicsMemory(8196, (void *)0x1500); + WddmAllocation *allocationTriple = (WddmAllocation *)mm->allocateGraphicsMemory(8196, ptr); // whole allocation unused since previous trim allocationTriple->getResidencyData().lastFence = 0; @@ -1048,9 +1065,9 @@ HWTEST_F(WddmMemoryManagerResidencyTest, trimToBudgetReturnsFalseWhenNumBytesToT HWTEST_F(WddmMemoryManagerResidencyTest, trimToBudgetStopsEvictingWhenNumBytesToTrimIsZero) { SetUpMm(); - WddmAllocation allocation1((void *)(0x1000), 0x1000, (void *)(0x1000), 0x1000), - allocation2((void *)(0x1000), 0x3000, (void *)(0x1000), 0x3000), - allocation3((void *)(0x1000), 0x1000, (void *)(0x1000), 0x1000); + WddmAllocation allocation1((void *)(0x1000), 0x1000, (void *)(0x1000), 0x1000, nullptr), + allocation2((void *)(0x1000), 0x3000, (void *)(0x1000), 0x3000, nullptr), + allocation3((void *)(0x1000), 0x1000, (void *)(0x1000), 0x1000, nullptr); allocation1.getResidencyData().resident = true; allocation1.getResidencyData().lastFence = 0; @@ -1148,16 +1165,19 @@ HWTEST_F(WddmMemoryManagerResidencyTest, trimToBudgetWaitsFromCpuWhenLastFenceIs HWTEST_F(WddmMemoryManagerResidencyTest, trimToBudgetEvictsDoneFragmentsOnly) { SetUpMm(); + WddmMock *mockWddm = static_cast(wddm); gdi.setNonZeroNumBytesToTrimInEvict(); - - WddmAllocation allocation1((void *)0x1000, 0x1000, (void *)0x1000, 0x1000), allocation2((void *)0x1000, 0x1000, (void *)0x1000, 0x1000); + void *ptr = reinterpret_cast(mockWddm->virtualAllocAddress + 0x1000); + WddmAllocation allocation1(ptr, 0x1000, ptr, 0x1000, nullptr); + WddmAllocation allocation2(ptr, 0x1000, ptr, 0x1000, nullptr); allocation1.getResidencyData().resident = true; allocation1.getResidencyData().lastFence = 0; allocation2.getResidencyData().lastFence = 1; allocation2.getResidencyData().resident = true; - WddmAllocation *allocationTriple = (WddmAllocation *)mm->allocateGraphicsMemory(8196, (void *)0x1500); + void *ptrTriple = reinterpret_cast(reinterpret_cast(ptr) + 0x500); + WddmAllocation *allocationTriple = static_cast(mm->allocateGraphicsMemory(8196, ptrTriple)); allocationTriple->getResidencyData().lastFence = 1; allocationTriple->getResidencyData().resident = true; @@ -1260,7 +1280,8 @@ HWTEST_F(BufferWithWddmMemory, NullOsHandleStorageAskedForPopulationReturnsFille HWTEST_F(BufferWithWddmMemory, GivenMisalignedHostPtrAndMultiplePagesSizeWhenAskedForGraphicsAllcoationThenItContainsAllFragmentsWithProperGpuAdrresses) { SetUpMm(); - auto ptr = (void *)0x1001; + WddmMock *mockWddm = static_cast(wddm); + auto ptr = reinterpret_cast(mockWddm->virtualAllocAddress + 0x1001); auto size = MemoryConstants::pageSize * 10; auto graphicsAllocation = mm->allocateGraphicsMemory(size, ptr); @@ -1289,10 +1310,11 @@ HWTEST_F(BufferWithWddmMemory, GivenMisalignedHostPtrAndMultiplePagesSizeWhenAsk HWTEST_F(BufferWithWddmMemory, GivenPointerAndSizeWhenAskedToCreateGrahicsAllocationThenGraphicsAllocationIsCreated) { SetUpMm(); + WddmMock *mockWddm = static_cast(wddm); OsHandleStorage handleStorage; - auto ptr = (void *)0x1000; - auto ptr2 = (void *)0x1001; + auto ptr = reinterpret_cast(mockWddm->virtualAllocAddress + 0x1000); + auto ptr2 = reinterpret_cast(mockWddm->virtualAllocAddress + 0x1001); auto size = MemoryConstants::pageSize; handleStorage.fragmentStorageData[0].cpuPtr = ptr; @@ -1317,7 +1339,7 @@ HWTEST_F(BufferWithWddmMemory, GivenPointerAndSizeWhenAskedToCreateGrahicsAlloca EXPECT_EQ(size * 3, allocation->fragmentsStorage.fragmentStorageData[2].fragmentSize); EXPECT_NE(&allocation->fragmentsStorage, &handleStorage); - delete allocation; + mm->freeGraphicsMemory(allocation); } HWTEST_F(WddmMemoryManagerTest2, makeResidentResidencyAllocationsDoesNotMarkAllocationsResidentWhenMakeResidentFails) { @@ -1347,8 +1369,9 @@ HWTEST_F(WddmMemoryManagerTest2, makeResidentResidencyAllocationsDoesNotMarkAllo HWTEST_F(WddmMemoryManagerTest2, makeResidentResidencyAllocationsDoesNotMarkTripleAllocationsResidentWhenMakeResidentFails) { SetUpMm(); WddmAllocation allocation1, allocation2; - - WddmAllocation *allocationTriple = (WddmAllocation *)mm->allocateGraphicsMemory(8196, (void *)0x1500); + void *ptr = reinterpret_cast(wddm->getWddmMinAddress() + 0x1500); + WddmAllocation *allocationTriple = static_cast(mm->allocateGraphicsMemory(8196, ptr)); + ASSERT_NE(nullptr, allocationTriple); auto makeResidentWithOutBytesToTrim = [](D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim) -> bool { *numberOfBytesToTrim = 4 * 4096; return false; }; @@ -1440,9 +1463,9 @@ HWTEST_F(WddmMemoryManagerTest2, givenAllocationPackWhenTheyArePassedToMakeResid HWTEST_F(WddmMemoryManagerTest2, makeResidentResidencyAllocationsSucceedsWhenMakeResidentFailsAndTrimToBudgetSucceeds) { SetUpMm(); WddmAllocation allocation1; - + void *cpuPtr = reinterpret_cast(wddm->getWddmMinAddress() + 0x1000); size_t allocationSize = 0x1000; - WddmAllocation allocationToTrim((void *)0x1000, allocationSize, (void *)0x1000, allocationSize); + WddmAllocation allocationToTrim(cpuPtr, allocationSize, cpuPtr, allocationSize, nullptr); allocationToTrim.getResidencyData().lastFence = wddm->getMonitoredFence().lastSubmittedFence; @@ -1721,7 +1744,7 @@ HWTEST_F(MockWddmMemoryManagerTest, givenRenderCompressedAllocationWhenReleasein auto mockMngr = new NiceMock(); wddm->resetPageTableManager(mockMngr); - auto wddmAlloc = (WddmAllocation *)memoryManager.allocateGraphicsMemory(4096u, 4096u); + auto wddmAlloc = static_cast(memoryManager.allocateGraphicsMemory(4096u, 4096u)); wddmAlloc->gpuPtr = gpuVa; wddmAlloc->gmm->isRenderCompressed = true; @@ -1746,7 +1769,7 @@ HWTEST_F(MockWddmMemoryManagerTest, givenNonRenderCompressedAllocationWhenReleas auto mockMngr = new NiceMock(); wddm->resetPageTableManager(mockMngr); - auto wddmAlloc = (WddmAllocation *)memoryManager.allocateGraphicsMemory(4096u, 4096u); + auto wddmAlloc = static_cast(memoryManager.allocateGraphicsMemory(4096u, 4096u)); wddmAlloc->gmm->isRenderCompressed = false; EXPECT_CALL(*mockMngr, updateAuxTable(_)).Times(0); @@ -1794,7 +1817,7 @@ HWTEST_F(MockWddmMemoryManagerTest, givenRenderCompressedFlagSetWhenInternalIsUn myGmm->isRenderCompressed = false; myGmm->gmmResourceInfo->getResourceFlags()->Info.RenderCompressed = 1; - auto wddmAlloc = (WddmAllocation *)memoryManager.allocateGraphicsMemory(4096u, 4096u); + auto wddmAlloc = static_cast(memoryManager.allocateGraphicsMemory(4096u, 4096u)); delete wddmAlloc->gmm; wddmAlloc->gmm = myGmm; diff --git a/unit_tests/os_interface/windows/wddm_memory_manager_tests.h b/unit_tests/os_interface/windows/wddm_memory_manager_tests.h index 096d202bbf..0ec8e530d1 100644 --- a/unit_tests/os_interface/windows/wddm_memory_manager_tests.h +++ b/unit_tests/os_interface/windows/wddm_memory_manager_tests.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Intel Corporation + * Copyright (c) 2017 - 2018, Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -58,6 +58,8 @@ class WddmMemoryManagerFixture : public MemoryManagementFixture, public WddmFixt } virtual void TearDown() { + WddmMock *mockWddm = static_cast(this->wddm); + EXPECT_EQ(0, mockWddm->reservedAddresses.size()); delete mm; this->wddm = nullptr; WddmFixture::TearDown(); @@ -91,6 +93,8 @@ class MockWddmMemoryManagerFixture : public WddmFixture { } virtual void TearDown() { + WddmMock *mockWddm = static_cast(this->wddm); + EXPECT_EQ(0, mockWddm->reservedAddresses.size()); delete mm; this->wddm = nullptr; WddmFixture::TearDown(); @@ -105,9 +109,23 @@ class GmockWddm : public Wddm { using Wddm::device; GmockWddm() { + virtualAllocAddress = OCLRT::windowsMinAddress; } ~GmockWddm() = default; + bool virtualFreeWrapper(void *ptr, size_t size, uint32_t flags) { + return true; + } + + void *virtualAllocWrapper(void *inPtr, size_t size, uint32_t flags, uint32_t type) { + void *tmp = reinterpret_cast(virtualAllocAddress); + size += MemoryConstants::pageSize; + size -= size % MemoryConstants::pageSize; + virtualAllocAddress += size; + return tmp; + } + uintptr_t virtualAllocAddress; + MOCK_METHOD4(makeResident, bool(D3DKMT_HANDLE *handles, uint32_t count, bool cantTrimFurther, uint64_t *numberOfBytesToTrim)); }; diff --git a/unit_tests/os_interface/windows/wddm_tests.cpp b/unit_tests/os_interface/windows/wddm_tests.cpp index 842cf88807..d21997eaab 100644 --- a/unit_tests/os_interface/windows/wddm_tests.cpp +++ b/unit_tests/os_interface/windows/wddm_tests.cpp @@ -111,7 +111,7 @@ HWTEST_F(WddmTest, allocation) { wddm->init(); ASSERT_TRUE(wddm->isInitialized()); OsAgnosticMemoryManager mm(false); - WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100); + WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100, nullptr); Gmm *gmm; gmm = getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize()); @@ -143,7 +143,7 @@ HWTEST_F(WddmTest, createAllocation32bit) { void *alignedPtr = (void *)0x12000; size_t alignedSize = 0x2000; - WddmAllocation allocation(alignedPtr, alignedSize); + WddmAllocation allocation(alignedPtr, alignedSize, nullptr); Gmm *gmm; gmm = getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize()); @@ -206,7 +206,7 @@ HWTEST_F(WddmTest, mapAndFreeGpuVa) { ASSERT_TRUE(wddm->isInitialized()); OsAgnosticMemoryManager mm(false); - WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100); + WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100, nullptr); Gmm *gmm; gmm = getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize()); @@ -237,7 +237,7 @@ HWTEST_F(WddmTest, givenNullAllocationWhenCreateThenAllocateAndMap) { ASSERT_TRUE(wddm->isInitialized()); OsAgnosticMemoryManager mm(false); - WddmAllocation allocation(nullptr, 100); + WddmAllocation allocation(nullptr, 100, nullptr); Gmm *gmm; gmm = getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize()); @@ -258,7 +258,7 @@ HWTEST_F(WddmTest, makeResidentNonResident) { ASSERT_TRUE(wddm->isInitialized()); OsAgnosticMemoryManager mm(false); - WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100); + WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100, nullptr); Gmm *gmm; gmm = getGmm(allocation.getUnderlyingBuffer(), allocation.getUnderlyingBufferSize()); @@ -559,7 +559,7 @@ HWTEST_F(WddmWithMockGdiTest, makeResidentMultipleHandles) { ASSERT_TRUE(wddm->isInitialized()); OsAgnosticMemoryManager mm(false); - WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100); + WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100, nullptr); allocation.handle = ALLOCATION_HANDLE; D3DKMT_HANDLE handles[2] = {0}; @@ -584,7 +584,7 @@ HWTEST_F(WddmWithMockGdiTest, makeResidentMultipleHandlesWithReturnBytesToTrim) ASSERT_TRUE(wddm->isInitialized()); OsAgnosticMemoryManager mm(false); - WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100); + WddmAllocation allocation(mm.allocateSystemMemory(100, 0), 100, nullptr); allocation.handle = ALLOCATION_HANDLE; D3DKMT_HANDLE handles[2] = {0}; @@ -606,7 +606,6 @@ HWTEST_F(WddmWithMockGdiTest, makeResidentMultipleHandlesWithReturnBytesToTrim) } TEST_F(WddmWithMockGdiTest, makeNonResidentCallsEvict) { - MockGdi gdi; WddmMock wddm(&gdi); @@ -628,13 +627,12 @@ TEST_F(WddmWithMockGdiTest, makeNonResidentCallsEvict) { } HWTEST_F(WddmWithMockGdiTest, destroyAllocationWithLastFenceValueGreaterThanCurrentValueCallsWaitFromCpu) { - MockGdi gdi; WddmMock wddm(&gdi); wddm.init(); - WddmAllocation allocation((void *)0x23000, 0x1000); + WddmAllocation allocation((void *)0x23000, 0x1000, nullptr); allocation.getResidencyData().lastFence = 20; allocation.handle = ALLOCATION_HANDLE; @@ -667,13 +665,12 @@ HWTEST_F(WddmWithMockGdiTest, destroyAllocationWithLastFenceValueGreaterThanCurr } HWTEST_F(WddmWithMockGdiTest, destroyAllocationWithLastFenceValueLessEqualToCurrentValueDoesNotCallWaitFromCpu) { - MockGdi gdi; WddmMock wddm(&gdi); wddm.init(); - WddmAllocation allocation((void *)0x23000, 0x1000); + WddmAllocation allocation((void *)0x23000, 0x1000, nullptr); allocation.getResidencyData().lastFence = 10; allocation.handle = ALLOCATION_HANDLE; @@ -706,13 +703,12 @@ HWTEST_F(WddmWithMockGdiTest, destroyAllocationWithLastFenceValueLessEqualToCurr } HWTEST_F(WddmWithMockGdiTest, WhenLastFenceLessEqualThanMonitoredThenWaitFromCpuIsNotCalled) { - MockGdi gdi; WddmMock wddm(&gdi); wddm.init(); - WddmAllocation allocation((void *)0x23000, 0x1000); + WddmAllocation allocation((void *)0x23000, 0x1000, nullptr); allocation.getResidencyData().lastFence = 10; allocation.handle = ALLOCATION_HANDLE; @@ -735,13 +731,12 @@ HWTEST_F(WddmWithMockGdiTest, WhenLastFenceLessEqualThanMonitoredThenWaitFromCpu } HWTEST_F(WddmWithMockGdiTest, WhenLastFenceGreaterThanMonitoredThenWaitFromCpuIsCalled) { - MockGdi gdi; WddmMock wddm(&gdi); wddm.init(); - WddmAllocation allocation((void *)0x23000, 0x1000); + WddmAllocation allocation((void *)0x23000, 0x1000, nullptr); allocation.getResidencyData().lastFence = 10; allocation.handle = ALLOCATION_HANDLE; @@ -764,7 +759,6 @@ HWTEST_F(WddmWithMockGdiTest, WhenLastFenceGreaterThanMonitoredThenWaitFromCpuIs } HWTEST_F(WddmWithMockGdiTest, createMonitoredFenceIsInitializedWithFenceValueZeroAndCurrentFenceValueIsSetToOne) { - MockGdi gdi; WddmMock wddm(&gdi); @@ -786,7 +780,6 @@ NTSTATUS APIENTRY queryResourceInfoMock(D3DKMT_QUERYRESOURCEINFO *pData) { } HWTEST_F(WddmWithMockGdiTest, givenOpenSharedHandleWhenZeroAllocationsThenReturnNull) { - MockGdi gdi; WddmMock wddm(&gdi); @@ -800,3 +793,90 @@ HWTEST_F(WddmWithMockGdiTest, givenOpenSharedHandleWhenZeroAllocationsThenReturn EXPECT_EQ(false, ret); } + +using WddmReserveAddressTest = WddmTest; + +HWTEST_F(WddmReserveAddressTest, givenWddmWhenFirstIsSuccessfulThenReturnReserveAddress) { + std::unique_ptr wddmMockPtr(new WddmMockReserveAddress()); + WddmMockReserveAddress *wddmMock = wddmMockPtr.get(); + size_t size = 0x1000; + void *reserve = nullptr; + + bool ret = wddmMock->init(); + EXPECT_TRUE(ret); + + wddmMock->returnGood = 1; + uintptr_t expectedReserve = wddmMock->virtualAllocAddress; + ret = wddmMock->reserveValidAddressRange(size, reserve); + EXPECT_TRUE(ret); + EXPECT_EQ(expectedReserve, reinterpret_cast(reserve)); + wddmMock->releaseReservedAddress(reserve); +} + +HWTEST_F(WddmReserveAddressTest, givenWddmWhenFirstIsNullThenReturnNull) { + std::unique_ptr wddmMockPtr(new WddmMockReserveAddress()); + WddmMockReserveAddress *wddmMock = wddmMockPtr.get(); + size_t size = 0x1000; + void *reserve = nullptr; + + bool ret = wddmMock->init(); + EXPECT_TRUE(ret); + uintptr_t expectedReserve = 0; + ret = wddmMock->reserveValidAddressRange(size, reserve); + EXPECT_FALSE(ret); + EXPECT_EQ(expectedReserve, reinterpret_cast(reserve)); +} + +HWTEST_F(WddmReserveAddressTest, givenWddmWhenFirstIsInvalidSecondSuccessfulThenReturnSecond) { + std::unique_ptr wddmMockPtr(new WddmMockReserveAddress()); + WddmMockReserveAddress *wddmMock = wddmMockPtr.get(); + size_t size = 0x1000; + void *reserve = nullptr; + + bool ret = wddmMock->init(); + EXPECT_TRUE(ret); + + wddmMock->returnInvalidCount = 1; + uintptr_t expectedReserve = wddmMock->virtualAllocAddress; + + ret = wddmMock->reserveValidAddressRange(size, reserve); + EXPECT_TRUE(ret); + EXPECT_EQ(expectedReserve, reinterpret_cast(reserve)); + wddmMock->releaseReservedAddress(reserve); +} + +HWTEST_F(WddmReserveAddressTest, givenWddmWhenSecondIsInvalidThirdSuccessfulThenReturnThird) { + std::unique_ptr wddmMockPtr(new WddmMockReserveAddress()); + WddmMockReserveAddress *wddmMock = wddmMockPtr.get(); + size_t size = 0x1000; + void *reserve = nullptr; + + bool ret = wddmMock->init(); + EXPECT_TRUE(ret); + + wddmMock->returnInvalidCount = 2; + uintptr_t expectedReserve = wddmMock->virtualAllocAddress; + + ret = wddmMock->reserveValidAddressRange(size, reserve); + EXPECT_TRUE(ret); + EXPECT_EQ(expectedReserve, reinterpret_cast(reserve)); + wddmMock->releaseReservedAddress(reserve); +} + +HWTEST_F(WddmReserveAddressTest, givenWddmWhenFirstIsInvalidSecondNullThenReturnSecondNull) { + std::unique_ptr wddmMockPtr(new WddmMockReserveAddress()); + WddmMockReserveAddress *wddmMock = wddmMockPtr.get(); + size_t size = 0x1000; + void *reserve = nullptr; + + bool ret = wddmMock->init(); + EXPECT_TRUE(ret); + + wddmMock->returnInvalidCount = 2; + wddmMock->returnNullCount = 1; + uintptr_t expectedReserve = 0; + + ret = wddmMock->reserveValidAddressRange(size, reserve); + EXPECT_FALSE(ret); + EXPECT_EQ(expectedReserve, reinterpret_cast(reserve)); +}