From ff0add74e3db6e1b324e03ea4602ba17348046ba Mon Sep 17 00:00:00 2001 From: Lukasz Jobczyk Date: Thu, 2 Jul 2020 11:49:46 +0200 Subject: [PATCH] Add alternative residency model on Linux Related-To: NEO-4732 Change-Id: I79e165d2b647af200ca314e1183ecf05903de644 Signed-off-by: Lukasz Jobczyk --- level_zero/core/source/device/device_imp.cpp | 8 +- level_zero/core/source/event/event.cpp | 2 +- opencl/source/mem_obj/buffer.cpp | 9 +- .../os_interface/linux/drm_command_stream.inl | 10 +- .../test/unit_test/mem_obj/buffer_tests.cpp | 17 ---- .../mocks/mock_memory_operations_handler.h | 6 +- .../linux/drm_memory_manager_mt_tests.cpp | 4 +- .../linux/drm_command_stream_fixture.h | 17 ++-- .../linux/drm_command_stream_mm_tests.cpp | 4 +- .../linux/drm_command_stream_tests.cpp | 68 +------------- .../linux/drm_gem_close_worker_tests.cpp | 2 +- ..._manager_allocate_in_device_pool_tests.cpp | 94 ++++++++++++++++++- .../linux/drm_memory_manager_tests.cpp | 4 +- .../linux/drm_memory_manager_tests.h | 6 +- .../unit_test/os_interface/linux/drm_mock.cpp | 5 + .../linux/drm_residency_handler_tests.cpp | 37 +++----- ...te_command_queue_with_properties_tests.cpp | 2 +- .../windows/wddm_residency_handler_tests.cpp | 39 ++++---- shared/source/device/device.cpp | 4 + shared/source/device/device.h | 1 + .../windows/wddm_direct_submission.inl | 2 +- .../memory_manager/graphics_allocation.cpp | 1 + .../memory_manager/graphics_allocation.h | 8 +- .../memory_operations_handler.h | 10 +- .../aub_memory_operations_handler.cpp | 6 +- .../aub_memory_operations_handler.h | 6 +- .../source/os_interface/linux/CMakeLists.txt | 6 +- .../os_interface/linux/drm_allocation.cpp | 29 ++++-- .../os_interface/linux/drm_allocation.h | 6 +- .../linux/drm_allocation_extended.cpp | 4 +- .../os_interface/linux/drm_buffer_object.cpp | 14 +++ .../os_interface/linux/drm_buffer_object.h | 3 + .../os_interface/linux/drm_memory_manager.cpp | 5 +- .../linux/drm_memory_operations_handler.cpp | 51 ---------- .../linux/drm_memory_operations_handler.h | 18 ++-- .../drm_memory_operations_handler_bind.cpp | 94 +++++++++++++++++++ .../drm_memory_operations_handler_bind.h | 25 +++++ .../drm_memory_operations_handler_create.cpp | 16 ++++ .../drm_memory_operations_handler_default.cpp | 61 ++++++++++++ .../drm_memory_operations_handler_default.h | 31 ++++++ shared/source/os_interface/linux/drm_neo.h | 3 + .../source/os_interface/linux/drm_query.cpp | 8 ++ .../os_interface/linux/drm_query_dg1.cpp | 8 ++ .../os_interface/linux/os_context_linux.cpp | 4 + .../os_interface/linux/os_context_linux.h | 1 + .../os_interface/linux/os_interface.cpp | 2 +- .../windows/wddm_memory_manager.cpp | 2 +- .../wddm_memory_operations_handler.cpp | 6 +- .../windows/wddm_memory_operations_handler.h | 6 +- .../aub_memory_operations_handler_tests.cpp | 18 ++-- 50 files changed, 521 insertions(+), 272 deletions(-) delete mode 100644 shared/source/os_interface/linux/drm_memory_operations_handler.cpp create mode 100644 shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp create mode 100644 shared/source/os_interface/linux/drm_memory_operations_handler_bind.h create mode 100644 shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp create mode 100644 shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp create mode 100644 shared/source/os_interface/linux/drm_memory_operations_handler_default.h diff --git a/level_zero/core/source/device/device_imp.cpp b/level_zero/core/source/device/device_imp.cpp index b0a0fa5943..9c735cfd3f 100644 --- a/level_zero/core/source/device/device_imp.cpp +++ b/level_zero/core/source/device/device_imp.cpp @@ -181,7 +181,7 @@ ze_result_t DeviceImp::createModule(const ze_module_desc_t *desc, ze_module_hand ze_result_t DeviceImp::evictImage(ze_image_handle_t hImage) { auto alloc = Image::fromHandle(hImage)->getAllocation(); NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get(); - auto success = memoryOperationsIface->evict(*alloc); + auto success = memoryOperationsIface->evict(neoDevice, *alloc); return changeMemoryOperationStatusToL0ResultType(success); } @@ -191,7 +191,7 @@ ze_result_t DeviceImp::evictMemory(void *ptr, size_t size) { return ZE_RESULT_ERROR_INVALID_ARGUMENT; } NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get(); - auto success = memoryOperationsIface->evict(*alloc->gpuAllocations.getGraphicsAllocation(getRootDeviceIndex())); + auto success = memoryOperationsIface->evict(neoDevice, *alloc->gpuAllocations.getGraphicsAllocation(getRootDeviceIndex())); return changeMemoryOperationStatusToL0ResultType(success); } @@ -399,7 +399,7 @@ ze_result_t DeviceImp::getSubDevices(uint32_t *pCount, ze_device_handle_t *phSub ze_result_t DeviceImp::makeImageResident(ze_image_handle_t hImage) { auto alloc = Image::fromHandle(hImage)->getAllocation(); NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get(); - auto success = memoryOperationsIface->makeResident(ArrayRef(&alloc, 1)); + auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef(&alloc, 1)); return changeMemoryOperationStatusToL0ResultType(success); } @@ -410,7 +410,7 @@ ze_result_t DeviceImp::makeMemoryResident(void *ptr, size_t size) { } NEO::MemoryOperationsHandler *memoryOperationsIface = neoDevice->getRootDeviceEnvironment().memoryOperationsInterface.get(); auto gpuAllocation = alloc->gpuAllocations.getGraphicsAllocation(getRootDeviceIndex()); - auto success = memoryOperationsIface->makeResident(ArrayRef(&gpuAllocation, 1)); + auto success = memoryOperationsIface->makeResident(neoDevice, ArrayRef(&gpuAllocation, 1)); return changeMemoryOperationStatusToL0ResultType(success); } diff --git a/level_zero/core/source/event/event.cpp b/level_zero/core/source/event/event.cpp index 24a008ef39..c91d7964d6 100644 --- a/level_zero/core/source/event/event.cpp +++ b/level_zero/core/source/event/event.cpp @@ -214,7 +214,7 @@ void EventImp::makeAllocationResident() { if (memoryOperationsIface) { auto alloc = &(this->eventPool->getAllocation()); - memoryOperationsIface->makeResident(ArrayRef(&alloc, 1)); + memoryOperationsIface->makeResident(deviceImp->neoDevice, ArrayRef(&alloc, 1)); } } diff --git a/opencl/source/mem_obj/buffer.cpp b/opencl/source/mem_obj/buffer.cpp index 30e7ec7b14..c8dcb9a009 100644 --- a/opencl/source/mem_obj/buffer.cpp +++ b/opencl/source/mem_obj/buffer.cpp @@ -359,9 +359,12 @@ Buffer *Buffer::create(Context *context, } if (DebugManager.flags.MakeAllBuffersResident.get()) { - auto graphicsAllocation = pBuffer->getGraphicsAllocation(rootDeviceIndex); - auto rootDeviceEnvironment = pBuffer->executionEnvironment->rootDeviceEnvironments[rootDeviceIndex].get(); - rootDeviceEnvironment->memoryOperationsInterface->makeResident(ArrayRef(&graphicsAllocation, 1)); + for (size_t deviceNum = 0u; deviceNum < context->getNumDevices(); deviceNum++) { + auto device = context->getDevice(deviceNum); + auto graphicsAllocation = pBuffer->getGraphicsAllocation(device->getRootDeviceIndex()); + auto rootDeviceEnvironment = pBuffer->executionEnvironment->rootDeviceEnvironments[device->getRootDeviceIndex()].get(); + rootDeviceEnvironment->memoryOperationsInterface->makeResident(&device->getDevice(), ArrayRef(&graphicsAllocation, 1)); + } } return pBuffer; diff --git a/opencl/source/os_interface/linux/drm_command_stream.inl b/opencl/source/os_interface/linux/drm_command_stream.inl index e8e93a0583..86bfa48aaf 100644 --- a/opencl/source/os_interface/linux/drm_command_stream.inl +++ b/opencl/source/os_interface/linux/drm_command_stream.inl @@ -69,12 +69,8 @@ bool DrmCommandStreamReceiver::flush(BatchBuffer &batchBuffer, Reside auto memoryOperationsInterface = static_cast(this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface.get()); - std::unique_lock lock; - if (DebugManager.flags.MakeAllBuffersResident.get()) { - lock = memoryOperationsInterface->acquireLock(); - } - - memoryOperationsInterface->mergeWithResidencyContainer(allocationsForResidency); + auto lock = memoryOperationsInterface->lockHandlerForExecWA(); + memoryOperationsInterface->mergeWithResidencyContainer(this->osContext, allocationsForResidency); this->flushStamp->setStamp(bb->peekHandle()); this->flushInternal(batchBuffer, allocationsForResidency); @@ -120,7 +116,7 @@ template void DrmCommandStreamReceiver::processResidency(const ResidencyContainer &inputAllocationsForResidency, uint32_t handleId) { for (auto &alloc : inputAllocationsForResidency) { auto drmAlloc = static_cast(alloc); - drmAlloc->getBOsForResidency(osContext->getContextId(), handleId, this->residency); + drmAlloc->makeBOsResident(osContext->getContextId(), 0u, handleId, &this->residency, false); } } diff --git a/opencl/test/unit_test/mem_obj/buffer_tests.cpp b/opencl/test/unit_test/mem_obj/buffer_tests.cpp index 6f9e16ae7e..d4dabf4606 100644 --- a/opencl/test/unit_test/mem_obj/buffer_tests.cpp +++ b/opencl/test/unit_test/mem_obj/buffer_tests.cpp @@ -2467,23 +2467,6 @@ TEST(SharedBuffersTest, whenBuffersIsCreatedWithSharingHandlerThenItIsSharedBuff buffer->release(); } -TEST(ResidencyTests, whenBuffersIsCreatedWithMakeResidentFlagThenItSuccessfulyCreates) { - VariableBackup backup(&ultHwConfig); - ultHwConfig.useMockedPrepareDeviceEnvironmentsFunc = false; - ultHwConfig.forceOsAgnosticMemoryManager = false; - DebugManagerStateRestore restorer; - DebugManager.flags.MakeAllBuffersResident.set(true); - - initPlatform(); - auto device = platform()->getClDevice(0u); - - MockContext context(device, false); - auto retValue = CL_SUCCESS; - auto clBuffer = clCreateBuffer(&context, 0u, 4096u, nullptr, &retValue); - ASSERT_EQ(retValue, CL_SUCCESS); - clReleaseMemObject(clBuffer); -} - class BufferTests : public ::testing::Test { protected: void SetUp() override { diff --git a/opencl/test/unit_test/mocks/mock_memory_operations_handler.h b/opencl/test/unit_test/mocks/mock_memory_operations_handler.h index 89822cdfe7..5910b7b7a2 100644 --- a/opencl/test/unit_test/mocks/mock_memory_operations_handler.h +++ b/opencl/test/unit_test/mocks/mock_memory_operations_handler.h @@ -15,8 +15,8 @@ class GraphicsAllocation; class MockMemoryOperationsHandler : public MemoryOperationsHandler { public: MockMemoryOperationsHandler() {} - MemoryOperationsStatus makeResident(ArrayRef gfxAllocations) override { return MemoryOperationsStatus::UNSUPPORTED; } - MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::UNSUPPORTED; } - MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::UNSUPPORTED; } + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations) override { return MemoryOperationsStatus::UNSUPPORTED; } + MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::UNSUPPORTED; } + MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override { return MemoryOperationsStatus::UNSUPPORTED; } }; } // namespace NEO diff --git a/opencl/test/unit_test/mt_tests/os_interface/linux/drm_memory_manager_mt_tests.cpp b/opencl/test/unit_test/mt_tests/os_interface/linux/drm_memory_manager_mt_tests.cpp index 1106a97ebf..bdef561131 100644 --- a/opencl/test/unit_test/mt_tests/os_interface/linux/drm_memory_manager_mt_tests.cpp +++ b/opencl/test/unit_test/mt_tests/os_interface/linux/drm_memory_manager_mt_tests.cpp @@ -37,7 +37,7 @@ TEST(DrmMemoryManagerTest, givenDrmMemoryManagerWhenSharedAllocationIsCreatedFro }; MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); auto mock = new MockDrm(0, *executionEnvironment.rootDeviceEnvironments[0]); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(mock); auto memoryManager = std::make_unique(executionEnvironment); @@ -103,7 +103,7 @@ TEST(DrmMemoryManagerTest, givenMultipleThreadsWhenSharedAllocationIsCreatedThen MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); auto mock = new MockDrm(0, *executionEnvironment.rootDeviceEnvironments[0]); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(mock); auto memoryManager = std::make_unique(executionEnvironment); diff --git a/opencl/test/unit_test/os_interface/linux/drm_command_stream_fixture.h b/opencl/test/unit_test/os_interface/linux/drm_command_stream_fixture.h index 5b97785ade..0d106632d0 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_command_stream_fixture.h +++ b/opencl/test/unit_test/os_interface/linux/drm_command_stream_fixture.h @@ -35,7 +35,7 @@ class DrmCommandStreamTest : public ::testing::Test { executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(mock); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); auto hwInfo = executionEnvironment.rootDeviceEnvironments[0]->getHardwareInfo(); mock->createVirtualMemoryAddressSpace(HwHelper::getSubDevicesCount(hwInfo)); @@ -82,11 +82,12 @@ class DrmCommandStreamTest : public ::testing::Test { std::unique_ptr osContext; }; -class DrmCommandStreamEnhancedTest : public ::testing::Test { +template +class DrmCommandStreamEnhancedTemplate : public ::testing::Test { public: std::unique_ptr dbgState; MockExecutionEnvironment *executionEnvironment; - DrmMockCustom *mock; + T *mock; CommandStreamReceiver *csr = nullptr; const uint32_t rootDeviceIndex = 0u; @@ -102,10 +103,10 @@ class DrmCommandStreamEnhancedTest : public ::testing::Test { //make sure this is disabled, we don't want to test this now DebugManager.flags.EnableForcePin.set(false); - mock = new DrmMockCustom(); + mock = new T(); executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface = std::make_unique(); executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface->get()->setDrm(mock); - executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = std::make_unique(); + executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); csr = new TestedDrmCommandStreamReceiver(*executionEnvironment, rootDeviceIndex); ASSERT_NE(nullptr, csr); @@ -127,7 +128,7 @@ class DrmCommandStreamEnhancedTest : public ::testing::Test { template void makeResidentBufferObjects(DrmAllocation *drmAllocation) { - drmAllocation->appendBOs(0u, static_cast *>(csr)->residency); + drmAllocation->bindBOs(0u, 0u, &static_cast *>(csr)->residency, false); } template @@ -143,7 +144,7 @@ class DrmCommandStreamEnhancedTest : public ::testing::Test { protected: class MockBufferObject : public BufferObject { - friend DrmCommandStreamEnhancedTest; + friend DrmCommandStreamEnhancedTemplate; protected: MockBufferObject(Drm *drm, size_t size) : BufferObject(drm, 1, 0) { @@ -155,3 +156,5 @@ class DrmCommandStreamEnhancedTest : public ::testing::Test { return new MockBufferObject(this->mock, size); } }; + +using DrmCommandStreamEnhancedTest = DrmCommandStreamEnhancedTemplate; diff --git a/opencl/test/unit_test/os_interface/linux/drm_command_stream_mm_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_command_stream_mm_tests.cpp index 194e84bff8..71137a5b9c 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_command_stream_mm_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_command_stream_mm_tests.cpp @@ -32,7 +32,7 @@ HWTEST_F(DrmCommandStreamMMTest, MMwithPinBB) { executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(drm); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); DrmCommandStreamReceiver csr(executionEnvironment, 0, gemCloseWorkerMode::gemCloseWorkerInactive); @@ -69,7 +69,7 @@ HWTEST_F(DrmCommandStreamMMTest, givenExecutionEnvironmentWithMoreThanOneRootDev executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->setHwInfo(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface->get()->setDrm(new DrmMockCustom()); - executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); } auto memoryManager = new TestedDrmMemoryManager(false, true, false, executionEnvironment); diff --git a/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp index 7580900d90..33ef27b356 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp @@ -14,6 +14,7 @@ #include "shared/source/memory_manager/internal_allocation_storage.h" #include "shared/source/memory_manager/residency.h" #include "shared/source/os_interface/linux/drm_buffer_object.h" +#include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h" #include "shared/source/os_interface/linux/os_context_linux.h" #include "shared/source/os_interface/linux/os_interface.h" #include "shared/source/os_interface/os_context.h" @@ -592,73 +593,6 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenGemCloseWorkerInactiveMode mm->freeGraphicsMemory(dummyAllocation); } -HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocInMemoryOperationsInterfaceWhenFlushThenAllocIsResident) { - auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - LinearStream cs(commandBuffer); - CommandStreamReceiverHw::addBatchBufferEnd(cs, nullptr); - CommandStreamReceiverHw::alignToCacheLine(cs); - BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr}; - - auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(ArrayRef(&allocation, 1)); - - csr->flush(batchBuffer, csr->getResidencyAllocations()); - - const auto boRequirments = [&allocation](const auto &bo) { - return (static_cast(bo.handle) == static_cast(allocation)->getBO()->peekHandle() && - bo.offset == static_cast(allocation)->getBO()->peekAddress()); - }; - - auto &residency = static_cast *>(csr)->getExecStorage(); - EXPECT_TRUE(std::find_if(residency.begin(), residency.end(), boRequirments) != residency.end()); - EXPECT_EQ(residency.size(), 2u); - residency.clear(); - - csr->flush(batchBuffer, csr->getResidencyAllocations()); - EXPECT_TRUE(std::find_if(residency.begin(), residency.end(), boRequirments) != residency.end()); - EXPECT_EQ(residency.size(), 2u); - residency.clear(); - - csr->getResidencyAllocations().clear(); - executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->evict(*allocation); - - csr->flush(batchBuffer, csr->getResidencyAllocations()); - - EXPECT_FALSE(std::find_if(residency.begin(), residency.end(), boRequirments) != residency.end()); - EXPECT_EQ(residency.size(), 1u); - - mm->freeGraphicsMemory(allocation); - mm->freeGraphicsMemory(commandBuffer); -} - -HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMakeAllBuffersResidentSetWhenFlushThenDrmMemoryOperationHandlerIsLocked) { - struct MockDrmMemoryOperationsHandler : public DrmMemoryOperationsHandler { - using DrmMemoryOperationsHandler::mutex; - }; - - struct MockDrmCsr : public DrmCommandStreamReceiver { - using DrmCommandStreamReceiver::DrmCommandStreamReceiver; - void flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override { - auto memoryOperationsInterface = static_cast(this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface.get()); - EXPECT_FALSE(memoryOperationsInterface->mutex.try_lock()); - } - }; - - DebugManagerStateRestore restorer; - DebugManager.flags.MakeAllBuffersResident.set(true); - - auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); - LinearStream cs(commandBuffer); - CommandStreamReceiverHw::addBatchBufferEnd(cs, nullptr); - CommandStreamReceiverHw::alignToCacheLine(cs); - BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr}; - - MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, gemCloseWorkerMode::gemCloseWorkerInactive); - mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations()); - - mm->freeGraphicsMemory(commandBuffer); -} - HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, GivenTwoAllocationsWhenBackingStorageIsDifferentThenMakeResidentShouldAddTwoLocations) { auto allocation = static_cast(mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize})); auto allocation2 = static_cast(mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize})); diff --git a/opencl/test/unit_test/os_interface/linux/drm_gem_close_worker_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_gem_close_worker_tests.cpp index 66857002ca..dbf79cb7a4 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_gem_close_worker_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_gem_close_worker_tests.cpp @@ -72,7 +72,7 @@ class DrmGemCloseWorkerFixture { executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(drmMock); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); this->mm = new DrmMemoryManager(gemCloseWorkerMode::gemCloseWorkerInactive, false, diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp index b4b0fdd89c..f39fe381c5 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp @@ -9,9 +9,13 @@ #include "shared/source/os_interface/linux/drm_memory_manager.h" #include "shared/source/os_interface/linux/drm_memory_operations_handler.h" #include "shared/source/os_interface/linux/os_interface.h" +#include "shared/test/unit_test/helpers/ult_hw_config.h" #include "opencl/test/unit_test/mocks/linux/mock_drm_memory_manager.h" +#include "opencl/test/unit_test/mocks/mock_allocation_properties.h" +#include "opencl/test/unit_test/mocks/mock_context.h" #include "opencl/test/unit_test/mocks/mock_execution_environment.h" +#include "opencl/test/unit_test/os_interface/linux/drm_command_stream_fixture.h" #include "opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h" #include "test.h" @@ -22,7 +26,7 @@ using namespace NEO; TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenAllocateInDevicePoolIsCalledThenNullptrAndStatusRetryIsReturned) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(Drm::create(nullptr, *executionEnvironment.rootDeviceEnvironments[0])); TestedDrmMemoryManager memoryManager(executionEnvironment); MemoryManager::AllocationStatus status = MemoryManager::AllocationStatus::Success; @@ -39,7 +43,7 @@ TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenAllocateInDevicePoolIs TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenLockResourceIsCalledOnNullBufferObjectThenReturnNullPtr) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); TestedDrmMemoryManager memoryManager(executionEnvironment); DrmAllocation drmAllocation(0, GraphicsAllocation::AllocationType::UNKNOWN, nullptr, nullptr, 0u, 0u, MemoryPool::LocalMemory); @@ -52,7 +56,7 @@ TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenLockResourceIsCalledOn TEST(DrmMemoryManagerSimpleTest, givenDrmMemoryManagerWhenFreeGraphicsMemoryIsCalledOnAllocationWithNullBufferObjectThenEarlyReturn) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); TestedDrmMemoryManager memoryManager(executionEnvironment); auto drmAllocation = new DrmAllocation(0, GraphicsAllocation::AllocationType::UNKNOWN, nullptr, nullptr, 0u, 0u, MemoryPool::LocalMemory); @@ -95,3 +99,87 @@ TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenCopyMemoryToAllocationThen TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerWhenGetLocalMemoryIsCalledThenSizeOfLocalMemoryIsReturned) { EXPECT_EQ(0 * GB, memoryManager->getLocalMemorySize(rootDeviceIndex)); } + +HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenMakeAllBuffersResidentSetWhenFlushThenDrmMemoryOperationHandlerIsLocked) { + struct MockDrmMemoryOperationsHandler : public DrmMemoryOperationsHandler { + using DrmMemoryOperationsHandler::mutex; + }; + + struct MockDrmCsr : public DrmCommandStreamReceiver { + using DrmCommandStreamReceiver::DrmCommandStreamReceiver; + void flushInternal(const BatchBuffer &batchBuffer, const ResidencyContainer &allocationsForResidency) override { + auto memoryOperationsInterface = static_cast(this->executionEnvironment.rootDeviceEnvironments[this->rootDeviceIndex]->memoryOperationsInterface.get()); + EXPECT_FALSE(memoryOperationsInterface->mutex.try_lock()); + } + }; + + DebugManagerStateRestore restorer; + DebugManager.flags.MakeAllBuffersResident.set(true); + + auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); + LinearStream cs(commandBuffer); + CommandStreamReceiverHw::addBatchBufferEnd(cs, nullptr); + CommandStreamReceiverHw::alignToCacheLine(cs); + BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr}; + + MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, gemCloseWorkerMode::gemCloseWorkerInactive); + mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations()); + + mm->freeGraphicsMemory(commandBuffer); +} + +HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocInMemoryOperationsInterfaceWhenFlushThenAllocIsResident) { + auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); + LinearStream cs(commandBuffer); + CommandStreamReceiverHw::addBatchBufferEnd(cs, nullptr); + CommandStreamReceiverHw::alignToCacheLine(cs); + BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr}; + + auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1)); + + csr->flush(batchBuffer, csr->getResidencyAllocations()); + + const auto boRequirments = [&allocation](const auto &bo) { + return (static_cast(bo.handle) == static_cast(allocation)->getBO()->peekHandle() && + bo.offset == static_cast(allocation)->getBO()->peekAddress()); + }; + + auto &residency = static_cast *>(csr)->getExecStorage(); + EXPECT_TRUE(std::find_if(residency.begin(), residency.end(), boRequirments) != residency.end()); + EXPECT_EQ(residency.size(), 2u); + residency.clear(); + + csr->flush(batchBuffer, csr->getResidencyAllocations()); + EXPECT_TRUE(std::find_if(residency.begin(), residency.end(), boRequirments) != residency.end()); + EXPECT_EQ(residency.size(), 2u); + residency.clear(); + + csr->getResidencyAllocations().clear(); + executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->evict(device.get(), *allocation); + + csr->flush(batchBuffer, csr->getResidencyAllocations()); + + EXPECT_FALSE(std::find_if(residency.begin(), residency.end(), boRequirments) != residency.end()); + EXPECT_EQ(residency.size(), 1u); + + mm->freeGraphicsMemory(allocation); + mm->freeGraphicsMemory(commandBuffer); +} + +TEST(ResidencyTests, whenBuffersIsCreatedWithMakeResidentFlagThenItSuccessfulyCreates) { + VariableBackup backup(&ultHwConfig); + ultHwConfig.useMockedPrepareDeviceEnvironmentsFunc = false; + ultHwConfig.forceOsAgnosticMemoryManager = false; + DebugManagerStateRestore restorer; + DebugManager.flags.MakeAllBuffersResident.set(true); + + initPlatform(); + auto device = platform()->getClDevice(0u); + + MockContext context(device, false); + auto retValue = CL_SUCCESS; + auto clBuffer = clCreateBuffer(&context, 0u, 4096u, nullptr, &retValue); + ASSERT_EQ(retValue, CL_SUCCESS); + clReleaseMemObject(clBuffer); +} \ No newline at end of file diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index c97e810d4d..06d329dc86 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -3671,7 +3671,7 @@ TEST(DrmMemoryManagerFreeGraphicsMemoryCallSequenceTest, givenDrmMemoryManagerAn MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); executionEnvironment.rootDeviceEnvironments[0]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[0]->osInterface->get()->setDrm(Drm::create(nullptr, *executionEnvironment.rootDeviceEnvironments[0])); - executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); GMockDrmMemoryManager gmockDrmMemoryManager(executionEnvironment); AllocationProperties properties{0, MemoryConstants::pageSize, GraphicsAllocation::AllocationType::BUFFER}; @@ -3691,7 +3691,7 @@ TEST(DrmMemoryManagerFreeGraphicsMemoryCallSequenceTest, givenDrmMemoryManagerAn TEST(DrmMemoryManagerFreeGraphicsMemoryUnreferenceTest, givenDrmMemoryManagerAndFreeGraphicsMemoryIsCalledForSharedAllocationThenUnreferenceBufferObjectIsCalledWithSynchronousDestroySetToFalse) { MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); const uint32_t rootDeviceIndex = 0u; - executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[rootDeviceIndex]->osInterface->get()->setDrm(Drm::create(nullptr, *executionEnvironment.rootDeviceEnvironments[rootDeviceIndex])); ::testing::NiceMock gmockDrmMemoryManager(executionEnvironment); diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h index 8d405f4892..cc75c1e09c 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h @@ -28,7 +28,7 @@ class DrmMemoryManagerBasic : public ::testing::Test { for (auto i = 0u; i < numRootDevices; i++) { executionEnvironment.rootDeviceEnvironments[i]->osInterface = std::make_unique(); executionEnvironment.rootDeviceEnvironments[i]->osInterface->get()->setDrm(Drm::create(nullptr, *executionEnvironment.rootDeviceEnvironments[i])); - executionEnvironment.rootDeviceEnvironments[i]->memoryOperationsInterface = std::make_unique(); + executionEnvironment.rootDeviceEnvironments[i]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); } } const uint32_t rootDeviceIndex = 1u; @@ -59,7 +59,7 @@ class DrmMemoryManagerFixture : public MemoryManagementFixture { auto rootDeviceEnvironment = executionEnvironment->rootDeviceEnvironments[i].get(); rootDeviceEnvironment->osInterface = std::make_unique(); rootDeviceEnvironment->osInterface->get()->setDrm(new DrmMockCustom()); - rootDeviceEnvironment->memoryOperationsInterface = std::make_unique(); + rootDeviceEnvironment->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); } rootDeviceEnvironment = executionEnvironment->rootDeviceEnvironments[rootDeviceIndex].get(); @@ -132,7 +132,7 @@ class DrmMemoryManagerFixtureWithoutQuietIoctlExpectation { rootDeviceEnvironment->setHwInfo(defaultHwInfo.get()); rootDeviceEnvironment->osInterface = std::make_unique(); rootDeviceEnvironment->osInterface->get()->setDrm(new DrmMockCustom); - rootDeviceEnvironment->memoryOperationsInterface = std::make_unique(); + rootDeviceEnvironment->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); } mock = static_cast(executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface->get()->getDrm()); memoryManager.reset(new TestedDrmMemoryManager(*executionEnvironment)); diff --git a/opencl/test/unit_test/os_interface/linux/drm_mock.cpp b/opencl/test/unit_test/os_interface/linux/drm_mock.cpp index 107c1e289f..2722dbbc6b 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_mock.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_mock.cpp @@ -114,6 +114,11 @@ int DrmMock::ioctl(unsigned long request, void *arg) { static_cast(arg)->value = this->StoredPersistentContextsSupport; return this->StoredRetValForPersistant; } + + if (receivedContextParamRequest.param == I915_CONTEXT_PARAM_VM) { + static_cast(arg)->value = 1u; + return 0u; + } } if (request == DRM_IOCTL_I915_GEM_EXECBUFFER2) { diff --git a/opencl/test/unit_test/os_interface/linux/drm_residency_handler_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_residency_handler_tests.cpp index e9214ddcac..b0fb742d9e 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_residency_handler_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_residency_handler_tests.cpp @@ -5,7 +5,7 @@ * */ -#include "shared/source/os_interface/linux/drm_memory_operations_handler.h" +#include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h" #include "opencl/test/unit_test/mocks/mock_graphics_allocation.h" #include "test.h" @@ -14,45 +14,36 @@ using namespace NEO; -#include "shared/source/os_interface/linux/drm_memory_operations_handler.h" - -#include "opencl/test/unit_test/mocks/mock_graphics_allocation.h" -#include "test.h" - -#include - -using namespace NEO; - -struct MockDrmMemoryOperationsHandler : public DrmMemoryOperationsHandler { - using DrmMemoryOperationsHandler::residency; +struct MockDrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandlerDefault { + using DrmMemoryOperationsHandlerDefault::residency; }; -struct DrmMemoryOperationsHandlerTest : public ::testing::Test { +struct DrmMemoryOperationsHandlerBaseTest : public ::testing::Test { void SetUp() override { - drmMemoryOperationsHandler = std::make_unique(); + drmMemoryOperationsHandler = std::make_unique(); allocationPtr = &graphicsAllocation; } MockGraphicsAllocation graphicsAllocation; GraphicsAllocation *allocationPtr; - std::unique_ptr drmMemoryOperationsHandler; + std::unique_ptr drmMemoryOperationsHandler; }; -TEST_F(DrmMemoryOperationsHandlerTest, whenMakingResidentAllocaionExpectMakeResidentFail) { - EXPECT_EQ(drmMemoryOperationsHandler->makeResident(ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); +TEST_F(DrmMemoryOperationsHandlerBaseTest, whenMakingResidentAllocaionExpectMakeResidentFail) { + EXPECT_EQ(drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandler->residency.find(allocationPtr) != drmMemoryOperationsHandler->residency.end()); - EXPECT_EQ(drmMemoryOperationsHandler->isResident(graphicsAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(drmMemoryOperationsHandler->isResident(nullptr, graphicsAllocation), MemoryOperationsStatus::SUCCESS); } -TEST_F(DrmMemoryOperationsHandlerTest, whenEvictingResidentAllocationExpectEvictFalse) { +TEST_F(DrmMemoryOperationsHandlerBaseTest, whenEvictingResidentAllocationExpectEvictFalse) { EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 0u); - EXPECT_EQ(drmMemoryOperationsHandler->makeResident(ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(drmMemoryOperationsHandler->isResident(graphicsAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(drmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(drmMemoryOperationsHandler->isResident(nullptr, graphicsAllocation), MemoryOperationsStatus::SUCCESS); EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 1u); EXPECT_TRUE(drmMemoryOperationsHandler->residency.find(allocationPtr) != drmMemoryOperationsHandler->residency.end()); - EXPECT_EQ(drmMemoryOperationsHandler->evict(graphicsAllocation), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(drmMemoryOperationsHandler->isResident(graphicsAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); + EXPECT_EQ(drmMemoryOperationsHandler->evict(nullptr, graphicsAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(drmMemoryOperationsHandler->isResident(nullptr, graphicsAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); EXPECT_EQ(drmMemoryOperationsHandler->residency.size(), 0u); } diff --git a/opencl/test/unit_test/os_interface/linux/linux_create_command_queue_with_properties_tests.cpp b/opencl/test/unit_test/os_interface/linux/linux_create_command_queue_with_properties_tests.cpp index d5f0bceb9c..0f0eabe480 100644 --- a/opencl/test/unit_test/os_interface/linux/linux_create_command_queue_with_properties_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/linux_create_command_queue_with_properties_tests.cpp @@ -26,7 +26,7 @@ struct clCreateCommandQueueWithPropertiesLinux : public UltCommandStreamReceiver auto osInterface = new OSInterface(); osInterface->get()->setDrm(drm); executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface.reset(osInterface); - executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = std::make_unique(); + executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(); executionEnvironment->memoryManager.reset(new TestedDrmMemoryManager(*executionEnvironment)); mdevice = std::make_unique(MockDevice::create(executionEnvironment, rootDeviceIndex)); diff --git a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp index 418d00154b..183fb8b999 100644 --- a/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp +++ b/opencl/test/unit_test/os_interface/windows/wddm_residency_handler_tests.cpp @@ -10,6 +10,7 @@ #include "shared/test/unit_test/helpers/debug_manager_state_restore.h" #include "shared/test/unit_test/helpers/ult_hw_config.h" #include "shared/test/unit_test/helpers/variable_backup.h" +#include "shared/test/unit_test/mocks/mock_device.h" #include "opencl/source/cl_device/cl_device.h" #include "opencl/test/unit_test/mocks/mock_allocation_properties.h" @@ -53,43 +54,43 @@ struct WddmMemoryOperationsHandlerTest : public WddmTest { }; TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocatioWhenMakingResidentAllocaionExpectMakeResidentCalled) { - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmAllocation), MemoryOperationsStatus::SUCCESS); } TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenMakingResidentAllocaionExpectMakeResidentCalled) { allocationPtr = &wddmFragmentedAllocation; - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); } TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenMakingResidentAllocaionExpectMakeResidentCalled) { - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(ArrayRef(allocationData)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmAllocation), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); } TEST_F(WddmMemoryOperationsHandlerTest, givenRegularAllocatioWhenEvictingResidentAllocationExpectEvictCalled) { - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->evict(wddmAllocation), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, wddmAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); } TEST_F(WddmMemoryOperationsHandlerTest, givenFragmentedAllocationWhenEvictingResidentAllocationExpectEvictCalled) { allocationPtr = &wddmFragmentedAllocation; - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->evict(wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmFragmentedAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(&allocationPtr, 1)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmFragmentedAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); } TEST_F(WddmMemoryOperationsHandlerTest, givenVariousAllocationsWhenEvictingResidentAllocationExpectEvictCalled) { - EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(ArrayRef(allocationData)), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->evict(wddmAllocation), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); - EXPECT_EQ(wddmMemoryOperationsHandler->evict(wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); - EXPECT_EQ(wddmMemoryOperationsHandler->isResident(wddmFragmentedAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); + EXPECT_EQ(wddmMemoryOperationsHandler->makeResident(nullptr, ArrayRef(allocationData)), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, wddmAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); + EXPECT_EQ(wddmMemoryOperationsHandler->evict(nullptr, wddmFragmentedAllocation), MemoryOperationsStatus::SUCCESS); + EXPECT_EQ(wddmMemoryOperationsHandler->isResident(nullptr, wddmFragmentedAllocation), MemoryOperationsStatus::MEMORY_NOT_FOUND); } TEST(WddmResidentBufferTests, whenBuffersIsCreatedWithMakeResidentFlagSetThenItIsMadeResidentUponCreation) { @@ -110,7 +111,7 @@ TEST(WddmResidentBufferTests, whenBuffersIsCreatedWithMakeResidentFlagSetThenItI auto memoryOperationsHandler = device->getRootDeviceEnvironment().memoryOperationsInterface.get(); auto neoBuffer = castToObject(clBuffer); auto bufferAllocation = neoBuffer->getGraphicsAllocation(device->getRootDeviceIndex()); - auto status = memoryOperationsHandler->isResident(*bufferAllocation); + auto status = memoryOperationsHandler->isResident(nullptr, *bufferAllocation); EXPECT_EQ(status, MemoryOperationsStatus::SUCCESS); diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp index 971995ed76..f8ab8a92cf 100644 --- a/shared/source/device/device.cpp +++ b/shared/source/device/device.cpp @@ -253,4 +253,8 @@ NEO::SourceLevelDebugger *Device::getSourceLevelDebugger() { return nullptr; } +const std::vector &Device::getEngines() const { + return this->engines; +} + } // namespace NEO diff --git a/shared/source/device/device.h b/shared/source/device/device.h index e84f00c0d2..167ab0fb40 100644 --- a/shared/source/device/device.h +++ b/shared/source/device/device.h @@ -61,6 +61,7 @@ class Device : public ReferenceTrackedObject { MOCKABLE_VIRTUAL bool isDebuggerActive() const; Debugger *getDebugger() { return getRootDeviceEnvironment().debugger.get(); } NEO::SourceLevelDebugger *getSourceLevelDebugger(); + const std::vector &getEngines() const; ExecutionEnvironment *getExecutionEnvironment() const { return executionEnvironment; } const RootDeviceEnvironment &getRootDeviceEnvironment() const { return *executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]; } diff --git a/shared/source/direct_submission/windows/wddm_direct_submission.inl b/shared/source/direct_submission/windows/wddm_direct_submission.inl index 11673a0261..abbcae1e9b 100644 --- a/shared/source/direct_submission/windows/wddm_direct_submission.inl +++ b/shared/source/direct_submission/windows/wddm_direct_submission.inl @@ -59,7 +59,7 @@ bool WddmDirectSubmission::allocateOsResources(DirectSubm bool ret = wddm->getWddmInterface()->createMonitoredFence(ringFence); ringFence.currentFenceValue = 1; if (ret) { - ret = memoryInterface->makeResident(ArrayRef(allocations)) == MemoryOperationsStatus::SUCCESS; + ret = memoryInterface->makeResident(&device, ArrayRef(allocations)) == MemoryOperationsStatus::SUCCESS; } perfLogResidencyVariadicLog(wddm->getResidencyLogger(), "ULLS resource allocation finished with: %d\n", ret); return ret; diff --git a/shared/source/memory_manager/graphics_allocation.cpp b/shared/source/memory_manager/graphics_allocation.cpp index 4a0a41bc4f..43361480e6 100644 --- a/shared/source/memory_manager/graphics_allocation.cpp +++ b/shared/source/memory_manager/graphics_allocation.cpp @@ -73,4 +73,5 @@ uint32_t GraphicsAllocation::getUsedPageSize() const { constexpr uint32_t GraphicsAllocation::objectNotUsed; constexpr uint32_t GraphicsAllocation::objectNotResident; +constexpr uint32_t GraphicsAllocation::objectAlwaysResident; } // namespace NEO diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index 252582555f..d1c9414aae 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -183,7 +183,12 @@ class GraphicsAllocation : public IDNode { void setInspectionId(uint32_t newInspectionId, uint32_t contextId) { usageInfos[contextId].inspectionId = newInspectionId; } bool isResident(uint32_t contextId) const { return GraphicsAllocation::objectNotResident != getResidencyTaskCount(contextId); } - void updateResidencyTaskCount(uint32_t newTaskCount, uint32_t contextId) { usageInfos[contextId].residencyTaskCount = newTaskCount; } + bool isAlwaysResident(uint32_t contextId) const { return GraphicsAllocation::objectAlwaysResident == getResidencyTaskCount(contextId); } + void updateResidencyTaskCount(uint32_t newTaskCount, uint32_t contextId) { + if (usageInfos[contextId].residencyTaskCount != GraphicsAllocation::objectAlwaysResident || newTaskCount == GraphicsAllocation::objectNotResident) { + usageInfos[contextId].residencyTaskCount = newTaskCount; + } + } uint32_t getResidencyTaskCount(uint32_t contextId) const { return usageInfos[contextId].residencyTaskCount; } void releaseResidencyInOsContext(uint32_t contextId) { updateResidencyTaskCount(objectNotResident, contextId); } bool isResidencyTaskCountBelow(uint32_t taskCount, uint32_t contextId) const { return !isResident(contextId) || getResidencyTaskCount(contextId) < taskCount; } @@ -242,6 +247,7 @@ class GraphicsAllocation : public IDNode { static constexpr uint32_t allBanks = 0xffffffff; constexpr static uint32_t objectNotResident = std::numeric_limits::max(); constexpr static uint32_t objectNotUsed = std::numeric_limits::max(); + constexpr static uint32_t objectAlwaysResident = std::numeric_limits::max() - 1; protected: struct UsageInfo { diff --git a/shared/source/memory_manager/memory_operations_handler.h b/shared/source/memory_manager/memory_operations_handler.h index c32832e451..2665be2625 100644 --- a/shared/source/memory_manager/memory_operations_handler.h +++ b/shared/source/memory_manager/memory_operations_handler.h @@ -9,8 +9,10 @@ #include "shared/source/memory_manager/memory_operations_status.h" #include "shared/source/utilities/arrayref.h" -namespace NEO { +#include +namespace NEO { +class Device; class GraphicsAllocation; class MemoryOperationsHandler { @@ -18,8 +20,8 @@ class MemoryOperationsHandler { MemoryOperationsHandler() = default; virtual ~MemoryOperationsHandler() = default; - virtual MemoryOperationsStatus makeResident(ArrayRef gfxAllocations) = 0; - virtual MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) = 0; - virtual MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) = 0; + virtual MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations) = 0; + virtual MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) = 0; + virtual MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) = 0; }; } // namespace NEO diff --git a/shared/source/os_interface/aub_memory_operations_handler.cpp b/shared/source/os_interface/aub_memory_operations_handler.cpp index 4931a9757a..788b16564e 100644 --- a/shared/source/os_interface/aub_memory_operations_handler.cpp +++ b/shared/source/os_interface/aub_memory_operations_handler.cpp @@ -19,7 +19,7 @@ AubMemoryOperationsHandler::AubMemoryOperationsHandler(aub_stream::AubManager *a this->aubManager = aubManager; } -MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(ArrayRef gfxAllocations) { +MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(Device *device, ArrayRef gfxAllocations) { if (!aubManager) { return MemoryOperationsStatus::DEVICE_UNINITIALIZED; } @@ -37,7 +37,7 @@ MemoryOperationsStatus AubMemoryOperationsHandler::makeResident(ArrayRef gfxAllocations) override; - MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override; - MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations) override; + MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; void setAubManager(aub_stream::AubManager *aubManager); protected: diff --git a/shared/source/os_interface/linux/CMakeLists.txt b/shared/source/os_interface/linux/CMakeLists.txt index 2f942b7d09..ff3ea6e208 100644 --- a/shared/source/os_interface/linux/CMakeLists.txt +++ b/shared/source/os_interface/linux/CMakeLists.txt @@ -23,8 +23,12 @@ set(NEO_CORE_OS_INTERFACE_LINUX ${CMAKE_CURRENT_SOURCE_DIR}/drm_neo.h ${CMAKE_CURRENT_SOURCE_DIR}/drm_neo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_null_device.h - ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler.h + ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_bind.h + ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/drm_memory_operations_handler_default.h + ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}/drm_memory_operations_handler_create.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_info_config.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id.h ${CMAKE_CURRENT_SOURCE_DIR}/hw_device_id_linux.cpp diff --git a/shared/source/os_interface/linux/drm_allocation.cpp b/shared/source/os_interface/linux/drm_allocation.cpp index 7314e03df9..e78e9f90c0 100644 --- a/shared/source/os_interface/linux/drm_allocation.cpp +++ b/shared/source/os_interface/linux/drm_allocation.cpp @@ -28,30 +28,39 @@ uint64_t DrmAllocation::peekInternalHandle(MemoryManager *memoryManager) { return static_cast((static_cast(memoryManager))->obtainFdFromHandle(getBO()->peekHandle(), this->rootDeviceIndex)); } -void DrmAllocation::getBOsForResidency(uint32_t osContextId, uint32_t handleId, std::vector &bufferObjects) { +void DrmAllocation::makeBOsResident(uint32_t osContextId, uint32_t drmContextId, uint32_t handleId, std::vector *bufferObjects, bool bind) { if (this->fragmentsStorage.fragmentCount) { for (unsigned int f = 0; f < this->fragmentsStorage.fragmentCount; f++) { if (!this->fragmentsStorage.fragmentStorageData[f].residency->resident[osContextId]) { - appendBO(this->fragmentsStorage.fragmentStorageData[f].osHandleStorage->bo, bufferObjects); + bindBO(this->fragmentsStorage.fragmentStorageData[f].osHandleStorage->bo, drmContextId, bufferObjects, bind); this->fragmentsStorage.fragmentStorageData[f].residency->resident[osContextId] = true; } } } else { - appendBOs(handleId, bufferObjects); + bindBOs(handleId, drmContextId, bufferObjects, bind); } } -void DrmAllocation::appendBO(BufferObject *bo, std::vector &bufferObjects) { +void DrmAllocation::bindBO(BufferObject *bo, uint32_t drmContextId, std::vector *bufferObjects, bool bind) { if (bo) { - if (bo->peekIsReusableAllocation()) { - for (auto bufferObject : bufferObjects) { - if (bufferObject == bo) { - return; + if (bufferObjects) { + if (bo->peekIsReusableAllocation()) { + for (auto bufferObject : *bufferObjects) { + if (bufferObject == bo) { + return; + } } } - } - bufferObjects.push_back(bo); + bufferObjects->push_back(bo); + + } else { + if (bind) { + bo->bind(drmContextId); + } else { + bo->unbind(drmContextId); + } + } } } diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 449917b7d4..583eb6c315 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -60,9 +60,9 @@ class DrmAllocation : public GraphicsAllocation { uint64_t peekInternalHandle(MemoryManager *memoryManager) override; - void getBOsForResidency(uint32_t osContextId, uint32_t handleId, std::vector &bufferObjects); - void appendBO(BufferObject *bo, std::vector &bufferObjects); - void appendBOs(uint32_t handleId, std::vector &bufferObjects); + void makeBOsResident(uint32_t osContextId, uint32_t drmContextId, uint32_t handleId, std::vector *bufferObjects, bool bind); + void bindBO(BufferObject *bo, uint32_t drmContextId, std::vector *bufferObjects, bool bind); + void bindBOs(uint32_t handleId, uint32_t drmContextId, std::vector *bufferObjects, bool bind); protected: BufferObjects bufferObjects{}; diff --git a/shared/source/os_interface/linux/drm_allocation_extended.cpp b/shared/source/os_interface/linux/drm_allocation_extended.cpp index 28c0d75b63..01d064fa9a 100644 --- a/shared/source/os_interface/linux/drm_allocation_extended.cpp +++ b/shared/source/os_interface/linux/drm_allocation_extended.cpp @@ -10,9 +10,9 @@ namespace NEO { -void DrmAllocation::appendBOs(uint32_t handleId, std::vector &bufferObjects) { +void DrmAllocation::bindBOs(uint32_t handleId, uint32_t drmContextId, std::vector *bufferObjects, bool bind) { auto bo = this->getBO(); - appendBO(bo, bufferObjects); + bindBO(bo, drmContextId, bufferObjects, bind); } } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_buffer_object.cpp b/shared/source/os_interface/linux/drm_buffer_object.cpp index 8741544749..725dd17697 100644 --- a/shared/source/os_interface/linux/drm_buffer_object.cpp +++ b/shared/source/os_interface/linux/drm_buffer_object.cpp @@ -128,6 +128,20 @@ int BufferObject::exec(uint32_t used, size_t startOffset, unsigned int flags, bo return err; } +void BufferObject::bind(uint32_t drmContextId) { + auto ret = this->drm->bindBufferObject(drmContextId, this); + auto err = this->drm->getErrno(); + printDebugString(DebugManager.flags.PrintDebugMessages.get(), stderr, "bind buffer object returned with %d. errno=%d(%s)\n", ret, err, strerror(err)); + UNRECOVERABLE_IF(ret != 0); +} + +void BufferObject::unbind(uint32_t drmContextId) { + auto ret = this->drm->unbindBufferObject(drmContextId, this); + auto err = this->drm->getErrno(); + printDebugString(DebugManager.flags.PrintDebugMessages.get(), stderr, "unbind buffer object returned with %d. errno=%d(%s)\n", ret, err, strerror(err)); + UNRECOVERABLE_IF(ret != 0); +} + void BufferObject::printExecutionBuffer(drm_i915_gem_execbuffer2 &execbuf, const size_t &residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage) { std::string logger = "drm_i915_gem_execbuffer2 {\n"; logger += " buffers_ptr: " + std::to_string(execbuf.buffers_ptr) + ",\n"; diff --git a/shared/source/os_interface/linux/drm_buffer_object.h b/shared/source/os_interface/linux/drm_buffer_object.h index 262896ee6f..5f1844b8a4 100644 --- a/shared/source/os_interface/linux/drm_buffer_object.h +++ b/shared/source/os_interface/linux/drm_buffer_object.h @@ -33,6 +33,9 @@ class BufferObject { int exec(uint32_t used, size_t startOffset, unsigned int flags, bool requiresCoherency, uint32_t drmContextId, BufferObject *const residency[], size_t residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage); + void bind(uint32_t drmContextId); + void unbind(uint32_t drmContextId); + void printExecutionBuffer(drm_i915_gem_execbuffer2 &execbuf, const size_t &residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage); int wait(int64_t timeoutNs); diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index bacb058f9a..b84fcfe666 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -589,7 +589,10 @@ void DrmMemoryManager::removeAllocationFromHostPtrManager(GraphicsAllocation *gf } void DrmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) { - executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface->evict(*gfxAllocation); + for (auto &engine : this->registeredEngines) { + auto memoryOperationsInterface = static_cast(executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface.get()); + memoryOperationsInterface->evictWithinOsContext(engine.osContext, *gfxAllocation); + } for (auto handleId = 0u; handleId < gfxAllocation->getNumGmms(); handleId++) { delete gfxAllocation->getGmm(handleId); diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler.cpp deleted file mode 100644 index 2dae547a39..0000000000 --- a/shared/source/os_interface/linux/drm_memory_operations_handler.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2019-2020 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/os_interface/linux/drm_memory_operations_handler.h" - -#include - -namespace NEO { - -DrmMemoryOperationsHandler::DrmMemoryOperationsHandler() = default; -DrmMemoryOperationsHandler::~DrmMemoryOperationsHandler() = default; - -MemoryOperationsStatus DrmMemoryOperationsHandler::makeResident(ArrayRef gfxAllocations) { - std::lock_guard lock(mutex); - this->residency.insert(gfxAllocations.begin(), gfxAllocations.end()); - return MemoryOperationsStatus::SUCCESS; -} - -MemoryOperationsStatus DrmMemoryOperationsHandler::evict(GraphicsAllocation &gfxAllocation) { - std::lock_guard lock(mutex); - this->residency.erase(&gfxAllocation); - return MemoryOperationsStatus::SUCCESS; -} - -MemoryOperationsStatus DrmMemoryOperationsHandler::isResident(GraphicsAllocation &gfxAllocation) { - std::lock_guard lock(mutex); - auto ret = this->residency.find(&gfxAllocation); - if (ret == this->residency.end()) { - return MemoryOperationsStatus::MEMORY_NOT_FOUND; - } - return MemoryOperationsStatus::SUCCESS; -} - -void DrmMemoryOperationsHandler::mergeWithResidencyContainer(ResidencyContainer &residencyContainer) { - for (auto gfxAllocation = this->residency.begin(); gfxAllocation != this->residency.end(); gfxAllocation++) { - auto ret = std::find(residencyContainer.begin(), residencyContainer.end(), *gfxAllocation); - if (ret == residencyContainer.end()) { - residencyContainer.push_back(*gfxAllocation); - } - } -} - -std::unique_lock DrmMemoryOperationsHandler::acquireLock() { - return std::unique_lock(this->mutex); -} - -} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler.h b/shared/source/os_interface/linux/drm_memory_operations_handler.h index afd8b6d824..9c27a359f5 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler.h @@ -9,25 +9,23 @@ #include "shared/source/memory_manager/memory_operations_handler.h" #include "shared/source/memory_manager/residency_container.h" +#include #include -#include namespace NEO { +class OsContext; class DrmMemoryOperationsHandler : public MemoryOperationsHandler { public: - DrmMemoryOperationsHandler(); - ~DrmMemoryOperationsHandler() override; + DrmMemoryOperationsHandler() = default; + ~DrmMemoryOperationsHandler() override = default; - MemoryOperationsStatus makeResident(ArrayRef gfxAllocations) override; - MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override; - MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override; + virtual MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) = 0; + virtual void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) = 0; + virtual std::unique_lock lockHandlerForExecWA() = 0; - void mergeWithResidencyContainer(ResidencyContainer &residencyContainer); - - std::unique_lock acquireLock(); + static std::unique_ptr create(); protected: - std::unordered_set residency; std::mutex mutex; }; } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp new file mode 100644 index 0000000000..4cdf4dbd47 --- /dev/null +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h" + +#include "shared/source/device/device.h" +#include "shared/source/os_interface/linux/drm_allocation.h" +#include "shared/source/os_interface/linux/drm_buffer_object.h" +#include "shared/source/os_interface/linux/drm_neo.h" +#include "shared/source/os_interface/linux/os_context_linux.h" +#include "shared/source/os_interface/linux/os_interface.h" + +namespace NEO { + +DrmMemoryOperationsHandlerBind::DrmMemoryOperationsHandlerBind() = default; +DrmMemoryOperationsHandlerBind::~DrmMemoryOperationsHandlerBind() = default; + +MemoryOperationsStatus DrmMemoryOperationsHandlerBind::makeResident(Device *device, ArrayRef gfxAllocations) { + std::lock_guard lock(mutex); + auto engines = device->getEngines(); + for (const auto &engine : engines) { + auto &drmContextIds = static_cast(engine.osContext)->getDrmContextIds(); + for (uint32_t drmIterator = 0u; drmIterator < drmContextIds.size(); drmIterator++) { + for (auto gfxAllocation = gfxAllocations.begin(); gfxAllocation != gfxAllocations.end(); gfxAllocation++) { + auto drmAllocation = static_cast(*gfxAllocation); + if (!drmAllocation->isAlwaysResident(engine.osContext->getContextId())) { + drmAllocation->makeBOsResident(engine.osContext->getContextId(), drmContextIds[drmIterator], drmIterator, nullptr, true); + drmAllocation->updateResidencyTaskCount(GraphicsAllocation::objectAlwaysResident, engine.osContext->getContextId()); + } + } + } + } + return MemoryOperationsStatus::SUCCESS; +} + +MemoryOperationsStatus DrmMemoryOperationsHandlerBind::evict(Device *device, GraphicsAllocation &gfxAllocation) { + auto engines = device->getEngines(); + auto retVal = MemoryOperationsStatus::SUCCESS; + for (const auto &engine : engines) { + retVal = this->evictWithinOsContext(engine.osContext, gfxAllocation); + if (retVal != MemoryOperationsStatus::SUCCESS) { + return retVal; + } + } + return MemoryOperationsStatus::SUCCESS; +} + +MemoryOperationsStatus DrmMemoryOperationsHandlerBind::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) { + std::lock_guard lock(mutex); + auto &drmContextIds = static_cast(osContext)->getDrmContextIds(); + for (uint32_t drmIterator = 0u; drmIterator < drmContextIds.size(); drmIterator++) { + auto drmAllocation = static_cast(&gfxAllocation); + if (drmAllocation->isAlwaysResident(osContext->getContextId())) { + drmAllocation->makeBOsResident(osContext->getContextId(), drmContextIds[drmIterator], drmIterator, nullptr, false); + drmAllocation->updateResidencyTaskCount(GraphicsAllocation::objectNotResident, osContext->getContextId()); + } + } + return MemoryOperationsStatus::SUCCESS; +} + +MemoryOperationsStatus DrmMemoryOperationsHandlerBind::isResident(Device *device, GraphicsAllocation &gfxAllocation) { + std::lock_guard lock(mutex); + bool isResident = true; + auto engines = device->getEngines(); + for (const auto &engine : engines) { + isResident &= gfxAllocation.isAlwaysResident(engine.osContext->getContextId()); + } + + if (isResident) { + return MemoryOperationsStatus::SUCCESS; + } + return MemoryOperationsStatus::MEMORY_NOT_FOUND; +} + +void DrmMemoryOperationsHandlerBind::mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) { + std::lock_guard lock(mutex); + for (auto gfxAllocation = residencyContainer.begin(); gfxAllocation != residencyContainer.end();) { + if ((*gfxAllocation)->isAlwaysResident(osContext->getContextId())) { + gfxAllocation = residencyContainer.erase(gfxAllocation); + } else { + gfxAllocation++; + } + } +} + +std::unique_lock DrmMemoryOperationsHandlerBind::lockHandlerForExecWA() { + return std::unique_lock(); +} + +} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h new file mode 100644 index 0000000000..e572f3a1e8 --- /dev/null +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/os_interface/linux/drm_memory_operations_handler.h" + +namespace NEO { +class DrmMemoryOperationsHandlerBind : public DrmMemoryOperationsHandler { + public: + DrmMemoryOperationsHandlerBind(); + ~DrmMemoryOperationsHandlerBind() override; + + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations) override; + MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; + + void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override; + std::unique_lock lockHandlerForExecWA() override; +}; +} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp new file mode 100644 index 0000000000..33fd83b4d7 --- /dev/null +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_create.cpp @@ -0,0 +1,16 @@ +/* + * Copyright (C) 2019-2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h" + +namespace NEO { + +std::unique_ptr DrmMemoryOperationsHandler::create() { + return std::make_unique(); +} + +} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp new file mode 100644 index 0000000000..b893a3a470 --- /dev/null +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/linux/drm_memory_operations_handler_default.h" + +#include "shared/source/debug_settings/debug_settings_manager.h" + +#include + +namespace NEO { + +DrmMemoryOperationsHandlerDefault::DrmMemoryOperationsHandlerDefault() = default; +DrmMemoryOperationsHandlerDefault::~DrmMemoryOperationsHandlerDefault() = default; + +MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::makeResident(Device *device, ArrayRef gfxAllocations) { + std::lock_guard lock(mutex); + this->residency.insert(gfxAllocations.begin(), gfxAllocations.end()); + return MemoryOperationsStatus::SUCCESS; +} + +MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) { + std::lock_guard lock(mutex); + this->residency.erase(&gfxAllocation); + return MemoryOperationsStatus::SUCCESS; +} + +MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::evict(Device *device, GraphicsAllocation &gfxAllocation) { + OsContext *osContext = nullptr; + return this->evictWithinOsContext(osContext, gfxAllocation); +} + +MemoryOperationsStatus DrmMemoryOperationsHandlerDefault::isResident(Device *device, GraphicsAllocation &gfxAllocation) { + std::lock_guard lock(mutex); + auto ret = this->residency.find(&gfxAllocation); + if (ret == this->residency.end()) { + return MemoryOperationsStatus::MEMORY_NOT_FOUND; + } + return MemoryOperationsStatus::SUCCESS; +} + +void DrmMemoryOperationsHandlerDefault::mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) { + for (auto gfxAllocation = this->residency.begin(); gfxAllocation != this->residency.end(); gfxAllocation++) { + auto ret = std::find(residencyContainer.begin(), residencyContainer.end(), *gfxAllocation); + if (ret == residencyContainer.end()) { + residencyContainer.push_back(*gfxAllocation); + } + } +} + +std::unique_lock DrmMemoryOperationsHandlerDefault::lockHandlerForExecWA() { + if (DebugManager.flags.MakeAllBuffersResident.get()) { + return std::unique_lock(this->mutex); + } + return std::unique_lock(); +} + +} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_default.h b/shared/source/os_interface/linux/drm_memory_operations_handler_default.h new file mode 100644 index 0000000000..600748a498 --- /dev/null +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_default.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include "shared/source/os_interface/linux/drm_memory_operations_handler.h" + +#include + +namespace NEO { +class OsContextLinux; +class DrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler { + public: + DrmMemoryOperationsHandlerDefault(); + ~DrmMemoryOperationsHandlerDefault() override; + + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations) override; + MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus evictWithinOsContext(OsContext *osContext, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; + + void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override; + std::unique_lock lockHandlerForExecWA() override; + + protected: + std::unordered_set residency; +}; +} // namespace NEO diff --git a/shared/source/os_interface/linux/drm_neo.h b/shared/source/os_interface/linux/drm_neo.h index 96dc7a377d..e348d4e03c 100644 --- a/shared/source/os_interface/linux/drm_neo.h +++ b/shared/source/os_interface/linux/drm_neo.h @@ -29,6 +29,7 @@ struct GT_SYSTEM_INFO; namespace NEO { #define I915_CONTEXT_PRIVATE_PARAM_BOOST 0x80000000 +class BufferObject; class DeviceFactory; struct HardwareInfo; struct RootDeviceEnvironment; @@ -86,6 +87,8 @@ class Drm { bool createVirtualMemoryAddressSpace(uint32_t vmCount); void destroyVirtualMemoryAddressSpace(); uint32_t getVirtualMemoryAddressSpace(uint32_t vmId); + int bindBufferObject(uint32_t drmContextId, BufferObject *bo); + int unbindBufferObject(uint32_t drmContextId, BufferObject *bo); int setupHardwareInfo(DeviceDescriptor *, bool); bool areNonPersistentContextsSupported() const { return nonPersistentContextsSupported; } diff --git a/shared/source/os_interface/linux/drm_query.cpp b/shared/source/os_interface/linux/drm_query.cpp index d5c5efe3d5..d38ac2c7aa 100644 --- a/shared/source/os_interface/linux/drm_query.cpp +++ b/shared/source/os_interface/linux/drm_query.cpp @@ -41,4 +41,12 @@ unsigned int Drm::bindDrmContext(uint32_t drmContextId, uint32_t deviceIndex, au return DrmEngineMapper::engineNodeMap(engineType); } +int Drm::bindBufferObject(uint32_t drmContextId, BufferObject *bo) { + return 0; +} + +int Drm::unbindBufferObject(uint32_t drmContextId, BufferObject *bo) { + return 0; +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_query_dg1.cpp b/shared/source/os_interface/linux/drm_query_dg1.cpp index 20f49c558a..bc75a2cc36 100644 --- a/shared/source/os_interface/linux/drm_query_dg1.cpp +++ b/shared/source/os_interface/linux/drm_query_dg1.cpp @@ -49,4 +49,12 @@ unsigned int Drm::bindDrmContext(uint32_t drmContextId, uint32_t deviceIndex, au return DrmEngineMapper::engineNodeMap(engineType); } +int Drm::bindBufferObject(uint32_t drmContextId, BufferObject *bo) { + return 0; +} + +int Drm::unbindBufferObject(uint32_t drmContextId, BufferObject *bo) { + return 0; +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/os_context_linux.cpp b/shared/source/os_interface/linux/os_context_linux.cpp index 7a6235bb7c..ed2963fd17 100644 --- a/shared/source/os_interface/linux/os_context_linux.cpp +++ b/shared/source/os_interface/linux/os_context_linux.cpp @@ -45,6 +45,10 @@ OsContextLinux::OsContextLinux(Drm &drm, uint32_t contextId, DeviceBitfield devi } } +Drm &OsContextLinux::getDrm() const { + return this->drm; +} + OsContextLinux::~OsContextLinux() { for (auto drmContextId : drmContextIds) { drm.destroyDrmContext(drmContextId); diff --git a/shared/source/os_interface/linux/os_context_linux.h b/shared/source/os_interface/linux/os_context_linux.h index 81b4f49655..d949b2c77a 100644 --- a/shared/source/os_interface/linux/os_context_linux.h +++ b/shared/source/os_interface/linux/os_context_linux.h @@ -24,6 +24,7 @@ class OsContextLinux : public OsContext { unsigned int getEngineFlag() const { return engineFlag; } const std::vector &getDrmContextIds() const { return drmContextIds; } + Drm &getDrm() const; protected: unsigned int engineFlag = 0; diff --git a/shared/source/os_interface/linux/os_interface.cpp b/shared/source/os_interface/linux/os_interface.cpp index b75f07ec68..4ecef0096f 100644 --- a/shared/source/os_interface/linux/os_interface.cpp +++ b/shared/source/os_interface/linux/os_interface.cpp @@ -46,7 +46,7 @@ bool RootDeviceEnvironment::initOsInterface(std::unique_ptr &&hwDevi return false; } - memoryOperationsInterface = std::make_unique(); + memoryOperationsInterface = DrmMemoryOperationsHandler::create(); osInterface.reset(new OSInterface()); osInterface->get()->setDrm(drm); auto hardwareInfo = getMutableHardwareInfo(); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index f624dde29f..52ee4b173e 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -392,7 +392,7 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation residencyController.removeFromTrimCandidateListIfUsed(input, true); } - executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface->evict(*input); + executionEnvironment.rootDeviceEnvironments[gfxAllocation->getRootDeviceIndex()]->memoryOperationsInterface->evict(nullptr, *input); auto defaultGmm = gfxAllocation->getDefaultGmm(); if (defaultGmm) { diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp index 43633fd043..ed05e700ad 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.cpp @@ -18,7 +18,7 @@ WddmMemoryOperationsHandler::WddmMemoryOperationsHandler(Wddm *wddm) : wddm(wddm residentAllocations = std::make_unique(wddm); } -MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRef gfxAllocations) { +MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(Device *device, ArrayRef gfxAllocations) { uint32_t totalHandlesCount = 0; constexpr uint32_t stackAllocations = 64; constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount * stackAllocations; @@ -44,7 +44,7 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::makeResident(ArrayRefmakeResidentResources(handlesForResidency.begin(), totalHandlesCount, totalSize); } -MemoryOperationsStatus WddmMemoryOperationsHandler::evict(GraphicsAllocation &gfxAllocation) { +MemoryOperationsStatus WddmMemoryOperationsHandler::evict(Device *device, GraphicsAllocation &gfxAllocation) { constexpr uint32_t stackHandlesCount = NEO::maxFragmentsCount * EngineLimits::maxHandleCount; StackVec handlesForEviction; WddmAllocation &wddmAllocation = reinterpret_cast(gfxAllocation); @@ -67,7 +67,7 @@ MemoryOperationsStatus WddmMemoryOperationsHandler::evict(GraphicsAllocation &gf return residentAllocations->evictResources(handlesForEviction.begin(), totalHandleCount); } -MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(GraphicsAllocation &gfxAllocation) { +MemoryOperationsStatus WddmMemoryOperationsHandler::isResident(Device *device, GraphicsAllocation &gfxAllocation) { WddmAllocation &wddmAllocation = reinterpret_cast(gfxAllocation); D3DKMT_HANDLE defaultHandle = 0u; if (wddmAllocation.fragmentsStorage.fragmentCount > 0) { diff --git a/shared/source/os_interface/windows/wddm_memory_operations_handler.h b/shared/source/os_interface/windows/wddm_memory_operations_handler.h index 3b6dfa41ab..65bdc93f40 100644 --- a/shared/source/os_interface/windows/wddm_memory_operations_handler.h +++ b/shared/source/os_interface/windows/wddm_memory_operations_handler.h @@ -20,9 +20,9 @@ class WddmMemoryOperationsHandler : public MemoryOperationsHandler { WddmMemoryOperationsHandler(Wddm *wddm); ~WddmMemoryOperationsHandler() override = default; - MemoryOperationsStatus makeResident(ArrayRef gfxAllocations) override; - MemoryOperationsStatus evict(GraphicsAllocation &gfxAllocation) override; - MemoryOperationsStatus isResident(GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus makeResident(Device *device, ArrayRef gfxAllocations) override; + MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override; + MemoryOperationsStatus isResident(Device *device, GraphicsAllocation &gfxAllocation) override; protected: Wddm *wddm; diff --git a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp index e45da8852d..b5a27f5e47 100644 --- a/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp +++ b/shared/test/unit_test/os_interface/aub_memory_operations_handler_tests.cpp @@ -13,7 +13,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenNullPtrAsAubManagerWhenMakeResidentCalledThenFalseReturned) { getMemoryOperationsHandler()->setAubManager(nullptr); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(ArrayRef(&allocPtr, 1)); + auto result = memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1)); EXPECT_EQ(result, MemoryOperationsStatus::DEVICE_UNINITIALIZED); } @@ -21,7 +21,7 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAubManagerWhenMakeResidentCalledThe MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->makeResident(ArrayRef(&allocPtr, 1)); + auto result = memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1)); EXPECT_EQ(result, MemoryOperationsStatus::SUCCESS); EXPECT_TRUE(aubManager.writeMemoryCalled); } @@ -30,36 +30,36 @@ TEST_F(AubMemoryOperationsHandlerTests, givenAllocationWhenMakeResidentCalledThe MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - memoryOperationsInterface->makeResident(ArrayRef(&allocPtr, 1)); + memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1)); EXPECT_EQ(aubManager.hintToWriteMemory, AubMemDump::DataTypeHintValues::TraceNotype); } TEST_F(AubMemoryOperationsHandlerTests, givenNonResidentAllocationWhenIsResidentCalledThenFalseReturned) { MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->isResident(allocation); + auto result = memoryOperationsInterface->isResident(nullptr, allocation); EXPECT_EQ(result, MemoryOperationsStatus::MEMORY_NOT_FOUND); } TEST_F(AubMemoryOperationsHandlerTests, givenResidentAllocationWhenIsResidentCalledThenTrueReturned) { MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - memoryOperationsInterface->makeResident(ArrayRef(&allocPtr, 1)); - auto result = memoryOperationsInterface->isResident(allocation); + memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1)); + auto result = memoryOperationsInterface->isResident(nullptr, allocation); EXPECT_EQ(result, MemoryOperationsStatus::SUCCESS); } TEST_F(AubMemoryOperationsHandlerTests, givenNonResidentAllocationWhenEvictCalledThenFalseReturned) { MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - auto result = memoryOperationsInterface->evict(allocation); + auto result = memoryOperationsInterface->evict(nullptr, allocation); EXPECT_EQ(result, MemoryOperationsStatus::MEMORY_NOT_FOUND); } TEST_F(AubMemoryOperationsHandlerTests, givenResidentAllocationWhenEvictCalledThenTrueReturned) { MockAubManager aubManager; getMemoryOperationsHandler()->setAubManager(&aubManager); auto memoryOperationsInterface = getMemoryOperationsHandler(); - memoryOperationsInterface->makeResident(ArrayRef(&allocPtr, 1)); - auto result = memoryOperationsInterface->evict(allocation); + memoryOperationsInterface->makeResident(nullptr, ArrayRef(&allocPtr, 1)); + auto result = memoryOperationsInterface->evict(nullptr, allocation); EXPECT_EQ(result, MemoryOperationsStatus::SUCCESS); } \ No newline at end of file