diff --git a/opencl/source/os_interface/linux/drm_command_stream.inl b/opencl/source/os_interface/linux/drm_command_stream.inl index 0c4bc2ad43..5d3a6878f4 100644 --- a/opencl/source/os_interface/linux/drm_command_stream.inl +++ b/opencl/source/os_interface/linux/drm_command_stream.inl @@ -67,6 +67,8 @@ bool DrmCommandStreamReceiver::flush(BatchBuffer &batchBuffer, Reside this->printDeviceIndex(); DrmAllocation *alloc = static_cast(batchBuffer.commandBufferAllocation); DEBUG_BREAK_IF(!alloc); + alloc->updateTaskCount(this->taskCount + 1, this->osContext->getContextId()); + alloc->updateResidencyTaskCount(this->taskCount + 1, this->osContext->getContextId()); BufferObject *bb = alloc->getBO(); if (bb == nullptr) { diff --git a/opencl/test/unit_test/os_interface/linux/device_command_stream_fixture.h b/opencl/test/unit_test/os_interface/linux/device_command_stream_fixture.h index 475c651690..908b7169f8 100644 --- a/opencl/test/unit_test/os_interface/linux/device_command_stream_fixture.h +++ b/opencl/test/unit_test/os_interface/linux/device_command_stream_fixture.h @@ -77,10 +77,10 @@ class DrmMockCustom : public Drm { using Drm::memoryInfo; struct IoctlResExt { - int32_t no; + std::vector no; int32_t res; - IoctlResExt(int32_t no, int32_t res) : no(no), res(res) {} + IoctlResExt(int32_t no, int32_t res) : no(1u, no), res(res) {} }; class Ioctls { @@ -319,7 +319,7 @@ class DrmMockCustom : public Drm { } } - if (ext->no != -1 && ext->no == ioctl_cnt.total.load()) { + if (!ext->no.empty() && std::find(ext->no.begin(), ext->no.end(), ioctl_cnt.total.load()) != ext->no.end()) { ioctl_cnt.total.fetch_add(1); return ext->res; } diff --git a/opencl/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp index fb83bbbcc2..94812ada76 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_buffer_object_tests.cpp @@ -54,6 +54,7 @@ class DrmBufferObjectFixture { void SetUp() { this->mock = std::make_unique(); ASSERT_NE(nullptr, this->mock); + constructPlatform()->peekExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*mock.get(), 0u); osContext.reset(new OsContextLinux(*this->mock, 0u, 1, EngineTypeUsage{aub_stream::ENGINE_RCS, EngineUsage::Regular}, PreemptionMode::Disabled, false)); this->mock->reset(); bo = new TestedBufferObject(this->mock.get()); @@ -84,7 +85,7 @@ TEST_F(DrmBufferObjectTest, WhenCallingExecThenReturnIsCorrect) { } TEST_F(DrmBufferObjectTest, GivenInvalidParamsWhenCallingExecThenEfaultIsReturned) { - mock->ioctl_expected.total = 1; + mock->ioctl_expected.total = 2; mock->ioctl_res = -1; mock->errnoValue = EFAULT; drm_i915_gem_exec_object2 execObjectsStorage = {}; @@ -143,7 +144,7 @@ TEST_F(DrmBufferObjectTest, givenAddressThatWhenSizeIsAddedWithin32BitBoundaryWh TEST_F(DrmBufferObjectTest, whenExecFailsThenPinFails) { std::unique_ptr buff(new uint32_t[1024]); - mock->ioctl_expected.total = 1; + mock->ioctl_expected.total = 2; mock->ioctl_res = -1; this->mock->errnoValue = EINVAL; @@ -159,7 +160,7 @@ TEST_F(DrmBufferObjectTest, whenExecFailsThenPinFails) { TEST_F(DrmBufferObjectTest, whenExecFailsThenValidateHostPtrFails) { std::unique_ptr buff(new uint32_t[1024]); - mock->ioctl_expected.total = 1; + mock->ioctl_expected.total = 2; mock->ioctl_res = -1; this->mock->errnoValue = EINVAL; @@ -233,6 +234,7 @@ TEST_F(DrmBufferObjectTest, whenPrintExecutionBufferIsSetToTrueThenMessageFoundI TEST(DrmBufferObjectSimpleTest, givenInvalidBoWhenValidateHostptrIsCalledThenErrorIsReturned) { std::unique_ptr buff(new uint32_t[256]); std::unique_ptr mock(new DrmMockCustom); + constructPlatform()->peekExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*mock.get(), 0u); OsContextLinux osContext(*mock, 0u, 1, EngineTypeUsage{aub_stream::ENGINE_RCS, EngineUsage::Regular}, PreemptionMode::Disabled, false); ASSERT_NE(nullptr, mock.get()); std::unique_ptr bo(new TestedBufferObject(mock.get())); @@ -256,6 +258,7 @@ TEST(DrmBufferObjectSimpleTest, givenInvalidBoWhenValidateHostptrIsCalledThenErr TEST(DrmBufferObjectSimpleTest, givenInvalidBoWhenPinIsCalledThenErrorIsReturned) { std::unique_ptr buff(new uint32_t[256]); std::unique_ptr mock(new DrmMockCustom); + constructPlatform()->peekExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*mock.get(), 0u); OsContextLinux osContext(*mock, 0u, 1, EngineTypeUsage{aub_stream::ENGINE_RCS, EngineUsage::Regular}, PreemptionMode::Disabled, false); ASSERT_NE(nullptr, mock.get()); std::unique_ptr bo(new TestedBufferObject(mock.get())); diff --git a/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp index f588874322..1c33e17956 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp @@ -807,7 +807,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamBatchingTests, givenRecordedCommandBufferWhen EXPECT_TRUE(cmdBuffers.peekIsEmpty()); auto commandBufferGraphicsAllocation = submittedCommandBuffer.getGraphicsAllocation(); - EXPECT_FALSE(commandBufferGraphicsAllocation->isResident(csr->getOsContext().getContextId())); + EXPECT_TRUE(commandBufferGraphicsAllocation->isResident(csr->getOsContext().getContextId())); //preemption allocation size_t csrSurfaceCount = (device->getPreemptionMode() == PreemptionMode::MidThread) ? 2 : 0; diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp index 5a3ea23bdf..cb45c317c0 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests.cpp @@ -119,6 +119,10 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenNoAllocsInMemoryOperationH } }; + auto osContext = std::make_unique(*mock, 0u, 1, HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*defaultHwInfo)[0], + PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo), + false); + auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); LinearStream cs(commandBuffer); CommandStreamReceiverHw::addBatchBufferEnd(cs, nullptr); @@ -126,6 +130,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenNoAllocsInMemoryOperationH BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false}; MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, 1, gemCloseWorkerMode::gemCloseWorkerInactive); + mockCsr.setupContext(*osContext.get()); mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations()); mm->freeGraphicsMemory(commandBuffer); @@ -144,6 +149,10 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocsInMemoryOperationHan } }; + auto osContext = std::make_unique(*mock, 0u, 1, HwHelper::get(defaultHwInfo->platform.eRenderCoreFamily).getGpgpuEngineInstances(*defaultHwInfo)[0], + PreemptionHelper::getDefaultPreemptionMode(*defaultHwInfo), + false); + auto allocation = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); executionEnvironment->rootDeviceEnvironments[csr->getRootDeviceIndex()]->memoryOperationsInterface->makeResident(device.get(), ArrayRef(&allocation, 1)); @@ -154,6 +163,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenAllocsInMemoryOperationHan BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false}; MockDrmCsr mockCsr(*executionEnvironment, rootDeviceIndex, 1, gemCloseWorkerMode::gemCloseWorkerInactive); + mockCsr.setupContext(*osContext.get()); mockCsr.flush(batchBuffer, mockCsr.getResidencyAllocations()); mm->freeGraphicsMemory(commandBuffer); diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index 57fd323e25..d5327a56cb 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -888,10 +888,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledHostMemoryValid mock->reset(); DrmMockCustom::IoctlResExt ioctlResExt = {1, -1}; + ioctlResExt.no.push_back(2); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = EFAULT; mock->ioctl_expected.gemUserptr = 1; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; MemoryManager::AllocationStatus result = memoryManager->populateOsHandles(storage, rootDeviceIndex); @@ -922,10 +923,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledHostMemoryValid size_t dummySize = 13u; DrmMockCustom::IoctlResExt ioctlResExt = {1, -1}; + ioctlResExt.no.push_back(2); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = EFAULT; mock->ioctl_expected.gemUserptr = 1; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; mock->ioctl_expected.gemClose = 1; AllocationData allocationData; @@ -965,10 +967,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledHostMemoryValid mock->reset(); DrmMockCustom::IoctlResExt ioctlResExt = {1, -1}; + ioctlResExt.no.push_back(2); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = 0; mock->ioctl_expected.gemUserptr = 1; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; AllocationData allocationData; allocationData.size = 13u; @@ -999,10 +1002,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledHostMemoryValid mock->reset(); DrmMockCustom::IoctlResExt ioctlResExt = {1, -1}; + ioctlResExt.no.push_back(2); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = 0; mock->ioctl_expected.gemUserptr = 1; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; AllocationData allocationData; allocationData.size = 13u; @@ -1036,10 +1040,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledHostMemoryValid mock->reset(); DrmMockCustom::IoctlResExt ioctlResExt = {1, -1}; + ioctlResExt.no.push_back(2); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = ENOMEM; mock->ioctl_expected.gemUserptr = 1; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; MemoryManager::AllocationStatus result = memoryManager->populateOsHandles(storage, rootDeviceIndex); @@ -3549,10 +3554,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledValidateHostMem mock->reset(); DrmMockCustom::IoctlResExt ioctlResExt = {2, -1}; + ioctlResExt.no.push_back(3); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = EFAULT; mock->ioctl_expected.gemUserptr = 2; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; OsHandleStorage handleStorage; OsHandleLinux handle1; @@ -3599,10 +3605,11 @@ TEST_F(DrmMemoryManagerWithExplicitExpectationsTest, givenEnabledValidateHostMem mock->reset(); DrmMockCustom::IoctlResExt ioctlResExt = {2, -1}; + ioctlResExt.no.push_back(3); mock->ioctl_res_ext = &ioctlResExt; mock->errnoValue = EFAULT; mock->ioctl_expected.gemUserptr = 2; - mock->ioctl_expected.execbuffer2 = 1; + mock->ioctl_expected.execbuffer2 = 2; OsHandleStorage handleStorage; OsHandleLinux handle1; diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h index 801193aeaa..f3cdd1a447 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.h @@ -159,6 +159,7 @@ class DrmMemoryManagerFixtureWithoutQuietIoctlExpectation { i++; } mock = static_cast(executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->osInterface->getDriverModel()->as()); + constructPlatform()->peekExecutionEnvironment()->rootDeviceEnvironments[0]->memoryOperationsInterface = DrmMemoryOperationsHandler::create(*mock, 0u); memoryManager.reset(new TestedDrmMemoryManager(*executionEnvironment)); ASSERT_NE(nullptr, memoryManager); diff --git a/shared/source/os_interface/linux/drm_buffer_object.cpp b/shared/source/os_interface/linux/drm_buffer_object.cpp index 5ee9df7cd7..c5ae75bddb 100644 --- a/shared/source/os_interface/linux/drm_buffer_object.cpp +++ b/shared/source/os_interface/linux/drm_buffer_object.cpp @@ -10,7 +10,7 @@ #include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/debug_helpers.h" #include "shared/source/os_interface/linux/drm_memory_manager.h" -#include "shared/source/os_interface/linux/drm_memory_operations_handler_bind.h" +#include "shared/source/os_interface/linux/drm_memory_operations_handler.h" #include "shared/source/os_interface/linux/drm_neo.h" #include "shared/source/os_interface/linux/os_time_linux.h" #include "shared/source/os_interface/os_context.h" @@ -146,6 +146,11 @@ int BufferObject::exec(uint32_t used, size_t startOffset, unsigned int flags, bo int ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf); + if (ret != 0) { + static_cast(this->drm->getRootDeviceEnvironment().memoryOperationsInterface.get())->evictUnusedAllocations(); + ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf); + } + if (ret == 0) { return 0; } diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler.h b/shared/source/os_interface/linux/drm_memory_operations_handler.h index 8923f7ddc1..2a5b9c692e 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -25,6 +25,8 @@ class DrmMemoryOperationsHandler : public MemoryOperationsHandler { virtual void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) = 0; virtual std::unique_lock lockHandlerIfUsed() = 0; + virtual void evictUnusedAllocations() = 0; + static std::unique_ptr create(Drm &drm, uint32_t rootDeviceIndex); protected: diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h index 532d3e7eaa..21f138b17b 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_bind.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 Intel Corporation + * Copyright (C) 2020-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -25,7 +25,7 @@ class DrmMemoryOperationsHandlerBind : public DrmMemoryOperationsHandler { void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override; std::unique_lock lockHandlerIfUsed() override; - MOCKABLE_VIRTUAL void evictUnusedAllocations(); + void evictUnusedAllocations() override; protected: void evictImpl(OsContext *osContext, GraphicsAllocation &gfxAllocation, DeviceBitfield deviceBitfield); diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp b/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp index f80f4ba89a..0655131242 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_default.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 Intel Corporation + * Copyright (C) 2020-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -64,4 +64,7 @@ std::unique_lock DrmMemoryOperationsHandlerDefault::lockHandlerIfUse return std::unique_lock(); } +void DrmMemoryOperationsHandlerDefault::evictUnusedAllocations() { +} + } // namespace NEO diff --git a/shared/source/os_interface/linux/drm_memory_operations_handler_default.h b/shared/source/os_interface/linux/drm_memory_operations_handler_default.h index e79b62a48b..8134d53d5b 100644 --- a/shared/source/os_interface/linux/drm_memory_operations_handler_default.h +++ b/shared/source/os_interface/linux/drm_memory_operations_handler_default.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 Intel Corporation + * Copyright (C) 2020-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -26,6 +26,8 @@ class DrmMemoryOperationsHandlerDefault : public DrmMemoryOperationsHandler { void mergeWithResidencyContainer(OsContext *osContext, ResidencyContainer &residencyContainer) override; std::unique_lock lockHandlerIfUsed() override; + void evictUnusedAllocations() override; + protected: std::unordered_set residency; };