diff --git a/level_zero/core/source/event/event.cpp b/level_zero/core/source/event/event.cpp index 6e7bf87814..40445cae92 100644 --- a/level_zero/core/source/event/event.cpp +++ b/level_zero/core/source/event/event.cpp @@ -71,9 +71,10 @@ ze_result_t EventPoolImp::initialize(DriverHandle *driver, uint32_t numDevices, deviceBitfield}; unifiedMemoryProperties.alignment = eventAlignment; - void *eventPoolPtr = driver->getMemoryManager()->createMultiGraphicsAllocation(rootDeviceIndices, - unifiedMemoryProperties, - *eventPoolAllocations); + void *eventPoolPtr = driver->getMemoryManager()->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, + unifiedMemoryProperties, + *eventPoolAllocations); + if (!eventPoolPtr) { return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; } diff --git a/level_zero/core/test/unit_tests/sources/device/test_device.cpp b/level_zero/core/test/unit_tests/sources/device/test_device.cpp index 6091c7351c..263b811a09 100644 --- a/level_zero/core/test/unit_tests/sources/device/test_device.cpp +++ b/level_zero/core/test/unit_tests/sources/device/test_device.cpp @@ -12,6 +12,7 @@ #include "shared/test/common/mocks/mock_device.h" #include "shared/test/common/mocks/ult_device_factory.h" +#include "opencl/test/unit_test/mocks/mock_memory_manager.h" #include "test.h" #include "level_zero/core/source/cmdqueue/cmdqueue_imp.h" @@ -143,14 +144,6 @@ TEST_F(DeviceTest, givenEmptySVmAllocStorageWhenAllocateMemoryFromHostPtrThenVal neoDevice->getMemoryManager()->freeGraphicsMemory(allocation); } -struct MemoryManagerHostPointer : public NEO::OsAgnosticMemoryManager { - MemoryManagerHostPointer(NEO::ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(const_cast(executionEnvironment)) {} - GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties, - const void *ptr) override { - return nullptr; - } -}; - struct DeviceHostPointerTest : public ::testing::Test { void SetUp() override { executionEnvironment = new NEO::ExecutionEnvironment(); @@ -159,9 +152,6 @@ struct DeviceHostPointerTest : public ::testing::Test { executionEnvironment->rootDeviceEnvironments[i]->setHwInfo(NEO::defaultHwInfo.get()); } - memoryManager = new MemoryManagerHostPointer(*executionEnvironment); - executionEnvironment->memoryManager.reset(memoryManager); - neoDevice = NEO::MockDevice::create(executionEnvironment, rootDeviceIndex); std::vector> devices; devices.push_back(std::unique_ptr(neoDevice)); @@ -169,6 +159,9 @@ struct DeviceHostPointerTest : public ::testing::Test { driverHandle = std::make_unique>(); driverHandle->initialize(std::move(devices)); + static_cast(driverHandle.get()->getMemoryManager())->isMockHostMemoryManager = true; + static_cast(driverHandle.get()->getMemoryManager())->forceFailureInAllocationWithHostPointer = true; + device = driverHandle->devices[0]; } void TearDown() override { @@ -178,7 +171,6 @@ struct DeviceHostPointerTest : public ::testing::Test { std::unique_ptr> driverHandle; NEO::MockDevice *neoDevice = nullptr; L0::Device *device = nullptr; - MemoryManagerHostPointer *memoryManager = nullptr; const uint32_t rootDeviceIndex = 1u; const uint32_t numRootDevices = 2u; }; diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp index 583c4213c4..fa245b6bb9 100644 --- a/level_zero/core/test/unit_tests/sources/event/test_event.cpp +++ b/level_zero/core/test/unit_tests/sources/event/test_event.cpp @@ -5,6 +5,7 @@ * */ +#include "opencl/test/unit_test/mocks/mock_memory_manager.h" #include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h" #include "test.h" @@ -319,15 +320,6 @@ TEST_F(EventPoolCreateMultiDevice, whenCreatingEventPoolWithNoDevicesThenEventPo EXPECT_EQ(allocation->getGraphicsAllocations().size(), 1u); } -struct MemoryManagerEventPoolCreateNegativeTest : public NEO::OsAgnosticMemoryManager { - MemoryManagerEventPoolCreateNegativeTest(NEO::ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(const_cast(executionEnvironment)) {} - void *createMultiGraphicsAllocation(std::vector &rootDeviceIndices, - NEO::AllocationProperties &properties, - NEO::MultiGraphicsAllocation &multiGraphicsAllocation) override { - return nullptr; - } -}; - struct EventPoolCreateNegativeTest : public ::testing::Test { void SetUp() override { executionEnvironment = new NEO::ExecutionEnvironment(); @@ -336,9 +328,6 @@ struct EventPoolCreateNegativeTest : public ::testing::Test { executionEnvironment->rootDeviceEnvironments[i]->setHwInfo(NEO::defaultHwInfo.get()); } - memoryManager = new MemoryManagerEventPoolCreateNegativeTest(*executionEnvironment); - executionEnvironment->memoryManager.reset(memoryManager); - std::vector> devices; for (uint32_t i = 0; i < numRootDevices; i++) { neoDevice = NEO::MockDevice::create(executionEnvironment, i); @@ -347,6 +336,7 @@ struct EventPoolCreateNegativeTest : public ::testing::Test { driverHandle = std::make_unique>(); driverHandle->initialize(std::move(devices)); + static_cast(driverHandle.get()->getMemoryManager())->isMockEventPoolCreateMemoryManager = true; device = driverHandle->devices[0]; } @@ -357,7 +347,6 @@ struct EventPoolCreateNegativeTest : public ::testing::Test { std::unique_ptr> driverHandle; NEO::MockDevice *neoDevice = nullptr; L0::Device *device = nullptr; - MemoryManagerEventPoolCreateNegativeTest *memoryManager = nullptr; const uint32_t numRootDevices = 2u; }; diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 55dfe81afe..abac5a2747 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -718,32 +718,6 @@ TEST(MemoryBitfieldTests, givenDeviceWithValidBitfieldWhenAllocatingSharedMemory ASSERT_EQ(result, ZE_RESULT_SUCCESS); } -class MockHostMemoryManager : public NEO::MockMemoryManager { - public: - MockHostMemoryManager(const NEO::ExecutionEnvironment &executionEnvironment) : NEO::MockMemoryManager(const_cast(executionEnvironment)){}; - - GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties) override { - allocateGraphicsMemoryWithPropertiesCount++; - if (forceFailureInPrimaryAllocation) { - return nullptr; - } - return NEO::MemoryManager::allocateGraphicsMemoryWithProperties(properties); - } - - GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties, - const void *ptr) override { - allocateGraphicsMemoryWithPropertiesCount++; - if (forceFailureInAllocationWithHostPointer) { - return nullptr; - } - return NEO::MemoryManager::allocateGraphicsMemoryWithProperties(properties, ptr); - } - - uint32_t allocateGraphicsMemoryWithPropertiesCount = 0; - bool forceFailureInPrimaryAllocation = false; - bool forceFailureInAllocationWithHostPointer = false; -}; - struct AllocHostMemoryTest : public ::testing::Test { void SetUp() override { std::vector> devices; @@ -758,16 +732,12 @@ struct AllocHostMemoryTest : public ::testing::Test { executionEnvironment, i))); } - memoryManager = new MockHostMemoryManager(*devices[0].get()->getExecutionEnvironment()); - devices[0].get()->getExecutionEnvironment()->memoryManager.reset(memoryManager); - driverHandle = std::make_unique>(); driverHandle->initialize(std::move(devices)); } DebugManagerStateRestore restorer; std::unique_ptr> driverHandle; - MockHostMemoryManager *memoryManager = nullptr; const uint32_t numRootDevices = 2u; }; @@ -775,12 +745,13 @@ TEST_F(AllocHostMemoryTest, whenCallingAllocHostMemThenAllocateGraphicsMemoryWithPropertiesIsCalledTheNumberOfTimesOfRootDevices) { void *ptr = nullptr; - memoryManager->allocateGraphicsMemoryWithPropertiesCount = 0; + static_cast(driverHandle.get()->getMemoryManager())->isMockHostMemoryManager = true; + static_cast(driverHandle.get()->getMemoryManager())->allocateGraphicsMemoryWithPropertiesCount = 0; ze_host_mem_alloc_desc_t hostDesc = {}; ze_result_t result = driverHandle->allocHostMem(&hostDesc, 4096u, 0u, &ptr); - EXPECT_EQ(memoryManager->allocateGraphicsMemoryWithPropertiesCount, numRootDevices); + EXPECT_EQ(static_cast(driverHandle.get()->getMemoryManager())->allocateGraphicsMemoryWithPropertiesCount, numRootDevices); EXPECT_EQ(ZE_RESULT_SUCCESS, result); EXPECT_NE(nullptr, ptr); @@ -790,16 +761,17 @@ TEST_F(AllocHostMemoryTest, TEST_F(AllocHostMemoryTest, whenCallingAllocHostMemAndFailingOnCreatingGraphicsAllocationThenNullIsReturned) { - memoryManager->forceFailureInPrimaryAllocation = true; + static_cast(driverHandle.get()->getMemoryManager())->isMockHostMemoryManager = true; + static_cast(driverHandle.get()->getMemoryManager())->forceFailureInPrimaryAllocation = true; void *ptr = nullptr; - memoryManager->allocateGraphicsMemoryWithPropertiesCount = 0; + static_cast(driverHandle.get()->getMemoryManager())->allocateGraphicsMemoryWithPropertiesCount = 0; ze_host_mem_alloc_desc_t hostDesc = {}; ze_result_t result = driverHandle->allocHostMem(&hostDesc, 4096u, 0u, &ptr); - EXPECT_EQ(memoryManager->allocateGraphicsMemoryWithPropertiesCount, 1u); + EXPECT_EQ(static_cast(driverHandle.get()->getMemoryManager())->allocateGraphicsMemoryWithPropertiesCount, 1u); EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, result); EXPECT_EQ(nullptr, ptr); } @@ -807,16 +779,17 @@ TEST_F(AllocHostMemoryTest, TEST_F(AllocHostMemoryTest, whenCallingAllocHostMemAndFailingOnCreatingGraphicsAllocationWithHostPointerThenNullIsReturned) { - memoryManager->forceFailureInAllocationWithHostPointer = true; + static_cast(driverHandle.get()->getMemoryManager())->isMockHostMemoryManager = true; + static_cast(driverHandle.get()->getMemoryManager())->forceFailureInAllocationWithHostPointer = true; void *ptr = nullptr; - memoryManager->allocateGraphicsMemoryWithPropertiesCount = 0; + static_cast(driverHandle.get()->getMemoryManager())->allocateGraphicsMemoryWithPropertiesCount = 0; ze_host_mem_alloc_desc_t hostDesc = {}; ze_result_t result = driverHandle->allocHostMem(&hostDesc, 4096u, 0u, &ptr); - EXPECT_EQ(memoryManager->allocateGraphicsMemoryWithPropertiesCount, numRootDevices); + EXPECT_EQ(static_cast(driverHandle.get()->getMemoryManager())->allocateGraphicsMemoryWithPropertiesCount, numRootDevices); EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, result); EXPECT_EQ(nullptr, ptr); } diff --git a/opencl/test/unit_test/api/cl_create_buffer_tests.cpp b/opencl/test/unit_test/api/cl_create_buffer_tests.cpp index e403eacc41..021fc23a42 100644 --- a/opencl/test/unit_test/api/cl_create_buffer_tests.cpp +++ b/opencl/test/unit_test/api/cl_create_buffer_tests.cpp @@ -514,34 +514,16 @@ INSTANTIATE_TEST_CASE_P( clCreateBufferWithMultiDeviceContextTests, testing::ValuesIn(validFlagsForMultiDeviceContextBuffer)); -class MockMemoryManagerWithFailures2 : public OsAgnosticMemoryManager { - public: - MockMemoryManagerWithFailures2(ExecutionEnvironment &executionEnvironment) : OsAgnosticMemoryManager(executionEnvironment){}; - - GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override { - if (allocationData.rootDeviceIndex == forbiddenRootDeviceIndex) { - return nullptr; - } - return OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool(allocationData, status); - } - uint32_t forbiddenRootDeviceIndex = 1u; -}; - using clCreateBufferWithMultiDeviceContextFaillingAllocationTests = clCreateBufferTemplateTests; TEST_F(clCreateBufferWithMultiDeviceContextFaillingAllocationTests, GivenContextdWithMultiDeviceFailingAllocationThenBufferAllocateFails) { UltClDeviceFactory deviceFactory{3, 0}; DebugManager.flags.EnableMultiRootDeviceContexts.set(true); - auto mockMemoryManager = new MockMemoryManagerWithFailures2(*deviceFactory.rootDevices[0]->getExecutionEnvironment()); - deviceFactory.rootDevices[0]->injectMemoryManager(mockMemoryManager); - cl_device_id devices[] = {deviceFactory.rootDevices[0], deviceFactory.rootDevices[1], deviceFactory.rootDevices[2]}; MockContext pContext(ClDeviceVector(devices, 3)); - pContext.memoryManager = mockMemoryManager; - EXPECT_EQ(2u, pContext.getMaxRootDeviceIndex()); constexpr auto bufferSize = 64u; @@ -551,6 +533,9 @@ TEST_F(clCreateBufferWithMultiDeviceContextFaillingAllocationTests, GivenContext cl_mem_flags flags = CL_MEM_USE_HOST_PTR; + static_cast(pContext.memoryManager)->successAllocatedGraphicsMemoryIndex = 0u; + static_cast(pContext.memoryManager)->maxSuccessAllocatedGraphicsMemoryIndex = 2u; + auto buffer = clCreateBuffer(&pContext, flags, bufferSize, ptrHostBuffer, &retVal); ASSERT_EQ(CL_OUT_OF_HOST_MEMORY, retVal); EXPECT_EQ(nullptr, buffer); diff --git a/opencl/test/unit_test/api/cl_unified_shared_memory_tests.inl b/opencl/test/unit_test/api/cl_unified_shared_memory_tests.inl index 762980a280..089161663a 100644 --- a/opencl/test/unit_test/api/cl_unified_shared_memory_tests.inl +++ b/opencl/test/unit_test/api/cl_unified_shared_memory_tests.inl @@ -27,6 +27,9 @@ TEST(clUnifiedSharedMemoryTests, whenClHostMemAllocINTELisCalledWithoutContextTh TEST(clUnifiedSharedMemoryTests, whenClHostMemAllocIntelIsCalledThenItAllocatesHostUnifiedMemoryAllocation) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; auto unifiedMemoryHostAllocation = clHostMemAllocINTEL(&mockContext, nullptr, 4, 0, &retVal); EXPECT_EQ(CL_SUCCESS, retVal); @@ -172,6 +175,9 @@ TEST(clUnifiedSharedMemoryTests, whenClMemBlockingFreeINTELisCalledWithNullPoint TEST(clUnifiedSharedMemoryTests, whenClMemFreeINTELisCalledWithValidUmPointerThenMemoryIsFreed) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; auto unifiedMemoryHostAllocation = clHostMemAllocINTEL(&mockContext, nullptr, 4, 0, &retVal); @@ -186,6 +192,9 @@ TEST(clUnifiedSharedMemoryTests, whenClMemFreeINTELisCalledWithValidUmPointerThe TEST(clUnifiedSharedMemoryTests, whenClMemFreeINTELisCalledWithInvalidUmPointerThenMemoryIsNotFreed) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; auto unifiedMemoryHostAllocation = clHostMemAllocINTEL(&mockContext, nullptr, 4, 0, &retVal); @@ -276,6 +285,9 @@ TEST(clUnifiedSharedMemoryTests, whenClGetMemAllocInfoINTELisCalledWithAllocatio TEST(clUnifiedSharedMemoryTests, whenClGetMemAllocInfoINTELisCalledWithValidUnifiedMemoryHostAllocationThenProperFieldsAreSet) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; size_t paramValueSize = sizeof(cl_int); cl_int paramValue = 0; @@ -308,6 +320,9 @@ TEST(clUnifiedSharedMemoryTests, whenHostMemAllocWithInvalidPropertiesTokenThenE TEST(clUnifiedSharedMemoryTests, whenHostMemAllocWithInvalidWriteCombinedTokenThenSuccessIsReturned) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; cl_mem_properties_intel properties[] = {CL_MEM_ALLOC_FLAGS_INTEL, CL_MEM_ALLOC_WRITE_COMBINED_INTEL, 0}; @@ -360,6 +375,9 @@ TEST(clUnifiedSharedMemoryTests, whenSharedMemAllocWithInvalidWriteCombinedToken TEST(clUnifiedSharedMemoryTests, givenUnifiedMemoryAllocWithoutPropertiesWhenGetMemAllocFlagsThenDefaultValueIsReturned) { uint64_t defaultValue = CL_MEM_ALLOC_DEFAULT_INTEL; MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; size_t paramValueSize = sizeof(cl_mem_properties_intel); cl_mem_properties_intel paramValue = 0; @@ -377,6 +395,9 @@ TEST(clUnifiedSharedMemoryTests, givenUnifiedMemoryAllocWithoutPropertiesWhenGet TEST(clUnifiedSharedMemoryTests, whenClGetMemAllocTypeIsCalledWithValidUnifiedMemoryHostAllocationThenProperTypeIsReturned) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; size_t paramValueSize = sizeof(cl_mem_properties_intel); cl_mem_properties_intel paramValue = 0; @@ -533,6 +554,9 @@ TEST(clUnifiedSharedMemoryTests, givenSharedAllocationWithoutDeviceWhenItIsQueri TEST(clUnifiedSharedMemoryTests, givenHostAllocationWhenItIsQueriedForDeviceThenProperDeviceIsReturned) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; size_t paramValueSizeRet = 0; auto unifiedMemoryHostAllocation = clHostMemAllocINTEL(&mockContext, nullptr, 4, 0, &retVal); @@ -573,6 +597,9 @@ TEST(clUnifiedSharedMemoryTests, whenClGetMemAllocInfoINTELisCalledWithAllocatio TEST(clUnifiedSharedMemoryTests, whenClGetMemAllocInfoINTELisCalledWithAllocationSizeParamNameThenProperFieldsAreSet) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_int retVal = CL_SUCCESS; size_t paramValueSize = sizeof(size_t); size_t paramValue = 0; @@ -1051,6 +1078,9 @@ TEST(clUnifiedSharedMemoryTests, givenUnifiedMemoryAllocationSizeGreaterThanMaxM using MultiRootDeviceClUnifiedSharedMemoryTests = MultiRootDeviceFixture; TEST_F(MultiRootDeviceClUnifiedSharedMemoryTests, WhenClHostMemAllocIntelIsCalledInMultiRootDeviceEnvironmentThenItAllocatesHostUnifiedMemoryAllocations) { + REQUIRE_SVM_OR_SKIP(device1); + REQUIRE_SVM_OR_SKIP(device2); + cl_int retVal = CL_SUCCESS; auto unifiedMemoryHostAllocation = clHostMemAllocINTEL(context.get(), nullptr, 4, 0, &retVal); @@ -1085,6 +1115,9 @@ TEST_F(MultiRootDeviceClUnifiedSharedMemoryTests, WhenClHostMemAllocIntelIsCalle } TEST_F(MultiRootDeviceClUnifiedSharedMemoryTests, WhenClSharedMemAllocIntelIsCalledWithoutDeviceInMultiRootDeviceEnvironmentThenItAllocatesHostUnifiedMemoryAllocations) { + REQUIRE_SVM_OR_SKIP(device1); + REQUIRE_SVM_OR_SKIP(device2); + cl_int retVal = CL_SUCCESS; auto unifiedMemorySharedAllocation = clSharedMemAllocINTEL(context.get(), nullptr, nullptr, 4, 0, &retVal); @@ -1118,6 +1151,9 @@ TEST_F(MultiRootDeviceClUnifiedSharedMemoryTests, WhenClSharedMemAllocIntelIsCal EXPECT_EQ(CL_SUCCESS, retVal); } TEST_F(MultiRootDeviceClUnifiedSharedMemoryTests, WhenClSharedMemAllocIntelIsCalledWithoutDeviceInMultiRootDeviceEnvironmentThenItWaitsForAllGpuAllocations) { + REQUIRE_SVM_OR_SKIP(device1); + REQUIRE_SVM_OR_SKIP(device2); + mockMemoryManager->waitAllocations.reset(new MultiGraphicsAllocation(2u)); cl_int retVal = CL_SUCCESS; diff --git a/opencl/test/unit_test/command_queue/enqueue_svm_tests.cpp b/opencl/test/unit_test/command_queue/enqueue_svm_tests.cpp index ff897d7d13..4697c3f0de 100644 --- a/opencl/test/unit_test/command_queue/enqueue_svm_tests.cpp +++ b/opencl/test/unit_test/command_queue/enqueue_svm_tests.cpp @@ -1423,6 +1423,7 @@ HWTEST_F(EnqueueSvmTest, whenInternalAllocationIsTriedToBeAddedTwiceToResidencyC struct createHostUnifiedMemoryAllocationTest : public ::testing::Test { void SetUp() override { + REQUIRE_SVM_OR_SKIP(defaultHwInfo); device0 = context.pRootDevice0; device1 = context.pRootDevice1; device2 = context.pRootDevice2; @@ -1489,7 +1490,7 @@ HWTEST_F(createHostUnifiedMemoryAllocationTest, SvmAllocationData allocData(maxRootDeviceIndex); - void *unifiedMemoryPtr = memoryManager->createMultiGraphicsAllocation(rootDeviceIndices, allocationProperties, allocData.gpuAllocations); + void *unifiedMemoryPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, allocationProperties, allocData.gpuAllocations); EXPECT_NE(nullptr, unifiedMemoryPtr); EXPECT_EQ(numDevices, allocData.gpuAllocations.getGraphicsAllocations().size()); @@ -1535,7 +1536,7 @@ HWTEST_F(createHostUnifiedMemoryAllocationTest, SvmAllocationData allocData(maxRootDeviceIndex); - void *unifiedMemoryPtr = memoryManager->createMultiGraphicsAllocation(rootDeviceIndices, allocationProperties, allocData.gpuAllocations); + void *unifiedMemoryPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, allocationProperties, allocData.gpuAllocations); EXPECT_NE(nullptr, unifiedMemoryPtr); EXPECT_EQ(numDevices, allocData.gpuAllocations.getGraphicsAllocations().size()); diff --git a/opencl/test/unit_test/command_stream/aub_file_stream_tests.cpp b/opencl/test/unit_test/command_stream/aub_file_stream_tests.cpp index 5fd3d6f523..50bbab9897 100644 --- a/opencl/test/unit_test/command_stream/aub_file_stream_tests.cpp +++ b/opencl/test/unit_test/command_stream/aub_file_stream_tests.cpp @@ -586,8 +586,9 @@ HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenFlushIsCalledThenI auto mockHardwareContext = static_cast(aubCsr.hardwareContextController->hardwareContexts[0].get()); auto commandBuffer = pDevice->executionEnvironment->memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), MemoryConstants::pageSize}); - auto tagAllocation = pDevice->executionEnvironment->memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), MemoryConstants::pageSize}); - aubCsr.setTagAllocation(tagAllocation); + + aubCsr.initializeTagAllocation(); + LinearStream cs(commandBuffer); BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 1, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false}; ResidencyContainer allocationsForResidency; @@ -611,8 +612,9 @@ HWTEST_F(AubFileStreamTests, givenAubCommandStreamReceiverWhenFlushIsCalledWithZ auto mockHardwareContext = static_cast(aubCsr.hardwareContextController->hardwareContexts[0].get()); auto commandBuffer = pDevice->executionEnvironment->memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), MemoryConstants::pageSize}); - auto tagAllocation = pDevice->executionEnvironment->memoryManager->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), MemoryConstants::pageSize}); - aubCsr.setTagAllocation(tagAllocation); + + aubCsr.initializeTagAllocation(); + LinearStream cs(commandBuffer); BatchBuffer batchBuffer{cs.getGraphicsAllocation(), 0, 0, nullptr, false, false, QueueThrottle::MEDIUM, QueueSliceCount::defaultSliceCount, cs.getUsed(), &cs, nullptr, false}; ResidencyContainer allocationsForResidency; diff --git a/opencl/test/unit_test/command_stream/command_stream_receiver_tests.cpp b/opencl/test/unit_test/command_stream/command_stream_receiver_tests.cpp index a19c84f794..b261a6486c 100644 --- a/opencl/test/unit_test/command_stream/command_stream_receiver_tests.cpp +++ b/opencl/test/unit_test/command_stream/command_stream_receiver_tests.cpp @@ -670,7 +670,13 @@ TEST(CommandStreamReceiverSimpleTest, givenCommandStreamReceiverWhenItIsDestroye MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); auto csr = std::make_unique(executionEnvironment, 0, 1); executionEnvironment.memoryManager.reset(new OsAgnosticMemoryManager(executionEnvironment)); - csr->setTagAllocation(mockGraphicsAllocation); + + auto allocations = new MultiGraphicsAllocation(0u); + allocations->addAllocation(mockGraphicsAllocation); + + csr->tagsMultiAllocation = allocations; + csr->setTagAllocation(allocations->getGraphicsAllocation(0u)); + EXPECT_FALSE(destructorCalled); csr.reset(nullptr); EXPECT_TRUE(destructorCalled); @@ -689,6 +695,31 @@ TEST(CommandStreamReceiverSimpleTest, givenCommandStreamReceiverWhenInitializeTa EXPECT_EQ(*csr->getTagAddress(), initialHardwareTag); } +TEST(CommandStreamReceiverSimpleTest, givenCommandStreamReceiverWhenInitializeTagAllocationIsCalledInMultiRootDeviceEnvironmentThenTagAllocationIsBeingAllocated) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get(), true, 10u); + auto csr = std::make_unique(executionEnvironment, 0, 1); + executionEnvironment.memoryManager.reset(new OsAgnosticMemoryManager(executionEnvironment)); + + EXPECT_EQ(nullptr, csr->getTagAllocation()); + EXPECT_TRUE(csr->getTagAddress() == nullptr); + + csr->initializeTagAllocation(); + + EXPECT_NE(nullptr, csr->getTagAllocation()); + EXPECT_EQ(GraphicsAllocation::AllocationType::TAG_BUFFER, csr->getTagAllocation()->getAllocationType()); + EXPECT_TRUE(csr->getTagAddress() != nullptr); + EXPECT_EQ(*csr->getTagAddress(), initialHardwareTag); + + auto tagsMultiAllocation = csr->getTagsMultiAllocation(); + auto graphicsAllocation0 = tagsMultiAllocation->getGraphicsAllocation(0); + + for (auto graphicsAllocation : tagsMultiAllocation->getGraphicsAllocations()) { + if (graphicsAllocation != graphicsAllocation0) { + EXPECT_EQ(graphicsAllocation->getUnderlyingBuffer(), graphicsAllocation0->getUnderlyingBuffer()); + } + } +} + HWTEST_F(CommandStreamReceiverTest, givenCommandStreamReceiverWhenFenceAllocationIsRequiredAndCreateGlobalFenceAllocationIsCalledThenFenceAllocationIsAllocated) { RAIIHwHelperFactory> hwHelperBackup{pDevice->getHardwareInfo().platform.eRenderCoreFamily}; diff --git a/opencl/test/unit_test/command_stream/tbx_command_stream_tests.cpp b/opencl/test/unit_test/command_stream/tbx_command_stream_tests.cpp index e8d7d0d505..dde6b6853b 100644 --- a/opencl/test/unit_test/command_stream/tbx_command_stream_tests.cpp +++ b/opencl/test/unit_test/command_stream/tbx_command_stream_tests.cpp @@ -344,9 +344,10 @@ HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingMakeSurfacePackNonResi HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingWaitForTaskCountWithKmdNotifyFallbackThenTagAllocationAndScheduledAllocationsAreDownloaded) { MockTbxCsrRegisterDownloadedAllocations tbxCsr{*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield()}; MockOsContext osContext(0, pDevice->getDeviceBitfield(), aub_stream::ENGINE_RCS, PreemptionMode::Disabled, false, false, false); - uint32_t tag = 0u; + tbxCsr.setupContext(osContext); - tbxCsr.setTagAllocation(pDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), false, sizeof(tag), pDevice->getDeviceBitfield()}, &tag)); + tbxCsr.initializeTagAllocation(); + *tbxCsr.getTagAddress() = 0u; tbxCsr.latestFlushedTaskCount = 1u; MockGraphicsAllocation allocation1, allocation2, allocation3; @@ -369,9 +370,10 @@ HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingWaitForTaskCountWithKm HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingWaitForCompletionWithTimeoutThenFlushIsCalledAndTagAllocationAndScheduledAllocationsAreDownloaded) { MockTbxCsrRegisterDownloadedAllocations tbxCsr{*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield()}; MockOsContext osContext(0, pDevice->getDeviceBitfield(), aub_stream::ENGINE_RCS, PreemptionMode::Disabled, false, false, false); - uint32_t tag = 0u; + tbxCsr.setupContext(osContext); - tbxCsr.setTagAllocation(pDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), false, sizeof(tag), pDevice->getDeviceBitfield()}, &tag)); + tbxCsr.initializeTagAllocation(); + *tbxCsr.getTagAddress() = 0u; tbxCsr.latestFlushedTaskCount = 1u; MockGraphicsAllocation allocation1, allocation2, allocation3; @@ -395,9 +397,10 @@ HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenCallingWaitForCompletionWithT HWTEST_F(TbxCommandSteamSimpleTest, givenTbxCsrWhenDownloadAllocatoinsCalledThenTagAndScheduledAllocationsAreDownloadedAndRemovedFromContainer) { MockTbxCsrRegisterDownloadedAllocations tbxCsr{*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield()}; MockOsContext osContext(0, pDevice->getDeviceBitfield(), aub_stream::ENGINE_RCS, PreemptionMode::Disabled, false, false, false); - uint32_t tag = 0u; + tbxCsr.setupContext(osContext); - tbxCsr.setTagAllocation(pDevice->getMemoryManager()->allocateGraphicsMemoryWithProperties(MockAllocationProperties{pDevice->getRootDeviceIndex(), false, sizeof(tag), pDevice->getDeviceBitfield()}, &tag)); + tbxCsr.initializeTagAllocation(); + *tbxCsr.getTagAddress() = 0u; MockGraphicsAllocation allocation1, allocation2, allocation3; tbxCsr.allocationsForDownload = {&allocation1, &allocation2, &allocation3}; diff --git a/opencl/test/unit_test/mem_obj/buffer_bcs_tests.cpp b/opencl/test/unit_test/mem_obj/buffer_bcs_tests.cpp index c01077518f..5c34ce3034 100644 --- a/opencl/test/unit_test/mem_obj/buffer_bcs_tests.cpp +++ b/opencl/test/unit_test/mem_obj/buffer_bcs_tests.cpp @@ -143,15 +143,14 @@ HWTEST_TEMPLATED_F(BcsBufferTests, givenBufferWithInitializationDataAndBcsCsrWhe } HWTEST_TEMPLATED_F(BcsBufferTests, givenBufferWithNotDefaultRootDeviceIndexAndBcsCsrWhenCreatingThenUseBlitOperation) { - - std::unique_ptr newDevice = std::make_unique(MockDevice::createWithNewExecutionEnvironment(nullptr, 1u)); + auto rootDeviceIndex = 1u; + std::unique_ptr newDevice = std::make_unique(MockDevice::createWithNewExecutionEnvironment(nullptr, rootDeviceIndex)); std::unique_ptr newBcsMockContext = std::make_unique(newDevice.get()); auto bcsCsr = static_cast *>(newBcsMockContext->bcsCsr.get()); - auto newMemoryManager = new MockMemoryManager(true, true, *newDevice->getExecutionEnvironment()); - newDevice->getExecutionEnvironment()->memoryManager.reset(newMemoryManager); - newBcsMockContext->memoryManager = newMemoryManager; + static_cast(newDevice->getExecutionEnvironment()->memoryManager.get())->enable64kbpages[rootDeviceIndex] = true; + static_cast(newDevice->getExecutionEnvironment()->memoryManager.get())->localMemorySupported[rootDeviceIndex] = true; EXPECT_EQ(0u, bcsCsr->blitBufferCalled); auto bufferForBlt = clUniquePtr(Buffer::create(newBcsMockContext.get(), CL_MEM_COPY_HOST_PTR, 2000, &hostPtr, retVal)); diff --git a/opencl/test/unit_test/memory_manager/memory_manager_multi_device_tests.cpp b/opencl/test/unit_test/memory_manager/memory_manager_multi_device_tests.cpp index a70df71591..cc3fc44d59 100644 --- a/opencl/test/unit_test/memory_manager/memory_manager_multi_device_tests.cpp +++ b/opencl/test/unit_test/memory_manager/memory_manager_multi_device_tests.cpp @@ -58,3 +58,30 @@ TEST_P(MemoryManagerMultiDeviceTest, givenRootDeviceIndexSpecifiedWhenAllocateGr } INSTANTIATE_TEST_CASE_P(MemoryManagerType, MemoryManagerMultiDeviceTest, ::testing::Bool()); + +TEST_P(MemoryManagerMultiDeviceTest, givenRootDeviceIndexSpecifiedWhenAllocateGraphicsMemoryIsCalledThenGraphicsAllocationHasProperGpuAddress) { + std::vector rootDeviceIndices; + + for (uint32_t rootDeviceIndex = 0; rootDeviceIndex < getNumRootDevices(); ++rootDeviceIndex) { + rootDeviceIndices.push_back(rootDeviceIndex); + } + + auto maxRootDeviceIndex = *std::max_element(rootDeviceIndices.begin(), rootDeviceIndices.end(), std::less()); + auto tagsMultiAllocation = new MultiGraphicsAllocation(maxRootDeviceIndex); + + AllocationProperties unifiedMemoryProperties{rootDeviceIndices.at(0), MemoryConstants::pageSize, GraphicsAllocation::AllocationType::TAG_BUFFER, systemMemoryBitfield}; + + memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, unifiedMemoryProperties, *tagsMultiAllocation); + EXPECT_NE(nullptr, tagsMultiAllocation); + + auto graphicsAllocation0 = tagsMultiAllocation->getGraphicsAllocation(0); + + for (auto graphicsAllocation : tagsMultiAllocation->getGraphicsAllocations()) { + EXPECT_EQ(graphicsAllocation->getUnderlyingBuffer(), graphicsAllocation0->getUnderlyingBuffer()); + } + + for (auto graphicsAllocation : tagsMultiAllocation->getGraphicsAllocations()) { + memoryManager->freeGraphicsMemory(graphicsAllocation); + } + delete tagsMultiAllocation; +} diff --git a/opencl/test/unit_test/memory_manager/unified_memory_manager_tests.cpp b/opencl/test/unit_test/memory_manager/unified_memory_manager_tests.cpp index 031a7369c1..14d1c3e219 100644 --- a/opencl/test/unit_test/memory_manager/unified_memory_manager_tests.cpp +++ b/opencl/test/unit_test/memory_manager/unified_memory_manager_tests.cpp @@ -23,6 +23,7 @@ #include "opencl/test/unit_test/mocks/mock_memory_manager.h" #include "opencl/test/unit_test/mocks/mock_platform.h" #include "opencl/test/unit_test/mocks/mock_svm_manager.h" +#include "opencl/test/unit_test/test_macros/test_checks_ocl.h" #include "test.h" #include "gtest/gtest.h" @@ -771,6 +772,9 @@ TEST_F(ShareableUnifiedMemoryManagerPropertiesTest, givenShareableUnifiedPropert TEST(UnfiedSharedMemoryTransferCalls, givenHostUsmAllocationWhenPointerIsUsedForTransferCallsThenUSMAllocationIsReused) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_context clContext = &mockContext; auto status = CL_INVALID_PLATFORM; @@ -939,6 +943,9 @@ TEST(UnfiedSharedMemoryTransferCalls, givenHostUsmAllocationWhenPtrIsUsedForTran TEST(UnfiedSharedMemoryTransferCalls, givenHostAllocationThatIsSmallerThenTransferRequirementsThenErrorIsReturned) { MockContext mockContext; + auto device = mockContext.getDevice(0u); + REQUIRE_SVM_OR_SKIP(device); + cl_context clContext = &mockContext; auto status = CL_SUCCESS; diff --git a/opencl/test/unit_test/mocks/mock_gfx_partition.h b/opencl/test/unit_test/mocks/mock_gfx_partition.h index dda844988d..5573e5ff9e 100644 --- a/opencl/test/unit_test/mocks/mock_gfx_partition.h +++ b/opencl/test/unit_test/mocks/mock_gfx_partition.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2020 Intel Corporation + * Copyright (C) 2019-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -41,6 +41,13 @@ class MockGfxPartition : public GfxPartition { OSMemory::ReservedCpuAddressRange reservedCpuAddressRange; }; +class MockGfxPartitionBasic : public GfxPartition { + public: + MockGfxPartitionBasic() : GfxPartition(reservedCpuAddressRange) {} + + OSMemory::ReservedCpuAddressRange reservedCpuAddressRange; +}; + class FailedInitGfxPartition : public MockGfxPartition { public: virtual bool init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToReserve, uint32_t rootDeviceIndex, size_t numRootDevices, bool useFrontWindowPool) override { diff --git a/opencl/test/unit_test/mocks/mock_memory_manager.cpp b/opencl/test/unit_test/mocks/mock_memory_manager.cpp index b103142355..122e01fa2c 100644 --- a/opencl/test/unit_test/mocks/mock_memory_manager.cpp +++ b/opencl/test/unit_test/mocks/mock_memory_manager.cpp @@ -37,12 +37,31 @@ void *MockMemoryManager::allocateSystemMemory(size_t size, size_t alignment) { } GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryWithProperties(const AllocationProperties &properties) { + if (isMockHostMemoryManager) { + allocateGraphicsMemoryWithPropertiesCount++; + if (forceFailureInPrimaryAllocation) { + return nullptr; + } + return NEO::MemoryManager::allocateGraphicsMemoryWithProperties(properties); + } + recentlyPassedDeviceBitfield = properties.subDevicesBitfield; AllocationProperties adjustedProperties(properties); adjustedProperties.size = redundancyRatio * properties.size; return OsAgnosticMemoryManager::allocateGraphicsMemoryWithProperties(adjustedProperties); } +GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryWithProperties(const AllocationProperties &properties, const void *ptr) { + if (isMockHostMemoryManager) { + allocateGraphicsMemoryWithPropertiesCount++; + if (forceFailureInAllocationWithHostPointer) { + return nullptr; + } + return NEO::MemoryManager::allocateGraphicsMemoryWithProperties(properties, ptr); + } + return OsAgnosticMemoryManager::allocateGraphicsMemoryWithProperties(properties, ptr); +} + GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryForImage(const AllocationData &allocationData) { allocateForImageCalled = true; auto *allocation = MemoryManager::allocateGraphicsMemoryForImage(allocationData); @@ -77,15 +96,20 @@ GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryInDevicePool(const status = AllocationStatus::Error; return nullptr; } + if (successAllocatedGraphicsMemoryIndex >= maxSuccessAllocatedGraphicsMemoryIndex) { + return nullptr; - auto allocation = OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool(allocationData, status); - if (allocation) { - allocationInDevicePoolCreated = true; - if (localMemorySupported[allocation->getRootDeviceIndex()]) { - static_cast(allocation)->overrideMemoryPool(MemoryPool::LocalMemory); + } else { + auto allocation = OsAgnosticMemoryManager::allocateGraphicsMemoryInDevicePool(allocationData, status); + if (allocation) { + allocationInDevicePoolCreated = true; + if (localMemorySupported[allocation->getRootDeviceIndex()]) { + static_cast(allocation)->overrideMemoryPool(MemoryPool::LocalMemory); + } } + successAllocatedGraphicsMemoryIndex++; + return allocation; } - return allocation; } GraphicsAllocation *MockMemoryManager::allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) { diff --git a/opencl/test/unit_test/mocks/mock_memory_manager.h b/opencl/test/unit_test/mocks/mock_memory_manager.h index 80ddeacb1b..8d49ae0ee1 100644 --- a/opencl/test/unit_test/mocks/mock_memory_manager.h +++ b/opencl/test/unit_test/mocks/mock_memory_manager.h @@ -48,7 +48,9 @@ class MockMemoryManager : public MemoryManagerCreate { using MemoryManager::useNonSvmHostPtrAlloc; using OsAgnosticMemoryManager::allocateGraphicsMemoryForImageFromHostPtr; using MemoryManagerCreate::MemoryManagerCreate; + using MemoryManager::enable64kbpages; using MemoryManager::isCopyRequired; + using MemoryManager::localMemorySupported; using MemoryManager::reservedMemory; MockMemoryManager(ExecutionEnvironment &executionEnvironment) : MockMemoryManager(false, executionEnvironment) {} @@ -73,6 +75,7 @@ class MockMemoryManager : public MemoryManagerCreate { GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override; GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override; GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties) override; + GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties, const void *ptr) override; void *allocateSystemMemory(size_t size, size_t alignment) override; @@ -113,6 +116,15 @@ class MockMemoryManager : public MemoryManagerCreate { return OsAgnosticMemoryManager::reserveCpuAddressRange(size, rootDeviceIndex); } + void *createMultiGraphicsAllocationInSystemMemoryPool(std::vector &rootDeviceIndices, + AllocationProperties &properties, + MultiGraphicsAllocation &multiGraphicsAllocation) override { + if (isMockEventPoolCreateMemoryManager) { + return nullptr; + } + return OsAgnosticMemoryManager::createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, properties, multiGraphicsAllocation); + } + bool isCpuCopyRequired(const void *ptr) override { return cpuCopyRequired; } GraphicsAllocation *allocate32BitGraphicsMemory(uint32_t rootDeviceIndex, size_t size, const void *ptr, GraphicsAllocation::AllocationType allocationType); @@ -124,9 +136,12 @@ class MockMemoryManager : public MemoryManagerCreate { uint32_t unlockResourceCalled = 0u; uint32_t lockResourceCalled = 0u; AllocationData alignAllocationData; + uint32_t successAllocatedGraphicsMemoryIndex = 0u; + uint32_t maxSuccessAllocatedGraphicsMemoryIndex = std::numeric_limits::max(); std::vector lockResourcePointers; uint32_t handleFenceCompletionCalled = 0u; uint32_t waitForEnginesCompletionCalled = 0u; + uint32_t allocateGraphicsMemoryWithPropertiesCount = 0; bool allocationCreated = false; bool allocation64kbPageCreated = false; bool allocationInDevicePoolCreated = false; @@ -141,6 +156,10 @@ class MockMemoryManager : public MemoryManagerCreate { bool failAllocate32Bit = false; bool cpuCopyRequired = false; bool forceRenderCompressed = false; + bool forceFailureInPrimaryAllocation = false; + bool forceFailureInAllocationWithHostPointer = false; + bool isMockHostMemoryManager = false; + bool isMockEventPoolCreateMemoryManager = false; std::unique_ptr mockExecutionEnvironment; DeviceBitfield recentlyPassedDeviceBitfield{}; std::unique_ptr waitAllocations = nullptr; diff --git a/opencl/test/unit_test/mocks/mock_tbx_csr.h b/opencl/test/unit_test/mocks/mock_tbx_csr.h index 57e171c68d..23eeee2e60 100644 --- a/opencl/test/unit_test/mocks/mock_tbx_csr.h +++ b/opencl/test/unit_test/mocks/mock_tbx_csr.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2021 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -74,6 +74,7 @@ class MockTbxCsr : public TbxCommandStreamReceiverHw { template struct MockTbxCsrRegisterDownloadedAllocations : TbxCommandStreamReceiverHw { using CommandStreamReceiver::latestFlushedTaskCount; + using CommandStreamReceiver::tagsMultiAllocation; using TbxCommandStreamReceiverHw::TbxCommandStreamReceiverHw; void downloadAllocation(GraphicsAllocation &gfxAllocation) override { *reinterpret_cast(CommandStreamReceiver::getTagAllocation()->getUnderlyingBuffer()) = this->latestFlushedTaskCount; diff --git a/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp index 47fefd27e5..6e8e5bfb71 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_command_stream_tests.cpp @@ -657,13 +657,11 @@ HWTEST_TEMPLATED_F(DrmCommandStreamEnhancedTest, givenDrmCsrCreatedWithInactiveG class DrmCommandStreamBatchingTests : public DrmCommandStreamEnhancedTest { public: - DrmAllocation *tagAllocation; DrmAllocation *preemptionAllocation; template void SetUpT() { DrmCommandStreamEnhancedTest::SetUpT(); - tagAllocation = static_cast(device->getDefaultEngine().commandStreamReceiver->getTagAllocation()); preemptionAllocation = static_cast(device->getDefaultEngine().commandStreamReceiver->getPreemptionAllocation()); } @@ -705,6 +703,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamBatchingTests, givenCSRWhenFlushIsCalledThenP HWTEST_TEMPLATED_F(DrmCommandStreamBatchingTests, givenCsrWhenDispatchPolicyIsSetToBatchingThenCommandBufferIsNotSubmitted) { mock->reset(); + csr->overrideDispatchPolicy(DispatchMode::BatchedDispatch); auto mockedSubmissionsAggregator = new mockSubmissionsAggregator(); @@ -721,7 +720,10 @@ HWTEST_TEMPLATED_F(DrmCommandStreamBatchingTests, givenCsrWhenDispatchPolicyIsSe csr->makeResident(*dummyAllocation); - csr->setTagAllocation(tagAllocation); + auto allocations = device->getDefaultEngine().commandStreamReceiver->getTagsMultiAllocation(); + + csr->setTagAllocation(static_cast(allocations->getGraphicsAllocation(csr->getRootDeviceIndex()))); + DispatchFlags dispatchFlags = DispatchFlagsHelper::createDefaultDispatchFlags(); dispatchFlags.preemptionMode = PreemptionHelper::getDefaultPreemptionMode(device->getHardwareInfo()); @@ -747,7 +749,7 @@ HWTEST_TEMPLATED_F(DrmCommandStreamBatchingTests, givenCsrWhenDispatchPolicyIsSe elementInVector = std::find(recordedCmdBuffer->surfaces.begin(), recordedCmdBuffer->surfaces.end(), commandBuffer); EXPECT_NE(elementInVector, recordedCmdBuffer->surfaces.end()); - elementInVector = std::find(recordedCmdBuffer->surfaces.begin(), recordedCmdBuffer->surfaces.end(), tagAllocation); + elementInVector = std::find(recordedCmdBuffer->surfaces.begin(), recordedCmdBuffer->surfaces.end(), allocations->getGraphicsAllocation(0u)); EXPECT_NE(elementInVector, recordedCmdBuffer->surfaces.end()); EXPECT_EQ(testedCsr->commandStream.getGraphicsAllocation(), recordedCmdBuffer->batchBuffer.commandBufferAllocation); @@ -779,7 +781,10 @@ HWTEST_TEMPLATED_F(DrmCommandStreamBatchingTests, givenRecordedCommandBufferWhen auto commandBuffer = mm->allocateGraphicsMemoryWithProperties(MockAllocationProperties{csr->getRootDeviceIndex(), MemoryConstants::pageSize}); IndirectHeap cs(commandBuffer); - csr->setTagAllocation(tagAllocation); + auto allocations = device->getDefaultEngine().commandStreamReceiver->getTagsMultiAllocation(); + + csr->setTagAllocation(static_cast(allocations->getGraphicsAllocation(csr->getRootDeviceIndex()))); + auto &submittedCommandBuffer = csr->getCS(1024); //use some bytes submittedCommandBuffer.getSpace(4); diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests_dg1.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests_dg1.cpp index 63c0fbe6b8..c0408d9317 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests_dg1.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_allocate_in_device_pool_tests_dg1.cpp @@ -277,7 +277,7 @@ TEST_F(DrmMemoryManagerLocalMemoryTest, givenMultiRootDeviceEnvironmentAndMemory static_cast(executionEnvironment->rootDeviceEnvironments[0]->osInterface->get()->getDrm())->outputFd = 7; - auto ptr = memoryManager->createMultiGraphicsAllocation(rootDeviceIndices, properties, multiGraphics); + auto ptr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, properties, multiGraphics); EXPECT_NE(ptr, nullptr); EXPECT_NE(static_cast(multiGraphics.getDefaultGraphicsAllocation())->getMmapPtr(), nullptr); @@ -321,7 +321,7 @@ TEST_F(DrmMemoryManagerLocalMemoryTest, givenMultiRootDeviceEnvironmentAndMemory size_t size = 4096u; AllocationProperties properties(rootDeviceIndex, true, size, GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY, false, {}); - auto ptr = memoryManager->createMultiGraphicsAllocation(rootDeviceIndices, properties, multiGraphics); + auto ptr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, properties, multiGraphics); EXPECT_EQ(ptr, nullptr); @@ -353,7 +353,7 @@ TEST_F(DrmMemoryManagerLocalMemoryTest, givenMultiRootDeviceEnvironmentAndNoMemo size_t size = 4096u; AllocationProperties properties(rootDeviceIndex, true, size, GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY, false, {}); - auto ptr = memoryManager->createMultiGraphicsAllocation(rootDeviceIndices, properties, multiGraphics); + auto ptr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, properties, multiGraphics); EXPECT_NE(ptr, nullptr); EXPECT_EQ(static_cast(multiGraphics.getDefaultGraphicsAllocation())->getMmapPtr(), nullptr); diff --git a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index e06ddec221..c68329b0c7 100644 --- a/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/opencl/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -306,6 +306,9 @@ TEST_F(DrmMemoryManagerTest, givenDefaultDrmMemoryManagerWhenItIsCreatedAndGfxPa memoryManager->gfxPartitions[0].reset(failedInitGfxPartition.release()); memoryManager->initialize(gemCloseWorkerMode::gemCloseWorkerInactive); EXPECT_FALSE(memoryManager->isInitialized()); + + auto mockGfxPartitionBasic = std::make_unique(); + memoryManager->overrideGfxPartition(mockGfxPartitionBasic.release()); } TEST_F(DrmMemoryManagerTest, WhenMemoryManagerIsCreatedThenPinBbIsCreated) { @@ -3872,6 +3875,9 @@ TEST_F(DrmMemoryManagerTest, givenDrmMemoryManagerAndReleaseGpuRangeIsCalledThen memoryManager->overrideGfxPartition(mockGfxPartition.release()); memoryManager->releaseGpuRange(reinterpret_cast(gpuAddressCanonized), size, 0); + + auto mockGfxPartitionBasic = std::make_unique(); + memoryManager->overrideGfxPartition(mockGfxPartitionBasic.release()); } class GMockDrmMemoryManager : public TestedDrmMemoryManager { diff --git a/shared/source/command_stream/command_stream_receiver.cpp b/shared/source/command_stream/command_stream_receiver.cpp index 057ab969f5..df346d43a1 100644 --- a/shared/source/command_stream/command_stream_receiver.cpp +++ b/shared/source/command_stream/command_stream_receiver.cpp @@ -213,8 +213,12 @@ void CommandStreamReceiver::cleanupResources() { commandStream.replaceBuffer(nullptr, 0); } - if (tagAllocation) { - getMemoryManager()->freeGraphicsMemory(tagAllocation); + if (tagsMultiAllocation) { + for (auto graphicsAllocation : tagsMultiAllocation->getGraphicsAllocations()) { + getMemoryManager()->freeGraphicsMemory(graphicsAllocation); + } + delete tagsMultiAllocation; + tagsMultiAllocation = nullptr; tagAllocation = nullptr; tagAddress = nullptr; } @@ -280,6 +284,25 @@ void CommandStreamReceiver::setTagAllocation(GraphicsAllocation *allocation) { reinterpret_cast(allocation->getUnderlyingBuffer()) + debugPauseStateAddressOffset); } +MultiGraphicsAllocation &CommandStreamReceiver::createTagsMultiAllocation(ExecutionEnvironment &executionEnvironment, MemoryManager &memoryManager, uint32_t currentRootDeviceIndex) { + std::vector rootDeviceIndices; + + for (auto index = 0u; index < executionEnvironment.rootDeviceEnvironments.size(); index++) { + if (executionEnvironment.rootDeviceEnvironments[index].get()->getHardwareInfo()->platform.eProductFamily == + executionEnvironment.rootDeviceEnvironments[currentRootDeviceIndex].get()->getHardwareInfo()->platform.eProductFamily) { + rootDeviceIndices.push_back(index); + } + } + + auto maxRootDeviceIndex = *std::max_element(rootDeviceIndices.begin(), rootDeviceIndices.end(), std::less()); + auto allocations = new MultiGraphicsAllocation(maxRootDeviceIndex); + + AllocationProperties unifiedMemoryProperties{rootDeviceIndices.at(0), MemoryConstants::pageSize, GraphicsAllocation::AllocationType::TAG_BUFFER, systemMemoryBitfield}; + + memoryManager.createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, unifiedMemoryProperties, *allocations); + return *allocations; +} + FlushStamp CommandStreamReceiver::obtainCurrentFlushStamp() const { return flushStamp->peekStamp(); } @@ -473,7 +496,9 @@ void *CommandStreamReceiver::asyncDebugBreakConfirmation(void *arg) { } bool CommandStreamReceiver::initializeTagAllocation() { - auto tagAllocation = getMemoryManager()->allocateGraphicsMemoryWithProperties({rootDeviceIndex, MemoryConstants::pageSize, GraphicsAllocation::AllocationType::TAG_BUFFER, systemMemoryBitfield}); + this->tagsMultiAllocation = &this->createTagsMultiAllocation(this->executionEnvironment, *this->getMemoryManager(), rootDeviceIndex); + + auto tagAllocation = tagsMultiAllocation->getGraphicsAllocation(rootDeviceIndex); if (!tagAllocation) { return false; } diff --git a/shared/source/command_stream/command_stream_receiver.h b/shared/source/command_stream/command_stream_receiver.h index dc1f1d4095..411a53b2a9 100644 --- a/shared/source/command_stream/command_stream_receiver.h +++ b/shared/source/command_stream/command_stream_receiver.h @@ -39,6 +39,7 @@ class IndirectHeap; class InternalAllocationStorage; class LinearStream; class MemoryManager; +class MultiGraphicsAllocation; class OsContext; class OSInterface; class ScratchSpaceController; @@ -112,6 +113,10 @@ class CommandStreamReceiver { GraphicsAllocation *getTagAllocation() const { return tagAllocation; } + MultiGraphicsAllocation *getTagsMultiAllocation() const { + return tagsMultiAllocation; + } + MultiGraphicsAllocation &createTagsMultiAllocation(ExecutionEnvironment &executionEnvironment, MemoryManager &memoryManager, uint32_t currentRootDeviceIndex); MOCKABLE_VIRTUAL volatile uint32_t *getTagAddress() const { return tagAddress; } uint64_t getDebugPauseStateGPUAddress() const { return tagAllocation->getGpuAddress() + debugPauseStateAddressOffset; } @@ -283,6 +288,8 @@ class CommandStreamReceiver { GraphicsAllocation *clearColorAllocation = nullptr; GraphicsAllocation *workPartitionAllocation = nullptr; + MultiGraphicsAllocation *tagsMultiAllocation = nullptr; + IndirectHeap *indirectHeap[IndirectHeap::NUM_TYPES]; OsContext *osContext = nullptr; diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp index dd3dc558c1..21dd578a29 100644 --- a/shared/source/memory_manager/memory_manager.cpp +++ b/shared/source/memory_manager/memory_manager.cpp @@ -139,7 +139,7 @@ GraphicsAllocation *MemoryManager::createPaddedAllocation(GraphicsAllocation *in return allocateGraphicsMemoryWithProperties({inputGraphicsAllocation->getRootDeviceIndex(), sizeWithPadding, GraphicsAllocation::AllocationType::INTERNAL_HOST_MEMORY, systemMemoryBitfield}); } -void *MemoryManager::createMultiGraphicsAllocation(std::vector &rootDeviceIndices, AllocationProperties &properties, MultiGraphicsAllocation &multiGraphicsAllocation) { +void *MemoryManager::createMultiGraphicsAllocationInSystemMemoryPool(std::vector &rootDeviceIndices, AllocationProperties &properties, MultiGraphicsAllocation &multiGraphicsAllocation) { void *ptr = nullptr; for (auto &rootDeviceIndex : rootDeviceIndices) { @@ -152,7 +152,7 @@ void *MemoryManager::createMultiGraphicsAllocation(std::vector &rootDe return nullptr; } multiGraphicsAllocation.addAllocation(graphicsAllocation); - ptr = reinterpret_cast(graphicsAllocation->getGpuAddress()); + ptr = reinterpret_cast(graphicsAllocation->getUnderlyingBuffer()); } else { properties.flags.allocateMemory = false; properties.flags.isUSMHostAllocation = true; diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index 657c356b57..8b7da52d52 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -99,7 +99,7 @@ class MemoryManager { GraphicsAllocation *createGraphicsAllocationWithPadding(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding); virtual GraphicsAllocation *createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding); - MOCKABLE_VIRTUAL void *createMultiGraphicsAllocation(std::vector &rootDeviceIndices, AllocationProperties &properties, MultiGraphicsAllocation &multiGraphicsAllocation); + MOCKABLE_VIRTUAL void *createMultiGraphicsAllocationInSystemMemoryPool(std::vector &rootDeviceIndices, AllocationProperties &properties, MultiGraphicsAllocation &multiGraphicsAllocation); virtual GraphicsAllocation *createGraphicsAllocationFromExistingStorage(AllocationProperties &properties, void *ptr, MultiGraphicsAllocation &multiGraphicsAllocation); virtual AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) = 0; diff --git a/shared/source/memory_manager/unified_memory_manager.cpp b/shared/source/memory_manager/unified_memory_manager.cpp index fc4bca3fd4..5c6c9267c7 100644 --- a/shared/source/memory_manager/unified_memory_manager.cpp +++ b/shared/source/memory_manager/unified_memory_manager.cpp @@ -146,7 +146,7 @@ void *SVMAllocsManager::createHostUnifiedMemoryAllocation(size_t size, auto maxRootDeviceIndex = *std::max_element(rootDeviceIndicesVector.begin(), rootDeviceIndicesVector.end(), std::less()); SvmAllocationData allocData(maxRootDeviceIndex); - void *usmPtr = memoryManager->createMultiGraphicsAllocation(rootDeviceIndicesVector, unifiedMemoryProperties, allocData.gpuAllocations); + void *usmPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector, unifiedMemoryProperties, allocData.gpuAllocations); if (!usmPtr) { return nullptr; } diff --git a/shared/test/common/mocks/mock_command_stream_receiver.h b/shared/test/common/mocks/mock_command_stream_receiver.h index 25a88fb7e4..31142e0e1b 100644 --- a/shared/test/common/mocks/mock_command_stream_receiver.h +++ b/shared/test/common/mocks/mock_command_stream_receiver.h @@ -35,6 +35,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver { using CommandStreamReceiver::newResources; using CommandStreamReceiver::requiredThreadArbitrationPolicy; using CommandStreamReceiver::tagAddress; + using CommandStreamReceiver::tagsMultiAllocation; using CommandStreamReceiver::taskCount; using CommandStreamReceiver::useGpuIdleImplicitFlush; using CommandStreamReceiver::useNewResourceImplicitFlush;