From e8579794064ddb9b006d8e5cfd5586eb7a8b5674 Mon Sep 17 00:00:00 2001 From: Zbigniew Zdanowicz Date: Thu, 25 Nov 2021 13:04:28 +0000 Subject: [PATCH] Refactor multi context device checker Related-To: NEO-6262 Signed-off-by: Zbigniew Zdanowicz --- level_zero/core/source/device/device.h | 7 ++- level_zero/core/source/device/device_imp.cpp | 6 +- level_zero/core/source/device/device_imp.h | 1 - .../core/test/unit_tests/mocks/mock_device.h | 7 ++- .../unit_tests/sources/device/test_device.cpp | 57 ++++++++++++------- .../unit_tests/sources/memory/test_memory.cpp | 40 +++++++++---- 6 files changed, 80 insertions(+), 38 deletions(-) diff --git a/level_zero/core/source/device/device.h b/level_zero/core/source/device/device.h index 7a83f3a350..3b3d27a34c 100644 --- a/level_zero/core/source/device/device.h +++ b/level_zero/core/source/device/device.h @@ -88,7 +88,9 @@ struct Device : _ze_device_handle_t { virtual uint32_t getMaxNumHwThreads() const = 0; virtual NEO::HwHelper &getHwHelper() = 0; - virtual bool isMultiDeviceCapable() const = 0; + bool isMultiDeviceCapable() const { + return multiDeviceCapable; + } virtual const NEO::HardwareInfo &getHwInfo() const = 0; virtual NEO::OSInterface &getOsInterface() = 0; virtual uint32_t getPlatformInfo() const = 0; @@ -137,6 +139,9 @@ struct Device : _ze_device_handle_t { virtual ze_result_t mapOrdinalForAvailableEngineGroup(uint32_t *ordinal) = 0; virtual NEO::GraphicsAllocation *obtainReusableAllocation(size_t requiredSize, NEO::GraphicsAllocation::AllocationType type) = 0; virtual void storeReusableAllocation(NEO::GraphicsAllocation &alloc) = 0; + + protected: + bool multiDeviceCapable = false; }; } // namespace L0 diff --git a/level_zero/core/source/device/device_imp.cpp b/level_zero/core/source/device/device_imp.cpp index a90899b83e..074e9c9a19 100644 --- a/level_zero/core/source/device/device_imp.cpp +++ b/level_zero/core/source/device/device_imp.cpp @@ -779,6 +779,7 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool neoDevice->incRefInternal(); device->execEnvironment = (void *)neoDevice->getExecutionEnvironment(); + device->multiDeviceCapable = NEO::ImplicitScalingHelper::isImplicitScalingEnabled(neoDevice->getDeviceBitfield(), true); device->metricContext = MetricContext::create(*device); device->builtins = BuiltinFunctionsLib::create( device, neoDevice->getBuiltIns()); @@ -861,7 +862,6 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, bool device->getSourceLevelDebugger() ->notifyNewDevice(osInterface ? osInterface->getDriverModel()->getDeviceHandle() : 0); } - device->createSysmanHandle(isSubDevice); device->resourcesReleased = false; return device; @@ -1149,8 +1149,4 @@ NEO::Device *DeviceImp::getActiveDevice() const { return this->neoDevice; } -bool DeviceImp::isMultiDeviceCapable() const { - return NEO::ImplicitScalingHelper::isImplicitScalingEnabled(this->neoDevice->getDeviceBitfield(), true); -} - } // namespace L0 diff --git a/level_zero/core/source/device/device_imp.h b/level_zero/core/source/device/device_imp.h index 6be8ee2ec9..a80cd52a99 100644 --- a/level_zero/core/source/device/device_imp.h +++ b/level_zero/core/source/device/device_imp.h @@ -66,7 +66,6 @@ struct DeviceImp : public Device { BuiltinFunctionsLib *getBuiltinFunctionsLib() override; uint32_t getMOCS(bool l3enabled, bool l1enabled) override; NEO::HwHelper &getHwHelper() override; - bool isMultiDeviceCapable() const override; const NEO::HardwareInfo &getHwInfo() const override; NEO::OSInterface &getOsInterface() override; uint32_t getPlatformInfo() const override; diff --git a/level_zero/core/test/unit_tests/mocks/mock_device.h b/level_zero/core/test/unit_tests/mocks/mock_device.h index 237e2f495a..9e21bf1f0c 100644 --- a/level_zero/core/test/unit_tests/mocks/mock_device.h +++ b/level_zero/core/test/unit_tests/mocks/mock_device.h @@ -17,7 +17,10 @@ namespace L0 { namespace ult { template <> -struct WhiteBox<::L0::Device> : public ::L0::Device {}; +struct WhiteBox<::L0::Device> : public ::L0::Device { + using Base = L0::Device; + using Base::multiDeviceCapable; +}; using Device = WhiteBox<::L0::Device>; @@ -52,7 +55,6 @@ struct Mock : public Device { // Runtime internal methods ADDMETHOD_NOBASE(getExecEnvironment, void *, nullptr, ()); ADDMETHOD_NOBASE_REFRETURN(getHwHelper, NEO::HwHelper &, ()); - ADDMETHOD_CONST_NOBASE(isMultiDeviceCapable, bool, false, ()); ADDMETHOD_NOBASE(getBuiltinFunctionsLib, BuiltinFunctionsLib *, nullptr, ()); ADDMETHOD_CONST_NOBASE(getMaxNumHwThreads, uint32_t, 16u, ()); ADDMETHOD_NOBASE(activateMetricGroupsDeferred, ze_result_t, ZE_RESULT_SUCCESS, (uint32_t count, zet_metric_group_handle_t *phMetricGroups)); @@ -97,6 +99,7 @@ template <> struct Mock : public L0::DeviceImp { using Base = L0::DeviceImp; using Base::debugSession; + using Base::multiDeviceCapable; explicit Mock(NEO::Device *device, NEO::ExecutionEnvironment *execEnv) { device->incRefInternal(); diff --git a/level_zero/core/test/unit_tests/sources/device/test_device.cpp b/level_zero/core/test/unit_tests/sources/device/test_device.cpp index 6257544f27..da20cf0253 100644 --- a/level_zero/core/test/unit_tests/sources/device/test_device.cpp +++ b/level_zero/core/test/unit_tests/sources/device/test_device.cpp @@ -1387,9 +1387,11 @@ struct MockMemoryManagerMultiDevice : public MemoryManagerMock { MockMemoryManagerMultiDevice(NEO::ExecutionEnvironment &executionEnvironment) : MemoryManagerMock(const_cast(executionEnvironment)) {} }; -struct MultipleDevicesTest : public ::testing::Test { +template +struct MultipleDevicesFixture : public ::testing::Test { void SetUp() override { NEO::MockCompilerEnableGuard mock(true); + DebugManager.flags.EnableWalkerPartition.set(enablePartitionWalker); DebugManager.flags.CreateMultipleSubDevices.set(numSubDevices); VariableBackup mockDeviceFlagBackup(&MockDevice::createSingleDevice, false); @@ -1431,25 +1433,33 @@ struct MultipleDevicesTest : public ::testing::Test { const uint32_t numSubDevices = 2u; }; -TEST_F(MultipleDevicesTest, whenCallingGetMemoryPropertiesWithSubDevicesThenCorrectSizeReturned) { +using MultipleDevicesTest = MultipleDevicesFixture<-1>; +using MultipleDevicesDisabledImplicitScalingTest = MultipleDevicesFixture<0>; +using MultipleDevicesEnabledImplicitScalingTest = MultipleDevicesFixture<1>; + +TEST_F(MultipleDevicesDisabledImplicitScalingTest, whenCallingGetMemoryPropertiesWithSubDevicesThenCorrectSizeReturned) { L0::Device *device0 = driverHandle->devices[0]; uint32_t count = 1; - DebugManager.flags.EnableWalkerPartition.set(0); ze_device_memory_properties_t memProperties = {}; ze_result_t res = device0->getMemoryProperties(&count, &memProperties); EXPECT_EQ(res, ZE_RESULT_SUCCESS); EXPECT_EQ(1u, count); EXPECT_EQ(memProperties.totalSize, device0->getNEODevice()->getDeviceInfo().globalMemSize / numSubDevices); +} - DebugManager.flags.EnableWalkerPartition.set(1); - res = device0->getMemoryProperties(&count, &memProperties); +TEST_F(MultipleDevicesEnabledImplicitScalingTest, whenCallingGetMemoryPropertiesWithSubDevicesThenCorrectSizeReturned) { + L0::Device *device0 = driverHandle->devices[0]; + uint32_t count = 1; + + ze_device_memory_properties_t memProperties = {}; + ze_result_t res = device0->getMemoryProperties(&count, &memProperties); EXPECT_EQ(res, ZE_RESULT_SUCCESS); EXPECT_EQ(1u, count); EXPECT_EQ(memProperties.totalSize, device0->getNEODevice()->getDeviceInfo().globalMemSize); } -TEST_F(MultipleDevicesTest, WhenGettingDevicePropertiesGetSubslicesPerSliceWhenImplicitScalingDisabledandEnabled) { +TEST_F(MultipleDevicesDisabledImplicitScalingTest, GivenImplicitScalingDisabledWhenGettingDevicePropertiesGetSubslicesPerSliceThenCorrectValuesReturned) { L0::Device *device = driverHandle->devices[0]; ze_device_properties_t deviceProperties = {ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES}; deviceProperties.type = ZE_DEVICE_TYPE_GPU; @@ -1459,11 +1469,20 @@ TEST_F(MultipleDevicesTest, WhenGettingDevicePropertiesGetSubslicesPerSliceWhenI device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SubSliceCount = 8; device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SliceCount = 1; - DebugManager.flags.EnableWalkerPartition.set(0); device->getProperties(&deviceProperties); EXPECT_EQ(((device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SubSliceCount * device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SliceCount) / numSubDevices), deviceProperties.numSubslicesPerSlice); +} + +TEST_F(MultipleDevicesEnabledImplicitScalingTest, GivenImplicitScalingEnabledWhenGettingDevicePropertiesGetSubslicesPerSliceThenCorrectValuesReturned) { + L0::Device *device = driverHandle->devices[0]; + ze_device_properties_t deviceProperties = {ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES}; + deviceProperties.type = ZE_DEVICE_TYPE_GPU; + + device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.MaxSubSlicesSupported = 48; + device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.MaxSlicesSupported = 3; + device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SubSliceCount = 8; + device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SliceCount = 1; - DebugManager.flags.EnableWalkerPartition.set(1); device->getProperties(&deviceProperties); EXPECT_EQ((device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SubSliceCount * device->getNEODevice()->getRootDeviceEnvironment().getMutableHardwareInfo()->gtSystemInfo.SliceCount), deviceProperties.numSubslicesPerSlice); } @@ -2445,11 +2464,13 @@ TEST_F(zeDeviceCacheReservationTest, WhenCallingZeDeviceSetCacheAdviceExtFailsTo } } +template struct MultiSubDeviceFixture : public DeviceFixture { void SetUp() { DebugManager.flags.CreateMultipleSubDevices.set(2); - osLocalMemoryBackup = std::make_unique>(&NEO::OSInterface::osEnableLocalMemory, true); - apiSupportBackup = std::make_unique>(&NEO::ImplicitScaling::apiSupport, true); + DebugManager.flags.EnableWalkerPartition.set(enablePartitionWalker); + osLocalMemoryBackup = std::make_unique>(&NEO::OSInterface::osEnableLocalMemory, osLocalMemory); + apiSupportBackup = std::make_unique>(&NEO::ImplicitScaling::apiSupport, apiSupport); DeviceFixture::SetUp(); @@ -2464,28 +2485,26 @@ struct MultiSubDeviceFixture : public DeviceFixture { std::unique_ptr> apiSupportBackup; }; -using MultiSubDeviceTest = Test; - +using MultiSubDeviceTest = Test>; TEST_F(MultiSubDeviceTest, GivenApiSupportAndLocalMemoryEnabledWhenDeviceContainsSubDevicesThenItIsMultiDeviceCapable) { EXPECT_TRUE(device->isMultiDeviceCapable()); EXPECT_EQ(neoDevice, deviceImp->getActiveDevice()); } -TEST_F(MultiSubDeviceTest, GivenNoApiSupportAndLocalMemoryEnabledWhenDeviceContainsSubDevicesThenItIsNotMultiDeviceCapable) { - NEO::ImplicitScaling::apiSupport = false; +using MultiSubDeviceTestNoApi = Test>; +TEST_F(MultiSubDeviceTestNoApi, GivenNoApiSupportAndLocalMemoryEnabledWhenDeviceContainsSubDevicesThenItIsNotMultiDeviceCapable) { EXPECT_FALSE(device->isMultiDeviceCapable()); EXPECT_EQ(subDevice, deviceImp->getActiveDevice()); } -TEST_F(MultiSubDeviceTest, GivenApiSupportAndLocalMemoryDisabledWhenDeviceContainsSubDevicesThenItIsNotMultiDeviceCapable) { - NEO::OSInterface::osEnableLocalMemory = false; +using MultiSubDeviceTestNoLocalMemory = Test>; +TEST_F(MultiSubDeviceTestNoLocalMemory, GivenApiSupportAndLocalMemoryDisabledWhenDeviceContainsSubDevicesThenItIsNotMultiDeviceCapable) { EXPECT_FALSE(device->isMultiDeviceCapable()); EXPECT_EQ(subDevice, deviceImp->getActiveDevice()); } -TEST_F(MultiSubDeviceTest, GivenNoApiSupportAndLocalMemoryEnabledWhenForcedImplicitScalingThenItIsMultiDeviceCapable) { - NEO::ImplicitScaling::apiSupport = false; - DebugManager.flags.EnableWalkerPartition.set(1); +using MultiSubDeviceTestNoApiForceOn = Test>; +TEST_F(MultiSubDeviceTestNoApiForceOn, GivenNoApiSupportAndLocalMemoryEnabledWhenForcedImplicitScalingThenItIsMultiDeviceCapable) { EXPECT_TRUE(device->isMultiDeviceCapable()); EXPECT_EQ(neoDevice, deviceImp->getActiveDevice()); } diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp index 03226fc136..d163f9a795 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory.cpp @@ -3055,9 +3055,11 @@ TEST_F(SharedAllocMultiDeviceTests, whenAllocatinSharedMemoryWithNonNullDeviceIn EXPECT_EQ(currSvmAllocsManager->createHostUnifiedMemoryAllocationTimes, 0u); } +template struct MemAllocMultiSubDeviceTests : public ::testing::Test { void SetUp() override { NEO::MockCompilerEnableGuard mock(true); + DebugManager.flags.EnableWalkerPartition.set(enableWalkerPartition); DebugManager.flags.CreateMultipleSubDevices.set(numSubDevices); auto executionEnvironment = new NEO::ExecutionEnvironment; auto devices = NEO::DeviceFactory::createDevices(*executionEnvironment); @@ -3094,7 +3096,10 @@ struct MemAllocMultiSubDeviceTests : public ::testing::Test { const uint32_t numRootDevices = 1u; }; -TEST_F(MemAllocMultiSubDeviceTests, whenAllocatingDeviceMemorySubDeviceMemorySizeUsedWhenImplicitScalingDisabled) { +using MemAllocMultiSubDeviceTestsDisabledImplicitScaling = MemAllocMultiSubDeviceTests<0>; +using MemAllocMultiSubDeviceTestsEnabledImplicitScaling = MemAllocMultiSubDeviceTests<1>; + +TEST_F(MemAllocMultiSubDeviceTestsDisabledImplicitScaling, GivenImplicitScalingDisabledWhenAllocatingDeviceMemorySubDeviceMemorySizeUsedThenExpectCorrectErrorReturned) { ze_device_mem_alloc_desc_t deviceDesc = {}; void *ptr = nullptr; size_t size = driverHandle->devices[0]->getNEODevice()->getDeviceInfo().globalMemSize; @@ -3104,18 +3109,25 @@ TEST_F(MemAllocMultiSubDeviceTests, whenAllocatingDeviceMemorySubDeviceMemorySiz relaxedSizeDesc.flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE; deviceDesc.pNext = &relaxedSizeDesc; - DebugManager.flags.EnableWalkerPartition.set(0); - ze_result_t res = context->allocDeviceMem(driverHandle->devices[0]->toHandle(), &deviceDesc, size, 0u, &ptr); EXPECT_EQ(res, ZE_RESULT_ERROR_UNSUPPORTED_SIZE); +} - DebugManager.flags.EnableWalkerPartition.set(1); +TEST_F(MemAllocMultiSubDeviceTestsEnabledImplicitScaling, GivenImplicitScalingEnabledWhenAllocatingDeviceMemorySubDeviceMemorySizeUsedThenExpectCorrectErrorReturned) { + ze_device_mem_alloc_desc_t deviceDesc = {}; + void *ptr = nullptr; + size_t size = driverHandle->devices[0]->getNEODevice()->getDeviceInfo().globalMemSize; + deviceDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + ze_relaxed_allocation_limits_exp_desc_t relaxedSizeDesc = {}; + relaxedSizeDesc.stype = ZE_STRUCTURE_TYPE_RELAXED_ALLOCATION_LIMITS_EXP_DESC; + relaxedSizeDesc.flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE; + deviceDesc.pNext = &relaxedSizeDesc; - res = context->allocDeviceMem(driverHandle->devices[0]->toHandle(), &deviceDesc, size, 0u, &ptr); + ze_result_t res = context->allocDeviceMem(driverHandle->devices[0]->toHandle(), &deviceDesc, size, 0u, &ptr); EXPECT_EQ(res, ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY); } -TEST_F(MemAllocMultiSubDeviceTests, whenAllocatingSharedMemorySubDeviceMemorySizeUsedWhenImplicitScalingDisabled) { +TEST_F(MemAllocMultiSubDeviceTestsDisabledImplicitScaling, GivenImplicitScalingDisabledWhenAllocatingSharedMemorySubDeviceMemorySizeUsedThenExpectCorrectErrorReturned) { ze_device_mem_alloc_desc_t deviceDesc = {}; ze_host_mem_alloc_desc_t hostDesc = {}; void *ptr = nullptr; @@ -3126,14 +3138,22 @@ TEST_F(MemAllocMultiSubDeviceTests, whenAllocatingSharedMemorySubDeviceMemorySiz relaxedSizeDesc.flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE; deviceDesc.pNext = &relaxedSizeDesc; - DebugManager.flags.EnableWalkerPartition.set(0); - ze_result_t res = context->allocSharedMem(driverHandle->devices[0]->toHandle(), &deviceDesc, &hostDesc, size, 0u, &ptr); EXPECT_EQ(res, ZE_RESULT_ERROR_UNSUPPORTED_SIZE); +} - DebugManager.flags.EnableWalkerPartition.set(1); +TEST_F(MemAllocMultiSubDeviceTestsEnabledImplicitScaling, GivenImplicitScalingDisabledWhenAllocatingSharedMemorySubDeviceMemorySizeUsedThenExpectCorrectErrorReturned) { + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_host_mem_alloc_desc_t hostDesc = {}; + void *ptr = nullptr; + size_t size = driverHandle->devices[0]->getNEODevice()->getDeviceInfo().globalMemSize; + deviceDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + ze_relaxed_allocation_limits_exp_desc_t relaxedSizeDesc = {}; + relaxedSizeDesc.stype = ZE_STRUCTURE_TYPE_RELAXED_ALLOCATION_LIMITS_EXP_DESC; + relaxedSizeDesc.flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE; + deviceDesc.pNext = &relaxedSizeDesc; - res = context->allocSharedMem(driverHandle->devices[0]->toHandle(), &deviceDesc, &hostDesc, size, 0u, &ptr); + ze_result_t res = context->allocSharedMem(driverHandle->devices[0]->toHandle(), &deviceDesc, &hostDesc, size, 0u, &ptr); EXPECT_EQ(res, ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY); }