Add reg key for enabling cross device access

Change-Id: Iede7bc8c6fc2ea7fd8594b6b3e2ffb40820d303b
Signed-off: Jaime Arteaga <jaime.a.arteaga.molina@intel.com>

level_zero/core/source/device/device_imp.cpp

@@ -24,8 +24,10 @@
 #include "shared/source/os_interface/os_interface.h"
 #include "shared/source/os_interface/os_time.h"
 #include "shared/source/source_level_debugger/source_level_debugger.h"
+#include "shared/source/utilities/debug_settings_reader_creator.h"
 
 #include "opencl/source/mem_obj/mem_obj.h"
+#include "opencl/source/os_interface/ocl_reg_path.h"
 #include "opencl/source/program/program.h"
 
 #include "level_zero/core/source/builtin/builtin_functions_lib.h"
@@ -61,18 +63,21 @@ void DeviceImp::setDriverHandle(DriverHandle *driverHandle) {
 }
 
 ze_result_t DeviceImp::canAccessPeer(ze_device_handle_t hPeerDevice, ze_bool_t *value) {
-    *value = true;
+    *value = false;
 
     DeviceImp *pPeerDevice = reinterpret_cast<DeviceImp *>(Device::fromHandle(hPeerDevice));
+    if (this->getNEODevice()->getRootDeviceIndex() == pPeerDevice->getNEODevice()->getRootDeviceIndex()) {
+        *value = true;
+    }
 
-    NEO::MemoryManager *memoryManager = this->getDriverHandle()->getMemoryManager();
+    auto settingsReader = NEO::SettingsReaderCreator::create(NEO::oclRegPath);
-    bool isLocalMemorySupportedinDevice =
+    int64_t accessOverride = settingsReader->getSetting("EnableCrossDeviceAcesss", -1);
-        memoryManager->isLocalMemorySupported(this->getNEODevice()->getRootDeviceIndex());
+
-    bool isLocalMemorySupportedinPeer =
+    if ((accessOverride == 1) || (NEO::DebugManager.flags.EnableCrossDeviceAccess.get() == 1)) {
-        memoryManager->isLocalMemorySupported(pPeerDevice->getNEODevice()->getRootDeviceIndex());
+        *value = true;
-    if (isLocalMemorySupportedinDevice && isLocalMemorySupportedinPeer &&
+    }
-        (this->getNEODevice()->getHardwareInfo().platform.eProductFamily !=
+
-         pPeerDevice->getNEODevice()->getHardwareInfo().platform.eProductFamily)) {
+    if ((accessOverride == 0) || (NEO::DebugManager.flags.EnableCrossDeviceAccess.get() == 0)) {
         *value = false;
     }
 

level_zero/core/test/unit_tests/sources/device/test_device.cpp

@@ -265,7 +265,22 @@ TEST_F(MultipleDevicesTest, givenTheSameDeviceThenCanAccessPeerReturnsTrue) {
     EXPECT_TRUE(canAccess);
 }
 
-TEST_F(MultipleDevicesTest, givenTwoDevicesFromSameFamilyThenCanAccessPeerReturnsTrue) {
+TEST_F(MultipleDevicesTest, givenTwoRootDevicesFromSameFamilyThenCanAccessPeerReturnsFalse) {
+    L0::Device *device0 = driverHandle->devices[0];
+    L0::Device *device1 = driverHandle->devices[1];
+
+    GFXCORE_FAMILY device0Family = device0->getNEODevice()->getHardwareInfo().platform.eRenderCoreFamily;
+    GFXCORE_FAMILY device1Family = device1->getNEODevice()->getHardwareInfo().platform.eRenderCoreFamily;
+    EXPECT_EQ(device0Family, device1Family);
+
+    ze_bool_t canAccess = true;
+    ze_result_t res = device0->canAccessPeer(device1->toHandle(), &canAccess);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+    EXPECT_FALSE(canAccess);
+}
+
+TEST_F(MultipleDevicesTest, givenTwoRootDevicesFromSameFamilyThenCanAccessPeerReturnsTrueIfEnableCrossDeviceAccessIsSetToOne) {
+    DebugManager.flags.EnableCrossDeviceAccess.set(1);
     L0::Device *device0 = driverHandle->devices[0];
     L0::Device *device1 = driverHandle->devices[1];
 
@@ -312,6 +327,40 @@ TEST_F(MultipleDevicesTest, givenTwoSubDevicesFromTheSameRootDeviceThenCanAccess
     EXPECT_TRUE(canAccess);
 }
 
+TEST_F(MultipleDevicesTest, givenTwoSubDevicesFromTheSameRootDeviceThenCanAccessPeerReturnsFalseIfEnableCrossDeviceAccessIsSetToZero) {
+    DebugManager.flags.EnableCrossDeviceAccess.set(0);
+    L0::Device *device0 = driverHandle->devices[0];
+    L0::Device *device1 = driverHandle->devices[1];
+
+    uint32_t subDeviceCount = 0;
+    ze_result_t res = device0->getSubDevices(&subDeviceCount, nullptr);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+    EXPECT_EQ(numSubDevices, subDeviceCount);
+
+    std::vector<ze_device_handle_t> subDevices0(subDeviceCount);
+    res = device0->getSubDevices(&subDeviceCount, subDevices0.data());
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+
+    subDeviceCount = 0;
+    res = device1->getSubDevices(&subDeviceCount, nullptr);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+    EXPECT_EQ(numSubDevices, subDeviceCount);
+
+    std::vector<ze_device_handle_t> subDevices1(subDeviceCount);
+    res = device1->getSubDevices(&subDeviceCount, subDevices1.data());
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+
+    ze_bool_t canAccess = true;
+    L0::Device *subDevice0_0 = Device::fromHandle(subDevices0[0]);
+    subDevice0_0->canAccessPeer(subDevices0[1], &canAccess);
+    EXPECT_FALSE(canAccess);
+
+    canAccess = true;
+    L0::Device *subDevice1_0 = Device::fromHandle(subDevices1[0]);
+    subDevice1_0->canAccessPeer(subDevices1[1], &canAccess);
+    EXPECT_FALSE(canAccess);
+}
+
 struct MultipleDevicesDifferentLocalMemorySupportTest : public MultipleDevicesTest {
     void SetUp() override {
         MultipleDevicesTest::SetUp();
@@ -325,11 +374,11 @@ struct MultipleDevicesDifferentLocalMemorySupportTest : public MultipleDevicesTe
     L0::Device *deviceWithoutLocalMemory = nullptr;
 };
 
-TEST_F(MultipleDevicesDifferentLocalMemorySupportTest, givenTwoDevicesWithDifferentLocalMemorySupportThenCanAccessPeerReturnsTrue) {
+TEST_F(MultipleDevicesDifferentLocalMemorySupportTest, givenTwoDevicesWithDifferentLocalMemorySupportThenCanAccessPeerReturnsFalse) {
-    ze_bool_t canAccess = false;
+    ze_bool_t canAccess = true;
     ze_result_t res = deviceWithLocalMemory->canAccessPeer(deviceWithoutLocalMemory->toHandle(), &canAccess);
     EXPECT_EQ(ZE_RESULT_SUCCESS, res);
-    EXPECT_TRUE(canAccess);
+    EXPECT_FALSE(canAccess);
 }
 
 struct MultipleDevicesDifferentFamilyAndLocalMemorySupportTest : public MultipleDevicesTest {
@@ -366,6 +415,32 @@ TEST_F(MultipleDevicesDifferentFamilyAndLocalMemorySupportTest, givenTwoDevicesF
     EXPECT_FALSE(canAccess);
 }
 
+struct MultipleDevicesSameFamilyAndLocalMemorySupportTest : public MultipleDevicesTest {
+    void SetUp() override {
+        MultipleDevicesTest::SetUp();
+
+        memoryManager->localMemorySupported[0] = 1;
+        memoryManager->localMemorySupported[1] = 1;
+
+        device0 = driverHandle->devices[0];
+        device1 = driverHandle->devices[1];
+    }
+
+    L0::Device *device0 = nullptr;
+    L0::Device *device1 = nullptr;
+};
+
+TEST_F(MultipleDevicesSameFamilyAndLocalMemorySupportTest, givenTwoDevicesFromSameFamilyThenCanAccessPeerReturnsFalse) {
+    PRODUCT_FAMILY device0Family = device0->getNEODevice()->getHardwareInfo().platform.eProductFamily;
+    PRODUCT_FAMILY device1Family = device1->getNEODevice()->getHardwareInfo().platform.eProductFamily;
+    EXPECT_EQ(device0Family, device1Family);
+
+    ze_bool_t canAccess = true;
+    ze_result_t res = device0->canAccessPeer(device1->toHandle(), &canAccess);
+    EXPECT_EQ(ZE_RESULT_SUCCESS, res);
+    EXPECT_FALSE(canAccess);
+}
+
 TEST_F(DeviceTest, givenBlitterSupportAndCopyOnlyFlagWhenCopyOnlyDebugFlagIsDefaultThenUseBliterIsTrueAndSuccessIsReturned) {
     NEO::HardwareInfo hwInfo = *NEO::defaultHwInfo.get();
     hwInfo.capabilityTable.blitterOperationsSupported = true;

opencl/test/unit_test/test_files/igdrcl.config

@@ -166,3 +166,4 @@ OverrideLeastOccupiedBank = -1
 UseAsyncDrmExec = -1
 EnableMultiStorageResources = -1
 PrintExecutionBuffer = 0
+EnableCrossDeviceAccess = -1

shared/source/debug_settings/debug_variables_base.inl

@@ -168,6 +168,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, EnableSharedSystemUsmSupport, -1, "-1: default,
 DECLARE_DEBUG_VARIABLE(int32_t, EnablePassInlineData, -1, "-1: default, 0: Do not allow to pass inline data 1: Enable passing of inline data")
 DECLARE_DEBUG_VARIABLE(int32_t, ForceFineGrainedSVMSupport, -1, "-1: default, 0: Do not report Fine Grained SVM capabilties 1: Report SVM Fine Grained capabilities if device supports SVM")
 DECLARE_DEBUG_VARIABLE(int32_t, UseAsyncDrmExec, -1, "-1: default, 0: Disabled 1: Enabled. If enabled, pass EXEC_OBJECT_ASYNC to exec ioctl.")
+DECLARE_DEBUG_VARIABLE(int32_t, EnableCrossDeviceAccess, -1, "-1: default behavior, 0: disabled, 1: enabled, Allows one device to access another device's memory")
 
 /*DRIVER TOGGLES*/
 DECLARE_DEBUG_VARIABLE(int64_t, ForceSystemMemoryPlacement, 0, "0: default,  >0: (bitmask) for given Graphics Allocation Type, force system memory placement")