Add debug flag to create multiple sub devices

Related-To: NEO-3691

Change-Id: Ia3554424221e102094c40cdd5941680c6a253b18
Signed-off-by: Mateusz Jablonski <mateusz.jablonski@intel.com>

runtime/device/CMakeLists.txt

@@ -16,6 +16,7 @@ set(RUNTIME_SRCS_DEVICE
   ${CMAKE_CURRENT_SOURCE_DIR}/driver_info.h
   ${CMAKE_CURRENT_SOURCE_DIR}/root_device.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/root_device.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/sub_device.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/sub_device.h
 )
 target_sources(${NEO_STATIC_LIB_NAME} PRIVATE ${RUNTIME_SRCS_DEVICE})

runtime/device/device.cpp

@@ -183,16 +183,6 @@ unsigned int Device::getSupportedClVersion() const {
     return getHardwareInfo().capabilityTable.clVersionSupport;
 }
 
-/* We hide the retain and release function of BaseObject. */
-void Device::retain() {
-    DEBUG_BREAK_IF(!isValid());
-}
-
-unique_ptr_if_unused<Device> Device::release() {
-    DEBUG_BREAK_IF(!isValid());
-    return unique_ptr_if_unused<Device>(this, false);
-}
-
 bool Device::isSimulation() const {
     auto &hwInfo = getHardwareInfo();
 

runtime/device/device.h

@@ -72,9 +72,6 @@ class Device : public BaseObject<_cl_device_id> {
     MemoryManager *getMemoryManager() const;
     GmmHelper *getGmmHelper() const;
 
-    /* We hide the retain and release function of BaseObject. */
-    void retain() override;
-    unique_ptr_if_unused<Device> release() override;
     OSTime *getOSTime() const { return osTime.get(); };
     double getProfilingTimerResolution();
     unsigned int getEnabledClVersion() const { return enabledClVersion; };
@@ -113,7 +110,7 @@ class Device : public BaseObject<_cl_device_id> {
         return device;
     }
 
-    bool createDeviceImpl();
+    virtual bool createDeviceImpl();
     bool createEngines();
     bool createEngine(uint32_t deviceIndex, uint32_t deviceCsrIndex, aub_stream::EngineType engineType);
 

runtime/device/root_device.cpp

@@ -8,10 +8,43 @@
 #include "runtime/device/root_device.h"
 
 #include "runtime/device/sub_device.h"
+#include "runtime/os_interface/debug_settings_manager.h"
 
 namespace NEO {
 
 RootDevice::~RootDevice() = default;
+
+uint32_t RootDevice::getNumSubDevices() const {
+    return static_cast<uint32_t>(subdevices.size());
+}
+
 RootDevice::RootDevice(ExecutionEnvironment *executionEnvironment, uint32_t deviceIndex) : Device(executionEnvironment, deviceIndex) {}
+bool RootDevice::createDeviceImpl() {
+    auto status = Device::createDeviceImpl();
+    if (!status) {
+        return status;
+    }
+    auto numSubDevices = DebugManager.flags.CreateMultipleSubDevices.get();
+    subdevices.reserve(numSubDevices);
+    for (int i = 0; i < numSubDevices; i++) {
+
+        auto subDevice = Device::createDeviceInternals(new SubDevice(executionEnvironment, deviceIndex + i + 1, *this));
+        if (!subDevice) {
+            return false;
+        }
+        subdevices.push_back(std::unique_ptr<SubDevice>(subDevice));
+    }
+    return true;
+}
+
+/* We hide the retain and release function of BaseObject. */
+void RootDevice::retain() {
+    DEBUG_BREAK_IF(!isValid());
+}
+
+unique_ptr_if_unused<Device> RootDevice::release() {
+    DEBUG_BREAK_IF(!isValid());
+    return unique_ptr_if_unused<Device>(this, false);
+}
 
 } // namespace NEO

runtime/device/root_device.h

@@ -16,8 +16,14 @@ class RootDevice : public Device {
   public:
     RootDevice(ExecutionEnvironment *executionEnvironment, uint32_t deviceIndex);
     ~RootDevice() override;
+    bool createDeviceImpl() override;
+    uint32_t getNumSubDevices() const;
+
+    /* We hide the retain and release function of BaseObject. */
+    void retain() override;
+    unique_ptr_if_unused<Device> release() override;
 
   protected:
-    std::vector<SubDevice *> subdevices;
+    std::vector<std::unique_ptr<SubDevice>> subdevices;
 };
 } // namespace NEO

runtime/device/sub_device.cpp

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2019 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "runtime/device/sub_device.h"
+
+#include "runtime/device/root_device.h"
+
+namespace NEO {
+
+SubDevice::SubDevice(ExecutionEnvironment *executionEnvironment, uint32_t deviceIndex, RootDevice &rootDevice) : Device(executionEnvironment, deviceIndex), rootDevice(rootDevice) {}
+void SubDevice::retain() {
+    rootDevice.incRefInternal();
+    Device::retain();
+};
+unique_ptr_if_unused<Device> SubDevice::release() {
+    rootDevice.decRefInternal();
+    return Device::release();
+};
+void SubDevice::retainInternal() {
+    rootDevice.incRefInternal();
+}
+void SubDevice::releaseInternal() {
+    rootDevice.decRefInternal();
+}
+
+} // namespace NEO

runtime/device/sub_device.h

@@ -12,9 +12,13 @@ namespace NEO {
 class RootDevice;
 class SubDevice : public Device {
   public:
-    using Device::Device;
+    SubDevice(ExecutionEnvironment *executionEnvironment, uint32_t deviceIndex, RootDevice &rootDevice);
+    void retain() override;
+    unique_ptr_if_unused<Device> release() override;
+    void retainInternal();
+    void releaseInternal();
 
   protected:
-    RootDevice *rootDevice = nullptr;
+    RootDevice &rootDevice;
 };
 } // namespace NEO

runtime/os_interface/debug_variables_base.inl

@@ -109,6 +109,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, EnableCacheFlushAfterWalker, -1, "-1: platform b
 DECLARE_DEBUG_VARIABLE(int32_t, EnableLocalMemory, -1, "-1: default behavior, 0: disabled, 1: enabled, Allows allocating graphics memory in Local Memory")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableStatelessToStatefulBufferOffsetOpt, -1, "-1: dont override, 0: disable, 1: enable, Enables buffer-offset improvement of the stateless to stateful optimization")
 DECLARE_DEBUG_VARIABLE(int32_t, CreateMultipleDevices, 0, "0: default - disable, 1+: Driver will create multiple (N) devices during initialization.")
+DECLARE_DEBUG_VARIABLE(int32_t, CreateMultipleSubDevices, 0, "0: default - disable, 1+: Driver will create multiple (N) sub devices during initialization.")
 DECLARE_DEBUG_VARIABLE(int32_t, LimitAmountOfReturnedDevices, 0, "0: default - disable, 1+: Driver will limit the number of devices returned from clGetDeviceIds to N.")
 DECLARE_DEBUG_VARIABLE(int32_t, Enable64kbpages, -1, "-1: default behaviour, 0 Disables, 1 Enables support for 64KB pages for driver allocated fine grain svm buffers")
 DECLARE_DEBUG_VARIABLE(int32_t, OverrideEnableKmdNotify, -1, "-1: dont override, 0: disable, 1: enable")

runtime/platform/platform.cpp

@@ -146,10 +146,12 @@ bool Platform::initialize() {
     this->platformInfo.reset(new PlatformInfo);
 
     this->devices.resize(numDevicesReturned);
+    auto deviceIndex = 0u;
     for (uint32_t deviceOrdinal = 0; deviceOrdinal < numDevicesReturned; ++deviceOrdinal) {
-        auto pDevice = Device::create<RootDevice>(executionEnvironment, deviceOrdinal);
+        auto pDevice = Device::create<RootDevice>(executionEnvironment, deviceIndex++);
         DEBUG_BREAK_IF(!pDevice);
         if (pDevice) {
+            deviceIndex += pDevice->getNumSubDevices();
             this->devices[deviceOrdinal] = pDevice;
 
             this->platformInfo->extensions = pDevice->getDeviceInfo().deviceExtensions;