Missing support for OCL debugging with L0

- make resident debugSurface
- allocate debugSurface with correct allocation type and size
- notify cmdQ create/destroy

Related-To: NEO-7075

Signed-off-by: Mateusz Hoppe <mateusz.hoppe@intel.com>

opencl/source/command_queue/command_queue.cpp

@@ -8,6 +8,7 @@
 #include "opencl/source/command_queue/command_queue.h"
 
 #include "shared/source/command_stream/command_stream_receiver.h"
+#include "shared/source/debugger/debugger_l0.h"
 #include "shared/source/helpers/aligned_memory.h"
 #include "shared/source/helpers/array_count.h"
 #include "shared/source/helpers/engine_node_helper.h"
@@ -60,7 +61,7 @@ CommandQueue *CommandQueue::create(Context *context,
 }
 
 CommandQueue::CommandQueue(Context *context, ClDevice *device, const cl_queue_properties *properties, bool internalUsage)
-    : context(context), device(device) {
+    : context(context), device(device), isInternalUsage(internalUsage) {
     if (context) {
         context->incRefInternal();
     }
@@ -90,6 +91,10 @@ CommandQueue::CommandQueue(Context *context, ClDevice *device, const cl_queue_pr
         if (!deferCmdQBcsInitialization) {
             this->constructBcsEngine(internalUsage);
         }
+
+        if (NEO::Debugger::isDebugEnabled(internalUsage) && device->getDevice().getL0Debugger()) {
+            device->getDevice().getL0Debugger()->notifyCommandQueueCreated(&device->getDevice());
+        }
     }
 
     storeProperties(properties);
@@ -114,6 +119,10 @@ CommandQueue::~CommandQueue() {
         }
 
         this->releaseMainCopyEngine();
+
+        if (NEO::Debugger::isDebugEnabled(isInternalUsage) && device->getDevice().getL0Debugger()) {
+            device->getDevice().getL0Debugger()->notifyCommandQueueDestroyed(&device->getDevice());
+        }
     }
 
     timestampPacketContainer.reset();
@@ -159,11 +168,10 @@ void CommandQueue::initializeGpgpu() const {
 
 void CommandQueue::initializeGpgpuInternals() const {
     auto &hwInfo = device->getDevice().getHardwareInfo();
-    auto &hwHelper = NEO::HwHelper::get(hwInfo.platform.eRenderCoreFamily);
     const auto &hwInfoConfig = *HwInfoConfig::get(hwInfo.platform.eProductFamily);
 
     if (device->getDevice().getDebugger() && !this->gpgpuEngine->commandStreamReceiver->getDebugSurfaceAllocation()) {
-        auto maxDbgSurfaceSize = hwHelper.getSipKernelMaxDbgSurfaceSize(hwInfo);
+        auto maxDbgSurfaceSize = NEO::SipKernel::getSipKernel(device->getDevice()).getStateSaveAreaSize(&device->getDevice());
         auto debugSurface = this->gpgpuEngine->commandStreamReceiver->allocateDebugSurface(maxDbgSurfaceSize);
         memset(debugSurface->getUnderlyingBuffer(), 0, debugSurface->getUnderlyingBufferSize());
 

opencl/source/command_queue/command_queue.h

@@ -416,7 +416,7 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
     std::array<CopyEngineState, bcsInfoMaskSize> bcsStates = {};
 
     bool perfCountersEnabled = false;
-
+    bool isInternalUsage = false;
     bool isCopyOnly = false;
     bool bcsAllowed = false;
     bool bcsInitialized = false;

opencl/test/unit_test/debugger/CMakeLists.txt

@@ -0,0 +1,13 @@
+#
+# Copyright (C) 2022 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+set(IGDRCL_SRCS_tests_debugger
+    ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+    ${CMAKE_CURRENT_SOURCE_DIR}/ocl_with_l0_debugger_tests.cpp
+)
+
+target_sources(igdrcl_tests PRIVATE ${IGDRCL_SRCS_tests_debugger})
+

opencl/test/unit_test/debugger/ocl_with_l0_debugger_tests.cpp

@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2022 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/common/helpers/debug_manager_state_restore.h"
+#include "shared/test/common/helpers/variable_backup.h"
+#include "shared/test/common/libult/ult_command_stream_receiver.h"
+#include "shared/test/common/mocks/mock_csr.h"
+#include "shared/test/common/mocks/mock_graphics_allocation.h"
+#include "shared/test/common/mocks/mock_l0_debugger.h"
+#include "shared/test/common/test_macros/hw_test.h"
+
+#include "opencl/source/command_queue/command_queue_hw.h"
+#include "opencl/test/unit_test/command_queue/command_queue_fixture.h"
+#include "opencl/test/unit_test/fixtures/cl_device_fixture.h"
+#include "opencl/test/unit_test/fixtures/context_fixture.h"
+#include "opencl/test/unit_test/mocks/mock_command_queue.h"
+#include "opencl/test/unit_test/mocks/mock_context.h"
+#include "opencl/test/unit_test/mocks/mock_kernel.h"
+
+using namespace NEO;
+
+struct DebuggerClFixture
+    : public ContextFixture,
+      public CommandQueueHwFixture {
+
+    void setUp() {
+        hardwareInfo = *NEO::defaultHwInfo.get();
+        auto executionEnvironment = MockClDevice::prepareExecutionEnvironment(&hardwareInfo, rootDeviceIndex);
+        executionEnvironment->setDebuggingEnabled();
+        device = MockDevice::createWithExecutionEnvironment<MockDevice>(&hardwareInfo, executionEnvironment, rootDeviceIndex);
+        ASSERT_NE(nullptr, device);
+        clExecutionEnvironment = static_cast<MockClExecutionEnvironment *>(device->getExecutionEnvironment());
+        clDevice = new MockClDevice{device};
+        ASSERT_NE(nullptr, clDevice);
+
+        executionEnvironment->rootDeviceEnvironments[rootDeviceIndex]->initDebuggerL0(device);
+        this->osContext = device->getDefaultEngine().osContext;
+
+        cl_device_id deviceId = clDevice;
+        ContextFixture::setUp(1, &deviceId);
+        CommandQueueHwFixture::setUp(clDevice, 0);
+    }
+
+    void tearDown() {
+        ContextFixture::tearDown();
+        CommandQueueHwFixture::tearDown();
+        delete clDevice;
+        clDevice = nullptr;
+        device = nullptr;
+    }
+
+    MockDevice *device = nullptr;
+    MockClDevice *clDevice = nullptr;
+    HardwareInfo hardwareInfo = {};
+    OsContext *osContext = nullptr;
+    const uint32_t rootDeviceIndex = 0u;
+    MockClExecutionEnvironment *clExecutionEnvironment = nullptr;
+};
+
+using DebuggerCmdQTest = Test<DebuggerClFixture>;
+
+HWTEST_F(DebuggerCmdQTest, GivenDebuggingEnabledWhenCommandQueueIsCreatedAndReleasedThenDebuggerL0IsNotified) {
+    auto debuggerL0Hw = static_cast<MockDebuggerL0Hw<FamilyType> *>(device->getL0Debugger());
+    debuggerL0Hw->commandQueueCreatedCount = 0;
+    debuggerL0Hw->commandQueueDestroyedCount = 0;
+    auto mockCmdQ = new MockCommandQueueHw<FamilyType>(context, clDevice, 0);
+
+    EXPECT_EQ(1u, debuggerL0Hw->commandQueueCreatedCount);
+    cl_int retVal = 0;
+    releaseQueue(mockCmdQ, retVal);
+    EXPECT_EQ(1u, debuggerL0Hw->commandQueueDestroyedCount);
+}
+
+HWTEST_F(DebuggerCmdQTest, GivenDebuggingEnabledWhenInternalCommandQueueIsCreatedAndReleasedThenDebuggerL0IsNotNotified) {
+    auto debuggerL0Hw = static_cast<MockDebuggerL0Hw<FamilyType> *>(device->getL0Debugger());
+    debuggerL0Hw->commandQueueCreatedCount = 0;
+    debuggerL0Hw->commandQueueDestroyedCount = 0;
+    auto mockCmdQ = new MockCommandQueueHw<FamilyType>(context, clDevice, 0, true);
+
+    EXPECT_EQ(0u, debuggerL0Hw->commandQueueCreatedCount);
+    cl_int retVal = 0;
+    releaseQueue(mockCmdQ, retVal);
+    EXPECT_EQ(0u, debuggerL0Hw->commandQueueDestroyedCount);
+}
+
+using Gen12Plus = IsAtLeastGfxCore<IGFX_GEN12_CORE>;
+HWTEST2_F(DebuggerCmdQTest, GivenDebuggingEnabledWhenEnqueueingKernelThenDebugSurfaceIsResident, Gen12Plus) {
+    int32_t executionStamp = 0;
+    auto mockCSR = new MockCsr<FamilyType>(executionStamp, *device->executionEnvironment, device->getRootDeviceIndex(), device->getDeviceBitfield());
+    device->resetCommandStreamReceiver(mockCSR);
+
+    MockKernelWithInternals mockKernelWithInternals(*clDevice);
+    auto mockKernel = mockKernelWithInternals.mockKernel;
+
+    size_t offset = 0;
+    size_t size = 1;
+    pCmdQ->initializeGpgpu();
+    pCmdQ->enqueueKernel(mockKernel, 1, &offset, &size, &size, 0, nullptr, nullptr);
+
+    EXPECT_TRUE(mockCSR->isMadeResident(mockCSR->getDebugSurfaceAllocation()));
+}

opencl/test/unit_test/mocks/mock_command_queue.h

@@ -256,7 +256,11 @@ class MockCommandQueueHw : public CommandQueueHw<GfxFamily> {
 
     MockCommandQueueHw(Context *context,
                        ClDevice *device,
-                       cl_queue_properties *properties) : BaseClass(context, device, properties, false) {
+                       cl_queue_properties *properties) : MockCommandQueueHw(context, device, properties, false) {}
+
+    MockCommandQueueHw(Context *context,
+                       ClDevice *device,
+                       cl_queue_properties *properties, bool isInternal) : BaseClass(context, device, properties, isInternal) {
         this->constructBcsEngine(false);
     }
 

shared/source/command_stream/command_stream_receiver.cpp

@@ -502,7 +502,7 @@ void CommandStreamReceiver::startControllingDirectSubmissions() {
 
 GraphicsAllocation *CommandStreamReceiver::allocateDebugSurface(size_t size) {
     UNRECOVERABLE_IF(debugSurface != nullptr);
-    debugSurface = getMemoryManager()->allocateGraphicsMemoryWithProperties({rootDeviceIndex, size, AllocationType::INTERNAL_HOST_MEMORY, getOsContext().getDeviceBitfield()});
+    debugSurface = getMemoryManager()->allocateGraphicsMemoryWithProperties({rootDeviceIndex, size, AllocationType::DEBUG_CONTEXT_SAVE_AREA, getOsContext().getDeviceBitfield()});
     return debugSurface;
 }
 

shared/source/command_stream/command_stream_receiver_hw_base.inl

@@ -537,7 +537,7 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
         makeResident(*SipKernel::getSipKernel(device).getSipAllocation());
     }
 
-    if (sourceLevelDebuggerActive && debugSurface) {
+    if (debuggingEnabled && debugSurface) {
         makeResident(*debugSurface);
     }
 

shared/test/unit_test/command_stream/command_stream_receiver_tests.cpp

@@ -404,6 +404,11 @@ TEST_F(CommandStreamReceiverTest, GivenNoParamatersWhenMakingResidentThenResiden
     EXPECT_EQ(0u, residencyAllocations.size());
 }
 
+TEST_F(CommandStreamReceiverTest, WhenDebugSurfaceIsAllocatedThenCorrectTypeIsSet) {
+    auto allocation = commandStreamReceiver->allocateDebugSurface(1024);
+    EXPECT_EQ(AllocationType::DEBUG_CONTEXT_SAVE_AREA, allocation->getAllocationType());
+}
+
 TEST_F(CommandStreamReceiverTest, givenForced32BitAddressingWhenDebugSurfaceIsAllocatedThenRegularAllocationIsReturned) {
     auto *memoryManager = commandStreamReceiver->getMemoryManager();
     memoryManager->setForce32BitAllocations(true);