Add tool for tracking events

Change-Id: Id61d814e4629a41a279d46097ec8b4f94a224234

runtime/event/CMakeLists.txt

@@ -26,6 +26,8 @@ set(RUNTIME_SRCS_EVENT
   ${CMAKE_CURRENT_SOURCE_DIR}/event.h
   ${CMAKE_CURRENT_SOURCE_DIR}/event_builder.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/event_builder.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/event_tracker.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/event_tracker.h
   ${CMAKE_CURRENT_SOURCE_DIR}/user_event.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/user_event.h
   ${CMAKE_CURRENT_SOURCE_DIR}/hw_timestamps.h

runtime/event/event.cpp

@@ -27,6 +27,7 @@
 #include "runtime/context/context.h"
 #include "runtime/device/device.h"
 #include "runtime/event/event.h"
+#include "runtime/event/event_tracker.h"
 #include "runtime/helpers/aligned_memory.h"
 #include "runtime/helpers/get_info.h"
 #include "runtime/api/cl_types.h"
@@ -59,6 +60,9 @@ Event::Event(
       perfCounterNode(nullptr),
       perfConfigurationData(nullptr),
       taskCount(taskCount) {
+    if (OCLRT::DebugManager.flags.EventsTrackerEnable.get()) {
+        EventsTracker::getEventsTracker().notifyCreation(this);
+    }
     parentCount = 0;
     executionStatus = CL_QUEUED;
     flushStamp.reset(new FlushStampTracker(true));
@@ -103,6 +107,10 @@ Event::Event(
 }
 
 Event::~Event() {
+    if (OCLRT::DebugManager.flags.EventsTrackerEnable.get()) {
+        EventsTracker::getEventsTracker().notifyDestruction(this);
+    }
+
     DBG_LOG(EventsDebugEnable, "~Event()", this);
     //no commands should be registred
     DEBUG_BREAK_IF(this->cmdToSubmit.load());
@@ -425,6 +433,18 @@ bool Event::setStatus(cl_int status) {
     return true;
 }
 
+void Event::transitionExecutionStatus(int32_t newExecutionStatus) const {
+    int32_t prevStatus = executionStatus;
+    DBG_LOG(EventsDebugEnable, "transitionExecutionStatus event", this, " new status", newExecutionStatus, "previousStatus", prevStatus);
+
+    while (prevStatus > newExecutionStatus) {
+        executionStatus.compare_exchange_weak(prevStatus, newExecutionStatus);
+    }
+    if (OCLRT::DebugManager.flags.EventsTrackerEnable.get()) {
+        EventsTracker::getEventsTracker().notifyTransitionedExecutionStatus();
+    }
+}
+
 void Event::submitCommand(bool abortTasks) {
     std::unique_ptr<Command> cmdToProcess(cmdToSubmit.exchange(nullptr));
     if (cmdToProcess.get() != nullptr) {

runtime/event/event.h

@@ -339,14 +339,7 @@ class Event : public BaseObject<_cl_event>, public IDNode<Event> {
 
     // transitions event to new execution state
     // guarantees that newStatus <= oldStatus
-    void transitionExecutionStatus(int32_t newExecutionStatus) const {
-        int32_t prevStatus = executionStatus;
-        DBG_LOG(EventsDebugEnable, "transitionExecutionStatus event", this, " new status", newExecutionStatus, "previousStatus", prevStatus);
-
-        while (prevStatus > newExecutionStatus) {
-            executionStatus.compare_exchange_weak(prevStatus, newExecutionStatus);
-        }
-    }
+    void transitionExecutionStatus(int32_t newExecutionStatus) const;
 
     //vector storing events that needs to be notified when this event is ready to go
     IFRefList<Event, true, true> childEventsToNotify;

runtime/event/event_tracker.cpp

@@ -0,0 +1,274 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "runtime/command_queue/command_queue.h"
+#include "runtime/event/event.h"
+#include "runtime/event/event_tracker.h"
+#include "runtime/utilities/iflist.h"
+#include "runtime/helpers/cl_helper.h"
+
+namespace OCLRT {
+
+std::unique_ptr<EventsTracker> EventsTracker::globalEvTracker = nullptr;
+
+EventsTracker &EventsTracker::getEventsTracker() {
+    static std::mutex initMutex;
+    std::lock_guard<std::mutex> autolock(initMutex);
+
+    if (!EventsTracker::globalEvTracker)
+        EventsTracker::globalEvTracker = std::unique_ptr<EventsTracker>{new EventsTracker()};
+    return *EventsTracker::globalEvTracker;
+}
+
+void EventsTracker::shutdownGlobalEvTracker() {
+    EventsTracker::globalEvTracker.reset();
+}
+
+std::string EventsTracker::label(Event *node, const EventIdMap &eventsIdMapping) {
+    std::string retLabel("e");
+
+    auto eventTag = eventsIdMapping.find(node);
+    if (eventTag != eventsIdMapping.end()) {
+        auto id = eventTag->second;
+        retLabel += std::to_string(id);
+    }
+    return retLabel;
+}
+
+std::string EventsTracker::label(CommandQueue *cmdQ) {
+    return "cq" + std::to_string(reinterpret_cast<uintptr_t>(cmdQ));
+}
+
+void EventsTracker::dumpQueue(CommandQueue *cmdQ, std::ostream &out, CmdqSet &dumpedCmdQs) {
+    if ((cmdQ == nullptr) || (dumpedCmdQs.find(cmdQ) != dumpedCmdQs.end())) {
+        return;
+    }
+
+    out << label(cmdQ) << "[label=\"{------CmdQueue, ptr=" << cmdQ << "------|task count=";
+    auto taskCount = cmdQ->taskCount;
+    auto taskLevel = cmdQ->taskLevel;
+    if (taskCount == Event::eventNotReady) {
+        out << "NOT_READY";
+    } else {
+        out << taskCount;
+    }
+
+    out << ", level=";
+    if (taskLevel == Event::eventNotReady) {
+        out << "NOT_READY";
+    } else {
+        out << taskLevel;
+    }
+    out << "}\",color=blue];\n";
+    dumpedCmdQs.insert(cmdQ);
+}
+
+void EventsTracker::dumpNode(Event *node, std::ostream &out, const EventIdMap &eventsIdMapping) {
+    if (node == nullptr) {
+        out << "eNULL[label=\"{ptr=nullptr}\",color=red];\n";
+        return;
+    }
+
+    bool isUserEvent = node->isUserEvent();
+
+    uint32_t statusId = static_cast<uint32_t>(node->peekExecutionStatus());
+    // clamp to aborted
+    statusId = (statusId > CL_QUEUED) ? (CL_QUEUED + 1) : statusId;
+
+    const char *color = ((statusId == CL_COMPLETE) || (statusId > CL_QUEUED)) ? "green" : (((statusId == CL_SUBMITTED) && (isUserEvent == false)) ? "yellow" : "red");
+
+    std::string eventType = isUserEvent ? "USER_EVENT" : (node->isCurrentCmdQVirtualEvent() ? "---V_EVENT " : "-----EVENT ");
+    std::string commandType = "";
+    if (isUserEvent == false) {
+        commandType = OCLRT::cmdTypetoString(node->getCommandType());
+    }
+
+    static const char *status[] = {
+        "CL_COMPLETE",
+        "CL_RUNNING",
+        "CL_SUBMITTED",
+        "CL_QUEUED",
+        "ABORTED"};
+
+    auto taskCount = node->peekTaskCount();
+    auto taskLevel = node->taskLevel.load();
+
+    out << label(node, eventsIdMapping) << "[label=\"{------" << eventType << " ptr=" << node << "------"
+                                                                                                 "|"
+        << commandType << "|" << status[statusId] << "|"
+                                                     "task count=";
+    if (taskCount == Event::eventNotReady) {
+        out << "NOT_READY";
+    } else {
+        out << taskCount;
+    }
+
+    out << ", level=";
+    if (taskLevel == Event::eventNotReady) {
+        out << "NOT_READY";
+    } else {
+        out << taskLevel;
+    }
+
+    out << "|CALLBACKS=" << (node->peekHasCallbacks() ? "TRUE" : "FALSE") << "}\",color=" << color << "];\n";
+
+    if (node->isCurrentCmdQVirtualEvent()) {
+        out << label(node->getCommandQueue()) << "->" << label(node, eventsIdMapping);
+        out << "[label=\"VIRTUAL_EVENT\"]";
+        out << ";\n";
+    }
+}
+
+void EventsTracker::dumpEdge(Event *leftNode, Event *rightNode, std::ostream &out, const EventIdMap &eventsIdMapping) {
+    out << label(leftNode, eventsIdMapping) << "->" << label(rightNode, eventsIdMapping) << ";\n";
+}
+
+// walk in DFS manner
+void EventsTracker::dumpGraph(Event *node, std::ostream &out, CmdqSet &dumpedCmdQs, std::set<Event *> &dumpedEvents,
+                              const EventIdMap &eventsIdMapping) {
+    if ((node == nullptr) || (dumpedEvents.find(node) != dumpedEvents.end())) {
+        return;
+    }
+
+    dumpedEvents.insert(node);
+
+    if (node->getCommandQueue() != nullptr) {
+        dumpQueue(node->getCommandQueue(), out, dumpedCmdQs);
+    }
+    dumpNode(node, out, eventsIdMapping);
+
+    auto *childNode = node->peekChildEvents();
+    while (childNode != nullptr) {
+        dumpGraph(childNode->ref, out, dumpedCmdQs, dumpedEvents, eventsIdMapping);
+        dumpEdge(node, childNode->ref, out, eventsIdMapping);
+        childNode = childNode->next;
+    }
+}
+
+IFList<TrackedEvent, true, true> *EventsTracker::getList() {
+    return &trackedEvents;
+}
+
+TrackedEvent *EventsTracker::getNodes() {
+    return trackedEvents.detachNodes();
+}
+
+void EventsTracker::dump() {
+    static std::mutex mutex;
+    std::lock_guard<std::mutex> lock(mutex);
+    auto time = std::chrono::system_clock::now();
+    std::string dumpFileName = "eg_"
+                               "reg" +
+                               std::to_string(reinterpret_cast<uintptr_t>(this)) + "_" + std::to_string(time.time_since_epoch().count()) + ".gv";
+    std::shared_ptr<std::ostream> out = createDumpStream(dumpFileName);
+
+    *out << "digraph events_registry_" << this << " {\n";
+    *out << "node [shape=record]\n";
+    *out << "//pragma: somePragmaData"
+         << "\n";
+    auto allNodes = getNodes();
+    EventIdMap deadNodeTags;
+    auto curr = allNodes;
+    TrackedEvent *prev = nullptr;
+    EventIdMap eventsIdMapping;
+
+    while (curr != nullptr) {
+        auto next = curr->next;
+        bool eraseNode = false;
+        if (curr->eventId < 0) {
+            auto prevTag = deadNodeTags.find(curr->ev);
+            if (prevTag == deadNodeTags.end()) {
+                deadNodeTags[curr->ev] = -curr->eventId;
+            }
+            eraseNode = true;
+        } else if ((deadNodeTags.find(curr->ev) != deadNodeTags.end()) && (deadNodeTags[curr->ev] > curr->eventId)) {
+            eraseNode = true;
+        }
+
+        if (eraseNode) {
+            if (prev != nullptr) {
+                prev->next = next;
+            }
+            if (allNodes == curr) {
+                allNodes = nullptr;
+            }
+            delete curr;
+        } else {
+            if (allNodes == nullptr) {
+                allNodes = curr;
+            }
+            prev = curr;
+            eventsIdMapping[curr->ev] = curr->eventId;
+        }
+        curr = next;
+    }
+
+    auto node = allNodes;
+    CmdqSet dumpedCmdQs;
+    std::set<Event *> dumpedEvents;
+    while (node != nullptr) {
+        if (node->ev->peekNumEventsBlockingThis() != 0) {
+            node = node->next;
+            continue;
+        }
+        dumpGraph(node->ev, *out, dumpedCmdQs, dumpedEvents, eventsIdMapping);
+        node = node->next;
+    }
+    *out << "\n}\n";
+
+    if (allNodes == nullptr) {
+        return;
+    }
+
+    if (trackedEvents.peekHead() != nullptr) {
+        trackedEvents.peekHead()->getTail()->insertAllNext(*allNodes);
+    } else {
+        auto rest = allNodes->next;
+        trackedEvents.pushFrontOne(*allNodes);
+        if (rest != nullptr) {
+            allNodes->insertAllNext(*rest);
+        }
+    }
+}
+
+void EventsTracker::notifyCreation(Event *eventToTrack) {
+    dump();
+    auto trackedE = new TrackedEvent{eventToTrack, eventId++};
+    trackedEvents.pushFrontOne(*trackedE);
+}
+
+void EventsTracker::notifyDestruction(Event *eventToDestroy) {
+    auto trackedE = new TrackedEvent{eventToDestroy, -(eventId++)};
+    trackedEvents.pushFrontOne(*trackedE);
+    dump();
+}
+
+void EventsTracker::notifyTransitionedExecutionStatus() {
+    dump();
+}
+
+std::shared_ptr<std::ostream> EventsTracker::createDumpStream(const std::string &filename) {
+    std::shared_ptr<std::fstream> out{new std::fstream(filename, std::ios::binary | std::ios::out)};
+    return out;
+}
+
+} // namespace OCLRT

runtime/event/event_tracker.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#pragma once
+#include <unordered_map>
+#include <set>
+
+namespace OCLRT {
+
+struct TrackedEvent : IFNode<TrackedEvent> {
+    TrackedEvent(Event *ev, int64_t eventId)
+        : ev(ev), eventId(eventId) {
+    }
+    Event *ev = nullptr;
+    int64_t eventId = 1;
+};
+
+class EventsTracker {
+
+    using EventIdMap = std::unordered_map<Event *, int64_t>;
+    using CmdqSet = std::set<CommandQueue *>;
+
+  protected:
+    std::atomic<int64_t> eventId{0};
+    static std::unique_ptr<EventsTracker> globalEvTracker;
+    IFList<TrackedEvent, true, true> trackedEvents;
+    EventsTracker() = default;
+
+  public:
+    MOCKABLE_VIRTUAL ~EventsTracker() = default;
+    IFList<TrackedEvent, true, true> *getList();
+    MOCKABLE_VIRTUAL TrackedEvent *getNodes();
+    void dump();
+    void notifyCreation(Event *eventToTrack);
+    void notifyDestruction(Event *eventToDestroy);
+    void notifyTransitionedExecutionStatus();
+    MOCKABLE_VIRTUAL std::shared_ptr<std::ostream> createDumpStream(const std::string &filename);
+    static EventsTracker &getEventsTracker();
+    static void shutdownGlobalEvTracker();
+    static std::string label(Event *node, const EventIdMap &eventsIdMapping);
+    static std::string label(CommandQueue *cmdQ);
+    static void dumpQueue(CommandQueue *cmdQ, std::ostream &out, CmdqSet &dumpedCmdQs);
+    static void dumpEdge(Event *leftNode, Event *rightNode, std::ostream &out, const EventIdMap &eventsIdMapping);
+    static void dumpNode(Event *node, std::ostream &out, const EventIdMap &eventsIdMapping);
+    static void dumpGraph(Event *node, std::ostream &out, CmdqSet &dumpedCmdQs, std::set<Event *> &dumpedEvents,
+                          const EventIdMap &eventsIdMapping);
+};
+
+} // namespace OCLRT

runtime/helpers/CMakeLists.txt

@@ -31,6 +31,7 @@ set(RUNTIME_SRCS_HELPERS_BASE
   ${CMAKE_CURRENT_SOURCE_DIR}/built_ins_helper.h
   ${CMAKE_CURRENT_SOURCE_DIR}/cache_policy.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cache_policy.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/cl_helper.h
   ${CMAKE_CURRENT_SOURCE_DIR}/completion_stamp.h
   ${CMAKE_CURRENT_SOURCE_DIR}/convert_color.h
   ${CMAKE_CURRENT_SOURCE_DIR}/debug_helpers.h

runtime/helpers/cl_helper.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "CL/cl.h"
+#include "CL/cl_gl_ext.h"
+
+namespace OCLRT {
+inline const std::string cmdTypetoString(cl_command_type cmd) {
+    switch (cmd) {
+    case CL_COMMAND_NDRANGE_KERNEL:
+        return "CL_COMMAND_NDRANGE_KERNEL";
+    case CL_COMMAND_TASK:
+        return "CL_COMMAND_TASK";
+    case CL_COMMAND_NATIVE_KERNEL:
+        return "CL_COMMAND_NATIVE_KERNEL";
+    case CL_COMMAND_READ_BUFFER:
+        return "CL_COMMAND_READ_BUFFER";
+    case CL_COMMAND_WRITE_BUFFER:
+        return "CL_COMMAND_WRITE_BUFFER";
+    case CL_COMMAND_COPY_BUFFER:
+        return "CL_COMMAND_COPY_BUFFER";
+    case CL_COMMAND_READ_IMAGE:
+        return "CL_COMMAND_READ_IMAGE";
+    case CL_COMMAND_WRITE_IMAGE:
+        return "CL_COMMAND_WRITE_IMAGE";
+    case CL_COMMAND_COPY_IMAGE:
+        return "CL_COMMAND_COPY_IMAGE";
+    case CL_COMMAND_COPY_IMAGE_TO_BUFFER:
+        return "CL_COMMAND_COPY_IMAGE_TO_BUFFER";
+    case CL_COMMAND_COPY_BUFFER_TO_IMAGE:
+        return "CL_COMMAND_COPY_BUFFER_TO_IMAGE";
+    case CL_COMMAND_MAP_BUFFER:
+        return "CL_COMMAND_MAP_BUFFER";
+    case CL_COMMAND_MAP_IMAGE:
+        return "CL_COMMAND_MAP_IMAGE";
+    case CL_COMMAND_UNMAP_MEM_OBJECT:
+        return "CL_COMMAND_UNMAP_MEM_OBJECT";
+    case CL_COMMAND_MARKER:
+        return "CL_COMMAND_MARKER";
+    case CL_COMMAND_ACQUIRE_GL_OBJECTS:
+        return "CL_COMMAND_ACQUIRE_GL_OBJECTS";
+    case CL_COMMAND_RELEASE_GL_OBJECTS:
+        return "CL_COMMAND_RELEASE_GL_OBJECTS";
+    case CL_COMMAND_READ_BUFFER_RECT:
+        return "CL_COMMAND_READ_BUFFER_RECT";
+    case CL_COMMAND_WRITE_BUFFER_RECT:
+        return "CL_COMMAND_WRITE_BUFFER_RECT";
+    case CL_COMMAND_COPY_BUFFER_RECT:
+        return "CL_COMMAND_COPY_BUFFER_RECT";
+    case CL_COMMAND_USER:
+        return "CL_COMMAND_USER";
+    case CL_COMMAND_BARRIER:
+        return "CL_COMMAND_BARRIER";
+    case CL_COMMAND_MIGRATE_MEM_OBJECTS:
+        return "CL_COMMAND_MIGRATE_MEM_OBJECTS";
+    case CL_COMMAND_FILL_BUFFER:
+        return "CL_COMMAND_FILL_BUFFER";
+    case CL_COMMAND_FILL_IMAGE:
+        return "CL_COMMAND_FILL_IMAGE";
+    case CL_COMMAND_SVM_FREE:
+        return "CL_COMMAND_SVM_FREE";
+    case CL_COMMAND_SVM_MEMCPY:
+        return "CL_COMMAND_SVM_MEMCPY";
+    case CL_COMMAND_SVM_MEMFILL:
+        return "CL_COMMAND_SVM_MEMFILL";
+    case CL_COMMAND_SVM_MAP:
+        return "CL_COMMAND_SVM_MAP";
+    case CL_COMMAND_SVM_UNMAP:
+        return "CL_COMMAND_SVM_UNMAP";
+    case CL_COMMAND_GL_FENCE_SYNC_OBJECT_KHR:
+        return "CL_COMMAND_GL_FENCE_SYNC_OBJECT_KHR";
+    default: {
+        std::string returnString("CMD_UNKNOWN:" + std::to_string((cl_command_type)cmd));
+        return returnString;
+    }
+    }
+}
+} // namespace OCLRT

runtime/os_interface/DebugVariables.inl

@@ -57,6 +57,7 @@ DECLARE_DEBUG_VARIABLE(bool, LogAlignedAllocations, false, "Logs alignedMalloc a
 DECLARE_DEBUG_VARIABLE(bool, LogMemoryObject, false, "Logs memory object ptrs, sizes and operations")
 DECLARE_DEBUG_VARIABLE(bool, ResidencyDebugEnable, 0, "enables debug messages and checks for Residency Model")
 DECLARE_DEBUG_VARIABLE(bool, EventsDebugEnable, 0, "enables debug messages for events, virtual events, blocked enqueues, events trees etc.")
+DECLARE_DEBUG_VARIABLE(bool, EventsTrackerEnable, false, "enables event graphs dumping")
 DECLARE_DEBUG_VARIABLE(bool, PrintEMDebugInformation, false, "prints execution model related debug information")
 DECLARE_DEBUG_VARIABLE(bool, PrintLWSSizes, false, "prints driver choosen local workgroup sizes")
 DECLARE_DEBUG_VARIABLE(bool, PrintDispatchParameters, false, "prints dispatch paramters of kernels passed to clEnqueueNDRangeKernel")