Multiple AUB CSRs to operate on a single file stream

This commit introduces AUB stream provider class to produce
a single file stream in multi device scenarios with multiple AUB CSRs.

Change-Id: Id70e0d680459d34f4291b9e56aa39d960f025ac6

runtime/aub_mem_dump/aub_mem_dump.h

@@ -9,6 +9,7 @@
 #include <cstdio>
 #include <cstdint>
 #include <fstream>
+#include <mutex>
 
 #ifndef BIT
 #define BIT(x) (((uint64_t)1) << (x))
@@ -137,9 +138,11 @@ struct AubFileStream : public AubStream {
     MOCKABLE_VIRTUAL void flush();
     MOCKABLE_VIRTUAL void expectMemory(uint64_t physAddress, const void *memory, size_t size, uint32_t addressSpace);
     MOCKABLE_VIRTUAL bool addComment(const char *message);
+    MOCKABLE_VIRTUAL std::unique_lock<std::mutex> lockStream();
 
     std::ofstream fileHandle;
     std::string fileName;
+    std::mutex mutex;
 };
 
 template <int addressingBits>

runtime/command_stream/CMakeLists.txt

@@ -10,6 +10,7 @@ set(RUNTIME_SRCS_COMMAND_STREAM
   ${CMAKE_CURRENT_SOURCE_DIR}/aub_command_stream_receiver.h
   ${CMAKE_CURRENT_SOURCE_DIR}/aub_command_stream_receiver_hw.h
   ${CMAKE_CURRENT_SOURCE_DIR}/aub_command_stream_receiver_hw.inl
+  ${CMAKE_CURRENT_SOURCE_DIR}/aub_stream_provider.h
   ${CMAKE_CURRENT_SOURCE_DIR}/aub_subcapture.h
   ${CMAKE_CURRENT_SOURCE_DIR}/aub_subcapture.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/command_stream_receiver.cpp

runtime/command_stream/aub_command_stream_receiver.cpp

@@ -137,7 +137,7 @@ void AubFileStream::writeMemoryWriteHeader(uint64_t physAddress, size_t size, ui
 }
 
 void AubFileStream::writeGTT(uint32_t gttOffset, uint64_t entry) {
-    fileHandle.write(reinterpret_cast<char *>(&entry), sizeof(entry));
+    write(reinterpret_cast<char *>(&entry), sizeof(entry));
 }
 
 void AubFileStream::writePTE(uint64_t physAddress, uint64_t entry) {
@@ -247,4 +247,8 @@ bool AubFileStream::addComment(const char *message) {
     return true;
 }
 
+std::unique_lock<std::mutex> AubFileStream::lockStream() {
+    return std::unique_lock<std::mutex>(mutex);
+}
+
 } // namespace AubMemDump

runtime/command_stream/aub_command_stream_receiver_hw.h

@@ -54,6 +54,8 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverHw<GfxFamily> {
     AUBCommandStreamReceiverHw(const AUBCommandStreamReceiverHw &) = delete;
     AUBCommandStreamReceiverHw &operator=(const AUBCommandStreamReceiverHw &) = delete;
 
+    MOCKABLE_VIRTUAL void openFile(const std::string &fileName);
+    MOCKABLE_VIRTUAL bool reopenFile(const std::string &fileName);
     MOCKABLE_VIRTUAL void initFile(const std::string &fileName);
     MOCKABLE_VIRTUAL void closeFile();
     MOCKABLE_VIRTUAL bool isFileOpen();
@@ -81,7 +83,7 @@ class AUBCommandStreamReceiverHw : public CommandStreamReceiverHw<GfxFamily> {
         uint32_t tailRingBuffer;
     } engineInfoTable[EngineType::NUM_ENGINES] = {};
 
-    std::unique_ptr<AUBCommandStreamReceiver::AubFileStream> stream;
+    AUBCommandStreamReceiver::AubFileStream *stream;
     std::unique_ptr<AubSubCaptureManager> subCaptureManager;
     uint32_t aubDeviceId;
     bool standalone;

runtime/command_stream/aub_command_stream_receiver_hw.inl

@@ -8,7 +8,9 @@
 #include "hw_cmds.h"
 #include "runtime/aub/aub_helper.h"
 #include "runtime/aub_mem_dump/page_table_entry_bits.h"
+#include "runtime/command_stream/aub_stream_provider.h"
 #include "runtime/command_stream/aub_subcapture.h"
+#include "runtime/execution_environment/execution_environment.h"
 #include "runtime/gmm_helper/gmm.h"
 #include "runtime/gmm_helper/gmm_helper.h"
 #include "runtime/gmm_helper/resource_info.h"
@@ -27,7 +29,6 @@ namespace OCLRT {
 template <typename GfxFamily>
 AUBCommandStreamReceiverHw<GfxFamily>::AUBCommandStreamReceiverHw(const HardwareInfo &hwInfoIn, const std::string &fileName, bool standalone, ExecutionEnvironment &executionEnvironment)
     : BaseClass(hwInfoIn, executionEnvironment),
-      stream(std::make_unique<AUBCommandStreamReceiver::AubFileStream>()),
       subCaptureManager(std::make_unique<AubSubCaptureManager>(fileName)),
       standalone(standalone) {
 
@@ -40,6 +41,8 @@ AUBCommandStreamReceiverHw<GfxFamily>::AUBCommandStreamReceiverHw(const Hardware
     if (DebugManager.flags.CsrDispatchMode.get()) {
         this->dispatchMode = (DispatchMode)DebugManager.flags.CsrDispatchMode.get();
     }
+    executionEnvironment.initAubStreamProvider();
+    stream = executionEnvironment.aubStreamProvider->getStream();
     if (DebugManager.flags.AUBDumpSubCaptureMode.get()) {
         this->subCaptureManager->subCaptureMode = static_cast<AubSubCaptureManager::SubCaptureMode>(DebugManager.flags.AUBDumpSubCaptureMode.get());
         this->subCaptureManager->subCaptureFilter.dumpKernelStartIdx = static_cast<uint32_t>(DebugManager.flags.AUBDumpFilterKernelStartIdx.get());
@@ -58,7 +61,6 @@ AUBCommandStreamReceiverHw<GfxFamily>::AUBCommandStreamReceiverHw(const Hardware
 
 template <typename GfxFamily>
 AUBCommandStreamReceiverHw<GfxFamily>::~AUBCommandStreamReceiverHw() {
-    AUBCommandStreamReceiverHw<GfxFamily>::closeFile();
     freeEngineInfoTable();
 }
 
@@ -85,19 +87,41 @@ void AUBCommandStreamReceiverHw<GfxFamily>::initEngineMMIO(EngineType engineType
 }
 
 template <typename GfxFamily>
-void AUBCommandStreamReceiverHw<GfxFamily>::initFile(const std::string &fileName) {
-    stream.reset(new AUBCommandStreamReceiver::AubFileStream());
+void AUBCommandStreamReceiverHw<GfxFamily>::openFile(const std::string &fileName) {
+    auto streamLocked = stream->lockStream();
+    initFile(fileName);
+}
 
-    // Open our file
-    stream->open(fileName.c_str());
-
-    if (!stream->isOpen()) {
-        // This DEBUG_BREAK_IF most probably means you are not executing aub tests with correct current directory (containing aub_out folder)
-        // try adding <familycodename>_aub
-        DEBUG_BREAK_IF(true);
+template <typename GfxFamily>
+bool AUBCommandStreamReceiverHw<GfxFamily>::reopenFile(const std::string &fileName) {
+    auto streamLocked = stream->lockStream();
+    if (isFileOpen()) {
+        if (fileName != getFileName()) {
+            closeFile();
+            freeEngineInfoTable();
+        }
+    }
+    if (!isFileOpen()) {
+        initFile(fileName);
+        return true;
+    }
+    return false;
+}
+
+template <typename GfxFamily>
+void AUBCommandStreamReceiverHw<GfxFamily>::initFile(const std::string &fileName) {
+    if (!stream->isOpen()) {
+        // Open our file
+        stream->open(fileName.c_str());
+
+        if (!stream->isOpen()) {
+            // This DEBUG_BREAK_IF most probably means you are not executing aub tests with correct current directory (containing aub_out folder)
+            // try adding <familycodename>_aub
+            DEBUG_BREAK_IF(true);
+        }
+        // Add the file header
+        stream->init(AubMemDump::SteppingValues::A, aubDeviceId);
     }
-    // Add the file header
-    stream->init(AubMemDump::SteppingValues::A, aubDeviceId);
 }
 
 template <typename GfxFamily>
@@ -235,7 +259,7 @@ CommandStreamReceiver *AUBCommandStreamReceiverHw<GfxFamily>::create(const Hardw
     auto csr = new AUBCommandStreamReceiverHw<GfxFamily>(hwInfoIn, fileName, standalone, executionEnvironment);
 
     if (!csr->subCaptureManager->isSubCaptureMode()) {
-        csr->initFile(fileName);
+        csr->openFile(fileName);
     }
 
     return csr;
@@ -253,6 +277,7 @@ FlushStamp AUBCommandStreamReceiverHw<GfxFamily>::flush(BatchBuffer &batchBuffer
         }
     }
 
+    auto streamLocked = stream->lockStream();
     uint32_t mmioBase = getCsTraits(engineType).mmioBase;
     auto &engineInfo = engineInfoTable[engineType];
 
@@ -621,14 +646,8 @@ void AUBCommandStreamReceiverHw<GfxFamily>::activateAubSubCapture(const MultiDis
     bool active = subCaptureManager->activateSubCapture(dispatchInfo);
     if (active) {
         std::string subCaptureFile = subCaptureManager->getSubCaptureFileName(dispatchInfo);
-        if (isFileOpen()) {
-            if (subCaptureFile != getFileName()) {
-                closeFile();
-                freeEngineInfoTable();
-            }
-        }
-        if (!isFileOpen()) {
-            initFile(subCaptureFile);
+        auto isReopened = reopenFile(subCaptureFile);
+        if (isReopened) {
             dumpAubNonWritable = true;
         }
     }

runtime/command_stream/aub_stream_provider.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2018 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+#include "runtime/aub_mem_dump/aub_mem_dump.h"
+#include <string>
+
+namespace OCLRT {
+
+class AubStreamProvider {
+  public:
+    virtual ~AubStreamProvider() = default;
+
+    virtual AubMemDump::AubFileStream *getStream() = 0;
+};
+
+class AubFileStreamProvider : public AubStreamProvider {
+  public:
+    AubMemDump::AubFileStream *getStream() override {
+        return &stream;
+    };
+
+  protected:
+    AubMemDump::AubFileStream stream;
+};
+} // namespace OCLRT

runtime/execution_environment/execution_environment.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "runtime/execution_environment/execution_environment.h"
+#include "runtime/command_stream/aub_stream_provider.h"
 #include "runtime/command_stream/command_stream_receiver.h"
 #include "runtime/compiler_interface/compiler_interface.h"
 #include "runtime/source_level_debugger/source_level_debugger.h"
@@ -21,6 +22,11 @@ ExecutionEnvironment::ExecutionEnvironment() = default;
 ExecutionEnvironment::~ExecutionEnvironment() = default;
 extern CommandStreamReceiver *createCommandStream(const HardwareInfo *pHwInfo, ExecutionEnvironment &executionEnvironment);
 
+void ExecutionEnvironment::initAubStreamProvider() {
+    if (!aubStreamProvider) {
+        aubStreamProvider.reset(new AubFileStreamProvider());
+    }
+}
 void ExecutionEnvironment::initGmm(const HardwareInfo *hwInfo) {
     if (!gmmHelper) {
         gmmHelper.reset(new GmmHelper(hwInfo));

runtime/execution_environment/execution_environment.h

@@ -13,6 +13,7 @@
 #include <vector>
 
 namespace OCLRT {
+class AubStreamProvider;
 class GmmHelper;
 class CommandStreamReceiver;
 class MemoryManager;
@@ -34,6 +35,7 @@ class ExecutionEnvironment : public ReferenceTrackedObject<ExecutionEnvironment>
     ExecutionEnvironment();
     ~ExecutionEnvironment() override;
 
+    void initAubStreamProvider();
     void initGmm(const HardwareInfo *hwInfo);
     bool initializeCommandStreamReceiver(const HardwareInfo *pHwInfo, uint32_t deviceIndex);
     void initializeMemoryManager(bool enable64KBpages, bool enableLocalMemory, uint32_t deviceIndex);
@@ -45,6 +47,7 @@ class ExecutionEnvironment : public ReferenceTrackedObject<ExecutionEnvironment>
 
     std::unique_ptr<OSInterface> osInterface;
     std::unique_ptr<MemoryManager> memoryManager;
+    std::unique_ptr<AubStreamProvider> aubStreamProvider;
     std::vector<std::unique_ptr<CommandStreamReceiver>> commandStreamReceivers;
     std::unique_ptr<BuiltIns> builtins;
     std::unique_ptr<CompilerInterface> compilerInterface;