Fixed Global Driver to be void * with library unload driver cleanup

- Changed Global Driver to be a void * to avoid auto add of Global
Driver Destructor to run before destruction of other L0 data structures
that might be enqueued to destory in static object destructors.
- Added register of library unload driverdestructor to cleanup
driver/device as the last destructor run.

Change-Id: I8ba6c5c27424b942a86a2613edd52fc682ab1c64
Signed-off-by: Spruit, Neil R <neil.r.spruit@intel.com>

level_zero/core/source/driver/driver.cpp

@@ -20,7 +20,7 @@
 
 namespace L0 {
 
-std::unique_ptr<_ze_driver_handle_t> GlobalDriver;
+_ze_driver_handle_t *GlobalDriverHandle;
 uint32_t driverCount = 1;
 
 void DriverImp::initialize(bool *result) {
@@ -33,8 +33,8 @@ void DriverImp::initialize(bool *result) {
     auto neoDevices = NEO::DeviceFactory::createDevices(*executionEnvironment);
     executionEnvironment->decRefInternal();
     if (!neoDevices.empty()) {
-        GlobalDriver.reset(DriverHandle::create(std::move(neoDevices)));
-        if (GlobalDriver != nullptr) {
+        GlobalDriverHandle = DriverHandle::create(std::move(neoDevices));
+        if (GlobalDriverHandle != nullptr) {
             *result = true;
         }
     }
@@ -66,7 +66,7 @@ ze_result_t driverHandleGet(uint32_t *pCount, ze_driver_handle_t *phDriverHandle
     }
 
     for (uint32_t i = 0; i < *pCount; i++) {
-        phDriverHandles[i] = GlobalDriver.get();
+        phDriverHandles[i] = GlobalDriverHandle;
     }
 
     return ZE_RESULT_SUCCESS;

level_zero/core/source/driver/driver.h

@@ -26,5 +26,5 @@ ze_result_t init(_ze_init_flag_t);
 ze_result_t driverHandleGet(uint32_t *pCount, ze_driver_handle_t *phDrivers);
 
 extern uint32_t driverCount;
-extern std::unique_ptr<_ze_driver_handle_t> GlobalDriver;
+extern _ze_driver_handle_t *GlobalDriverHandle;
 } // namespace L0

level_zero/core/source/driver/driver_handle_imp.cpp

@@ -24,6 +24,8 @@
 
 namespace L0 {
 
+struct DriverHandleImp *GlobalDriver;
+
 NEO::MemoryManager *DriverHandleImp::getMemoryManager() {
     return this->memoryManager;
 }
@@ -190,7 +192,10 @@ DriverHandle *DriverHandle::create(std::vector<std::unique_ptr<NEO::Device>> dev
         return nullptr;
     }
 
+    GlobalDriver = driverHandle;
+
     driverHandle->memoryManager->setForceNonSvmForExternalHostPtr(true);
+
     return driverHandle;
 }
 

level_zero/core/source/driver/driver_handle_imp.h

@@ -69,4 +69,6 @@ struct DriverHandleImp : public DriverHandle {
     bool enableProgramDebugging = false;
 };
 
+extern struct DriverHandleImp *GlobalDriver;
+
 } // namespace L0

level_zero/core/source/linux/CMakeLists.txt

@@ -8,6 +8,7 @@ if(UNIX)
     set(L0_SOURCES_LINUX
         ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
         ${CMAKE_CURRENT_SOURCE_DIR}/registry_path.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/driver_teardown.cpp
     )
     set_property(GLOBAL PROPERTY L0_SOURCES_LINUX ${L0_SOURCES_LINUX})
 endif()

level_zero/core/source/linux/driver_teardown.cpp

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/driver/driver_handle_imp.h"
+
+using namespace L0;
+
+void __attribute__((destructor)) driverHandleDestructor() {
+    if (GlobalDriver != nullptr) {
+        delete GlobalDriver;
+        GlobalDriver = nullptr;
+    }
+}

level_zero/core/source/windows/CMakeLists.txt

@@ -8,6 +8,7 @@ if(WIN32)
     set(L0_SOURCES_WINDOWS
         ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
         ${CMAKE_CURRENT_SOURCE_DIR}/registry_path.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/driver_teardown.cpp
     )
     set_property(GLOBAL PROPERTY L0_SOURCES_WINDOWS ${L0_SOURCES_WINDOWS})
 endif()

level_zero/core/source/windows/driver_teardown.cpp

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/driver/driver_handle_imp.h"
+
+using namespace L0;
+
+BOOL APIENTRY DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
+    if (fdwReason == DLL_PROCESS_DETACH) {
+        if (GlobalDriver != nullptr) {
+            delete GlobalDriver;
+            GlobalDriver = nullptr;
+        }
+    }
+    return TRUE;
+}

level_zero/core/test/unit_tests/gen11/test_cmdqueue_thread_arbitration_policy_gen11.cpp

@@ -57,6 +57,7 @@ struct CommandQueueThreadArbitrationPolicyTests : public ::testing::Test {
     void TearDown() override {
         commandList->destroy();
         commandQueue->destroy();
+        L0::GlobalDriver = nullptr;
     }
 
     DebugManagerStateRestore restorer;

level_zero/core/test/unit_tests/gen9/test_cmdqueue_thread_arbitration_policy_gen9.cpp

@@ -57,6 +57,7 @@ struct CommandQueueThreadArbitrationPolicyTests : public ::testing::Test {
     void TearDown() override {
         commandList->destroy();
         commandQueue->destroy();
+        L0::GlobalDriver = nullptr;
     }
 
     DebugManagerStateRestore restorer;

level_zero/core/test/unit_tests/sources/debugger/active_debugger_fixture.h

@@ -13,6 +13,7 @@
 #include "opencl/test/unit_test/mocks/mock_source_level_debugger.h"
 
 #include "level_zero/core/source/cmdqueue/cmdqueue_hw.h"
+#include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/fence/fence.h"
 #include "level_zero/core/source/module/module.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_built_ins.h"
@@ -54,6 +55,7 @@ struct ActiveDebuggerFixture {
         ASSERT_NE(nullptr, deviceL0);
     }
     void TearDown() { // NOLINT(readability-identifier-naming)
+        L0::GlobalDriver = nullptr;
     }
 
     std::unique_ptr<L0::ult::WhiteBox<L0::DriverHandle>> driverHandle;

level_zero/core/test/unit_tests/sources/driver/test_driver.cpp

@@ -47,6 +47,7 @@ TEST(DriverTestFamilySupport, whenInitializingDriverOnSupportedFamilyThenDriverI
     auto driverHandle = DriverHandle::create(std::move(devices));
     EXPECT_NE(nullptr, driverHandle);
     delete driverHandle;
+    L0::GlobalDriver = nullptr;
 }
 
 TEST(DriverTestFamilySupport, whenInitializingDriverOnNotSupportedFamilyThenDriverIsNotCreated) {
@@ -75,6 +76,7 @@ TEST(DriverTest, givenNullEnvVariableWhenCreatingDriverThenEnableProgramDebuggin
     EXPECT_FALSE(driverHandle->enableProgramDebugging);
 
     delete driverHandle;
+    L0::GlobalDriver = nullptr;
 }
 
 TEST(DriverTest, givenEnvVariableNonZeroWhenCreatingDriverThenEnableProgramDebuggingIsSetTrue) {
@@ -93,6 +95,7 @@ TEST(DriverTest, givenEnvVariableNonZeroWhenCreatingDriverThenEnableProgramDebug
     EXPECT_TRUE(driverHandle->enableProgramDebugging);
 
     delete driverHandle;
+    L0::GlobalDriver = nullptr;
 }
 
 struct DriverTestMultipleFamilySupport : public ::testing::Test {
@@ -134,6 +137,7 @@ TEST_F(DriverTestMultipleFamilySupport, whenInitializingDriverWithArrayOfDevices
     }
 
     delete driverHandle;
+    L0::GlobalDriver = nullptr;
 }
 
 struct DriverTestMultipleFamilyNoSupport : public ::testing::Test {
@@ -224,6 +228,7 @@ TEST_F(DriverTestMultipleDeviceWithAffinityMask,
     }
 
     delete driverHandle;
+    L0::GlobalDriver = nullptr;
 }
 
 TEST_P(DriverTestMultipleDeviceWithAffinityMask,
@@ -284,6 +289,7 @@ TEST_P(DriverTestMultipleDeviceWithAffinityMask,
     }
 
     delete driverHandle;
+    L0::GlobalDriver = nullptr;
 }
 
 MaskArray maskArray;

level_zero/tools/source/metrics/metric.cpp

@@ -169,7 +169,7 @@ void MetricContext::enableMetricApi(ze_result_t &result) {
         return;
     }
 
-    DriverHandle *driverHandle = L0::DriverHandle::fromHandle(GlobalDriver.get());
+    DriverHandle *driverHandle = L0::DriverHandle::fromHandle(GlobalDriverHandle);
 
     uint32_t count = 0;
     result = driverHandle->getDevice(&count, nullptr);

level_zero/tools/source/sysman/sysman.cpp

@@ -24,7 +24,7 @@ void SysmanHandleContext::init(ze_init_flag_t flag) {
 }
 
 SysmanHandleContext::SysmanHandleContext() {
-    DriverHandle *dH = L0::DriverHandle::fromHandle(GlobalDriver.get());
+    DriverHandle *dH = L0::DriverHandle::fromHandle(GlobalDriverHandle);
     uint32_t count = 0;
     dH->getDevice(&count, nullptr);
     std::vector<ze_device_handle_t> devices(count);