From d31b950b9a76f8b8b19d89d434cffb58a9ca2396 Mon Sep 17 00:00:00 2001 From: Neil R Spruit Date: Wed, 22 Mar 2023 01:22:13 +0000 Subject: [PATCH] l0_feature: Use L0 Loader teardown callback Related-To: LOCI-4174 - Call zelSetDriverTeardown during L0 Driver teardown to prevent users from calling into destroyed functions and encountering crashes during teardown. Signed-off-by: Neil R Spruit --- level_zero/core/source/CMakeLists.txt | 2 + level_zero/core/source/global_teardown.cpp | 38 +++++++++++++++++++ level_zero/core/source/global_teardown.h | 19 ++++++++++ .../core/source/linux/driver_teardown.cpp | 14 ++----- .../core/source/windows/driver_teardown.cpp | 14 ++----- level_zero/core/test/common/CMakeLists.txt | 1 + .../common/test_l0_loader_lib/CMakeLists.txt | 21 ++++++++++ .../test_l0_loader_lib_lin.cpp | 12 ++++++ .../test_l0_loader_lib_win.cpp | 12 ++++++ .../core/test/unit_tests/CMakeLists.txt | 2 +- level_zero/core/test/unit_tests/main.cpp | 19 ++++++++++ .../unit_tests/os_interface/CMakeLists.txt | 4 +- .../os_interface/global_teardown_tests.cpp | 34 +++++++++++++++++ .../os_interface/global_teardown_tests.h | 15 ++++++++ .../os_interface/linux/CMakeLists.txt | 3 +- .../linux/global_teardown_test_values.cpp | 13 +++++++ .../os_interface/windows/CMakeLists.txt | 1 + .../windows/global_teardown_test_values.cpp | 13 +++++++ .../test/common/mocks/linux/mock_dlopen.cpp | 5 ++- 19 files changed, 217 insertions(+), 25 deletions(-) create mode 100644 level_zero/core/source/global_teardown.cpp create mode 100644 level_zero/core/source/global_teardown.h create mode 100644 level_zero/core/test/common/test_l0_loader_lib/CMakeLists.txt create mode 100644 level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_lin.cpp create mode 100644 level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_win.cpp create mode 100644 level_zero/core/test/unit_tests/os_interface/global_teardown_tests.cpp create mode 100644 level_zero/core/test/unit_tests/os_interface/global_teardown_tests.h create mode 100644 level_zero/core/test/unit_tests/os_interface/linux/global_teardown_test_values.cpp create mode 100644 level_zero/core/test/unit_tests/os_interface/windows/global_teardown_test_values.cpp diff --git a/level_zero/core/source/CMakeLists.txt b/level_zero/core/source/CMakeLists.txt index df12c9241a..d3bb5db815 100644 --- a/level_zero/core/source/CMakeLists.txt +++ b/level_zero/core/source/CMakeLists.txt @@ -8,6 +8,8 @@ target_sources(${L0_STATIC_LIB_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/get_extension_function_lookup_map.h + ${CMAKE_CURRENT_SOURCE_DIR}/global_teardown.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/global_teardown.h ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}get_extension_function_lookup_map.cpp ) diff --git a/level_zero/core/source/global_teardown.cpp b/level_zero/core/source/global_teardown.cpp new file mode 100644 index 0000000000..3dea7d2e45 --- /dev/null +++ b/level_zero/core/source/global_teardown.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/core/source/global_teardown.h" + +#include "shared/source/os_interface/os_library.h" + +#include "level_zero/core/source/driver/driver_handle_imp.h" +#include "level_zero/sysman/source/sysman_driver_handle_imp.h" + +namespace L0 { + +ze_result_t loaderDriverTeardown(std::string loaderLibraryName) { + std::unique_ptr loaderLibrary = std::unique_ptr{NEO::OsLibrary::load(loaderLibraryName.c_str())}; + if (loaderLibrary) { + zelSetDriverTeardown_fn setDriverTeardown = reinterpret_cast(loaderLibrary->getProcAddress("zelSetDriverTeardown")); + if (setDriverTeardown) { + return setDriverTeardown(); + } + } + return ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE; +} + +void globalDriverTeardown() { + if (GlobalDriver != nullptr) { + delete GlobalDriver; + GlobalDriver = nullptr; + } + if (Sysman::GlobalSysmanDriver != nullptr) { + delete Sysman::GlobalSysmanDriver; + Sysman::GlobalSysmanDriver = nullptr; + } +} +} // namespace L0 diff --git a/level_zero/core/source/global_teardown.h b/level_zero/core/source/global_teardown.h new file mode 100644 index 0000000000..5fb6cc0f78 --- /dev/null +++ b/level_zero/core/source/global_teardown.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once +#include + +#include + +namespace L0 { +using zelSetDriverTeardown_fn = ze_result_t (*)(); +static const std::string loaderLibraryFilename = "ze_loader"; + +void globalDriverTeardown(); +ze_result_t loaderDriverTeardown(std::string loaderLibraryName); +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/source/linux/driver_teardown.cpp b/level_zero/core/source/linux/driver_teardown.cpp index 6bd2fae3aa..9e80430c88 100644 --- a/level_zero/core/source/linux/driver_teardown.cpp +++ b/level_zero/core/source/linux/driver_teardown.cpp @@ -5,18 +5,12 @@ * */ -#include "level_zero/core/source/driver/driver_handle_imp.h" -#include "level_zero/sysman/source/sysman_driver_handle_imp.h" +#include "level_zero/core/source/global_teardown.h" using namespace L0; void __attribute__((destructor)) driverHandleDestructor() { - if (GlobalDriver != nullptr) { - delete GlobalDriver; - GlobalDriver = nullptr; - } - if (Sysman::GlobalSysmanDriver != nullptr) { - delete Sysman::GlobalSysmanDriver; - Sysman::GlobalSysmanDriver = nullptr; - } + std::string loaderLibraryName = "lib" + L0::loaderLibraryFilename + ".so.1"; + loaderDriverTeardown(loaderLibraryName); + globalDriverTeardown(); } \ No newline at end of file diff --git a/level_zero/core/source/windows/driver_teardown.cpp b/level_zero/core/source/windows/driver_teardown.cpp index 11e38443ad..f1fa59ce75 100644 --- a/level_zero/core/source/windows/driver_teardown.cpp +++ b/level_zero/core/source/windows/driver_teardown.cpp @@ -5,8 +5,7 @@ * */ -#include "level_zero/core/source/driver/driver_handle_imp.h" -#include "level_zero/sysman/source/sysman_driver_handle_imp.h" +#include "level_zero/core/source/global_teardown.h" #include "level_zero/tools/source/sysman/os_sysman_driver.h" #include "level_zero/tools/source/sysman/sysman.h" @@ -16,14 +15,9 @@ using namespace L0; BOOL APIENTRY DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { if (fdwReason == DLL_PROCESS_DETACH) { - if (GlobalDriver != nullptr) { - delete GlobalDriver; - GlobalDriver = nullptr; - } - if (Sysman::GlobalSysmanDriver != nullptr) { - delete Sysman::GlobalSysmanDriver; - Sysman::GlobalSysmanDriver = nullptr; - } + std::string loaderLibraryName = L0::loaderLibraryFilename + ".dll"; + loaderDriverTeardown(loaderLibraryName); + globalDriverTeardown(); if (GlobalOsSysmanDriver != nullptr) { delete GlobalOsSysmanDriver; GlobalOsSysmanDriver = nullptr; diff --git a/level_zero/core/test/common/CMakeLists.txt b/level_zero/core/test/common/CMakeLists.txt index d88b666732..fb26ed0e24 100644 --- a/level_zero/core/test/common/CMakeLists.txt +++ b/level_zero/core/test/common/CMakeLists.txt @@ -40,3 +40,4 @@ apply_macro_for_each_core_type("TESTED") add_custom_target(l0_common_test_kernels DEPENDS ${l0_test_kernel_outputs} ${l0_bindless_test_kernel_outputs} copy_compiler_files) set_target_properties(l0_common_test_kernels PROPERTIES FOLDER ${TARGET_NAME_L0}) add_dependencies(prepare_test_kernels_for_l0 l0_common_test_kernels) +add_subdirectories() \ No newline at end of file diff --git a/level_zero/core/test/common/test_l0_loader_lib/CMakeLists.txt b/level_zero/core/test/common/test_l0_loader_lib/CMakeLists.txt new file mode 100644 index 0000000000..1c052540c1 --- /dev/null +++ b/level_zero/core/test/common/test_l0_loader_lib/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# Copyright (C) 2023 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +project(test_l0_loader_lib) + +if(WIN32) + set(OS_SUFFIX win) +else() + set(OS_SUFFIX lin) +endif() + +add_library(test_l0_loader_lib SHARED test_l0_loader_lib_${OS_SUFFIX}.cpp) + +create_project_source_tree(test_l0_loader_lib) + +set_target_properties(test_l0_loader_lib PROPERTIES FOLDER "test mocks") +set_property(TARGET test_l0_loader_lib APPEND_STRING PROPERTY COMPILE_FLAGS ${ASAN_FLAGS}) + diff --git a/level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_lin.cpp b/level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_lin.cpp new file mode 100644 index 0000000000..ec14828299 --- /dev/null +++ b/level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_lin.cpp @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include + +extern "C" __attribute__((visibility("default"))) ze_result_t zelSetDriverTeardown() { + return ZE_RESULT_SUCCESS; +} diff --git a/level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_win.cpp b/level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_win.cpp new file mode 100644 index 0000000000..4fd39bd043 --- /dev/null +++ b/level_zero/core/test/common/test_l0_loader_lib/test_l0_loader_lib_win.cpp @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include + +extern "C" __declspec(dllexport) ze_result_t zelSetDriverTeardown() { + return ZE_RESULT_SUCCESS; +} diff --git a/level_zero/core/test/unit_tests/CMakeLists.txt b/level_zero/core/test/unit_tests/CMakeLists.txt index ff57f5aba0..561398ad77 100644 --- a/level_zero/core/test/unit_tests/CMakeLists.txt +++ b/level_zero/core/test/unit_tests/CMakeLists.txt @@ -102,6 +102,6 @@ if(MSVC) ) endif() -add_dependencies(unit_tests ${TARGET_NAME} prepare_test_kernels_for_l0) +add_dependencies(unit_tests ${TARGET_NAME} prepare_test_kernels_for_l0 test_l0_loader_lib) create_source_tree(${TARGET_NAME} ${L0_ROOT_DIR}/..) diff --git a/level_zero/core/test/unit_tests/main.cpp b/level_zero/core/test/unit_tests/main.cpp index c4f62c811c..de78f66bf9 100644 --- a/level_zero/core/test/unit_tests/main.cpp +++ b/level_zero/core/test/unit_tests/main.cpp @@ -173,6 +173,25 @@ int main(int argc, char **argv) { sysmanUltsEnable = (strcmp(sysmanUltsEnableEnv, "1") == 0); } +#if defined(__linux__) + if (getenv("NEO_L0_TEST_SELF_EXEC") == nullptr) { + std::string wd = getRunPath(argv[0]); + char *ldLibraryPath = getenv("LD_LIBRARY_PATH"); + + if (ldLibraryPath == nullptr) { + setenv("LD_LIBRARY_PATH", wd.c_str(), 1); + } else { + std::string ldLibraryPathConcat = wd + ":" + std::string(ldLibraryPath); + setenv("LD_LIBRARY_PATH", ldLibraryPathConcat.c_str(), 1); + } + + setenv("NEO_L0_TEST_SELF_EXEC", wd.c_str(), 1); + execv(argv[0], argv); + printf("FATAL ERROR: cannot self-exec test: %s!, errno: %d\n", argv[0], errno); + return -1; + } +#endif + applyWorkarounds(); testing::InitGoogleMock(&argc, argv); diff --git a/level_zero/core/test/unit_tests/os_interface/CMakeLists.txt b/level_zero/core/test/unit_tests/os_interface/CMakeLists.txt index a228a53164..603bb97d16 100644 --- a/level_zero/core/test/unit_tests/os_interface/CMakeLists.txt +++ b/level_zero/core/test/unit_tests/os_interface/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2021-2023 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -7,6 +7,8 @@ set(NEO_L0_OS_INTERFACE_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/os_context_l0_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/global_teardown_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/global_teardown_tests.h ) add_subdirectories() diff --git a/level_zero/core/test/unit_tests/os_interface/global_teardown_tests.cpp b/level_zero/core/test/unit_tests/os_interface/global_teardown_tests.cpp new file mode 100644 index 0000000000..c295bbc5a4 --- /dev/null +++ b/level_zero/core/test/unit_tests/os_interface/global_teardown_tests.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "level_zero/core/test/unit_tests/os_interface/global_teardown_tests.h" + +#include "shared/test/common/test_macros/test.h" + +#include "level_zero/core/source/driver/driver_handle_imp.h" +#include "level_zero/core/source/global_teardown.h" +#include "level_zero/sysman/source/sysman_driver_handle_imp.h" + +namespace L0 { +namespace ult { + +TEST(GlobalTearDownCallbackTests, givenL0LoaderThenGlobalTeardownCallbackIsCalled) { + EXPECT_EQ(ZE_RESULT_SUCCESS, loaderDriverTeardown(L0::ult::testLoaderDllName)); +} +TEST(GlobalTearDownCallbackTests, givenL0LoaderIsMissingThenGlobalTeardownCallbackIsNotCalled) { + EXPECT_EQ(ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE, loaderDriverTeardown("invalid.so")); +} +TEST(GlobalTearDownCallbackTests, givenL0LoaderWithoutGlobalTeardownCallbackThenGlobalTeardownCallbackIsNotCalled) { + EXPECT_EQ(ZE_RESULT_ERROR_DEPENDENCY_UNAVAILABLE, loaderDriverTeardown(L0::ult::testDllName)); +} +TEST(GlobalTearDownTests, givenCallToGlobalTearDownFunctionThenGlobalDriversAreNull) { + globalDriverTeardown(); + EXPECT_EQ(GlobalDriver, nullptr); + EXPECT_EQ(Sysman::GlobalSysmanDriver, nullptr); +} +} // namespace ult +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/os_interface/global_teardown_tests.h b/level_zero/core/test/unit_tests/os_interface/global_teardown_tests.h new file mode 100644 index 0000000000..94ece2fde9 --- /dev/null +++ b/level_zero/core/test/unit_tests/os_interface/global_teardown_tests.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +namespace L0 { +namespace ult { +extern const char *testLoaderDllName; +extern const char *testDllName; +} // namespace ult +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/os_interface/linux/CMakeLists.txt b/level_zero/core/test/unit_tests/os_interface/linux/CMakeLists.txt index cf9bea6cd6..dedc2f2c00 100644 --- a/level_zero/core/test/unit_tests/os_interface/linux/CMakeLists.txt +++ b/level_zero/core/test/unit_tests/os_interface/linux/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -8,5 +8,6 @@ if(UNIX) target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/cmdqueue_linux_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/global_teardown_test_values.cpp ) endif() diff --git a/level_zero/core/test/unit_tests/os_interface/linux/global_teardown_test_values.cpp b/level_zero/core/test/unit_tests/os_interface/linux/global_teardown_test_values.cpp new file mode 100644 index 0000000000..da8c436c14 --- /dev/null +++ b/level_zero/core/test/unit_tests/os_interface/linux/global_teardown_test_values.cpp @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +namespace L0 { +namespace ult { +const char *testLoaderDllName = "libtest_l0_loader_lib.so"; +const char *testDllName = "libtest_dynamic_lib.so"; +} // namespace ult +} // namespace L0 \ No newline at end of file diff --git a/level_zero/core/test/unit_tests/os_interface/windows/CMakeLists.txt b/level_zero/core/test/unit_tests/os_interface/windows/CMakeLists.txt index b1e2769ac4..04c3857180 100644 --- a/level_zero/core/test/unit_tests/os_interface/windows/CMakeLists.txt +++ b/level_zero/core/test/unit_tests/os_interface/windows/CMakeLists.txt @@ -8,5 +8,6 @@ if(WIN32) target_sources(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/wddm_l0_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/global_teardown_test_values.cpp ) endif() diff --git a/level_zero/core/test/unit_tests/os_interface/windows/global_teardown_test_values.cpp b/level_zero/core/test/unit_tests/os_interface/windows/global_teardown_test_values.cpp new file mode 100644 index 0000000000..1584bcb777 --- /dev/null +++ b/level_zero/core/test/unit_tests/os_interface/windows/global_teardown_test_values.cpp @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +namespace L0 { +namespace ult { +const char *testLoaderDllName = "test_l0_loader_lib.dll"; +const char *testDllName = "test_dynamic_lib.dll"; +} // namespace ult +} // namespace L0 \ No newline at end of file diff --git a/shared/test/common/mocks/linux/mock_dlopen.cpp b/shared/test/common/mocks/linux/mock_dlopen.cpp index 0d427155c2..2ad7d98930 100644 --- a/shared/test/common/mocks/linux/mock_dlopen.cpp +++ b/shared/test/common/mocks/linux/mock_dlopen.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Intel Corporation + * Copyright (C) 2021-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -25,7 +25,8 @@ void *dlopen(const char *filename, int flags) { dlopenError = -1; if (filename == nullptr || - strcmp(filename, "libtest_dynamic_lib.so") == 0) { + (strcmp(filename, "libtest_dynamic_lib.so") == 0) || + (strcmp(filename, "libtest_l0_loader_lib.so") == 0)) { return dlopenFunc(filename, flags); } if (filename[0] == '_') {