diff --git a/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl b/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl index 9af9e75d92..a068bbcf0b 100644 --- a/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl +++ b/level_zero/core/source/cmdqueue/cmdqueue_hw_base.inl @@ -15,7 +15,7 @@ #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/hw_helper.h" #include "shared/source/helpers/hw_info.h" -#include "shared/source/helpers/interlocked_max.h" +#include "shared/source/helpers/mt_helpers.h" #include "shared/source/helpers/preamble.h" #include "shared/source/helpers/state_base_address.h" #include "shared/source/helpers/state_base_address_bdw_and_later.inl" diff --git a/opencl/source/event/event.cpp b/opencl/source/event/event.cpp index cfd93e3504..90eaae3cf2 100644 --- a/opencl/source/event/event.cpp +++ b/opencl/source/event/event.cpp @@ -11,6 +11,7 @@ #include "shared/source/device/device.h" #include "shared/source/helpers/aligned_memory.h" #include "shared/source/helpers/get_info.h" +#include "shared/source/helpers/mt_helpers.h" #include "shared/source/helpers/timestamp_packet.h" #include "shared/source/memory_manager/internal_allocation_storage.h" #include "shared/source/utilities/perf_counter.h" @@ -563,7 +564,9 @@ void Event::transitionExecutionStatus(int32_t newExecutionStatus) const { DBG_LOG(EventsDebugEnable, "transitionExecutionStatus event", this, " new status", newExecutionStatus, "previousStatus", prevStatus); while (prevStatus > newExecutionStatus) { - executionStatus.compare_exchange_weak(prevStatus, newExecutionStatus); + if (NEO::MultiThreadHelpers::atomicCompareExchangeWeakSpin(executionStatus, prevStatus, newExecutionStatus)) { + break; + } } if (NEO::DebugManager.flags.EventsTrackerEnable.get()) { EventsTracker::getEventsTracker().notifyTransitionedExecutionStatus(); diff --git a/opencl/test/unit_test/mt_tests/helpers/CMakeLists.txt b/opencl/test/unit_test/mt_tests/helpers/CMakeLists.txt index ead9069869..e355b3902c 100644 --- a/opencl/test/unit_test/mt_tests/helpers/CMakeLists.txt +++ b/opencl/test/unit_test/mt_tests/helpers/CMakeLists.txt @@ -8,7 +8,7 @@ set(IGDRCL_SRCS_mt_tests_helpers # local files ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}/base_object_tests_mt.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/interlocked_max_mt_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/map_operations_handler_mt_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mt_helpers_tests.cpp ) target_sources(igdrcl_mt_tests PRIVATE ${IGDRCL_SRCS_mt_tests_helpers}) diff --git a/opencl/test/unit_test/mt_tests/helpers/interlocked_max_mt_tests.cpp b/opencl/test/unit_test/mt_tests/helpers/interlocked_max_mt_tests.cpp deleted file mode 100644 index 7ab310fe32..0000000000 --- a/opencl/test/unit_test/mt_tests/helpers/interlocked_max_mt_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2019-2020 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#include "shared/source/helpers/interlocked_max.h" - -#include "gtest/gtest.h" - -#include - -TEST(MtTestInterlockedMaxFixture, givenCurrentPagingFenceValueWhenValueChangedThenValueIsSet) { - std::atomic currentPagingFenceValue; - std::atomic testCount; - std::atomic maxValue; - currentPagingFenceValue.store(0); - testCount.store(100); - maxValue.store(0); - int threadsCount = 8; - std::thread threads[8]; - for (int i = 0; i < threadsCount; i++) { - threads[i] = std::thread([&]() { - while (testCount-- > 0) { - uint64_t newVal = ++maxValue; - interlockedMax(currentPagingFenceValue, newVal); - } - }); - } - for (int i = 0; i < threadsCount; i++) { - threads[i].join(); - } - uint64_t endValue = currentPagingFenceValue.load(); - EXPECT_EQ(endValue, 100u); -} \ No newline at end of file diff --git a/opencl/test/unit_test/mt_tests/helpers/mt_helpers_tests.cpp b/opencl/test/unit_test/mt_tests/helpers/mt_helpers_tests.cpp new file mode 100644 index 0000000000..64a4af3bb5 --- /dev/null +++ b/opencl/test/unit_test/mt_tests/helpers/mt_helpers_tests.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2019-2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/helpers/mt_helpers.h" + +#include "gtest/gtest.h" + +TEST(MtTestInterlockedMaxFixture, givenCurrentPagingFenceValueWhenValueChangedThenValueIsSet) { + std::atomic currentPagingFenceValue; + std::atomic testCount; + std::atomic maxValue; + currentPagingFenceValue.store(0); + testCount.store(100); + maxValue.store(0); + int threadsCount = 8; + std::thread threads[8]; + for (int i = 0; i < threadsCount; i++) { + threads[i] = std::thread([&]() { + while (testCount-- > 0) { + uint64_t newVal = ++maxValue; + NEO::MultiThreadHelpers::interlockedMax(currentPagingFenceValue, newVal); + } + }); + } + for (int i = 0; i < threadsCount; i++) { + threads[i].join(); + } + uint64_t endValue = currentPagingFenceValue.load(); + EXPECT_EQ(endValue, 100u); +} + +TEST(AtomicCompareExchangeWeakSpinTest, givenCurrentEqualsExpectedWhenAtomicCompareExchangeWeakSpinCalledThenReturnsTrue) { + using TestedType = int; + std::atomic current = 0; + TestedType expected = 0; + const TestedType desired = 1; + + EXPECT_TRUE(NEO::MultiThreadHelpers::atomicCompareExchangeWeakSpin(current, expected, desired)); + + EXPECT_EQ(current, 1); + EXPECT_EQ(expected, 0); +} + +TEST(AtomicCompareExchangeWeakSpinTest, givenCurrentNotEqualsExpectedWhenAtomicCompareExchangeWeakSpinCalledThenReturnsFalseAndUpdatesExpected) { + using TestedType = int; + std::atomic current = 0; + TestedType expected = 1; + const TestedType desired = 2; + + EXPECT_FALSE(NEO::MultiThreadHelpers::atomicCompareExchangeWeakSpin(current, expected, desired)); + + EXPECT_EQ(current, 0); + EXPECT_EQ(expected, current); +} diff --git a/shared/source/helpers/CMakeLists.txt b/shared/source/helpers/CMakeLists.txt index b264fe94b8..d20a72ba06 100644 --- a/shared/source/helpers/CMakeLists.txt +++ b/shared/source/helpers/CMakeLists.txt @@ -79,7 +79,6 @@ set(NEO_CORE_HELPERS ${CMAKE_CURRENT_SOURCE_DIR}/hw_info.cpp ${CMAKE_CURRENT_SOURCE_DIR}/hw_info.h ${CMAKE_CURRENT_SOURCE_DIR}/hw_walk_order.h - ${CMAKE_CURRENT_SOURCE_DIR}/interlocked_max.h ${CMAKE_CURRENT_SOURCE_DIR}/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel_helpers.h ${CMAKE_CURRENT_SOURCE_DIR}/kmd_notify_properties.cpp @@ -96,6 +95,7 @@ set(NEO_CORE_HELPERS ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}memory_properties_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory_properties_helpers.h ${CMAKE_CURRENT_SOURCE_DIR}/memory_properties_helpers_base.inl + ${CMAKE_CURRENT_SOURCE_DIR}/mt_helpers.h ${CMAKE_CURRENT_SOURCE_DIR}/neo_driver_version.h ${CMAKE_CURRENT_SOURCE_DIR}/non_copyable_or_moveable.h ${CMAKE_CURRENT_SOURCE_DIR}/options.h diff --git a/shared/source/helpers/interlocked_max.h b/shared/source/helpers/interlocked_max.h deleted file mode 100644 index 41d257fe21..0000000000 --- a/shared/source/helpers/interlocked_max.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2019-2020 Intel Corporation - * - * SPDX-License-Identifier: MIT - * - */ - -#pragma once - -#include -template -void interlockedMax(std::atomic &dest, Type newVal) { - Type oldVal = dest; - Type maxVal = oldVal < newVal ? newVal : oldVal; - while (!std::atomic_compare_exchange_weak(&dest, &oldVal, maxVal)) { - oldVal = dest; - maxVal = oldVal < newVal ? newVal : oldVal; - } -} \ No newline at end of file diff --git a/shared/source/helpers/mt_helpers.h b/shared/source/helpers/mt_helpers.h new file mode 100644 index 0000000000..49ed7fe268 --- /dev/null +++ b/shared/source/helpers/mt_helpers.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2019-2022 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#pragma once + +#include +#include + +namespace NEO { +namespace MultiThreadHelpers { + +template +inline bool atomicCompareExchangeWeakSpin(std::atomic &destination, + Type &expectedValue, + const Type desiredValue) { + const Type currentValue = destination; + if (currentValue == expectedValue) { + if (destination.compare_exchange_weak(expectedValue, desiredValue)) { + return true; + } + } else { + expectedValue = currentValue; + } + return false; +} + +template +void interlockedMax(std::atomic &dest, Type newVal) { + Type oldVal = dest; + Type maxVal = oldVal < newVal ? newVal : oldVal; + while (!atomicCompareExchangeWeakSpin(dest, oldVal, maxVal)) { + maxVal = oldVal < newVal ? newVal : oldVal; + } +} +} // namespace MultiThreadHelpers +} // namespace NEO \ No newline at end of file diff --git a/shared/source/os_interface/linux/os_context_linux.h b/shared/source/os_interface/linux/os_context_linux.h index 486a296568..0315001637 100644 --- a/shared/source/os_interface/linux/os_context_linux.h +++ b/shared/source/os_interface/linux/os_context_linux.h @@ -7,7 +7,7 @@ #pragma once -#include "shared/source/helpers/interlocked_max.h" +#include "shared/source/helpers/mt_helpers.h" #include "shared/source/os_interface/os_context.h" #include @@ -33,7 +33,7 @@ class OsContextLinux : public OsContext { uint32_t peekTlbFlushCounter() const { return tlbFlushCounter.load(); } void setTlbFlushed(uint32_t newCounter) { - interlockedMax(lastFlushedTlbFlushCounter, newCounter); + NEO::MultiThreadHelpers::interlockedMax(lastFlushedTlbFlushCounter, newCounter); }; bool isTlbFlushRequired() const { return (tlbFlushCounter.load() > lastFlushedTlbFlushCounter.load()); diff --git a/shared/source/os_interface/windows/wddm/wddm.cpp b/shared/source/os_interface/windows/wddm/wddm.cpp index 3d75ed5568..50b3f14018 100644 --- a/shared/source/os_interface/windows/wddm/wddm.cpp +++ b/shared/source/os_interface/windows/wddm/wddm.cpp @@ -18,7 +18,7 @@ #include "shared/source/gmm_helper/resource_info.h" #include "shared/source/helpers/api_specific_config.h" #include "shared/source/helpers/heap_assigner.h" -#include "shared/source/helpers/interlocked_max.h" +#include "shared/source/helpers/mt_helpers.h" #include "shared/source/helpers/string.h" #include "shared/source/helpers/windows/gmm_callbacks.h" #include "shared/source/os_interface/hw_info_config.h" @@ -1171,7 +1171,7 @@ void Wddm::waitOnPagingFenceFromCpu() { } void Wddm::updatePagingFenceValue(uint64_t newPagingFenceValue) { - interlockedMax(currentPagingFenceValue, newPagingFenceValue); + NEO::MultiThreadHelpers::interlockedMax(currentPagingFenceValue, newPagingFenceValue); } WddmVersion Wddm::getWddmVersion() { diff --git a/shared/source/utilities/iflist.h b/shared/source/utilities/iflist.h index c9b07e4c5d..50f126c573 100644 --- a/shared/source/utilities/iflist.h +++ b/shared/source/utilities/iflist.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2021 Intel Corporation + * Copyright (C) 2018-2022 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -7,6 +7,8 @@ #pragma once +#include "shared/source/helpers/mt_helpers.h" + #include #include #include @@ -166,7 +168,7 @@ class IFList { template typename std::enable_if::type compareExchangeHead(NodeObjectType *&expected, NodeObjectType *desired) { - while (!std::atomic_compare_exchange_weak(&head, &expected, desired)) { + while (!NEO::MultiThreadHelpers::atomicCompareExchangeWeakSpin(head, expected, desired)) { ; } } diff --git a/shared/test/unit_test/helpers/CMakeLists.txt b/shared/test/unit_test/helpers/CMakeLists.txt index 5597a5348c..cbce731fb9 100644 --- a/shared/test/unit_test/helpers/CMakeLists.txt +++ b/shared/test/unit_test/helpers/CMakeLists.txt @@ -14,6 +14,7 @@ target_sources(neo_shared_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bit_helpers_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/blit_commands_helper_tests_gen12lp.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/cache_policy_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/cmd_buffer_validator_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/compiler_hw_info_config_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/debug_helpers_tests.cpp @@ -43,7 +44,6 @@ target_sources(neo_shared_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/string_to_hash_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_debug_variables.inl ${CMAKE_CURRENT_SOURCE_DIR}/test_hw_info_config.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/cache_policy_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/timestamp_packet_tests.cpp )