intel/llvm
1
0
Fork 0
mirror of https://github.com/intel/llvm.git synced 2026-01-13 19:08:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[libc][NFC] refactor Cortex memcpy code" (#149035)

Browse Source 
Reverts llvm/llvm-project#148204

`libc-arm32-qemu-debian-dbg` is failing, reverting and investigating
This commit is contained in:
Guillaume Chatelet
2025-07-16 10:29:11 +02:00
committed by GitHub
parent 1c223829b8
commit 999a8fb812
4 changed files with 99 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
libc/src/string/memory_utils/CMakeLists.txt
View File

@@ -7,7 +7,6 @@ add_header_library(
aarch64/inline_memcpy.h
aarch64/inline_memmove.h
aarch64/inline_memset.h
arm/common.h
arm/inline_memcpy.h
generic/aligned_access.h
generic/byte_per_byte.h

52
libc/src/string/memory_utils/arm/common.h
View File

@@ -1,52 +0,0 @@
//===-- Common constants and defines for arm --------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_COMMON_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_COMMON_H
#include "src/__support/macros/attributes.h" // LIBC_INLINE_VAR
#include "src/string/memory_utils/utils.h" // CPtr, Ptr, distance_to_align
#include <stddef.h> // size_t
// https://libc.llvm.org/compiler_support.html
// Support for [[likely]] / [[unlikely]]
// [X] GCC 12.2
// [X] Clang 12
// [ ] Clang 11
#define LIBC_ATTR_LIKELY [[likely]]
#define LIBC_ATTR_UNLIKELY [[unlikely]]
#if defined(LIBC_COMPILER_IS_CLANG)
#if LIBC_COMPILER_CLANG_VER < 1200
#undef LIBC_ATTR_LIKELY
#undef LIBC_ATTR_UNLIKELY
#define LIBC_ATTR_LIKELY
#define LIBC_ATTR_UNLIKELY
#endif
#endif
namespace LIBC_NAMESPACE_DECL {
LIBC_INLINE_VAR constexpr size_t kWordSize = sizeof(uint32_t);
enum class AssumeAccess { kUnknown, kAligned };
enum class BlockOp { kFull, kByWord };
LIBC_INLINE auto misaligned(CPtr ptr) {
return distance_to_align_down<kWordSize>(ptr);
}
LIBC_INLINE CPtr bitwise_or(CPtr a, CPtr b) {
return cpp::bit_cast<CPtr>(cpp::bit_cast<uintptr_t>(a) |
cpp::bit_cast<uintptr_t>(b));
}
} // namespace LIBC_NAMESPACE_DECL
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_COMMON_H

197
libc/src/string/memory_utils/arm/inline_memcpy.h
View File

@@ -5,56 +5,63 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// The functions defined in this file give approximate code size. These sizes
// assume the following configuration options:
// - LIBC_CONF_KEEP_FRAME_POINTER = false
// - LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR = false
// - LIBC_ADD_NULL_CHECKS = false
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
#include "src/__support/macros/attributes.h" // LIBC_INLINE
#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
#include "src/string/memory_utils/arm/common.h" // LIBC_ATTR_LIKELY, LIBC_ATTR_UNLIKELY
#include "src/string/memory_utils/utils.h" // memcpy_inline, distance_to_align
#include <stddef.h> // size_t
// https://libc.llvm.org/compiler_support.html
// Support for [[likely]] / [[unlikely]]
// [X] GCC 12.2
// [X] Clang 12
// [ ] Clang 11
#define LIBC_ATTR_LIKELY [[likely]]
#define LIBC_ATTR_UNLIKELY [[unlikely]]
#if defined(LIBC_COMPILER_IS_CLANG)
#if LIBC_COMPILER_CLANG_VER < 1200
#undef LIBC_ATTR_LIKELY
#undef LIBC_ATTR_UNLIKELY
#define LIBC_ATTR_LIKELY
#define LIBC_ATTR_UNLIKELY
#endif
#endif
namespace LIBC_NAMESPACE_DECL {
namespace {
// Performs a copy of `bytes` byte from `src` to `dst`. This function has the
// semantics of `memcpy` where `src` and `dst` are `__restrict`. The compiler is
// free to use whatever instruction is best for the size and assumed access.
template <size_t bytes, AssumeAccess access>
LIBC_INLINE void copy(void *dst, const void *src) {
if constexpr (access == AssumeAccess::kAligned) {
constexpr size_t alignment = bytes > kWordSize ? kWordSize : bytes;
memcpy_inline<bytes>(assume_aligned<alignment>(dst),
assume_aligned<alignment>(src));
} else if constexpr (access == AssumeAccess::kUnknown) {
memcpy_inline<bytes>(dst, src);
} else {
static_assert(false);
}
}
LIBC_INLINE_VAR constexpr size_t kWordSize = sizeof(uint32_t);
template <size_t bytes, BlockOp block_op = BlockOp::kFull,
AssumeAccess access = AssumeAccess::kUnknown>
LIBC_INLINE void copy_block_and_bump_pointers(Ptr &dst, CPtr &src) {
if constexpr (block_op == BlockOp::kFull) {
copy<bytes, access>(dst, src);
} else if constexpr (block_op == BlockOp::kByWord) {
enum Strategy {
ForceWordLdStChain,
AssumeWordAligned,
AssumeUnaligned,
};
template <size_t bytes, Strategy strategy = AssumeUnaligned>
LIBC_INLINE void copy_and_bump_pointers(Ptr &dst, CPtr &src) {
if constexpr (strategy == AssumeUnaligned) {
memcpy_inline<bytes>(assume_aligned<1>(dst), assume_aligned<1>(src));
} else if constexpr (strategy == AssumeWordAligned) {
static_assert(bytes >= kWordSize);
memcpy_inline<bytes>(assume_aligned<kWordSize>(dst),
assume_aligned<kWordSize>(src));
} else if constexpr (strategy == ForceWordLdStChain) {
// We restrict loads/stores to 4 byte to prevent the use of load/store
// multiple (LDM, STM) and load/store double (LDRD, STRD).
// multiple (LDM, STM) and load/store double (LDRD, STRD). First, they may
// fault (see notes below) and second, they use more registers which in turn
// adds push/pop instructions in the hot path.
static_assert((bytes % kWordSize == 0) && (bytes >= kWordSize));
LIBC_LOOP_UNROLL
for (size_t offset = 0; offset < bytes; offset += kWordSize) {
copy<kWordSize, access>(dst + offset, src + offset);
for (size_t i = 0; i < bytes / kWordSize; ++i) {
const size_t offset = i * kWordSize;
memcpy_inline<kWordSize>(dst + offset, src + offset);
}
} else {
static_assert(false, "Invalid BlockOp");
}
// In the 1, 2, 4 byte copy case, the compiler can fold pointer offsetting
// into the load/store instructions.
@@ -65,27 +72,39 @@ LIBC_INLINE void copy_block_and_bump_pointers(Ptr &dst, CPtr &src) {
src += bytes;
}
template <size_t bytes, BlockOp block_op, AssumeAccess access>
LIBC_INLINE void consume_by_block(Ptr &dst, CPtr &src, size_t &size) {
LIBC_LOOP_NOUNROLL
for (size_t i = 0; i < size / bytes; ++i)
copy_block_and_bump_pointers<bytes, block_op, access>(dst, src);
size %= bytes;
}
[[maybe_unused]] LIBC_INLINE void
copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
LIBC_INLINE void copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src,
const size_t size) {
LIBC_LOOP_NOUNROLL
for (size_t i = 0; i < size; ++i)
*dst++ = *src++;
}
template <size_t block_size, Strategy strategy>
LIBC_INLINE void copy_blocks_and_update_args(Ptr &dst, CPtr &src,
size_t &size) {
LIBC_LOOP_NOUNROLL
for (size_t i = 0; i < size / block_size; ++i)
copy_and_bump_pointers<block_size, strategy>(dst, src);
// Update `size` once at the end instead of once per iteration.
size %= block_size;
}
LIBC_INLINE CPtr bitwise_or(CPtr a, CPtr b) {
return cpp::bit_cast<CPtr>(cpp::bit_cast<uintptr_t>(a) |
cpp::bit_cast<uintptr_t>(b));
}
LIBC_INLINE auto misaligned(CPtr a) {
return distance_to_align_down<kWordSize>(a);
}
} // namespace
// Implementation for Cortex-M0, M0+, M1 cores that do not allow for unaligned
// loads/stores. It compiles down to 208 bytes when used through `memcpy` that
// also needs to return the `dst` ptr.
// Note:
// Implementation for Cortex-M0, M0+, M1.
// Notes:
// - It compiles down to 196 bytes, but 220 bytes when used through `memcpy`
// that also needs to return the `dst` ptr.
// - These cores do not allow for unaligned loads/stores.
// - When `src` and `dst` are coaligned, we start by aligning them and perform
// bulk copies. We let the compiler know the pointers are aligned so it can
// use load/store multiple (LDM, STM). This significantly increase throughput
@@ -106,18 +125,9 @@ copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
if (src_alignment == 0)
LIBC_ATTR_LIKELY {
// Both `src` and `dst` are now word-aligned.
// We first copy by blocks of 64 bytes, the compiler will use 4
// load/store multiple (LDM, STM), each of 4 words. This requires more
// registers so additional push/pop are needed but the speedup is worth
// it.
consume_by_block<64, BlockOp::kFull, AssumeAccess::kAligned>(dst, src,
size);
// Then we use blocks of 4 word load/store.
consume_by_block<16, BlockOp::kByWord, AssumeAccess::kAligned>(dst, src,
size);
// Then we use word by word copy.
consume_by_block<4, BlockOp::kByWord, AssumeAccess::kAligned>(dst, src,
size);
copy_blocks_and_update_args<64, AssumeWordAligned>(dst, src, size);
copy_blocks_and_update_args<16, AssumeWordAligned>(dst, src, size);
copy_blocks_and_update_args<4, AssumeWordAligned>(dst, src, size);
}
else {
// `dst` is aligned but `src` is not.
@@ -128,7 +138,7 @@ copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
src_alignment == 2
? load_aligned<uint32_t, uint16_t, uint16_t>(src)
: load_aligned<uint32_t, uint8_t, uint16_t, uint8_t>(src);
copy<kWordSize, AssumeAccess::kAligned>(dst, &value);
memcpy_inline<kWordSize>(assume_aligned<kWordSize>(dst), &value);
dst += kWordSize;
src += kWordSize;
size -= kWordSize;
@@ -141,8 +151,17 @@ copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
}
// Implementation for Cortex-M3, M4, M7, M23, M33, M35P, M52 with hardware
// support for unaligned loads and stores. It compiles down to 272 bytes when
// used through `memcpy` that also needs to return the `dst` ptr.
// support for unaligned loads and stores.
// Notes:
// - It compiles down to 266 bytes.
// - `dst` and `src` are not `__restrict` to prevent the compiler from
// reordering loads/stores.
// - We keep state variables to a strict minimum to keep everything in the free
// registers and prevent costly push / pop.
// - If unaligned single loads/stores to normal memory are supported, unaligned
// accesses for load/store multiple (LDM, STM) and load/store double (LDRD,
// STRD) instructions are generally not supported and will still fault so we
// make sure to restrict unrolling to word loads/stores.
[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_mid_end(Ptr dst, CPtr src,
size_t size) {
if (misaligned(bitwise_or(src, dst)))
@@ -150,60 +169,38 @@ copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
if (size < 8)
LIBC_ATTR_UNLIKELY {
if (size & 1)
copy_block_and_bump_pointers<1>(dst, src);
copy_and_bump_pointers<1>(dst, src);
if (size & 2)
copy_block_and_bump_pointers<2>(dst, src);
copy_and_bump_pointers<2>(dst, src);
if (size & 4)
copy_block_and_bump_pointers<4>(dst, src);
copy_and_bump_pointers<4>(dst, src);
return;
}
if (misaligned(src))
LIBC_ATTR_UNLIKELY {
const size_t offset = distance_to_align_up<kWordSize>(dst);
if (offset & 1)
copy_block_and_bump_pointers<1>(dst, src);
copy_and_bump_pointers<1>(dst, src);
if (offset & 2)
copy_block_and_bump_pointers<2>(dst, src);
copy_and_bump_pointers<2>(dst, src);
size -= offset;
}
}
// `dst` and `src` are not necessarily both aligned at that point but this
// implementation assumes hardware support for unaligned loads and stores so
// it is still fast to perform unrolled word by word copy. Note that wider
// accesses through the use of load/store multiple (LDM, STM) and load/store
// double (LDRD, STRD) instructions are generally not supported and can fault.
// By forcing decomposition of 64 bytes copy into word by word copy, the
// compiler can use the first load to prefetch memory:
// ldr r3, [r1, #64]! <- prefetch next cache line
// str r3, [r0]
// ldr r3, [r1, #0x4]
// str r3, [r0, #0x4]
// ...
// ldr r3, [r1, #0x3c]
// str r3, [r0, #0x3c]
// This is a bit detrimental for sizes between 64 and 256 (less than 10%
// penalty) but the prefetch yields better throughput for larger copies.
consume_by_block<64, BlockOp::kByWord, AssumeAccess::kUnknown>(dst, src,
size);
consume_by_block<16, BlockOp::kByWord, AssumeAccess::kUnknown>(dst, src,
size);
consume_by_block<4, BlockOp::kByWord, AssumeAccess::kUnknown>(dst, src, size);
copy_blocks_and_update_args<64, ForceWordLdStChain>(dst, src, size);
copy_blocks_and_update_args<16, ForceWordLdStChain>(dst, src, size);
copy_blocks_and_update_args<4, AssumeUnaligned>(dst, src, size);
if (size & 1)
copy_block_and_bump_pointers<1>(dst, src);
copy_and_bump_pointers<1>(dst, src);
if (size & 2)
copy_block_and_bump_pointers<2>(dst, src);
LIBC_ATTR_UNLIKELY
copy_and_bump_pointers<2>(dst, src);
}
[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm(Ptr dst, CPtr src,
[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm(void *__restrict dst_,
const void *__restrict src_,
size_t size) {
// The compiler performs alias analysis and is able to prove that `dst` and
// `src` do not alias by propagating the `__restrict` keyword from the
// `memcpy` prototype. This allows the compiler to merge consecutive
// load/store (LDR, STR) instructions generated in
// `copy_block_and_bump_pointers` with `BlockOp::kByWord` into load/store
// double (LDRD, STRD) instructions, this is is undesirable so we prevent the
// compiler from inferring `__restrict` with the following line.
asm volatile("" : "+r"(dst), "+r"(src));
Ptr dst = cpp::bit_cast<Ptr>(dst_);
CPtr src = cpp::bit_cast<CPtr>(src_);
#ifdef __ARM_FEATURE_UNALIGNED
return inline_memcpy_arm_mid_end(dst, src, size);
#else
@@ -213,4 +210,8 @@ copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src, size_t size) {
} // namespace LIBC_NAMESPACE_DECL
// Cleanup local macros
#undef LIBC_ATTR_LIKELY
#undef LIBC_ATTR_UNLIKELY
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H

1
utils/bazel/llvm-project-overlay/libc/BUILD.bazel
View File

@@ -4448,7 +4448,6 @@ libc_support_library(
"src/string/memory_utils/aarch64/inline_memcpy.h",
"src/string/memory_utils/aarch64/inline_memmove.h",
"src/string/memory_utils/aarch64/inline_memset.h",
"src/string/memory_utils/arm/common.h",
"src/string/memory_utils/arm/inline_memcpy.h",
"src/string/memory_utils/generic/aligned_access.h",
"src/string/memory_utils/generic/byte_per_byte.h",