intel/llvm
1
0
Fork 0
mirror of https://github.com/intel/llvm.git synced 2026-01-20 01:58:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "[libc] Separate memcpy implementations per arch"

Browse Source 
Reverting as it broke downstream users.
This reverts commit 26541d4176.
This commit is contained in:
Guillaume Chatelet
2023-02-14 16:14:51 +00:00
parent bf987e108f
commit 9a8d64a66a
5 changed files with 100 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
libc/src/string/memory_utils/CMakeLists.txt
View File

@@ -2,7 +2,6 @@
add_header_library(
memory_utils
HDRS
aarch64/memcpy_implementations.h
bcmp_implementations.h
bzero_implementations.h
memcmp_implementations.h
@@ -14,12 +13,11 @@ add_header_library(
op_generic.h
op_x86.h
utils.h
x86_64/memcpy_implementations.h
DEPS
libc.src.__support.common
libc.src.__support.CPP.bit
libc.src.__support.CPP.cstddef
libc.src.__support.CPP.type_traits
libc.src.__support.macros.config
libc.src.__support.macros.optimization
)

48
libc/src/string/memory_utils/aarch64/memcpy_implementations.h
View File

@@ -1,48 +0,0 @@
//===-- Memcpy implementation for aarch64 -----------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
#include "src/__support/macros/config.h" // LIBC_INLINE
#include "src/string/memory_utils/op_builtin.h"
#include "src/string/memory_utils/utils.h"
#include <stddef.h> // size_t
namespace __llvm_libc {
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
if (count == 0)
return;
if (count == 1)
return builtin::Memcpy<1>::block(dst, src);
if (count == 2)
return builtin::Memcpy<2>::block(dst, src);
if (count == 3)
return builtin::Memcpy<3>::block(dst, src);
if (count == 4)
return builtin::Memcpy<4>::block(dst, src);
if (count < 8)
return builtin::Memcpy<4>::head_tail(dst, src, count);
if (count < 16)
return builtin::Memcpy<8>::head_tail(dst, src, count);
if (count < 32)
return builtin::Memcpy<16>::head_tail(dst, src, count);
if (count < 64)
return builtin::Memcpy<32>::head_tail(dst, src, count);
if (count < 128)
return builtin::Memcpy<64>::head_tail(dst, src, count);
builtin::Memcpy<16>::block(dst, src);
align_to_next_boundary<16, Arg::Src>(dst, src, count);
return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
}
} // namespace __llvm_libc
#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H

106
libc/src/string/memory_utils/memcpy_implementations.h
View File

@@ -9,19 +9,17 @@
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
#include "src/__support/macros/config.h" // LIBC_INLINE
#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
#include "src/__support/common.h"
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY LIBC_LOOP_NOUNROLL
#include "src/__support/macros/properties/architectures.h"
#include "src/string/memory_utils/op_aarch64.h"
#include "src/string/memory_utils/op_builtin.h"
#include "src/string/memory_utils/op_generic.h"
#include "src/string/memory_utils/op_x86.h"
#include "src/string/memory_utils/utils.h"
#include <stddef.h> // size_t
#if defined(LIBC_TARGET_ARCH_IS_X86)
#include "src/string/memory_utils/x86_64/memcpy_implementations.h"
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
#include "src/string/memory_utils/aarch64/memcpy_implementations.h"
#endif
namespace __llvm_libc {
[[maybe_unused]] LIBC_INLINE void
@@ -32,6 +30,98 @@ inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src,
builtin::Memcpy<1>::block(dst + offset, src + offset);
}
#if defined(LIBC_TARGET_ARCH_IS_X86)
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
if (count == 0)
return;
if (count == 1)
return builtin::Memcpy<1>::block(dst, src);
if (count == 2)
return builtin::Memcpy<2>::block(dst, src);
if (count == 3)
return builtin::Memcpy<3>::block(dst, src);
if (count == 4)
return builtin::Memcpy<4>::block(dst, src);
if (count < 8)
return builtin::Memcpy<4>::head_tail(dst, src, count);
if (count < 16)
return builtin::Memcpy<8>::head_tail(dst, src, count);
if (count < 32)
return builtin::Memcpy<16>::head_tail(dst, src, count);
if (count < 64)
return builtin::Memcpy<32>::head_tail(dst, src, count);
if (count < 128)
return builtin::Memcpy<64>::head_tail(dst, src, count);
if (x86::kAvx && count < 256)
return builtin::Memcpy<128>::head_tail(dst, src, count);
builtin::Memcpy<32>::block(dst, src);
align_to_next_boundary<32, Arg::Dst>(dst, src, count);
static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
}
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
CPtr __restrict src, size_t count) {
// Whether to use rep;movsb exclusively, not at all, or only above a certain
// threshold.
#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
#endif
#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
static constexpr size_t kRepMovsbThreshold =
LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
if constexpr (kRepMovsbThreshold == 0) {
return x86::Memcpy::repmovsb(dst, src, count);
} else if constexpr (kRepMovsbThreshold == size_t(-1)) {
return inline_memcpy_x86(dst, src, count);
} else {
if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
return x86::Memcpy::repmovsb(dst, src, count);
else
return inline_memcpy_x86(dst, src, count);
}
}
#endif // defined(LIBC_TARGET_ARCH_IS_X86)
#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
if (count == 0)
return;
if (count == 1)
return builtin::Memcpy<1>::block(dst, src);
if (count == 2)
return builtin::Memcpy<2>::block(dst, src);
if (count == 3)
return builtin::Memcpy<3>::block(dst, src);
if (count == 4)
return builtin::Memcpy<4>::block(dst, src);
if (count < 8)
return builtin::Memcpy<4>::head_tail(dst, src, count);
if (count < 16)
return builtin::Memcpy<8>::head_tail(dst, src, count);
if (count < 32)
return builtin::Memcpy<16>::head_tail(dst, src, count);
if (count < 64)
return builtin::Memcpy<32>::head_tail(dst, src, count);
if (count < 128)
return builtin::Memcpy<64>::head_tail(dst, src, count);
builtin::Memcpy<16>::block(dst, src);
align_to_next_boundary<16, Arg::Src>(dst, src, count);
return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
}
#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)
LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src,
size_t count) {
using namespace __llvm_libc::builtin;

84
libc/src/string/memory_utils/x86_64/memcpy_implementations.h
View File

@@ -1,84 +0,0 @@
//===-- Memcpy implementation for x86_64 ------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
#include "src/__support/macros/config.h" // LIBC_INLINE
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
#include "src/string/memory_utils/op_builtin.h"
#include "src/string/memory_utils/op_x86.h"
#include "src/string/memory_utils/utils.h"
#include <stddef.h> // size_t
namespace __llvm_libc {
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
if (count == 0)
return;
if (count == 1)
return builtin::Memcpy<1>::block(dst, src);
if (count == 2)
return builtin::Memcpy<2>::block(dst, src);
if (count == 3)
return builtin::Memcpy<3>::block(dst, src);
if (count == 4)
return builtin::Memcpy<4>::block(dst, src);
if (count < 8)
return builtin::Memcpy<4>::head_tail(dst, src, count);
if (count < 16)
return builtin::Memcpy<8>::head_tail(dst, src, count);
if (count < 32)
return builtin::Memcpy<16>::head_tail(dst, src, count);
if (count < 64)
return builtin::Memcpy<32>::head_tail(dst, src, count);
if (count < 128)
return builtin::Memcpy<64>::head_tail(dst, src, count);
if (x86::kAvx && count < 256)
return builtin::Memcpy<128>::head_tail(dst, src, count);
builtin::Memcpy<32>::block(dst, src);
align_to_next_boundary<32, Arg::Dst>(dst, src, count);
static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
}
[[maybe_unused]] LIBC_INLINE void
inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
CPtr __restrict src, size_t count) {
// Whether to use rep;movsb exclusively, not at all, or only above a certain
// threshold.
#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
#endif
#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
static constexpr size_t kRepMovsbThreshold =
LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
if constexpr (kRepMovsbThreshold == 0) {
return x86::Memcpy::repmovsb(dst, src, count);
} else if constexpr (kRepMovsbThreshold == size_t(-1)) {
return inline_memcpy_x86(dst, src, count);
} else {
if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
return x86::Memcpy::repmovsb(dst, src, count);
else
return inline_memcpy_x86(dst, src, count);
}
}
} // namespace __llvm_libc
#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H

3
utils/bazel/llvm-project-overlay/libc/BUILD.bazel
View File

@@ -1404,14 +1404,13 @@ libc_support_library(
"src/string/memory_utils/bzero_implementations.h",
"src/string/memory_utils/memcmp_implementations.h",
"src/string/memory_utils/memcpy_implementations.h",
"src/string/memory_utils/aarch64/memcpy_implementations.h",
"src/string/memory_utils/x86_64/memcpy_implementations.h",
"src/string/memory_utils/memmove_implementations.h",
"src/string/memory_utils/memset_implementations.h",
"src/string/memory_utils/strcmp_implementations.h",
"src/string/memory_utils/strstr_implementations.h",
],
deps = [
":__support_common",
":__support_cpp_array",
":__support_cpp_bit",
":__support_cpp_cstddef",