diff options
| author | H.J. Lu <hjl.tools@gmail.com> | 2021-01-12 05:15:49 -0800 |
|---|---|---|
| committer | H.J. Lu <hjl.tools@gmail.com> | 2021-01-12 07:04:55 -0800 |
| commit | 56fbd0ba218c59a1cc8c29464f153d45a4c887c0 (patch) | |
| tree | 5756e6ddd05ef6039c8c1363e1c364ce0d381702 | |
| parent | d81114e0aaedc30d82956d3a21ae6777b3fbba3c (diff) | |
| download | glibc-56fbd0ba218c59a1cc8c29464f153d45a4c887c0.tar glibc-56fbd0ba218c59a1cc8c29464f153d45a4c887c0.tar.gz glibc-56fbd0ba218c59a1cc8c29464f153d45a4c887c0.tar.bz2 glibc-56fbd0ba218c59a1cc8c29464f153d45a4c887c0.zip | |
x86-64: Avoid rep movsb with short distance [BZ #27130]
When copying with "rep movsb", if the distance between source and
destination is N*4GB + [1..63] with N >= 0, performance may be very
slow. This patch updates memmove-vec-unaligned-erms.S for AVX and
AVX512 versions with the distance in RCX:
cmpl $63, %ecx
// Don't use "rep movsb" if ECX <= 63
jbe L(Don't use rep movsb")
Use "rep movsb"
Benchtests data with bench-memcpy, bench-memcpy-large, bench-memcpy-random
and bench-memcpy-walk on Skylake, Ice Lake and Tiger Lake show that its
performance impact is within noise range as "rep movsb" is only used for
data size >= 4KB.
(cherry picked from commit 3ec5d83d2a237d39e7fd6ef7a0bc8ac4c171a4a5)
| -rw-r--r-- | NEWS | 1 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S | 21 |
2 files changed, 22 insertions, 0 deletions
@@ -76,6 +76,7 @@ The following bugs are resolved with this release: [25414] 'glob' use-after-free bug (CVE-2020-1752) [25423] Array overflow in backtrace on powerpc [25933] Off by one error in __strncmp_avx2 + [27130] "rep movsb" performance issue Security related changes: diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S index 9bab1147d5..5aaadc233f 100644 --- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S @@ -67,6 +67,13 @@ # define REP_MOVSB_THRESHOLD (2048 * (VEC_SIZE / 16)) #endif +/* Avoid short distance rep movsb only with non-SSE vector. */ +#ifndef AVOID_SHORT_DISTANCE_REP_MOVSB +# define AVOID_SHORT_DISTANCE_REP_MOVSB (VEC_SIZE > 16) +#else +# define AVOID_SHORT_DISTANCE_REP_MOVSB 0 +#endif + #ifndef PREFETCH # define PREFETCH(addr) prefetcht0 addr #endif @@ -257,7 +264,21 @@ L(movsb): # error Unsupported REP_MOVSB_THRESHOLD and VEC_SIZE! # endif jb L(more_8x_vec_backward) +# if AVOID_SHORT_DISTANCE_REP_MOVSB + movq %rdi, %rcx + subq %rsi, %rcx + jmp 2f +# endif 1: +# if AVOID_SHORT_DISTANCE_REP_MOVSB + movq %rsi, %rcx + subq %rdi, %rcx +2: +/* Avoid "rep movsb" if RCX, the distance between source and destination, + is N*4GB + [1..63] with N >= 0. */ + cmpl $63, %ecx + jbe L(more_2x_vec) /* Avoid "rep movsb" if ECX <= 63. */ +# endif mov %RDX_LP, %RCX_LP rep movsb L(nop): |