diff options
Diffstat (limited to 'sysdeps')
| -rw-r--r-- | sysdeps/s390/ifunc-memcpy.h | 23 | ||||
| -rw-r--r-- | sysdeps/s390/memcpy-z900.S | 55 | ||||
| -rw-r--r-- | sysdeps/s390/memmove.c | 16 | ||||
| -rw-r--r-- | sysdeps/s390/multiarch/ifunc-impl-list.c | 5 | ||||
| -rw-r--r-- | sysdeps/s390/multiarch/ifunc-resolve.h | 5 |
5 files changed, 99 insertions, 5 deletions
diff --git a/sysdeps/s390/ifunc-memcpy.h b/sysdeps/s390/ifunc-memcpy.h index b83ae73508..1badb30ed8 100644 --- a/sysdeps/s390/ifunc-memcpy.h +++ b/sysdeps/s390/ifunc-memcpy.h @@ -44,7 +44,7 @@ #endif #if defined SHARED && defined USE_MULTIARCH && IS_IN (libc) \ - && ! defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT + && ! defined HAVE_S390_MIN_ARCH13_ZARCH_ASM_SUPPORT # define HAVE_MEMMOVE_IFUNC 1 #else # define HAVE_MEMMOVE_IFUNC 0 @@ -56,14 +56,27 @@ # define HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT 0 #endif -#if defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT +#ifdef HAVE_S390_ARCH13_ASM_SUPPORT +# define HAVE_MEMMOVE_IFUNC_AND_ARCH13_SUPPORT HAVE_MEMMOVE_IFUNC +#else +# define HAVE_MEMMOVE_IFUNC_AND_ARCH13_SUPPORT 0 +#endif + +#if defined HAVE_S390_MIN_ARCH13_ZARCH_ASM_SUPPORT +# define MEMMOVE_DEFAULT MEMMOVE_ARCH13 +# define HAVE_MEMMOVE_C 0 +# define HAVE_MEMMOVE_Z13 0 +# define HAVE_MEMMOVE_ARCH13 1 +#elif defined HAVE_S390_MIN_Z13_ZARCH_ASM_SUPPORT # define MEMMOVE_DEFAULT MEMMOVE_Z13 # define HAVE_MEMMOVE_C 0 # define HAVE_MEMMOVE_Z13 1 +# define HAVE_MEMMOVE_ARCH13 HAVE_MEMMOVE_IFUNC_AND_ARCH13_SUPPORT #else # define MEMMOVE_DEFAULT MEMMOVE_C # define HAVE_MEMMOVE_C 1 # define HAVE_MEMMOVE_Z13 HAVE_MEMMOVE_IFUNC_AND_VX_SUPPORT +# define HAVE_MEMMOVE_ARCH13 HAVE_MEMMOVE_IFUNC_AND_ARCH13_SUPPORT #endif #if HAVE_MEMCPY_Z900_G5 @@ -101,3 +114,9 @@ #else # define MEMMOVE_Z13 NULL #endif + +#if HAVE_MEMMOVE_ARCH13 +# define MEMMOVE_ARCH13 __memmove_arch13 +#else +# define MEMMOVE_ARCH13 NULL +#endif diff --git a/sysdeps/s390/memcpy-z900.S b/sysdeps/s390/memcpy-z900.S index 90d5f7becc..307332fcf9 100644 --- a/sysdeps/s390/memcpy-z900.S +++ b/sysdeps/s390/memcpy-z900.S @@ -277,6 +277,61 @@ ENTRY(MEMMOVE_Z13) END(MEMMOVE_Z13) #endif /* HAVE_MEMMOVE_Z13 */ +#if HAVE_MEMMOVE_ARCH13 +ENTRY(MEMMOVE_ARCH13) + .machine "arch13" + .machinemode "zarch_nohighgprs" +# if ! defined __s390x__ + /* Note: The 31bit dst and src pointers are prefixed with zeroes. */ + llgfr %r4,%r4 + llgfr %r3,%r3 + llgfr %r2,%r2 +# endif /* ! defined __s390x__ */ + sgrk %r5,%r2,%r3 + aghik %r0,%r4,-1 /* Both vstl and mvcrl needs highest index. */ + clgijh %r4,16,.L_MEMMOVE_ARCH13_LARGE +.L_MEMMOVE_ARCH13_SMALL: + jl .L_MEMMOVE_ARCH13_END /* Return if len was zero (cc of aghik). */ + /* Store up to 16 bytes with vll/vstl (needs highest index). */ + vll %v16,%r0,0(%r3) + vstl %v16,%r0,0(%r2) +.L_MEMMOVE_ARCH13_END: + br %r14 +.L_MEMMOVE_ARCH13_LARGE: + lgr %r1,%r2 /* For memcpy: r1: Use as dest ; r2: Return dest */ + /* The unsigned comparison (dst - src >= len) determines if we can + execute the forward case with memcpy. */ +#if ! HAVE_MEMCPY_Z196 +# error The arch13 variant of memmove needs the z196 variant of memcpy! +#endif + /* Backward case. */ + clgrjhe %r5,%r4,.L_Z196_start2 + clgijh %r0,255,.L_MEMMOVE_ARCH13_LARGER_256B + /* Move up to 256bytes with mvcrl (move right to left). */ + mvcrl 0(%r1),0(%r3) /* Move (r0 + 1) bytes from r3 to r1. */ + br %r14 +.L_MEMMOVE_ARCH13_LARGER_256B: + /* First move the "remaining" block of up to 256 bytes at the end of + src/dst buffers. Then move blocks of 256bytes in a loop starting + with the block at the end. + (If src/dst pointers are aligned e.g. to 256 bytes, then the pointers + passed to mvcrl instructions are aligned, too) */ + risbgn %r5,%r0,8,128+63,56 /* r5 = r0 / 256 */ + risbgn %r0,%r0,56,128+63,0 /* r0 = r0 & 0xFF */ + slgr %r4,%r0 + lay %r1,-1(%r4,%r1) + lay %r3,-1(%r4,%r3) + mvcrl 0(%r1),0(%r3) /* Move (r0 + 1) bytes from r3 to r1. */ + lghi %r0,255 /* Always copy 256 bytes in the loop below! */ +.L_MEMMOVE_ARCH13_LARGE_256B_LOOP: + aghi %r1,-256 + aghi %r3,-256 + mvcrl 0(%r1),0(%r3) /* Move (r0 + 1) bytes from r3 to r1. */ + brctg %r5,.L_MEMMOVE_ARCH13_LARGE_256B_LOOP + br %r14 +END(MEMMOVE_ARCH13) +#endif /* HAVE_MEMMOVE_ARCH13 */ + #if ! HAVE_MEMCPY_IFUNC /* If we don't use ifunc, define an alias for mem[p]cpy here. Otherwise see sysdeps/s390/mem[p]cpy.c. */ diff --git a/sysdeps/s390/memmove.c b/sysdeps/s390/memmove.c index fd4da377a3..fb6b69ae2f 100644 --- a/sysdeps/s390/memmove.c +++ b/sysdeps/s390/memmove.c @@ -36,9 +36,19 @@ extern __typeof (__redirect_memmove) MEMMOVE_C attribute_hidden; extern __typeof (__redirect_memmove) MEMMOVE_Z13 attribute_hidden; # endif +# if HAVE_MEMMOVE_ARCH13 +extern __typeof (__redirect_memmove) MEMMOVE_ARCH13 attribute_hidden; +# endif + s390_libc_ifunc_expr (__redirect_memmove, memmove, - (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX)) - ? MEMMOVE_Z13 - : MEMMOVE_DEFAULT + ({ + s390_libc_ifunc_expr_stfle_init (); + (HAVE_MEMMOVE_ARCH13 + && S390_IS_ARCH13_MIE3 (stfle_bits)) + ? MEMMOVE_ARCH13 + : (HAVE_MEMMOVE_Z13 && (hwcap & HWCAP_S390_VX)) + ? MEMMOVE_Z13 + : MEMMOVE_DEFAULT; + }) ) #endif diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c index b54c52af36..d742d66a6a 100644 --- a/sysdeps/s390/multiarch/ifunc-impl-list.c +++ b/sysdeps/s390/multiarch/ifunc-impl-list.c @@ -169,6 +169,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, #if HAVE_MEMMOVE_IFUNC IFUNC_IMPL (i, name, memmove, +# if HAVE_MEMMOVE_ARCH13 + IFUNC_IMPL_ADD (array, i, memmove, + S390_IS_ARCH13_MIE3 (stfle_bits), + MEMMOVE_ARCH13) +# endif # if HAVE_MEMMOVE_Z13 IFUNC_IMPL_ADD (array, i, memmove, dl_hwcap & HWCAP_S390_VX, MEMMOVE_Z13) diff --git a/sysdeps/s390/multiarch/ifunc-resolve.h b/sysdeps/s390/multiarch/ifunc-resolve.h index b833dfef28..743de9e591 100644 --- a/sysdeps/s390/multiarch/ifunc-resolve.h +++ b/sysdeps/s390/multiarch/ifunc-resolve.h @@ -22,6 +22,11 @@ #define S390_STFLE_BITS_Z10 34 /* General instructions extension */ #define S390_STFLE_BITS_Z196 45 /* Distinct operands, pop ... */ +#define S390_STFLE_BITS_ARCH13_MIE3 61 /* Miscellaneous-Instruction-Extensions + Facility 3, e.g. mvcrl. */ + +#define S390_IS_ARCH13_MIE3(STFLE_BITS) \ + ((STFLE_BITS & (1ULL << (63 - S390_STFLE_BITS_ARCH13_MIE3))) != 0) #define S390_IS_Z196(STFLE_BITS) \ ((STFLE_BITS & (1ULL << (63 - S390_STFLE_BITS_Z196))) != 0) |