diff options
21 files changed, 450 insertions, 474 deletions
@@ -1,3 +1,35 @@ +2017-06-14 H.J. Lu <hongjiu.lu@intel.com> + Erich Elsen <eriche@google.com> + Adhemerval Zanella <adhemerval.zanella@linaro.org> + + * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add + memmove-sse2-unaligned-erms, memcpy_chk-nonshared, + mempcpy_chk-nonshared and memmove_chk-nonshared. + * sysdeps/x86_64/multiarch/ifunc-impl-list.c + (__libc_ifunc_impl_list): Add tests for __memmove_chk_erms, + __memcpy_chk_erms and __mempcpy_chk_erms. Update comments. + * sysdeps/x86_64/multiarch/ifunc-memmove.h: New file. + * sysdeps/x86_64/multiarch/memcpy.c: Likewise. + * sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S: Likewise. + * sysdeps/x86_64/multiarch/memcpy_chk.c: Likewise. + * sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S: Likewise. + * sysdeps/x86_64/multiarch/memmove.c: Likewise. + * sysdeps/x86_64/multiarch/memmove_chk-nonshared.S: Likewise. + * sysdeps/x86_64/multiarch/memmove_chk.c: Likewise. + * sysdeps/x86_64/multiarch/mempcpy.c: Likewise. + * sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S: Likewise. + * sysdeps/x86_64/multiarch/mempcpy_chk.c: Likewise. + * sysdeps/x86_64/multiarch/memcpy.S: Removed. + * sysdeps/x86_64/multiarch/memcpy_chk.S: Likewise. + * sysdeps/x86_64/multiarch/memmove.S: Likewise. + * sysdeps/x86_64/multiarch/memmove_chk.S: Likewise. + * sysdeps/x86_64/multiarch/mempcpy.S: Likewise. + * sysdeps/x86_64/multiarch/mempcpy_chk.S: Likewise. + * sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S + (__mempcpy_chk_erms): New function. + (__memmove_chk_erms): Likewise. + (__memcpy_chk_erms): New alias. + 2017-06-14 Florian Weimer <fweimer@redhat.com> * sysdeps/i386/i686/multiarch/strcspn-c.c: Add IS_IN (libc) guard. diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 202a19c492..6e70993a45 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -26,6 +26,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned \ strcspn-c strpbrk-c strspn-c varshift \ memset-avx512-no-vzeroupper \ + memmove-sse2-unaligned-erms \ memmove-avx-unaligned-erms \ memmove-avx512-unaligned-erms \ memset-avx2-unaligned-erms \ @@ -48,5 +49,7 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \ endif ifeq ($(subdir),debug) -sysdep_routines += wmemset_chk-nonshared +sysdep_routines += memcpy_chk-nonshared mempcpy_chk-nonshared \ + memmove_chk-nonshared \ + wmemset_chk-nonshared endif diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c index 1385b1f3b0..3462e67cc7 100644 --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c @@ -57,7 +57,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __memcmp_ssse3) IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2)) - /* Support sysdeps/x86_64/multiarch/memmove_chk.S. */ + /* Support sysdeps/x86_64/multiarch/memmove_chk.c. */ IFUNC_IMPL (i, name, __memmove_chk, IFUNC_IMPL_ADD (array, i, __memmove_chk, HAS_ARCH_FEATURE (AVX512F_Usable), @@ -83,9 +83,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, IFUNC_IMPL_ADD (array, i, __memmove_chk, 1, __memmove_chk_sse2_unaligned) IFUNC_IMPL_ADD (array, i, __memmove_chk, 1, - __memmove_chk_sse2_unaligned_erms)) + __memmove_chk_sse2_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memmove_chk, 1, + __memmove_chk_erms)) - /* Support sysdeps/x86_64/multiarch/memmove.S. */ + /* Support sysdeps/x86_64/multiarch/memmove.c. */ IFUNC_IMPL (i, name, memmove, IFUNC_IMPL_ADD (array, i, memmove, HAS_ARCH_FEATURE (AVX_Usable), @@ -410,7 +412,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __wmemset_avx512_unaligned)) #ifdef SHARED - /* Support sysdeps/x86_64/multiarch/memcpy_chk.S. */ + /* Support sysdeps/x86_64/multiarch/memcpy_chk.c. */ IFUNC_IMPL (i, name, __memcpy_chk, IFUNC_IMPL_ADD (array, i, __memcpy_chk, HAS_ARCH_FEATURE (AVX512F_Usable), @@ -436,9 +438,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1, __memcpy_chk_sse2_unaligned) IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1, - __memcpy_chk_sse2_unaligned_erms)) + __memcpy_chk_sse2_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1, + __memcpy_chk_erms)) - /* Support sysdeps/x86_64/multiarch/memcpy.S. */ + /* Support sysdeps/x86_64/multiarch/memcpy.c. */ IFUNC_IMPL (i, name, memcpy, IFUNC_IMPL_ADD (array, i, memcpy, HAS_ARCH_FEATURE (AVX_Usable), @@ -464,7 +468,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __memcpy_sse2_unaligned_erms) IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms)) - /* Support sysdeps/x86_64/multiarch/mempcpy_chk.S. */ + /* Support sysdeps/x86_64/multiarch/mempcpy_chk.c. */ IFUNC_IMPL (i, name, __mempcpy_chk, IFUNC_IMPL_ADD (array, i, __mempcpy_chk, HAS_ARCH_FEATURE (AVX512F_Usable), @@ -490,9 +494,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1, __mempcpy_chk_sse2_unaligned) IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1, - __mempcpy_chk_sse2_unaligned_erms)) + __mempcpy_chk_sse2_unaligned_erms) + IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1, + __mempcpy_chk_erms)) - /* Support sysdeps/x86_64/multiarch/mempcpy.S. */ + /* Support sysdeps/x86_64/multiarch/mempcpy.c. */ IFUNC_IMPL (i, name, mempcpy, IFUNC_IMPL_ADD (array, i, mempcpy, HAS_ARCH_FEATURE (AVX512F_Usable), diff --git a/sysdeps/x86_64/multiarch/ifunc-memmove.h b/sysdeps/x86_64/multiarch/ifunc-memmove.h new file mode 100644 index 0000000000..5951e156fa --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-memmove.h @@ -0,0 +1,80 @@ +/* Common definition for memcpy, mempcpy and memmove implementation. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) + attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms) + attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms) + attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned) + attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms) + attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper) + attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_ERMS)) + return OPTIMIZE (erms); + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable) + && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) + { + if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE (avx512_no_vzeroupper); + + if (CPU_FEATURES_CPU_P (cpu_features, ERMS)) + return OPTIMIZE (avx512_unaligned_erms); + + return OPTIMIZE (avx512_unaligned); + } + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { + if (CPU_FEATURES_CPU_P (cpu_features, ERMS)) + return OPTIMIZE (avx_unaligned_erms); + + return OPTIMIZE (avx_unaligned); + } + + if (!CPU_FEATURES_CPU_P (cpu_features, SSSE3) + || CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy)) + { + if (CPU_FEATURES_CPU_P (cpu_features, ERMS)) + return OPTIMIZE (sse2_unaligned_erms); + + return OPTIMIZE (sse2_unaligned); + } + + if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Copy_Backward)) + return OPTIMIZE (ssse3_back); + + return OPTIMIZE (ssse3); +} diff --git a/sysdeps/x86_64/multiarch/memcpy.S b/sysdeps/x86_64/multiarch/memcpy.S deleted file mode 100644 index af2770397c..0000000000 --- a/sysdeps/x86_64/multiarch/memcpy.S +++ /dev/null @@ -1,75 +0,0 @@ -/* Multiple versions of memcpy - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2010-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in lib and for - DSO. In static binaries we need memcpy before the initialization - happened. */ -#if defined SHARED && IS_IN (libc) - .text -ENTRY(__new_memcpy) - .type __new_memcpy, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - lea __memcpy_erms(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_ERMS) - jnz 2f - HAS_ARCH_FEATURE (Prefer_No_AVX512) - jnz 1f - HAS_ARCH_FEATURE (AVX512F_Usable) - jz 1f - lea __memcpy_avx512_no_vzeroupper(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 2f - lea __memcpy_avx512_unaligned_erms(%rip), %RAX_LP - HAS_CPU_FEATURE (ERMS) - jnz 2f - lea __memcpy_avx512_unaligned(%rip), %RAX_LP - ret -1: lea __memcpy_avx_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz L(Fast_Unaligned_Load) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memcpy_avx_unaligned_erms(%rip), %RAX_LP - ret -L(Fast_Unaligned_Load): - lea __memcpy_sse2_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Unaligned_Copy) - jz L(SSSE3) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memcpy_sse2_unaligned_erms(%rip), %RAX_LP - ret -L(SSSE3): - HAS_CPU_FEATURE (SSSE3) - jz 2f - lea __memcpy_ssse3_back(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Copy_Backward) - jnz 2f - lea __memcpy_ssse3(%rip), %RAX_LP -2: ret -END(__new_memcpy) - -# undef memcpy -# include <shlib-compat.h> -versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14); -#endif diff --git a/sysdeps/x86_64/multiarch/memcpy.c b/sysdeps/x86_64/multiarch/memcpy.c new file mode 100644 index 0000000000..6a2d3539e2 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcpy.c @@ -0,0 +1,39 @@ +/* Multiple versions of memcpy. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in lib and for + DSO. In static binaries we need memcpy before the initialization + happened. */ +#if defined SHARED && IS_IN (libc) +# define memcpy __redirect_memcpy +# include <string.h> +# undef memcpy + +# define SYMBOL_NAME memcpy +# include "ifunc-memmove.h" + +libc_ifunc_redirected (__redirect_memcpy, __new_memcpy, + IFUNC_SELECTOR ()); + +__hidden_ver1 (__new_memcpy, __GI_memcpy, __redirect_memcpy) + __attribute__ ((visibility ("hidden"))); + +# include <shlib-compat.h> +versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14); +#endif diff --git a/sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S b/sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S new file mode 100644 index 0000000000..41eb0e81f6 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S @@ -0,0 +1,21 @@ +/* Non-shared version of memcpy_chk for x86-64. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) && !defined SHARED +# include <sysdeps/x86_64/memcpy_chk.S> +#endif diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.S b/sysdeps/x86_64/multiarch/memcpy_chk.S deleted file mode 100644 index 8737fb9755..0000000000 --- a/sysdeps/x86_64/multiarch/memcpy_chk.S +++ /dev/null @@ -1,72 +0,0 @@ -/* Multiple versions of __memcpy_chk - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2010-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in lib and for - DSO. There are no multiarch memcpy functions for static binaries. - */ -#if IS_IN (libc) -# ifdef SHARED - .text -ENTRY(__memcpy_chk) - .type __memcpy_chk, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - HAS_ARCH_FEATURE (Prefer_No_AVX512) - jnz 1f - HAS_ARCH_FEATURE (AVX512F_Usable) - jz 1f - lea __memcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 2f - lea __memcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP - HAS_CPU_FEATURE (ERMS) - jnz 2f - lea __memcpy_chk_avx512_unaligned(%rip), %RAX_LP - ret -1: lea __memcpy_chk_avx_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz L(Fast_Unaligned_Load) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memcpy_chk_avx_unaligned_erms(%rip), %RAX_LP - ret -L(Fast_Unaligned_Load): - lea __memcpy_chk_sse2_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Unaligned_Copy) - jz L(SSSE3) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP - ret -L(SSSE3): - HAS_CPU_FEATURE (SSSE3) - jz 2f - lea __memcpy_chk_ssse3_back(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Copy_Backward) - jnz 2f - lea __memcpy_chk_ssse3(%rip), %RAX_LP -2: ret -END(__memcpy_chk) -# else -# include "../memcpy_chk.S" -# endif -#endif diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.c b/sysdeps/x86_64/multiarch/memcpy_chk.c new file mode 100644 index 0000000000..6347c21e3c --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcpy_chk.c @@ -0,0 +1,31 @@ +/* Multiple versions of __memcpy_chk + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc.so. */ +#if IS_IN (libc) && defined SHARED +# define __memcpy_chk __redirect_memcpy_chk +# include <string.h> +# undef __memcpy_chk + +# define SYMBOL_NAME memcpy_chk +# include "ifunc-memmove.h" + +libc_ifunc_redirected (__redirect_memcpy_chk, __memcpy_chk, + IFUNC_SELECTOR ()); +#endif diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S new file mode 100644 index 0000000000..743064b63d --- /dev/null +++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S @@ -0,0 +1,35 @@ +/* memmove with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s +#endif + +#if !defined SHARED || !IS_IN (libc) +weak_alias (__mempcpy, mempcpy) +#endif + +#include <sysdeps/x86_64/memmove.S> + +#if defined SHARED && IS_IN (libc) +# include <shlib-compat.h> +# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14) +/* Use __memmove_sse2_unaligned to support overlapping addresses. */ +compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S index dee3ec529c..d694e8b2be 100644 --- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S @@ -152,6 +152,11 @@ END (MEMMOVE_SYMBOL (__memmove, unaligned)) # if VEC_SIZE == 16 # if defined SHARED +ENTRY (__mempcpy_chk_erms) + cmpq %rdx, %rcx + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__mempcpy_chk_erms) + /* Only used to measure performance of REP MOVSB. */ ENTRY (__mempcpy_erms) movq %rdi, %rax @@ -160,6 +165,11 @@ ENTRY (__mempcpy_erms) END (__mempcpy_erms) # endif +ENTRY (__memmove_chk_erms) + cmpq %rdx, %rcx + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__memmove_chk_erms) + ENTRY (__memmove_erms) movq %rdi, %rax L(start_movsb): @@ -185,6 +195,7 @@ L(movsb_backward): END (__memmove_erms) # if defined SHARED strong_alias (__memmove_erms, __memcpy_erms) +strong_alias (__memmove_chk_erms, __memcpy_chk_erms) # endif # endif diff --git a/sysdeps/x86_64/multiarch/memmove.S b/sysdeps/x86_64/multiarch/memmove.S deleted file mode 100644 index 8c534e83e0..0000000000 --- a/sysdeps/x86_64/multiarch/memmove.S +++ /dev/null @@ -1,101 +0,0 @@ -/* Multiple versions of memmove - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2016-2017 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in lib and for - DSO. */ -#if IS_IN (libc) - .text -ENTRY(__libc_memmove) - .type __libc_memmove, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - lea __memmove_erms(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_ERMS) - jnz 2f - HAS_ARCH_FEATURE (Prefer_No_AVX512) - jnz 1f - HAS_ARCH_FEATURE (AVX512F_Usable) - jz 1f - lea __memmove_avx512_no_vzeroupper(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 2f - lea __memmove_avx512_unaligned_erms(%rip), %RAX_LP - HAS_CPU_FEATURE (ERMS) - jnz 2f - lea __memmove_avx512_unaligned(%rip), %RAX_LP - ret -1: lea __memmove_avx_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz L(Fast_Unaligned_Load) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memmove_avx_unaligned_erms(%rip), %RAX_LP - ret -L(Fast_Unaligned_Load): - lea __memmove_sse2_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Unaligned_Copy) - jz L(SSSE3) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memmove_sse2_unaligned_erms(%rip), %RAX_LP - ret -L(SSSE3): - HAS_CPU_FEATURE (SSSE3) - jz 2f - lea __memmove_ssse3_back(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Copy_Backward) - jnz 2f - lea __memmove_ssse3(%rip), %RAX_LP -2: ret -END(__libc_memmove) -#endif - -#if IS_IN (libc) -# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s - -# ifdef SHARED -libc_hidden_ver (__memmove_sse2_unaligned, memmove) -libc_hidden_ver (__memcpy_sse2_unaligned, memcpy) -libc_hidden_ver (__mempcpy_sse2_unaligned, mempcpy) -libc_hidden_ver (__mempcpy_sse2_unaligned, __mempcpy) - -# undef libc_hidden_builtin_def -/* It doesn't make sense to send libc-internal memmove calls through a PLT. - The speedup we get from using SSE2 instructions is likely eaten away - by the indirect call in the PLT. */ -# define libc_hidden_builtin_def -# endif -strong_alias (__libc_memmove, memmove) -#endif - -#if !defined SHARED || !IS_IN (libc) -weak_alias (__mempcpy, mempcpy) -#endif - -#include "../memmove.S" - -#if defined SHARED && IS_IN (libc) -# include <shlib-compat.h> -# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14) -/* Use __memmove_sse2_unaligned to support overlapping addresses. */ -compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5); -# endif -#endif diff --git a/sysdeps/x86_64/multiarch/memmove.c b/sysdeps/x86_64/multiarch/memmove.c new file mode 100644 index 0000000000..c615808cc1 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memmove.c @@ -0,0 +1,38 @@ +/* Multiple versions of memmmove. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2016-2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in lib and for + DSO. */ +#if IS_IN (libc) +# define memmove __redirect_memmove +# include <string.h> +# undef memmove + +# define SYMBOL_NAME memmove +# include "ifunc-memmove.h" + +libc_ifunc_redirected (__redirect_memmove, __libc_memmove, + IFUNC_SELECTOR ()); + +strong_alias (__libc_memmove, memmove); +# ifdef SHARED +__hidden_ver1 (__libc_memmove, __GI_memmove, __redirect_memmove) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/memmove_chk-nonshared.S b/sysdeps/x86_64/multiarch/memmove_chk-nonshared.S new file mode 100644 index 0000000000..704a20c408 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memmove_chk-nonshared.S @@ -0,0 +1,21 @@ +/* Non-shared version of memmove_chk for x86-64. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) && !defined SHARED +# include <sysdeps/x86_64/memmove_chk.S> +#endif diff --git a/sysdeps/x86_64/multiarch/memmove_chk.S b/sysdeps/x86_64/multiarch/memmove_chk.S deleted file mode 100644 index 7870dd0247..0000000000 --- a/sysdeps/x86_64/multiarch/memmove_chk.S +++ /dev/null @@ -1,71 +0,0 @@ -/* Multiple versions of __memmove_chk - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2016-2017 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in lib and for - DSO. There are no multiarch memmove functions for static binaries. - */ -#if IS_IN (libc) -# ifdef SHARED - .text -ENTRY(__memmove_chk) - .type __memmove_chk, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - HAS_ARCH_FEATURE (Prefer_No_AVX512) - jnz 1f - HAS_ARCH_FEATURE (AVX512F_Usable) - jz 1f - lea __memmove_chk_avx512_no_vzeroupper(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 2f - lea __memmove_chk_avx512_unaligned_erms(%rip), %RAX_LP - HAS_CPU_FEATURE (ERMS) - jnz 2f - lea __memmove_chk_avx512_unaligned(%rip), %RAX_LP - ret -1: lea __memmove_chk_avx_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz L(Fast_Unaligned_Load) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memmove_chk_avx_unaligned_erms(%rip), %RAX_LP - ret -L(Fast_Unaligned_Load): - lea __memmove_chk_sse2_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Unaligned_Copy) - jz L(SSSE3) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __memmove_chk_sse2_unaligned_erms(%rip), %RAX_LP - ret -L(SSSE3): - HAS_CPU_FEATURE (SSSE3) - jz 2f - lea __memmove_chk_ssse3_back(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Copy_Backward) - jnz 2f - lea __memmove_chk_ssse3(%rip), %RAX_LP -2: ret -END(__memmove_chk) -# else -# include "../memmove_chk.S" -# endif -#endif diff --git a/sysdeps/x86_64/multiarch/memmove_chk.c b/sysdeps/x86_64/multiarch/memmove_chk.c new file mode 100644 index 0000000000..44339d64ec --- /dev/null +++ b/sysdeps/x86_64/multiarch/memmove_chk.c @@ -0,0 +1,31 @@ +/* Multiple versions of __memmove_chk + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc.so. */ +#if IS_IN (libc) && defined SHARED +# define __memmove_chk __redirect_memmove_chk +# include <string.h> +# undef __memmove_chk + +# define SYMBOL_NAME memmove_chk +# include "ifunc-memmove.h" + +libc_ifunc_redirected (__redirect_memmove_chk, __memmove_chk, + IFUNC_SELECTOR ()); +#endif diff --git a/sysdeps/x86_64/multiarch/mempcpy.S b/sysdeps/x86_64/multiarch/mempcpy.S deleted file mode 100644 index b8b2b28094..0000000000 --- a/sysdeps/x86_64/multiarch/mempcpy.S +++ /dev/null @@ -1,73 +0,0 @@ -/* Multiple versions of mempcpy - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2010-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in lib and for - DSO. In static binaries we need mempcpy before the initialization - happened. */ -#if defined SHARED && IS_IN (libc) - .text -ENTRY(__mempcpy) - .type __mempcpy, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - lea __mempcpy_erms(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_ERMS) - jnz 2f - HAS_ARCH_FEATURE (Prefer_No_AVX512) - jnz 1f - HAS_ARCH_FEATURE (AVX512F_Usable) - jz 1f - lea __mempcpy_avx512_no_vzeroupper(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 2f - lea __mempcpy_avx512_unaligned_erms(%rip), %RAX_LP - HAS_CPU_FEATURE (ERMS) - jnz 2f - lea __mempcpy_avx512_unaligned(%rip), %RAX_LP - ret -1: lea __mempcpy_avx_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz L(Fast_Unaligned_Load) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __mempcpy_avx_unaligned_erms(%rip), %RAX_LP - ret -L(Fast_Unaligned_Load): - lea __mempcpy_sse2_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Unaligned_Copy) - jz L(SSSE3) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __mempcpy_sse2_unaligned_erms(%rip), %RAX_LP - ret -L(SSSE3): - HAS_CPU_FEATURE (SSSE3) - jz 2f - lea __mempcpy_ssse3_back(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Copy_Backward) - jnz 2f - lea __mempcpy_ssse3(%rip), %RAX_LP -2: ret -END(__mempcpy) - -weak_alias (__mempcpy, mempcpy) -#endif diff --git a/sysdeps/x86_64/multiarch/mempcpy.c b/sysdeps/x86_64/multiarch/mempcpy.c new file mode 100644 index 0000000000..f6e9b8acc0 --- /dev/null +++ b/sysdeps/x86_64/multiarch/mempcpy.c @@ -0,0 +1,40 @@ +/* Multiple versions of mempcpy. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in lib and for + DSO. In static binaries we need mempcpy before the initialization + happened. */ +#if defined SHARED && IS_IN (libc) +# define mempcpy __redirect_mempcpy +# define __mempcpy __redirect___mempcpy +# include <string.h> +# undef mempcpy +# undef __mempcpy + +# define SYMBOL_NAME mempcpy +# include "ifunc-memmove.h" + +libc_ifunc_redirected (__redirect_mempcpy, __mempcpy, IFUNC_SELECTOR ()); + +weak_alias (__mempcpy, mempcpy) +__hidden_ver1 (__mempcpy, __GI___mempcpy, __redirect___mempcpy) + __attribute__ ((visibility ("hidden"))); +__hidden_ver1 (mempcpy, __GI_mempcpy, __redirect_mempcpy) + __attribute__ ((visibility ("hidden"))); +#endif diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S b/sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S new file mode 100644 index 0000000000..01b7df7f8c --- /dev/null +++ b/sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S @@ -0,0 +1,21 @@ +/* Non-shared version of mempcpy_chk for x86-64. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) && !defined SHARED +# include <sysdeps/x86_64/mempcpy_chk.S> +#endif diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.S b/sysdeps/x86_64/multiarch/mempcpy_chk.S deleted file mode 100644 index 072b22c49f..0000000000 --- a/sysdeps/x86_64/multiarch/mempcpy_chk.S +++ /dev/null @@ -1,72 +0,0 @@ -/* Multiple versions of __mempcpy_chk - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2010-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in lib and for - DSO. There are no multiarch mempcpy functions for static binaries. - */ -#if IS_IN (libc) -# ifdef SHARED - .text -ENTRY(__mempcpy_chk) - .type __mempcpy_chk, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - HAS_ARCH_FEATURE (Prefer_No_AVX512) - jnz 1f - HAS_ARCH_FEATURE (AVX512F_Usable) - jz 1f - lea __mempcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 2f - lea __mempcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP - HAS_CPU_FEATURE (ERMS) - jnz 2f - lea __mempcpy_chk_avx512_unaligned(%rip), %RAX_LP - ret -1: lea __mempcpy_chk_avx_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz L(Fast_Unaligned_Load) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __mempcpy_chk_avx_unaligned_erms(%rip), %RAX_LP - ret -L(Fast_Unaligned_Load): - lea __mempcpy_chk_sse2_unaligned(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Unaligned_Copy) - jz L(SSSE3) - HAS_CPU_FEATURE (ERMS) - jz 2f - lea __mempcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP - ret -L(SSSE3): - HAS_CPU_FEATURE (SSSE3) - jz 2f - lea __mempcpy_chk_ssse3_back(%rip), %RAX_LP - HAS_ARCH_FEATURE (Fast_Copy_Backward) - jnz 2f - lea __mempcpy_chk_ssse3(%rip), %RAX_LP -2: ret -END(__mempcpy_chk) -# else -# include "../mempcpy_chk.S" -# endif -#endif diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.c b/sysdeps/x86_64/multiarch/mempcpy_chk.c new file mode 100644 index 0000000000..c73273ece3 --- /dev/null +++ b/sysdeps/x86_64/multiarch/mempcpy_chk.c @@ -0,0 +1,31 @@ +/* Multiple versions of __mempcpy_chk + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc.so. */ +#if IS_IN (libc) && defined SHARED +# define __mempcpy_chk __redirect_mempcpy_chk +# include <string.h> +# undef __mempcpy_chk + +# define SYMBOL_NAME mempcpy_chk +# include "ifunc-memmove.h" + +libc_ifunc_redirected (__redirect_mempcpy_chk, __mempcpy_chk, + IFUNC_SELECTOR ()); +#endif |