From 3afd5a3b555694361181f854f658f669105d0ad6 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 25 Feb 2010 23:52:07 -0800 Subject: sparc: Add multiarch support for memset/bzero/memcpy. 2010-02-25 David S. Miller * sysdeps/sparc/elf/rtld-global-offsets.sym: New file. * sysdeps/sparc/Makefile (csu): Add rtld-global-offsets.sym to gen-as-const-headers. * sysdeps/sparc/sparc32/sparcv9/multiarch/Makefile: New file. * sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy.S: New file. * sysdeps/sparc/sparc32/sparcv9/multiarch/memset.S: New file. * sysdeps/sparc/sparc32/sparcv9/sparcv9b/memcpy.S: Move to... * sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy-ultra3.S: ...here. * sysdeps/sparc/sparc32/sparcv9/sparcv9v/memcpy.S: Move to... * sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy-niagara1.S: ...here. * sysdeps/sparc/sparc32/sparcv9/sparcv9v/memset.S: Move to... * sysdeps/sparc/sparc32/sparcv9/multiarch/memset-niagara1.S: ...here. * sysdeps/sparc/sparc32/sparcv9/sparcv9v2/memcpy.S: Move to... * sysdeps/sparc/sparc32/sparcv9/multiarch/memcpy-niagara2.S: ...here. * sysdeps/sparc/sparc32/sparcv9/sparcv9v2/memset.S: Removed. * sysdeps/sparc/sparc64/multiarch/Makefile: New file. * sysdeps/sparc/sparc64/sparcv9v/memcpy.S: Move to... * sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S: ...here. * sysdeps/sparc/sparc64/sparcv9v2/memcpy.S: Move to... * sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S: ...here. * sysdeps/sparc/sparc64/sparcv9b/memcpy.S: Move to... * sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S: ...here. * sysdeps/sparc/sparc64/sparcv9v/memset.S: Move to... * sysdeps/sparc/sparc64/multiarch/memset-niagara1.S: ...here. * sysdeps/sparc/sparc64/sparcv9v2/memset.S: Removed. * sysdeps/sparc/sparc64/multiarch/memcpy.S: New file. * sysdeps/sparc/sparc64/multiarch/memset.S: New file. --- sysdeps/sparc/sparc64/multiarch/Makefile | 4 + sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S | 342 +++++++++++++++ sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S | 492 ++++++++++++++++++++++ sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S | 320 ++++++++++++++ sysdeps/sparc/sparc64/multiarch/memcpy.S | 107 +++++ sysdeps/sparc/sparc64/multiarch/memset-niagara1.S | 132 ++++++ sysdeps/sparc/sparc64/multiarch/memset.S | 145 +++++++ sysdeps/sparc/sparc64/sparcv9b/memcpy.S | 318 -------------- sysdeps/sparc/sparc64/sparcv9v/memcpy.S | 340 --------------- sysdeps/sparc/sparc64/sparcv9v/memset.S | 131 ------ sysdeps/sparc/sparc64/sparcv9v2/memcpy.S | 490 --------------------- sysdeps/sparc/sparc64/sparcv9v2/memset.S | 1 - 12 files changed, 1542 insertions(+), 1280 deletions(-) create mode 100644 sysdeps/sparc/sparc64/multiarch/Makefile create mode 100644 sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S create mode 100644 sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S create mode 100644 sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S create mode 100644 sysdeps/sparc/sparc64/multiarch/memcpy.S create mode 100644 sysdeps/sparc/sparc64/multiarch/memset-niagara1.S create mode 100644 sysdeps/sparc/sparc64/multiarch/memset.S delete mode 100644 sysdeps/sparc/sparc64/sparcv9b/memcpy.S delete mode 100644 sysdeps/sparc/sparc64/sparcv9v/memcpy.S delete mode 100644 sysdeps/sparc/sparc64/sparcv9v/memset.S delete mode 100644 sysdeps/sparc/sparc64/sparcv9v2/memcpy.S delete mode 100644 sysdeps/sparc/sparc64/sparcv9v2/memset.S (limited to 'sysdeps/sparc/sparc64') diff --git a/sysdeps/sparc/sparc64/multiarch/Makefile b/sysdeps/sparc/sparc64/multiarch/Makefile new file mode 100644 index 0000000000..4d45042a95 --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/Makefile @@ -0,0 +1,4 @@ +ifeq ($(subdir),string) +sysdep_routines += memcpy-ultra3 memcpy-niagara1 memcpy-niagara2 \ + memset-niagara1 +endif diff --git a/sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S new file mode 100644 index 0000000000..6a78295e81 --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara1.S @@ -0,0 +1,342 @@ +/* Copy SIZE bytes from SRC to DEST. For SUN4V Niagara. + Copyright (C) 2006, 2008 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by David S. Miller (davem@davemloft.net) + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define ASI_P 0x80 +#define ASI_PNF 0x82 + +#define LOAD(type,addr,dest) type##a [addr] ASI_P, dest +#define LOAD_TWIN(addr_reg,dest0,dest1) \ + ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_P, dest0 + +#define STORE(type,src,addr) type src, [addr] +#define STORE_INIT(src,addr) stxa src, [addr] %asi + +#ifndef XCC +#define USE_BPR +#define XCC xcc +#endif + +#if !defined NOT_IN_libc + + .register %g2,#scratch + .register %g3,#scratch + .register %g6,#scratch + + .text + + .align 32 +ENTRY(__memcpy_niagara1) +# ifndef USE_BPR + srl %o2, 0, %o2 +# endif +100: /* %o0=dst, %o1=src, %o2=len */ + mov %o0, %g5 + cmp %o2, 0 + be,pn %XCC, 85f +218: or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + /* 2 blocks (128 bytes) is the minimum we can do the block + * copy with. We need to ensure that we'll iterate at least + * once in the block copy loop. At worst we'll need to align + * the destination to a 64-byte boundary which can chew up + * to (64 - 1) bytes from the length before we perform the + * block copy loop. + */ + cmp %o2, (2 * 64) + blu,pt %XCC, 70f + andcc %o3, 0x7, %g0 + + /* %o0: dst + * %o1: src + * %o2: len (known to be >= 128) + * + * The block copy loops will use %o4/%o5,%g2/%g3 as + * temporaries while copying the data. + */ + + LOAD(prefetch, %o1, #one_read) + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + + /* Align destination on 64-byte boundary. */ + andcc %o0, (64 - 1), %o4 + be,pt %XCC, 2f + sub %o4, 64, %o4 + sub %g0, %o4, %o4 ! bytes to align dst + sub %o2, %o4, %o2 +1: subcc %o4, 1, %o4 + LOAD(ldub, %o1, %g1) + STORE(stb, %g1, %o0) + add %o1, 1, %o1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + + /* If the source is on a 16-byte boundary we can do + * the direct block copy loop. If it is 8-byte aligned + * we can do the 16-byte loads offset by -8 bytes and the + * init stores offset by one register. + * + * If the source is not even 8-byte aligned, we need to do + * shifting and masking (basically integer faligndata). + * + * The careful bit with init stores is that if we store + * to any part of the cache line we have to store the whole + * cacheline else we can end up with corrupt L2 cache line + * contents. Since the loop works on 64-bytes of 64-byte + * aligned store data at a time, this is easy to ensure. + */ +2: + andcc %o1, (16 - 1), %o4 + andn %o2, (64 - 1), %g1 ! block copy loop iterator + sub %o2, %g1, %o2 ! final sub-block copy bytes + be,pt %XCC, 50f + cmp %o4, 8 + be,a,pt %XCC, 10f + sub %o1, 0x8, %o1 + + /* Neither 8-byte nor 16-byte aligned, shift and mask. */ + mov %g1, %o4 + and %o1, 0x7, %g1 + sll %g1, 3, %g1 + mov 64, %o3 + andn %o1, 0x7, %o1 + LOAD(ldx, %o1, %g2) + sub %o3, %g1, %o3 + sllx %g2, %g1, %g2 + +#define SWIVEL_ONE_DWORD(SRC, TMP1, TMP2, PRE_VAL, PRE_SHIFT, POST_SHIFT, DST)\ + LOAD(ldx, SRC, TMP1); \ + srlx TMP1, PRE_SHIFT, TMP2; \ + or TMP2, PRE_VAL, TMP2; \ + STORE_INIT(TMP2, DST); \ + sllx TMP1, POST_SHIFT, PRE_VAL; + +1: add %o1, 0x8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x00) + add %o1, 0x8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x08) + add %o1, 0x8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x10) + add %o1, 0x8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x18) + add %o1, 32, %o1 + LOAD(prefetch, %o1, #one_read) + sub %o1, 32 - 8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x20) + add %o1, 8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x28) + add %o1, 8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x30) + add %o1, 8, %o1 + SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x38) + subcc %o4, 64, %o4 + bne,pt %XCC, 1b + add %o0, 64, %o0 + +#undef SWIVEL_ONE_DWORD + + srl %g1, 3, %g1 + ba,pt %XCC, 60f + add %o1, %g1, %o1 + +10: /* Destination is 64-byte aligned, source was only 8-byte + * aligned but it has been subtracted by 8 and we perform + * one twin load ahead, then add 8 back into source when + * we finish the loop. + */ + LOAD_TWIN(%o1, %o4, %o5) +1: add %o1, 16, %o1 + LOAD_TWIN(%o1, %g2, %g3) + add %o1, 16 + 32, %o1 + LOAD(prefetch, %o1, #one_read) + sub %o1, 32, %o1 + STORE_INIT(%o5, %o0 + 0x00) ! initializes cache line + STORE_INIT(%g2, %o0 + 0x08) + LOAD_TWIN(%o1, %o4, %o5) + add %o1, 16, %o1 + STORE_INIT(%g3, %o0 + 0x10) + STORE_INIT(%o4, %o0 + 0x18) + LOAD_TWIN(%o1, %g2, %g3) + add %o1, 16, %o1 + STORE_INIT(%o5, %o0 + 0x20) + STORE_INIT(%g2, %o0 + 0x28) + LOAD_TWIN(%o1, %o4, %o5) + STORE_INIT(%g3, %o0 + 0x30) + STORE_INIT(%o4, %o0 + 0x38) + subcc %g1, 64, %g1 + bne,pt %XCC, 1b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %o1, 0x8, %o1 + +50: /* Destination is 64-byte aligned, and source is 16-byte + * aligned. + */ +1: LOAD_TWIN(%o1, %o4, %o5) + add %o1, 16, %o1 + LOAD_TWIN(%o1, %g2, %g3) + add %o1, 16 + 32, %o1 + LOAD(prefetch, %o1, #one_read) + sub %o1, 32, %o1 + STORE_INIT(%o4, %o0 + 0x00) ! initializes cache line + STORE_INIT(%o5, %o0 + 0x08) + LOAD_TWIN(%o1, %o4, %o5) + add %o1, 16, %o1 + STORE_INIT(%g2, %o0 + 0x10) + STORE_INIT(%g3, %o0 + 0x18) + LOAD_TWIN(%o1, %g2, %g3) + add %o1, 16, %o1 + STORE_INIT(%o4, %o0 + 0x20) + STORE_INIT(%o5, %o0 + 0x28) + STORE_INIT(%g2, %o0 + 0x30) + STORE_INIT(%g3, %o0 + 0x38) + subcc %g1, 64, %g1 + bne,pt %XCC, 1b + add %o0, 64, %o0 + /* fall through */ + +60: + /* %o2 contains any final bytes still needed to be copied + * over. If anything is left, we copy it one byte at a time. + */ + wr %g0, ASI_PNF, %asi + brz,pt %o2, 85f + sub %o0, %o1, %o3 + ba,a,pt %XCC, 90f + + .align 64 +70: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: + andn %o2, 0xf, %o4 + and %o2, 0xf, %o2 +1: subcc %o4, 0x10, %o4 + LOAD(ldx, %o1, %o5) + add %o1, 0x08, %o1 + LOAD(ldx, %o1, %g1) + sub %o1, 0x08, %o1 + STORE(stx, %o5, %o1 + %o3) + add %o1, 0x8, %o1 + STORE(stx, %g1, %o1 + %o3) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x8, %o2 + LOAD(ldx, %o1, %o5) + STORE(stx, %o5, %o1 + %o3) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + LOAD(lduw, %o1, %o5) + STORE(stw, %o5, %o1 + %o3) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %XCC, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: subcc %g1, 1, %g1 + LOAD(ldub, %o1, %o5) + STORE(stb, %o5, %o1 + %o3) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %XCC, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + LOAD(ldx, %o1, %g2) + sub %o3, %g1, %o3 + andn %o2, 0x7, %o4 + sllx %g2, %g1, %g2 +1: add %o1, 0x8, %o1 + LOAD(ldx, %o1, %g3) + subcc %o4, 0x8, %o4 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + STORE(stx, %o5, %o0) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %XCC, 90f + sub %o0, %o1, %o3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + LOAD(lduw, %o1, %g1) + STORE(stw, %g1, %o1 + %o3) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov %g5, %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + LOAD(ldub, %o1, %g1) + STORE(stb, %g1, %o1 + %o3) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov %g5, %o0 + +END(__memcpy_niagara1) + +#endif diff --git a/sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S new file mode 100644 index 0000000000..35f6989aca --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/memcpy-niagara2.S @@ -0,0 +1,492 @@ +/* Copy SIZE bytes from SRC to DEST. For SUN4V Niagara-2. + Copyright (C) 2007, 2008 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by David S. Miller (davem@davemloft.net) + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define ASI_BLK_P 0xf0 +#define ASI_P 0x80 +#define ASI_PNF 0x82 + +#define FPRS_FEF 0x04 + +#define VISEntryHalf \ + rd %fprs, %o5; \ + wr %g0, FPRS_FEF, %fprs + +#define VISExitHalf \ + and %o5, FPRS_FEF, %o5; \ + wr %o5, 0x0, %fprs + +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P + +#define LOAD(type,addr,dest) type [addr], dest +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest +#define STORE(type,src,addr) type src, [addr] +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI + +#ifndef XCC +#define USE_BPR +#define XCC xcc +#endif + +#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ + faligndata %x0, %x1, %f0; \ + faligndata %x1, %x2, %f2; \ + faligndata %x2, %x3, %f4; \ + faligndata %x3, %x4, %f6; \ + faligndata %x4, %x5, %f8; \ + faligndata %x5, %x6, %f10; \ + faligndata %x6, %x7, %f12; \ + faligndata %x7, %x8, %f14; + +#define FREG_MOVE_1(x0) \ + fmovd %x0, %f0; +#define FREG_MOVE_2(x0, x1) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; +#define FREG_MOVE_3(x0, x1, x2) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; +#define FREG_MOVE_4(x0, x1, x2, x3) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; +#define FREG_MOVE_5(x0, x1, x2, x3, x4) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; +#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; \ + fmovd %x5, %f10; +#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; \ + fmovd %x5, %f10; \ + fmovd %x6, %f12; +#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; \ + fmovd %x5, %f10; \ + fmovd %x6, %f12; \ + fmovd %x7, %f14; +#define FREG_LOAD_1(base, x0) \ + LOAD(ldd, base + 0x00, %x0) +#define FREG_LOAD_2(base, x0, x1) \ + LOAD(ldd, base + 0x00, %x0); \ + LOAD(ldd, base + 0x08, %x1); +#define FREG_LOAD_3(base, x0, x1, x2) \ + LOAD(ldd, base + 0x00, %x0); \ + LOAD(ldd, base + 0x08, %x1); \ + LOAD(ldd, base + 0x10, %x2); +#define FREG_LOAD_4(base, x0, x1, x2, x3) \ + LOAD(ldd, base + 0x00, %x0); \ + LOAD(ldd, base + 0x08, %x1); \ + LOAD(ldd, base + 0x10, %x2); \ + LOAD(ldd, base + 0x18, %x3); +#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ + LOAD(ldd, base + 0x00, %x0); \ + LOAD(ldd, base + 0x08, %x1); \ + LOAD(ldd, base + 0x10, %x2); \ + LOAD(ldd, base + 0x18, %x3); \ + LOAD(ldd, base + 0x20, %x4); +#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ + LOAD(ldd, base + 0x00, %x0); \ + LOAD(ldd, base + 0x08, %x1); \ + LOAD(ldd, base + 0x10, %x2); \ + LOAD(ldd, base + 0x18, %x3); \ + LOAD(ldd, base + 0x20, %x4); \ + LOAD(ldd, base + 0x28, %x5); +#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ + LOAD(ldd, base + 0x00, %x0); \ + LOAD(ldd, base + 0x08, %x1); \ + LOAD(ldd, base + 0x10, %x2); \ + LOAD(ldd, base + 0x18, %x3); \ + LOAD(ldd, base + 0x20, %x4); \ + LOAD(ldd, base + 0x28, %x5); \ + LOAD(ldd, base + 0x30, %x6); + +#if !defined NOT_IN_libc + + .register %g2,#scratch + .register %g3,#scratch + .register %g6,#scratch + + .text + + .align 32 +ENTRY(__memcpy_niagara2) +# ifndef USE_BPR + srl %o2, 0, %o2 +# endif +100: /* %o0=dst, %o1=src, %o2=len */ + mov %o0, %g5 + cmp %o2, 0 + be,pn %XCC, 85f +218: or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + /* 2 blocks (128 bytes) is the minimum we can do the block + * copy with. We need to ensure that we'll iterate at least + * once in the block copy loop. At worst we'll need to align + * the destination to a 64-byte boundary which can chew up + * to (64 - 1) bytes from the length before we perform the + * block copy loop. + * + * However, the cut-off point, performance wise, is around + * 4 64-byte blocks. + */ + cmp %o2, (4 * 64) + blu,pt %XCC, 75f + andcc %o3, 0x7, %g0 + + /* %o0: dst + * %o1: src + * %o2: len (known to be >= 128) + * + * The block copy loops can use %o4, %g2, %g3 as + * temporaries while copying the data. %o5 must + * be preserved between VISEntryHalf and VISExitHalf + */ + + LOAD(prefetch, %o1 + 0x000, #one_read) + LOAD(prefetch, %o1 + 0x040, #one_read) + LOAD(prefetch, %o1 + 0x080, #one_read) + + /* Align destination on 64-byte boundary. */ + andcc %o0, (64 - 1), %o4 + be,pt %XCC, 2f + sub %o4, 64, %o4 + sub %g0, %o4, %o4 ! bytes to align dst + sub %o2, %o4, %o2 +1: subcc %o4, 1, %o4 + LOAD(ldub, %o1, %g1) + STORE(stb, %g1, %o0) + add %o1, 1, %o1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + +2: + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve + * o5 from here until we hit VISExitHalf. + */ + VISEntryHalf + + alignaddr %o1, %g0, %g0 + + add %o1, (64 - 1), %o4 + andn %o4, (64 - 1), %o4 + andn %o2, (64 - 1), %g1 + sub %o2, %g1, %o2 + + and %o1, (64 - 1), %g2 + add %o1, %g1, %o1 + sub %o0, %o4, %g3 + brz,pt %g2, 190f + cmp %g2, 32 + blu,a 5f + cmp %g2, 16 + cmp %g2, 48 + blu,a 4f + cmp %g2, 40 + cmp %g2, 56 + blu 170f + nop + ba,a,pt %xcc, 180f + +4: /* 32 <= low bits < 48 */ + blu 150f + nop + ba,a,pt %xcc, 160f +5: /* 0 < low bits < 32 */ + blu,a 6f + cmp %g2, 8 + cmp %g2, 24 + blu 130f + nop + ba,a,pt %xcc, 140f +6: /* 0 < low bits < 16 */ + bgeu 120f + nop + /* fall through for 0 < low bits < 8 */ +110: sub %o4, 64, %g2 + LOAD_BLK(%g2, %f0) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +120: sub %o4, 56, %g2 + FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +130: sub %o4, 48, %g2 + FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_6(f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +140: sub %o4, 40, %g2 + FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_5(f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +150: sub %o4, 32, %g2 + FREG_LOAD_4(%g2, f0, f2, f4, f6) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_4(f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +160: sub %o4, 24, %g2 + FREG_LOAD_3(%g2, f0, f2, f4) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_3(f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +170: sub %o4, 16, %g2 + FREG_LOAD_2(%g2, f0, f2) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_2(f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +180: sub %o4, 8, %g2 + FREG_LOAD_1(%g2, f0) +1: STORE_INIT(%g0, %o4 + %g3) + LOAD_BLK(%o4, %f16) + FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) + STORE_BLK(%f0, %o4 + %g3) + FREG_MOVE_1(f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +190: +1: STORE_INIT(%g0, %o4 + %g3) + subcc %g1, 64, %g1 + LOAD_BLK(%o4, %f0) + STORE_BLK(%f0, %o4 + %g3) + add %o4, 64, %o4 + bne,pt %XCC, 1b + LOAD(prefetch, %o4 + 64, #one_read) + +195: + add %o4, %g3, %o0 + membar #Sync + + VISExitHalf + + /* %o2 contains any final bytes still needed to be copied + * over. If anything is left, we copy it one byte at a time. + */ + brz,pt %o2, 85f + sub %o0, %o1, %o3 + ba,a,pt %XCC, 90f + + .align 64 +75: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: + andn %o2, 0xf, %o4 + and %o2, 0xf, %o2 +1: subcc %o4, 0x10, %o4 + LOAD(ldx, %o1, %o5) + add %o1, 0x08, %o1 + LOAD(ldx, %o1, %g1) + sub %o1, 0x08, %o1 + STORE(stx, %o5, %o1 + %o3) + add %o1, 0x8, %o1 + STORE(stx, %g1, %o1 + %o3) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x8, %o2 + LOAD(ldx, %o1, %o5) + STORE(stx, %o5, %o1 + %o3) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + LOAD(lduw, %o1, %o5) + STORE(stw, %o5, %o1 + %o3) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: subcc %g1, 1, %g1 + LOAD(ldub, %o1, %o5) + STORE(stb, %o5, %o1 + %o3) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + LOAD(ldx, %o1, %g2) + sub %o3, %g1, %o3 + andn %o2, 0x7, %o4 + sllx %g2, %g1, %g2 +1: add %o1, 0x8, %o1 + LOAD(ldx, %o1, %g3) + subcc %o4, 0x8, %o4 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + STORE(stx, %o5, %o0) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, %o3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + LOAD(lduw, %o1, %g1) + STORE(stw, %g1, %o1 + %o3) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov %g5, %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + LOAD(ldub, %o1, %g1) + STORE(stb, %g1, %o1 + %o3) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov %g5, %o0 + +END(__memcpy_niagara2) + +#endif diff --git a/sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S b/sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S new file mode 100644 index 0000000000..34ca089f93 --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S @@ -0,0 +1,320 @@ +/* Copy SIZE bytes from SRC to DEST. + For UltraSPARC-III. + Copyright (C) 2001, 2003 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by David S. Miller (davem@redhat.com) + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#define ASI_BLK_P 0xf0 +#define FPRS_FEF 0x04 +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs + +#ifndef XCC +#define USE_BPR +#define XCC xcc +#endif + +#if !defined NOT_IN_libc + + .register %g2,#scratch + .register %g3,#scratch + .register %g6,#scratch + + .text + + /* Special/non-trivial issues of this code: + * + * 1) %o5 is preserved from VISEntryHalf to VISExitHalf + * 2) Only low 32 FPU registers are used so that only the + * lower half of the FPU register set is dirtied by this + * code. This is especially important in the kernel. + * 3) This code never prefetches cachelines past the end + * of the source buffer. + * + * The cheetah's flexible spine, oversized liver, enlarged heart, + * slender muscular body, and claws make it the swiftest hunter + * in Africa and the fastest animal on land. Can reach speeds + * of up to 2.4GB per second. + */ + .align 32 +ENTRY(__memcpy_ultra3) + +100: /* %o0=dst, %o1=src, %o2=len */ + mov %o0, %g5 + cmp %o2, 0 + be,pn %XCC, out +218: or %o0, %o1, %o3 + cmp %o2, 16 + bleu,a,pn %XCC, small_copy + or %o3, %o2, %o3 + + cmp %o2, 256 + blu,pt %XCC, medium_copy + andcc %o3, 0x7, %g0 + + ba,pt %xcc, enter + andcc %o0, 0x3f, %g2 + + /* Here len >= 256 and condition codes reflect execution + * of "andcc %o0, 0x7, %g2", done by caller. + */ + .align 64 +enter: + /* Is 'dst' already aligned on an 64-byte boundary? */ + be,pt %XCC, 2f + + /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number + * of bytes to copy to make 'dst' 64-byte aligned. We pre- + * subtract this from 'len'. + */ + sub %g2, 0x40, %g2 + sub %g0, %g2, %g2 + sub %o2, %g2, %o2 + + /* Copy %g2 bytes from src to dst, one byte at a time. */ +1: ldub [%o1 + 0x00], %o3 + add %o1, 0x1, %o1 + add %o0, 0x1, %o0 + subcc %g2, 0x1, %g2 + + bg,pt %XCC, 1b + stb %o3, [%o0 + -1] + +2: VISEntryHalf + and %o1, 0x7, %g1 + ba,pt %xcc, begin + alignaddr %o1, %g0, %o1 + + .align 64 +begin: + prefetch [%o1 + 0x000], #one_read + prefetch [%o1 + 0x040], #one_read + andn %o2, (0x40 - 1), %o4 + prefetch [%o1 + 0x080], #one_read + prefetch [%o1 + 0x0c0], #one_read + ldd [%o1 + 0x000], %f0 + prefetch [%o1 + 0x100], #one_read + ldd [%o1 + 0x008], %f2 + prefetch [%o1 + 0x140], #one_read + ldd [%o1 + 0x010], %f4 + prefetch [%o1 + 0x180], #one_read + faligndata %f0, %f2, %f16 + ldd [%o1 + 0x018], %f6 + faligndata %f2, %f4, %f18 + ldd [%o1 + 0x020], %f8 + faligndata %f4, %f6, %f20 + ldd [%o1 + 0x028], %f10 + faligndata %f6, %f8, %f22 + + ldd [%o1 + 0x030], %f12 + faligndata %f8, %f10, %f24 + ldd [%o1 + 0x038], %f14 + faligndata %f10, %f12, %f26 + ldd [%o1 + 0x040], %f0 + + sub %o4, 0x80, %o4 + add %o1, 0x40, %o1 + ba,pt %xcc, loop + srl %o4, 6, %o3 + + .align 64 +loop: + ldd [%o1 + 0x008], %f2 + faligndata %f12, %f14, %f28 + ldd [%o1 + 0x010], %f4 + faligndata %f14, %f0, %f30 + stda %f16, [%o0] ASI_BLK_P + ldd [%o1 + 0x018], %f6 + faligndata %f0, %f2, %f16 + + ldd [%o1 + 0x020], %f8 + faligndata %f2, %f4, %f18 + ldd [%o1 + 0x028], %f10 + faligndata %f4, %f6, %f20 + ldd [%o1 + 0x030], %f12 + faligndata %f6, %f8, %f22 + ldd [%o1 + 0x038], %f14 + faligndata %f8, %f10, %f24 + + ldd [%o1 + 0x040], %f0 + prefetch [%o1 + 0x180], #one_read + faligndata %f10, %f12, %f26 + subcc %o3, 0x01, %o3 + add %o1, 0x40, %o1 + bg,pt %XCC, loop + add %o0, 0x40, %o0 + + /* Finally we copy the last full 64-byte block. */ +loopfini: + ldd [%o1 + 0x008], %f2 + faligndata %f12, %f14, %f28 + ldd [%o1 + 0x010], %f4 + faligndata %f14, %f0, %f30 + stda %f16, [%o0] ASI_BLK_P + ldd [%o1 + 0x018], %f6 + faligndata %f0, %f2, %f16 + ldd [%o1 + 0x020], %f8 + faligndata %f2, %f4, %f18 + ldd [%o1 + 0x028], %f10 + faligndata %f4, %f6, %f20 + ldd [%o1 + 0x030], %f12 + faligndata %f6, %f8, %f22 + ldd [%o1 + 0x038], %f14 + faligndata %f8, %f10, %f24 + cmp %g1, 0 + be,pt %XCC, 1f + add %o0, 0x40, %o0 + ldd [%o1 + 0x040], %f0 +1: faligndata %f10, %f12, %f26 + faligndata %f12, %f14, %f28 + faligndata %f14, %f0, %f30 + stda %f16, [%o0] ASI_BLK_P + add %o0, 0x40, %o0 + add %o1, 0x40, %o1 + membar #Sync + + /* Now we copy the (len modulo 64) bytes at the end. + * Note how we borrow the %f0 loaded above. + * + * Also notice how this code is careful not to perform a + * load past the end of the src buffer. + */ +loopend: + and %o2, 0x3f, %o2 + andcc %o2, 0x38, %g2 + be,pn %XCC, endcruft + subcc %g2, 0x8, %g2 + be,pn %XCC, endcruft + cmp %g1, 0 + + be,a,pt %XCC, 1f + ldd [%o1 + 0x00], %f0 + +1: ldd [%o1 + 0x08], %f2 + add %o1, 0x8, %o1 + sub %o2, 0x8, %o2 + subcc %g2, 0x8, %g2 + faligndata %f0, %f2, %f8 + std %f8, [%o0 + 0x00] + be,pn %XCC, endcruft + add %o0, 0x8, %o0 + ldd [%o1 + 0x08], %f0 + add %o1, 0x8, %o1 + sub %o2, 0x8, %o2 + subcc %g2, 0x8, %g2 + faligndata %f2, %f0, %f8 + std %f8, [%o0 + 0x00] + bne,pn %XCC, 1b + add %o0, 0x8, %o0 + + /* If anything is left, we copy it one byte at a time. + * Note that %g1 is (src & 0x3) saved above before the + * alignaddr was performed. + */ +endcruft: + cmp %o2, 0 + add %o1, %g1, %o1 + VISExitHalf + be,pn %XCC, out + sub %o0, %o1, %o3 + + andcc %g1, 0x7, %g0 + bne,pn %icc, small_copy_unaligned + andcc %o2, 0x8, %g0 + be,pt %icc, 1f + nop + ldx [%o1], %o5 + stx %o5, [%o1 + %o3] + add %o1, 0x8, %o1 + +1: andcc %o2, 0x4, %g0 + be,pt %icc, 1f + nop + lduw [%o1], %o5 + stw %o5, [%o1 + %o3] + add %o1, 0x4, %o1 + +1: andcc %o2, 0x2, %g0 + be,pt %icc, 1f + nop + lduh [%o1], %o5 + sth %o5, [%o1 + %o3] + add %o1, 0x2, %o1 + +1: andcc %o2, 0x1, %g0 + be,pt %icc, out + nop + ldub [%o1], %o5 + ba,pt %xcc, out + stb %o5, [%o1 + %o3] + +medium_copy: /* 16 < len <= 64 */ + bne,pn %XCC, small_copy_unaligned + sub %o0, %o1, %o3 + +medium_copy_aligned: + andn %o2, 0x7, %o4 + and %o2, 0x7, %o2 +1: subcc %o4, 0x8, %o4 + ldx [%o1], %o5 + stx %o5, [%o1 + %o3] + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 + andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + lduw [%o1], %o5 + stw %o5, [%o1 + %o3] + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, out + nop + ba,pt %xcc, small_copy_unaligned + nop + +small_copy: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, small_copy_unaligned + sub %o0, %o1, %o3 + +small_copy_aligned: + subcc %o2, 4, %o2 + lduw [%o1], %g1 + stw %g1, [%o1 + %o3] + bgu,pt %XCC, small_copy_aligned + add %o1, 4, %o1 + +out: retl + mov %g5, %o0 + + .align 32 +small_copy_unaligned: + subcc %o2, 1, %o2 + ldub [%o1], %g1 + stb %g1, [%o1 + %o3] + bgu,pt %XCC, small_copy_unaligned + add %o1, 1, %o1 + retl + mov %g5, %o0 + +END(__memcpy_ultra3) + +#endif \ No newline at end of file diff --git a/sysdeps/sparc/sparc64/multiarch/memcpy.S b/sysdeps/sparc/sparc64/multiarch/memcpy.S new file mode 100644 index 0000000000..a708de10e2 --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/memcpy.S @@ -0,0 +1,107 @@ +/* Multiple versions of memcpy + Copyright (C) 2010 Free Software Foundation, Inc. + Contributed by David S. Miller (davem@davemloft.net) + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include +#include + +#if !defined NOT_IN_libc + .text +ENTRY(memcpy) + .type memcpy, @gnu_indirect_function +# ifdef SHARED + mov %o7, %o5 + sethi %hi(_GLOBAL_OFFSET_TABLE_-4), %o3 + call 1f + or %o3, %lo(_GLOBAL_OFFSET_TABLE_+4), %o3 +1: add %o7, %o3, %o3 + mov %o5, %o7 + sethi %hi(_rtld_global_ro), %o2 + or %o2, %lo(_rtld_global_ro), %o2 +# ifdef __arch64__ + ldx [%o3 + %o2], %o2 + ldx [%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET], %o2 +# else + ld [%o3 + %o2], %o2 + ld [%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET + 4], %o2 +# endif +# else + set _dl_hwcap, %o3 +# ifdef __arch64__ + ldx [%o3], %o2 +# else + ld [%o3 + 4], %o2 +# endif +# endif + andcc %o2, 0x80, %g0 ! HWCAP_SPARC_N2 + be 1f + andcc %o2, 0x40, %g0 ! HWCAP_SPARC_BLKINIT +# ifdef SHARED + sethi %gdop_hix22(__memcpy_niagara2), %o1 + xor %o1, %gdop_lox10(__memcpy_niagara2), %o1 +# else + set __memcpy_niagara2, %o1 +# endif + ba 10f + nop +1: be 1f + andcc %o2, 0x20, %g0 ! HWCAP_SPARC_ULTRA3 +# ifdef SHARED + sethi %gdop_hix22(__memcpy_niagara1), %o1 + xor %o1, %gdop_lox10(__memcpy_niagara1), %o1 +# else + set __memcpy_niagara1, %o1 +# endif + ba 10f + nop +1: be 9f + nop +# ifdef SHARED + sethi %gdop_hix22(__memcpy_ultra3), %o1 + xor %o1, %gdop_lox10(__memcpy_ultra3), %o1 +# else + set __memcpy_ultra3, %o1 +# endif + ba 10f + nop +9: +# ifdef SHARED + sethi %gdop_hix22(__memcpy_ultra1), %o1 + xor %o1, %gdop_lox10(__memcpy_ultra1), %o1 +# else + set __memcpy_ultra1, %o1 +# endif +10: +# ifdef SHARED + add %o3, %o1, %o1 +# endif + retl + mov %o1, %o0 +END(memcpy) + +# undef libc_hidden_builtin_def +/* IFUNC doesn't work with the hidden functions in a shared library. */ +# define libc_hidden_builtin_def(name) \ + .globl __GI_memcpy; __GI_memcpy = __memcpy_ultra1 + +#define memcpy __memcpy_ultra1 + +#endif + +#include "../memcpy.S" diff --git a/sysdeps/sparc/sparc64/multiarch/memset-niagara1.S b/sysdeps/sparc/sparc64/multiarch/memset-niagara1.S new file mode 100644 index 0000000000..20ea056216 --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/memset-niagara1.S @@ -0,0 +1,132 @@ +/* Set a block of memory to some byte value. For SUN4V Niagara. + Copyright (C) 2006, 2008 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by David S. Miller (davem@davemloft.net) + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define ASI_P 0x80 +#define ASI_PNF 0x82 + +#ifndef XCC +#define USE_BPR +#define XCC xcc +#endif + +#if !defined NOT_IN_libc + + .register %g2,#scratch + + .text + .align 32 + +ENTRY(__memset_niagara1) + /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %XCC, 1f + or %g1, %o2, %o2 +END(__memset_niagara1) + +ENTRY(__bzero_niagara1) + clr %o2 +1: +# ifndef USE_BRP + srl %o1, 0, %o1 +# endif + brz,pn %o1, 90f + mov %o0, %o3 + + wr %g0, ASI_P, %asi + + cmp %o1, 15 + bl,pn %icc, 70f + andcc %o0, 0x7, %g1 + be,pt %XCC, 2f + mov 8, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: stba %o2, [%o0 + 0x00] %asi + subcc %g1, 1, %g1 + bne,pt %XCC, 1b + add %o0, 1, %o0 +2: cmp %o1, 128 + bl,pn %icc, 60f + andcc %o0, (64 - 1), %g1 + be,pt %XCC, 40f + mov 64, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: stxa %o2, [%o0 + 0x00] %asi + subcc %g1, 8, %g1 + bne,pt %XCC, 1b + add %o0, 8, %o0 + +40: + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + andn %o1, (64 - 1), %g1 + sub %o1, %g1, %o1 +50: + stxa %o2, [%o0 + 0x00] %asi + stxa %o2, [%o0 + 0x08] %asi + stxa %o2, [%o0 + 0x10] %asi + stxa %o2, [%o0 + 0x18] %asi + stxa %o2, [%o0 + 0x20] %asi + stxa %o2, [%o0 + 0x28] %asi + stxa %o2, [%o0 + 0x30] %asi + stxa %o2, [%o0 + 0x38] %asi + subcc %g1, 64, %g1 + bne,pt %XCC, 50b + add %o0, 64, %o0 + + wr %g0, ASI_P, %asi + brz,pn %o1, 80f +60: + andncc %o1, 0x7, %g1 + be,pn %XCC, 2f + sub %o1, %g1, %o1 +1: stxa %o2, [%o0 + 0x00] %asi + subcc %g1, 8, %g1 + bne,pt %XCC, 1b + add %o0, 8, %o0 +2: brz,pt %o1, 80f + nop + +70: +1: stba %o2, [%o0 + 0x00] %asi + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 + + /* fallthrough */ + +80: + wr %g0, ASI_PNF, %asi + +90: + retl + mov %o3, %o0 +END(__bzero_niagara1) + +#endif diff --git a/sysdeps/sparc/sparc64/multiarch/memset.S b/sysdeps/sparc/sparc64/multiarch/memset.S new file mode 100644 index 0000000000..23e513f18f --- /dev/null +++ b/sysdeps/sparc/sparc64/multiarch/memset.S @@ -0,0 +1,145 @@ +/* Multiple versions of memset and bzero + Copyright (C) 2010 Free Software Foundation, Inc. + Contributed by David S. Miller (davem@davemloft.net) + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include +#include + +#if !defined NOT_IN_libc + .text +ENTRY(memset) + .type memset, @gnu_indirect_function +# ifdef SHARED + mov %o7, %o5 + sethi %hi(_GLOBAL_OFFSET_TABLE_-4), %o3 + call 1f + or %o3, %lo(_GLOBAL_OFFSET_TABLE_+4), %o3 +1: add %o7, %o3, %o3 + mov %o5, %o7 + sethi %hi(_rtld_global_ro), %o2 + or %o2, %lo(_rtld_global_ro), %o2 +# ifdef __arch64__ + ldx [%o3 + %o2], %o2 + ldx [%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET], %o2 +# else + ld [%o3 + %o2], %o2 + ld [%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET + 4], %o2 +# endif +# else + set _dl_hwcap, %o3 +# ifdef __arch64__ + ldx [%o3], %o2 +# else + ld [%o3 + 4], %o2 +# endif +# endif + andcc %o2, 0x40, %g0 ! HWCAP_SPARC_BLKINIT + be 9f + nop +# ifdef SHARED + sethi %gdop_hix22(__memset_niagara1), %o1 + xor %o1, %gdop_lox10(__memset_niagara1), %o1 +# else + set __memset_niagara1, %o1 +# endif + ba 10f + nop +9: +# ifdef SHARED + sethi %gdop_hix22(__memset_ultra1), %o1 + xor %o1, %gdop_lox10(__memset_ultra1), %o1 +# else + set __memset_ultra1, %o1 +# endif +10: +# ifdef SHARED + add %o3, %o1, %o1 +# endif + retl + mov %o1, %o0 +END(memset) + +ENTRY(__bzero) + .type bzero, @gnu_indirect_function +# ifdef SHARED + mov %o7, %o5 + sethi %hi(_GLOBAL_OFFSET_TABLE_-4), %o3 + call 1f + or %o3, %lo(_GLOBAL_OFFSET_TABLE_+4), %o3 +1: add %o7, %o3, %o3 + mov %o5, %o7 + sethi %hi(_rtld_global_ro), %o2 + or %o2, %lo(_rtld_global_ro), %o2 +# ifdef __arch64__ + ldx [%o3 + %o2], %o2 + ldx [%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET], %o2 +# else + ld [%o3 + %o2], %o2 + ld [%o2 + RTLD_GLOBAL_RO_DL_HWCAP_OFFSET + 4], %o2 +# endif +# else + set _dl_hwcap, %o3 +# ifdef __arch64__ + ldx [%o3], %o2 +# else + ld [%o3 + 4], %o2 +# endif +# endif + andcc %o2, 0x40, %g0 ! HWCAP_SPARC_BLKINIT + be 9f + nop +# ifdef SHARED + sethi %gdop_hix22(__bzero_niagara1), %o1 + xor %o1, %gdop_lox10(__bzero_niagara1), %o1 +# else + set __bzero_niagara1, %o1 +# endif + ba 10f + nop +9: +# ifdef SHARED + sethi %gdop_hix22(__memset_ultra1), %o1 + xor %o1, %gdop_lox10(__memset_ultra1), %o1 +# else + set __bzero_ultra1, %o1 +# endif +10: +# ifdef SHARED + add %o3, %o1, %o1 +# endif + retl + mov %o1, %o0 +END(__bzero) + +weak_alias (__bzero, bzero) + +# undef weak_alias +# define weak_alias(a, b) + +# undef libc_hidden_builtin_def +/* IFUNC doesn't work with the hidden functions in a shared library. */ +# define libc_hidden_builtin_def(name) \ + .globl __GI_memset; __GI_memset = __memset_ultra1 + +#define memset __memset_ultra1 +#define __bzero __bzero_ultra1 + +#endif + +#include "../memset.S" diff --git a/sysdeps/sparc/sparc64/sparcv9b/memcpy.S b/sysdeps/sparc/sparc64/sparcv9b/memcpy.S deleted file mode 100644 index 389e09d820..0000000000 --- a/sysdeps/sparc/sparc64/sparcv9b/memcpy.S +++ /dev/null @@ -1,318 +0,0 @@ -/* Copy SIZE bytes from SRC to DEST. - For UltraSPARC-III. - Copyright (C) 2001, 2003 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by David S. Miller (davem@redhat.com) - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#include - -#define ASI_BLK_P 0xf0 -#define FPRS_FEF 0x04 -#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs -#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs - -#ifndef XCC -#define USE_BPR -#define XCC xcc -#endif - - .register %g2,#scratch - .register %g3,#scratch - .register %g6,#scratch - - .text - - /* Special/non-trivial issues of this code: - * - * 1) %o5 is preserved from VISEntryHalf to VISExitHalf - * 2) Only low 32 FPU registers are used so that only the - * lower half of the FPU register set is dirtied by this - * code. This is especially important in the kernel. - * 3) This code never prefetches cachelines past the end - * of the source buffer. - * - * The cheetah's flexible spine, oversized liver, enlarged heart, - * slender muscular body, and claws make it the swiftest hunter - * in Africa and the fastest animal on land. Can reach speeds - * of up to 2.4GB per second. - */ - .align 32 -ENTRY(memcpy) - -100: /* %o0=dst, %o1=src, %o2=len */ - mov %o0, %g5 - cmp %o2, 0 - be,pn %XCC, out -218: or %o0, %o1, %o3 - cmp %o2, 16 - bleu,a,pn %XCC, small_copy - or %o3, %o2, %o3 - - cmp %o2, 256 - blu,pt %XCC, medium_copy - andcc %o3, 0x7, %g0 - - ba,pt %xcc, enter - andcc %o0, 0x3f, %g2 - - /* Here len >= 256 and condition codes reflect execution - * of "andcc %o0, 0x7, %g2", done by caller. - */ - .align 64 -enter: - /* Is 'dst' already aligned on an 64-byte boundary? */ - be,pt %XCC, 2f - - /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number - * of bytes to copy to make 'dst' 64-byte aligned. We pre- - * subtract this from 'len'. - */ - sub %g2, 0x40, %g2 - sub %g0, %g2, %g2 - sub %o2, %g2, %o2 - - /* Copy %g2 bytes from src to dst, one byte at a time. */ -1: ldub [%o1 + 0x00], %o3 - add %o1, 0x1, %o1 - add %o0, 0x1, %o0 - subcc %g2, 0x1, %g2 - - bg,pt %XCC, 1b - stb %o3, [%o0 + -1] - -2: VISEntryHalf - and %o1, 0x7, %g1 - ba,pt %xcc, begin - alignaddr %o1, %g0, %o1 - - .align 64 -begin: - prefetch [%o1 + 0x000], #one_read - prefetch [%o1 + 0x040], #one_read - andn %o2, (0x40 - 1), %o4 - prefetch [%o1 + 0x080], #one_read - prefetch [%o1 + 0x0c0], #one_read - ldd [%o1 + 0x000], %f0 - prefetch [%o1 + 0x100], #one_read - ldd [%o1 + 0x008], %f2 - prefetch [%o1 + 0x140], #one_read - ldd [%o1 + 0x010], %f4 - prefetch [%o1 + 0x180], #one_read - faligndata %f0, %f2, %f16 - ldd [%o1 + 0x018], %f6 - faligndata %f2, %f4, %f18 - ldd [%o1 + 0x020], %f8 - faligndata %f4, %f6, %f20 - ldd [%o1 + 0x028], %f10 - faligndata %f6, %f8, %f22 - - ldd [%o1 + 0x030], %f12 - faligndata %f8, %f10, %f24 - ldd [%o1 + 0x038], %f14 - faligndata %f10, %f12, %f26 - ldd [%o1 + 0x040], %f0 - - sub %o4, 0x80, %o4 - add %o1, 0x40, %o1 - ba,pt %xcc, loop - srl %o4, 6, %o3 - - .align 64 -loop: - ldd [%o1 + 0x008], %f2 - faligndata %f12, %f14, %f28 - ldd [%o1 + 0x010], %f4 - faligndata %f14, %f0, %f30 - stda %f16, [%o0] ASI_BLK_P - ldd [%o1 + 0x018], %f6 - faligndata %f0, %f2, %f16 - - ldd [%o1 + 0x020], %f8 - faligndata %f2, %f4, %f18 - ldd [%o1 + 0x028], %f10 - faligndata %f4, %f6, %f20 - ldd [%o1 + 0x030], %f12 - faligndata %f6, %f8, %f22 - ldd [%o1 + 0x038], %f14 - faligndata %f8, %f10, %f24 - - ldd [%o1 + 0x040], %f0 - prefetch [%o1 + 0x180], #one_read - faligndata %f10, %f12, %f26 - subcc %o3, 0x01, %o3 - add %o1, 0x40, %o1 - bg,pt %XCC, loop - add %o0, 0x40, %o0 - - /* Finally we copy the last full 64-byte block. */ -loopfini: - ldd [%o1 + 0x008], %f2 - faligndata %f12, %f14, %f28 - ldd [%o1 + 0x010], %f4 - faligndata %f14, %f0, %f30 - stda %f16, [%o0] ASI_BLK_P - ldd [%o1 + 0x018], %f6 - faligndata %f0, %f2, %f16 - ldd [%o1 + 0x020], %f8 - faligndata %f2, %f4, %f18 - ldd [%o1 + 0x028], %f10 - faligndata %f4, %f6, %f20 - ldd [%o1 + 0x030], %f12 - faligndata %f6, %f8, %f22 - ldd [%o1 + 0x038], %f14 - faligndata %f8, %f10, %f24 - cmp %g1, 0 - be,pt %XCC, 1f - add %o0, 0x40, %o0 - ldd [%o1 + 0x040], %f0 -1: faligndata %f10, %f12, %f26 - faligndata %f12, %f14, %f28 - faligndata %f14, %f0, %f30 - stda %f16, [%o0] ASI_BLK_P - add %o0, 0x40, %o0 - add %o1, 0x40, %o1 - membar #Sync - - /* Now we copy the (len modulo 64) bytes at the end. - * Note how we borrow the %f0 loaded above. - * - * Also notice how this code is careful not to perform a - * load past the end of the src buffer. - */ -loopend: - and %o2, 0x3f, %o2 - andcc %o2, 0x38, %g2 - be,pn %XCC, endcruft - subcc %g2, 0x8, %g2 - be,pn %XCC, endcruft - cmp %g1, 0 - - be,a,pt %XCC, 1f - ldd [%o1 + 0x00], %f0 - -1: ldd [%o1 + 0x08], %f2 - add %o1, 0x8, %o1 - sub %o2, 0x8, %o2 - subcc %g2, 0x8, %g2 - faligndata %f0, %f2, %f8 - std %f8, [%o0 + 0x00] - be,pn %XCC, endcruft - add %o0, 0x8, %o0 - ldd [%o1 + 0x08], %f0 - add %o1, 0x8, %o1 - sub %o2, 0x8, %o2 - subcc %g2, 0x8, %g2 - faligndata %f2, %f0, %f8 - std %f8, [%o0 + 0x00] - bne,pn %XCC, 1b - add %o0, 0x8, %o0 - - /* If anything is left, we copy it one byte at a time. - * Note that %g1 is (src & 0x3) saved above before the - * alignaddr was performed. - */ -endcruft: - cmp %o2, 0 - add %o1, %g1, %o1 - VISExitHalf - be,pn %XCC, out - sub %o0, %o1, %o3 - - andcc %g1, 0x7, %g0 - bne,pn %icc, small_copy_unaligned - andcc %o2, 0x8, %g0 - be,pt %icc, 1f - nop - ldx [%o1], %o5 - stx %o5, [%o1 + %o3] - add %o1, 0x8, %o1 - -1: andcc %o2, 0x4, %g0 - be,pt %icc, 1f - nop - lduw [%o1], %o5 - stw %o5, [%o1 + %o3] - add %o1, 0x4, %o1 - -1: andcc %o2, 0x2, %g0 - be,pt %icc, 1f - nop - lduh [%o1], %o5 - sth %o5, [%o1 + %o3] - add %o1, 0x2, %o1 - -1: andcc %o2, 0x1, %g0 - be,pt %icc, out - nop - ldub [%o1], %o5 - ba,pt %xcc, out - stb %o5, [%o1 + %o3] - -medium_copy: /* 16 < len <= 64 */ - bne,pn %XCC, small_copy_unaligned - sub %o0, %o1, %o3 - -medium_copy_aligned: - andn %o2, 0x7, %o4 - and %o2, 0x7, %o2 -1: subcc %o4, 0x8, %o4 - ldx [%o1], %o5 - stx %o5, [%o1 + %o3] - bgu,pt %XCC, 1b - add %o1, 0x8, %o1 - andcc %o2, 0x4, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x4, %o2 - lduw [%o1], %o5 - stw %o5, [%o1 + %o3] - add %o1, 0x4, %o1 -1: cmp %o2, 0 - be,pt %XCC, out - nop - ba,pt %xcc, small_copy_unaligned - nop - -small_copy: /* 0 < len <= 16 */ - andcc %o3, 0x3, %g0 - bne,pn %XCC, small_copy_unaligned - sub %o0, %o1, %o3 - -small_copy_aligned: - subcc %o2, 4, %o2 - lduw [%o1], %g1 - stw %g1, [%o1 + %o3] - bgu,pt %XCC, small_copy_aligned - add %o1, 4, %o1 - -out: retl - mov %g5, %o0 - - .align 32 -small_copy_unaligned: - subcc %o2, 1, %o2 - ldub [%o1], %g1 - stb %g1, [%o1 + %o3] - bgu,pt %XCC, small_copy_unaligned - add %o1, 1, %o1 - retl - mov %g5, %o0 - -END(memcpy) - -libc_hidden_builtin_def (memcpy) diff --git a/sysdeps/sparc/sparc64/sparcv9v/memcpy.S b/sysdeps/sparc/sparc64/sparcv9v/memcpy.S deleted file mode 100644 index 116c7113ff..0000000000 --- a/sysdeps/sparc/sparc64/sparcv9v/memcpy.S +++ /dev/null @@ -1,340 +0,0 @@ -/* Copy SIZE bytes from SRC to DEST. For SUN4V Niagara. - Copyright (C) 2006, 2008 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by David S. Miller (davem@davemloft.net) - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#include - -#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 -#define ASI_P 0x80 -#define ASI_PNF 0x82 - -#define LOAD(type,addr,dest) type##a [addr] ASI_P, dest -#define LOAD_TWIN(addr_reg,dest0,dest1) \ - ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_P, dest0 - -#define STORE(type,src,addr) type src, [addr] -#define STORE_INIT(src,addr) stxa src, [addr] %asi - -#ifndef XCC -#define USE_BPR -#define XCC xcc -#endif - - .register %g2,#scratch - .register %g3,#scratch - .register %g6,#scratch - - .text - - .align 32 -ENTRY(memcpy) -#ifndef USE_BPR - srl %o2, 0, %o2 -#endif -100: /* %o0=dst, %o1=src, %o2=len */ - mov %o0, %g5 - cmp %o2, 0 - be,pn %XCC, 85f -218: or %o0, %o1, %o3 - cmp %o2, 16 - blu,a,pn %XCC, 80f - or %o3, %o2, %o3 - - /* 2 blocks (128 bytes) is the minimum we can do the block - * copy with. We need to ensure that we'll iterate at least - * once in the block copy loop. At worst we'll need to align - * the destination to a 64-byte boundary which can chew up - * to (64 - 1) bytes from the length before we perform the - * block copy loop. - */ - cmp %o2, (2 * 64) - blu,pt %XCC, 70f - andcc %o3, 0x7, %g0 - - /* %o0: dst - * %o1: src - * %o2: len (known to be >= 128) - * - * The block copy loops will use %o4/%o5,%g2/%g3 as - * temporaries while copying the data. - */ - - LOAD(prefetch, %o1, #one_read) - wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi - - /* Align destination on 64-byte boundary. */ - andcc %o0, (64 - 1), %o4 - be,pt %XCC, 2f - sub %o4, 64, %o4 - sub %g0, %o4, %o4 ! bytes to align dst - sub %o2, %o4, %o2 -1: subcc %o4, 1, %o4 - LOAD(ldub, %o1, %g1) - STORE(stb, %g1, %o0) - add %o1, 1, %o1 - bne,pt %XCC, 1b - add %o0, 1, %o0 - - /* If the source is on a 16-byte boundary we can do - * the direct block copy loop. If it is 8-byte aligned - * we can do the 16-byte loads offset by -8 bytes and the - * init stores offset by one register. - * - * If the source is not even 8-byte aligned, we need to do - * shifting and masking (basically integer faligndata). - * - * The careful bit with init stores is that if we store - * to any part of the cache line we have to store the whole - * cacheline else we can end up with corrupt L2 cache line - * contents. Since the loop works on 64-bytes of 64-byte - * aligned store data at a time, this is easy to ensure. - */ -2: - andcc %o1, (16 - 1), %o4 - andn %o2, (64 - 1), %g1 ! block copy loop iterator - sub %o2, %g1, %o2 ! final sub-block copy bytes - be,pt %XCC, 50f - cmp %o4, 8 - be,a,pt %XCC, 10f - sub %o1, 0x8, %o1 - - /* Neither 8-byte nor 16-byte aligned, shift and mask. */ - mov %g1, %o4 - and %o1, 0x7, %g1 - sll %g1, 3, %g1 - mov 64, %o3 - andn %o1, 0x7, %o1 - LOAD(ldx, %o1, %g2) - sub %o3, %g1, %o3 - sllx %g2, %g1, %g2 - -#define SWIVEL_ONE_DWORD(SRC, TMP1, TMP2, PRE_VAL, PRE_SHIFT, POST_SHIFT, DST)\ - LOAD(ldx, SRC, TMP1); \ - srlx TMP1, PRE_SHIFT, TMP2; \ - or TMP2, PRE_VAL, TMP2; \ - STORE_INIT(TMP2, DST); \ - sllx TMP1, POST_SHIFT, PRE_VAL; - -1: add %o1, 0x8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x00) - add %o1, 0x8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x08) - add %o1, 0x8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x10) - add %o1, 0x8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x18) - add %o1, 32, %o1 - LOAD(prefetch, %o1, #one_read) - sub %o1, 32 - 8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x20) - add %o1, 8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x28) - add %o1, 8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x30) - add %o1, 8, %o1 - SWIVEL_ONE_DWORD(%o1, %g3, %o5, %g2, %o3, %g1, %o0 + 0x38) - subcc %o4, 64, %o4 - bne,pt %XCC, 1b - add %o0, 64, %o0 - -#undef SWIVEL_ONE_DWORD - - srl %g1, 3, %g1 - ba,pt %XCC, 60f - add %o1, %g1, %o1 - -10: /* Destination is 64-byte aligned, source was only 8-byte - * aligned but it has been subtracted by 8 and we perform - * one twin load ahead, then add 8 back into source when - * we finish the loop. - */ - LOAD_TWIN(%o1, %o4, %o5) -1: add %o1, 16, %o1 - LOAD_TWIN(%o1, %g2, %g3) - add %o1, 16 + 32, %o1 - LOAD(prefetch, %o1, #one_read) - sub %o1, 32, %o1 - STORE_INIT(%o5, %o0 + 0x00) ! initializes cache line - STORE_INIT(%g2, %o0 + 0x08) - LOAD_TWIN(%o1, %o4, %o5) - add %o1, 16, %o1 - STORE_INIT(%g3, %o0 + 0x10) - STORE_INIT(%o4, %o0 + 0x18) - LOAD_TWIN(%o1, %g2, %g3) - add %o1, 16, %o1 - STORE_INIT(%o5, %o0 + 0x20) - STORE_INIT(%g2, %o0 + 0x28) - LOAD_TWIN(%o1, %o4, %o5) - STORE_INIT(%g3, %o0 + 0x30) - STORE_INIT(%o4, %o0 + 0x38) - subcc %g1, 64, %g1 - bne,pt %XCC, 1b - add %o0, 64, %o0 - - ba,pt %XCC, 60f - add %o1, 0x8, %o1 - -50: /* Destination is 64-byte aligned, and source is 16-byte - * aligned. - */ -1: LOAD_TWIN(%o1, %o4, %o5) - add %o1, 16, %o1 - LOAD_TWIN(%o1, %g2, %g3) - add %o1, 16 + 32, %o1 - LOAD(prefetch, %o1, #one_read) - sub %o1, 32, %o1 - STORE_INIT(%o4, %o0 + 0x00) ! initializes cache line - STORE_INIT(%o5, %o0 + 0x08) - LOAD_TWIN(%o1, %o4, %o5) - add %o1, 16, %o1 - STORE_INIT(%g2, %o0 + 0x10) - STORE_INIT(%g3, %o0 + 0x18) - LOAD_TWIN(%o1, %g2, %g3) - add %o1, 16, %o1 - STORE_INIT(%o4, %o0 + 0x20) - STORE_INIT(%o5, %o0 + 0x28) - STORE_INIT(%g2, %o0 + 0x30) - STORE_INIT(%g3, %o0 + 0x38) - subcc %g1, 64, %g1 - bne,pt %XCC, 1b - add %o0, 64, %o0 - /* fall through */ - -60: - /* %o2 contains any final bytes still needed to be copied - * over. If anything is left, we copy it one byte at a time. - */ - wr %g0, ASI_PNF, %asi - brz,pt %o2, 85f - sub %o0, %o1, %o3 - ba,a,pt %XCC, 90f - - .align 64 -70: /* 16 < len <= 64 */ - bne,pn %XCC, 75f - sub %o0, %o1, %o3 - -72: - andn %o2, 0xf, %o4 - and %o2, 0xf, %o2 -1: subcc %o4, 0x10, %o4 - LOAD(ldx, %o1, %o5) - add %o1, 0x08, %o1 - LOAD(ldx, %o1, %g1) - sub %o1, 0x08, %o1 - STORE(stx, %o5, %o1 + %o3) - add %o1, 0x8, %o1 - STORE(stx, %g1, %o1 + %o3) - bgu,pt %XCC, 1b - add %o1, 0x8, %o1 -73: andcc %o2, 0x8, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x8, %o2 - LOAD(ldx, %o1, %o5) - STORE(stx, %o5, %o1 + %o3) - add %o1, 0x8, %o1 -1: andcc %o2, 0x4, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x4, %o2 - LOAD(lduw, %o1, %o5) - STORE(stw, %o5, %o1 + %o3) - add %o1, 0x4, %o1 -1: cmp %o2, 0 - be,pt %XCC, 85f - nop - ba,pt %XCC, 90f - nop - -75: - andcc %o0, 0x7, %g1 - sub %g1, 0x8, %g1 - be,pn %icc, 2f - sub %g0, %g1, %g1 - sub %o2, %g1, %o2 - -1: subcc %g1, 1, %g1 - LOAD(ldub, %o1, %o5) - STORE(stb, %o5, %o1 + %o3) - bgu,pt %icc, 1b - add %o1, 1, %o1 - -2: add %o1, %o3, %o0 - andcc %o1, 0x7, %g1 - bne,pt %icc, 8f - sll %g1, 3, %g1 - - cmp %o2, 16 - bgeu,pt %icc, 72b - nop - ba,a,pt %XCC, 73b - -8: mov 64, %o3 - andn %o1, 0x7, %o1 - LOAD(ldx, %o1, %g2) - sub %o3, %g1, %o3 - andn %o2, 0x7, %o4 - sllx %g2, %g1, %g2 -1: add %o1, 0x8, %o1 - LOAD(ldx, %o1, %g3) - subcc %o4, 0x8, %o4 - srlx %g3, %o3, %o5 - or %o5, %g2, %o5 - STORE(stx, %o5, %o0) - add %o0, 0x8, %o0 - bgu,pt %icc, 1b - sllx %g3, %g1, %g2 - - srl %g1, 3, %g1 - andcc %o2, 0x7, %o2 - be,pn %icc, 85f - add %o1, %g1, %o1 - ba,pt %XCC, 90f - sub %o0, %o1, %o3 - - .align 64 -80: /* 0 < len <= 16 */ - andcc %o3, 0x3, %g0 - bne,pn %XCC, 90f - sub %o0, %o1, %o3 - -1: - subcc %o2, 4, %o2 - LOAD(lduw, %o1, %g1) - STORE(stw, %g1, %o1 + %o3) - bgu,pt %XCC, 1b - add %o1, 4, %o1 - -85: retl - mov %g5, %o0 - - .align 32 -90: - subcc %o2, 1, %o2 - LOAD(ldub, %o1, %g1) - STORE(stb, %g1, %o1 + %o3) - bgu,pt %XCC, 90b - add %o1, 1, %o1 - retl - mov %g5, %o0 - -END(memcpy) - -libc_hidden_builtin_def (memcpy) diff --git a/sysdeps/sparc/sparc64/sparcv9v/memset.S b/sysdeps/sparc/sparc64/sparcv9v/memset.S deleted file mode 100644 index 64817b8871..0000000000 --- a/sysdeps/sparc/sparc64/sparcv9v/memset.S +++ /dev/null @@ -1,131 +0,0 @@ -/* Set a block of memory to some byte value. For SUN4V Niagara. - Copyright (C) 2006, 2008 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by David S. Miller (davem@davemloft.net) - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#include - -#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 -#define ASI_P 0x80 -#define ASI_PNF 0x82 - -#ifndef XCC -#define USE_BPR -#define XCC xcc -#endif - - .register %g2,#scratch - - .text - .align 32 - -ENTRY(memset) - /* %o0=buf, %o1=pat, %o2=len */ - and %o1, 0xff, %o3 - mov %o2, %o1 - sllx %o3, 8, %g1 - or %g1, %o3, %o2 - sllx %o2, 16, %g1 - or %g1, %o2, %o2 - sllx %o2, 32, %g1 - ba,pt %XCC, 1f - or %g1, %o2, %o2 -END(memset) - -ENTRY(__bzero) - clr %o2 -1: -#ifndef USE_BRP - srl %o1, 0, %o1 -#endif - brz,pn %o1, 90f - mov %o0, %o3 - - wr %g0, ASI_P, %asi - - cmp %o1, 15 - bl,pn %icc, 70f - andcc %o0, 0x7, %g1 - be,pt %XCC, 2f - mov 8, %g2 - sub %g2, %g1, %g1 - sub %o1, %g1, %o1 -1: stba %o2, [%o0 + 0x00] %asi - subcc %g1, 1, %g1 - bne,pt %XCC, 1b - add %o0, 1, %o0 -2: cmp %o1, 128 - bl,pn %icc, 60f - andcc %o0, (64 - 1), %g1 - be,pt %XCC, 40f - mov 64, %g2 - sub %g2, %g1, %g1 - sub %o1, %g1, %o1 -1: stxa %o2, [%o0 + 0x00] %asi - subcc %g1, 8, %g1 - bne,pt %XCC, 1b - add %o0, 8, %o0 - -40: - wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi - andn %o1, (64 - 1), %g1 - sub %o1, %g1, %o1 -50: - stxa %o2, [%o0 + 0x00] %asi - stxa %o2, [%o0 + 0x08] %asi - stxa %o2, [%o0 + 0x10] %asi - stxa %o2, [%o0 + 0x18] %asi - stxa %o2, [%o0 + 0x20] %asi - stxa %o2, [%o0 + 0x28] %asi - stxa %o2, [%o0 + 0x30] %asi - stxa %o2, [%o0 + 0x38] %asi - subcc %g1, 64, %g1 - bne,pt %XCC, 50b - add %o0, 64, %o0 - - wr %g0, ASI_P, %asi - brz,pn %o1, 80f -60: - andncc %o1, 0x7, %g1 - be,pn %XCC, 2f - sub %o1, %g1, %o1 -1: stxa %o2, [%o0 + 0x00] %asi - subcc %g1, 8, %g1 - bne,pt %XCC, 1b - add %o0, 8, %o0 -2: brz,pt %o1, 80f - nop - -70: -1: stba %o2, [%o0 + 0x00] %asi - subcc %o1, 1, %o1 - bne,pt %icc, 1b - add %o0, 1, %o0 - - /* fallthrough */ - -80: - wr %g0, ASI_PNF, %asi - -90: - retl - mov %o3, %o0 -END(__bzero) - -libc_hidden_builtin_def (memset) -weak_alias (__bzero, bzero) diff --git a/sysdeps/sparc/sparc64/sparcv9v2/memcpy.S b/sysdeps/sparc/sparc64/sparcv9v2/memcpy.S deleted file mode 100644 index 300b12f427..0000000000 --- a/sysdeps/sparc/sparc64/sparcv9v2/memcpy.S +++ /dev/null @@ -1,490 +0,0 @@ -/* Copy SIZE bytes from SRC to DEST. For SUN4V Niagara-2. - Copyright (C) 2007, 2008 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by David S. Miller (davem@davemloft.net) - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#include - -#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 -#define ASI_BLK_P 0xf0 -#define ASI_P 0x80 -#define ASI_PNF 0x82 - -#define FPRS_FEF 0x04 - -#define VISEntryHalf \ - rd %fprs, %o5; \ - wr %g0, FPRS_FEF, %fprs - -#define VISExitHalf \ - and %o5, FPRS_FEF, %o5; \ - wr %o5, 0x0, %fprs - -#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P - -#define LOAD(type,addr,dest) type [addr], dest -#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest -#define STORE(type,src,addr) type src, [addr] -#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P -#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI - -#ifndef XCC -#define USE_BPR -#define XCC xcc -#endif - -#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ - faligndata %x0, %x1, %f0; \ - faligndata %x1, %x2, %f2; \ - faligndata %x2, %x3, %f4; \ - faligndata %x3, %x4, %f6; \ - faligndata %x4, %x5, %f8; \ - faligndata %x5, %x6, %f10; \ - faligndata %x6, %x7, %f12; \ - faligndata %x7, %x8, %f14; - -#define FREG_MOVE_1(x0) \ - fmovd %x0, %f0; -#define FREG_MOVE_2(x0, x1) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; -#define FREG_MOVE_3(x0, x1, x2) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; -#define FREG_MOVE_4(x0, x1, x2, x3) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; -#define FREG_MOVE_5(x0, x1, x2, x3, x4) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; -#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; -#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; \ - fmovd %x6, %f12; -#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; \ - fmovd %x6, %f12; \ - fmovd %x7, %f14; -#define FREG_LOAD_1(base, x0) \ - LOAD(ldd, base + 0x00, %x0) -#define FREG_LOAD_2(base, x0, x1) \ - LOAD(ldd, base + 0x00, %x0); \ - LOAD(ldd, base + 0x08, %x1); -#define FREG_LOAD_3(base, x0, x1, x2) \ - LOAD(ldd, base + 0x00, %x0); \ - LOAD(ldd, base + 0x08, %x1); \ - LOAD(ldd, base + 0x10, %x2); -#define FREG_LOAD_4(base, x0, x1, x2, x3) \ - LOAD(ldd, base + 0x00, %x0); \ - LOAD(ldd, base + 0x08, %x1); \ - LOAD(ldd, base + 0x10, %x2); \ - LOAD(ldd, base + 0x18, %x3); -#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ - LOAD(ldd, base + 0x00, %x0); \ - LOAD(ldd, base + 0x08, %x1); \ - LOAD(ldd, base + 0x10, %x2); \ - LOAD(ldd, base + 0x18, %x3); \ - LOAD(ldd, base + 0x20, %x4); -#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ - LOAD(ldd, base + 0x00, %x0); \ - LOAD(ldd, base + 0x08, %x1); \ - LOAD(ldd, base + 0x10, %x2); \ - LOAD(ldd, base + 0x18, %x3); \ - LOAD(ldd, base + 0x20, %x4); \ - LOAD(ldd, base + 0x28, %x5); -#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ - LOAD(ldd, base + 0x00, %x0); \ - LOAD(ldd, base + 0x08, %x1); \ - LOAD(ldd, base + 0x10, %x2); \ - LOAD(ldd, base + 0x18, %x3); \ - LOAD(ldd, base + 0x20, %x4); \ - LOAD(ldd, base + 0x28, %x5); \ - LOAD(ldd, base + 0x30, %x6); - - .register %g2,#scratch - .register %g3,#scratch - .register %g6,#scratch - - .text - - .align 32 -ENTRY(memcpy) -#ifndef USE_BPR - srl %o2, 0, %o2 -#endif -100: /* %o0=dst, %o1=src, %o2=len */ - mov %o0, %g5 - cmp %o2, 0 - be,pn %XCC, 85f -218: or %o0, %o1, %o3 - cmp %o2, 16 - blu,a,pn %XCC, 80f - or %o3, %o2, %o3 - - /* 2 blocks (128 bytes) is the minimum we can do the block - * copy with. We need to ensure that we'll iterate at least - * once in the block copy loop. At worst we'll need to align - * the destination to a 64-byte boundary which can chew up - * to (64 - 1) bytes from the length before we perform the - * block copy loop. - * - * However, the cut-off point, performance wise, is around - * 4 64-byte blocks. - */ - cmp %o2, (4 * 64) - blu,pt %XCC, 75f - andcc %o3, 0x7, %g0 - - /* %o0: dst - * %o1: src - * %o2: len (known to be >= 128) - * - * The block copy loops can use %o4, %g2, %g3 as - * temporaries while copying the data. %o5 must - * be preserved between VISEntryHalf and VISExitHalf - */ - - LOAD(prefetch, %o1 + 0x000, #one_read) - LOAD(prefetch, %o1 + 0x040, #one_read) - LOAD(prefetch, %o1 + 0x080, #one_read) - - /* Align destination on 64-byte boundary. */ - andcc %o0, (64 - 1), %o4 - be,pt %XCC, 2f - sub %o4, 64, %o4 - sub %g0, %o4, %o4 ! bytes to align dst - sub %o2, %o4, %o2 -1: subcc %o4, 1, %o4 - LOAD(ldub, %o1, %g1) - STORE(stb, %g1, %o0) - add %o1, 1, %o1 - bne,pt %XCC, 1b - add %o0, 1, %o0 - -2: - /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve - * o5 from here until we hit VISExitHalf. - */ - VISEntryHalf - - alignaddr %o1, %g0, %g0 - - add %o1, (64 - 1), %o4 - andn %o4, (64 - 1), %o4 - andn %o2, (64 - 1), %g1 - sub %o2, %g1, %o2 - - and %o1, (64 - 1), %g2 - add %o1, %g1, %o1 - sub %o0, %o4, %g3 - brz,pt %g2, 190f - cmp %g2, 32 - blu,a 5f - cmp %g2, 16 - cmp %g2, 48 - blu,a 4f - cmp %g2, 40 - cmp %g2, 56 - blu 170f - nop - ba,a,pt %xcc, 180f - -4: /* 32 <= low bits < 48 */ - blu 150f - nop - ba,a,pt %xcc, 160f -5: /* 0 < low bits < 32 */ - blu,a 6f - cmp %g2, 8 - cmp %g2, 24 - blu 130f - nop - ba,a,pt %xcc, 140f -6: /* 0 < low bits < 16 */ - bgeu 120f - nop - /* fall through for 0 < low bits < 8 */ -110: sub %o4, 64, %g2 - LOAD_BLK(%g2, %f0) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -120: sub %o4, 56, %g2 - FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -130: sub %o4, 48, %g2 - FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_6(f20, f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -140: sub %o4, 40, %g2 - FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_5(f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -150: sub %o4, 32, %g2 - FREG_LOAD_4(%g2, f0, f2, f4, f6) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_4(f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -160: sub %o4, 24, %g2 - FREG_LOAD_3(%g2, f0, f2, f4) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_3(f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -170: sub %o4, 16, %g2 - FREG_LOAD_2(%g2, f0, f2) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_2(f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -180: sub %o4, 8, %g2 - FREG_LOAD_1(%g2, f0) -1: STORE_INIT(%g0, %o4 + %g3) - LOAD_BLK(%o4, %f16) - FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) - STORE_BLK(%f0, %o4 + %g3) - FREG_MOVE_1(f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - ba,pt %xcc, 195f - nop - -190: -1: STORE_INIT(%g0, %o4 + %g3) - subcc %g1, 64, %g1 - LOAD_BLK(%o4, %f0) - STORE_BLK(%f0, %o4 + %g3) - add %o4, 64, %o4 - bne,pt %XCC, 1b - LOAD(prefetch, %o4 + 64, #one_read) - -195: - add %o4, %g3, %o0 - membar #Sync - - VISExitHalf - - /* %o2 contains any final bytes still needed to be copied - * over. If anything is left, we copy it one byte at a time. - */ - brz,pt %o2, 85f - sub %o0, %o1, %o3 - ba,a,pt %XCC, 90f - - .align 64 -75: /* 16 < len <= 64 */ - bne,pn %XCC, 75f - sub %o0, %o1, %o3 - -72: - andn %o2, 0xf, %o4 - and %o2, 0xf, %o2 -1: subcc %o4, 0x10, %o4 - LOAD(ldx, %o1, %o5) - add %o1, 0x08, %o1 - LOAD(ldx, %o1, %g1) - sub %o1, 0x08, %o1 - STORE(stx, %o5, %o1 + %o3) - add %o1, 0x8, %o1 - STORE(stx, %g1, %o1 + %o3) - bgu,pt %XCC, 1b - add %o1, 0x8, %o1 -73: andcc %o2, 0x8, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x8, %o2 - LOAD(ldx, %o1, %o5) - STORE(stx, %o5, %o1 + %o3) - add %o1, 0x8, %o1 -1: andcc %o2, 0x4, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x4, %o2 - LOAD(lduw, %o1, %o5) - STORE(stw, %o5, %o1 + %o3) - add %o1, 0x4, %o1 -1: cmp %o2, 0 - be,pt %XCC, 85f - nop - ba,pt %xcc, 90f - nop - -75: - andcc %o0, 0x7, %g1 - sub %g1, 0x8, %g1 - be,pn %icc, 2f - sub %g0, %g1, %g1 - sub %o2, %g1, %o2 - -1: subcc %g1, 1, %g1 - LOAD(ldub, %o1, %o5) - STORE(stb, %o5, %o1 + %o3) - bgu,pt %icc, 1b - add %o1, 1, %o1 - -2: add %o1, %o3, %o0 - andcc %o1, 0x7, %g1 - bne,pt %icc, 8f - sll %g1, 3, %g1 - - cmp %o2, 16 - bgeu,pt %icc, 72b - nop - ba,a,pt %xcc, 73b - -8: mov 64, %o3 - andn %o1, 0x7, %o1 - LOAD(ldx, %o1, %g2) - sub %o3, %g1, %o3 - andn %o2, 0x7, %o4 - sllx %g2, %g1, %g2 -1: add %o1, 0x8, %o1 - LOAD(ldx, %o1, %g3) - subcc %o4, 0x8, %o4 - srlx %g3, %o3, %o5 - or %o5, %g2, %o5 - STORE(stx, %o5, %o0) - add %o0, 0x8, %o0 - bgu,pt %icc, 1b - sllx %g3, %g1, %g2 - - srl %g1, 3, %g1 - andcc %o2, 0x7, %o2 - be,pn %icc, 85f - add %o1, %g1, %o1 - ba,pt %xcc, 90f - sub %o0, %o1, %o3 - - .align 64 -80: /* 0 < len <= 16 */ - andcc %o3, 0x3, %g0 - bne,pn %XCC, 90f - sub %o0, %o1, %o3 - -1: - subcc %o2, 4, %o2 - LOAD(lduw, %o1, %g1) - STORE(stw, %g1, %o1 + %o3) - bgu,pt %XCC, 1b - add %o1, 4, %o1 - -85: retl - mov %g5, %o0 - - .align 32 -90: - subcc %o2, 1, %o2 - LOAD(ldub, %o1, %g1) - STORE(stb, %g1, %o1 + %o3) - bgu,pt %XCC, 90b - add %o1, 1, %o1 - retl - mov %g5, %o0 - -END(memcpy) - -libc_hidden_builtin_def (memcpy) diff --git a/sysdeps/sparc/sparc64/sparcv9v2/memset.S b/sysdeps/sparc/sparc64/sparcv9v2/memset.S deleted file mode 100644 index 809d3ed9c6..0000000000 --- a/sysdeps/sparc/sparc64/sparcv9v2/memset.S +++ /dev/null @@ -1 +0,0 @@ -#include -- cgit v1.2.3