aboutsummaryrefslogtreecommitdiff
path: root/sysdeps/powerpc
diff options
context:
space:
mode:
authorUlrich Drepper <drepper@redhat.com>2004-07-25 04:27:09 +0000
committerUlrich Drepper <drepper@redhat.com>2004-07-25 04:27:09 +0000
commita8870a617d5f20fe588edd5ffb6ea14d80ebdbef (patch)
tree95d7285c8a7238c997387baff79bdd2eeb0b25fd /sysdeps/powerpc
parentdeddf8092de80946261279dd9669a801c24aad81 (diff)
downloadglibc-a8870a617d5f20fe588edd5ffb6ea14d80ebdbef.tar
glibc-a8870a617d5f20fe588edd5ffb6ea14d80ebdbef.tar.gz
glibc-a8870a617d5f20fe588edd5ffb6ea14d80ebdbef.tar.bz2
glibc-a8870a617d5f20fe588edd5ffb6ea14d80ebdbef.zip
Update.
2004-07-17 Steven Munroe <sjmunroe@us.ibm.com> * sysdeps/powerpc/powerpc64/memcpy.S: Improve instruction scheduling for POWER4 machines.
Diffstat (limited to 'sysdeps/powerpc')
-rw-r--r--sysdeps/powerpc/powerpc64/memcpy.S327
1 files changed, 241 insertions, 86 deletions
diff --git a/sysdeps/powerpc/powerpc64/memcpy.S b/sysdeps/powerpc/powerpc64/memcpy.S
index f3448e0b31..251e1fed80 100644
--- a/sysdeps/powerpc/powerpc64/memcpy.S
+++ b/sysdeps/powerpc/powerpc64/memcpy.S
@@ -24,12 +24,12 @@
/* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
Returns 'dst'.
- Memcpy handles short copies (< 32-bytes) using an unaligned
- word lwz/stw loop. The tail (remaining 1-3) bytes is handled with the
- appropriate combination of byte and halfword load/stores. There is no
- attempt to optimize the alignment of short moves. The 64-bit
- implementations of POWER3 and POWER4 do a reasonable job of handling
- unligned load/stores that do not cross 32-byte boundries.
+ Memcpy handles short copies (< 32-bytes) using a binary move blocks
+ (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
+ with the appropriate combination of byte and halfword load/stores.
+ There is minimal effort to optimize the alignment of short moves.
+ The 64-bit implementations of POWER3 and POWER4 do a reasonable job
+ of handling unligned load/stores that do not cross 32-byte boundries.
Longer moves (>= 32-bytes) justify the effort to get at least the
destination doubleword (8-byte) aligned. Further optimization is
@@ -39,18 +39,22 @@
EALIGN (BP_SYM (memcpy), 5, 0)
cmpldi cr1,5,31
neg 0,3
- std 30,-16(1)
+ std 3,-16(1)
std 31,-8(1)
- rldicl. 0,0,0,61
+ andi. 11,3,7 /* check alignement of dst. */
+ clrldi 0,0,61 /* Number of bytes until the 1st doubleword of dst. */
+ clrldi 10,4,61 /* check alignement of src. */
+ cmpldi cr6,5,8
+ ble- cr1,.L2 /* If move < 32 bytes use short move code. */
+ cmpld cr6,10,11
mr 12,4
+ srdi 9,5,3 /* Number of full double words remaining. */
+ mtcrf 0x01,0
mr 31,5
- mr 30,3
- ble- cr1,.L2
+ beq .L0
+
subf 31,0,5
-
/* Move 0-7 bytes as needed to get the destination doubleword alligned. */
- beq 0f
- mtcrf 0x01,0
1: bf 31,2f
lbz 6,0(12)
addi 12,12,1
@@ -67,88 +71,91 @@ EALIGN (BP_SYM (memcpy), 5, 0)
stw 6,0(3)
addi 3,3,4
0:
+ clrldi 10,12,61 /* check alignement of src again. */
+ srdi 9,31,3 /* Number of full double words remaining. */
+
/* Copy doublewords from source to destination, assumpting the
destination is aligned on a doubleword boundary.
- First verify that there is > 7 bytes to copy and check if the source
- is also doubleword aligned. If there are < 8 bytes to copy fall
- through to the tail byte copy code. Otherwise if the source and
- destination are both doubleword aligned use an optimized doubleword
- copy loop. Otherwise the source has a different alignment and we use
- a load, shift, store strategy. */
- rldicl. 0,12,0,61
- cmpldi cr6,31,7
- ble- cr6,.L2 /* less than 8 bytes left. */
- srdi 11,31,3
- andi. 10,12,7
- bne- 0,.L6 /* Source is not DW aligned. */
- srdi. 9,31,3
- mr 10,3
- mr 11,12
+ At this point we know there are at least 25 bytes left (32-7) to copy.
+ The next step is to determine if the source is also doubleword aligned.
+ If not branch to the unaligned move code at .L6. which uses
+ a load, shift, store strategy.
+
+ Otherwise source and destination are doubleword aligned, and we can
+ the optimized doubleword copy loop. */
+.L0:
+ clrldi 11,31,61
+ mtcrf 0x01,9
+ bne- cr6,.L6 /* If source is not DW aligned. */
- /* Move doublewords where destination and source are aligned.
+ /* Move doublewords where destination and source are DW aligned.
Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration.
- If the remainder is >0 and < 32 bytes copy 1-3 doublewords. */
+ If the the copy is not an exact multiple of 32 bytes, 1-3
+ doublewords are copied as needed to set up the main loop. After
+ the main loop exits there may be a tail of 1-7 bytes. These byte are
+ copied a word/halfword/byte at a time as needed to preserve alignment. */
+
+ srdi 8,31,5
cmpldi cr1,9,4
- beq 0f
- mtcrf 0x01,9
- blt cr1,2f
- ld 6,0(11)
+ cmpldi cr6,11,0
+ mr 11,12
+
+ bf 30,1f
+ ld 6,0(12)
+ ld 7,8(12)
+ addi 11,12,16
+ mtctr 8
+ std 6,0(3)
+ std 7,8(3)
+ addi 10,3,16
+ bf 31,4f
+ ld 0,16(12)
+ std 0,16(3)
+ blt cr1,3f
+ addi 11,12,24
+ addi 10,3,24
+ b 4f
+ .align 4
+1:
+ mr 10,3
+ mtctr 8
+ bf 31,4f
+ ld 6,0(12)
+ addi 11,12,8
+ std 6,0(3)
+ addi 10,3,8
+
.align 4
4:
- ld 7,8(11)
- addi 9,9,-4
- std 6,0(10)
- ld 6,16(11)
- std 7,8(10)
- ld 7,24(11)
- addi 11,11,32
- cmpldi cr1,9,4
- std 6,16(10)
- blt cr1,3f
- ld 6,0(11)
- std 7,24(10)
- addi 10,10,32
- b 4b
-3: std 7,24(10)
- addi 10,10,32
-2: bf 30,1f
ld 6,0(11)
ld 7,8(11)
- addi 11,11,16
+ ld 8,16(11)
+ ld 0,24(11)
+ addi 11,11,32
+2:
std 6,0(10)
std 7,8(10)
- addi 10,10,16
-1: bf 31,0f
- ld 6,0(11)
- addi 11,11,8
- std 6,0(10)
- addi 10,10,8
-0:
+ std 8,16(10)
+ std 0,24(10)
+ addi 10,10,32
+ bdnz 4b
+3:
-.L8:
rldicr 0,31,0,60
- rldicl 31,31,0,61
+ mtcrf 0x01,31
+ beq cr6,0f
+.L9:
add 3,3,0
add 12,12,0
-
- /* Copy the tail for up to 31 bytes. If this is the tail of a longer
- copy then the destination will be aligned and the length will be
- less than 8. So it is normally not worth the set-up overhead to
- get doubleword aligned and do doubleword load/store. */
-.L2:
- mr. 10,31
- cmpldi cr1,31,4
- beq 0f
- mtcrf 0x01,31
- blt cr1,2f
-4: lwz 6,0(12)
+
+/* At this point we have a tail of 0-7 bytes and we know that the
+ destiniation is double word aligned. */
+4: bf 29,2f
+ lwz 6,0(12)
addi 12,12,4
- addi 10,10,-4
stw 6,0(3)
- cmpldi cr1,10,4
addi 3,3,4
- bge cr1,4b
2: bf 30,1f
lhz 6,0(12)
addi 12,12,2
@@ -156,14 +163,153 @@ EALIGN (BP_SYM (memcpy), 5, 0)
addi 3,3,2
1: bf 31,0f
lbz 6,0(12)
- addi 12,12,1
stb 6,0(3)
- addi 3,3,1
0:
/* Return original dst pointer. */
ld 31,-8(1)
- mr 3,30
- ld 30,-16(1)
+ ld 3,-16(1)
+ blr
+
+/* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31
+ bytes. Each case is handled without loops, using binary (1,2,4,8)
+ tests.
+
+ In the short (0-8 byte) case no attempt is made to force alignment
+ of either source or destination. The hardware will handle the
+ unaligned load/stores with small delays for crossing 32- 64-byte, and
+ 4096-byte boundaries. Since these short moves are unlikely to be
+ unaligned or cross these boundaries, the overhead to force
+ alignment is not justified.
+
+ The longer (9-31 byte) move is more likely to cross 32- or 64-byte
+ boundaries. Since only loads are sensitive to the 32-/64-byte
+ boundaries it is more important to align the source then the
+ destination. If the source is not already word aligned, we first
+ move 1-3 bytes as needed. Since we are only word aligned we don't
+ use double word load/stores to insure that all loads are aligned.
+ While the destination and stores may still be unaligned, this
+ is only an issue for page (4096 byte boundary) crossing, which
+ should be rare for these short moves. The hardware handles this
+ case automatically with a small delay. */
+
+ .align 4
+.L2:
+ mtcrf 0x01,5
+ neg 8,4
+ clrrdi 11,4,2
+ andi. 0,8,3
+ ble cr6,.LE8 /* Handle moves of 0-8 bytes. */
+/* At least 9 bytes left. Get the source word aligned. */
+ cmpldi cr1,5,16
+ mr 10,5
+ mr 12,4
+ cmpldi cr6,0,2
+ beq .L3 /* If the source is already word aligned skip this. */
+/* Copy 1-3 bytes to get source address word aligned. */
+ lwz 6,0(11)
+ subf 10,0,5
+ add 12,4,0
+ blt cr6,5f
+ srdi 7,6,16
+ bgt cr6,3f
+ sth 6,0(3)
+ b 7f
+ .align 4
+3:
+ stb 7,0(3)
+ sth 6,1(3)
+ b 7f
+ .align 4
+5:
+ stb 6,0(3)
+7:
+ cmpldi cr1,10,16
+ add 3,3,0
+ mtcrf 0x01,10
+ .align 4
+.L3:
+/* At least 6 bytes left and the source is word aligned. */
+ blt cr1,8f
+16: /* Move 16 bytes. */
+ lwz 6,0(12)
+ lwz 7,4(12)
+ stw 6,0(3)
+ lwz 6,8(12)
+ stw 7,4(3)
+ lwz 7,12(12)
+ addi 12,12,16
+ stw 6,8(3)
+ stw 7,12(3)
+ addi 3,3,16
+8: /* Move 8 bytes. */
+ bf 28,4f
+ lwz 6,0(12)
+ lwz 7,4(12)
+ addi 12,12,8
+ stw 6,0(3)
+ stw 7,4(3)
+ addi 3,3,8
+4: /* Move 4 bytes. */
+ bf 29,2f
+ lwz 6,0(12)
+ addi 12,12,4
+ stw 6,0(3)
+ addi 3,3,4
+2: /* Move 2-3 bytes. */
+ bf 30,1f
+ lhz 6,0(12)
+ sth 6,0(3)
+ bf 31,0f
+ lbz 7,2(12)
+ stb 7,2(3)
+ ld 3,-16(1)
+ blr
+1: /* Move 1 byte. */
+ bf 31,0f
+ lbz 6,0(12)
+ stb 6,0(3)
+0:
+ /* Return original dst pointer. */
+ ld 3,-16(1)
+ blr
+
+/* Special case to copy 0-8 bytes. */
+ .align 4
+.LE8:
+ mr 12,4
+ bne cr6,4f
+/* Would have liked to use use ld/std here but the 630 processors are
+ slow for load/store doubles that are not at least word aligned.
+ Unaligned Load/Store word execute with only a 1 cycle penaltity. */
+ lwz 6,0(4)
+ lwz 7,4(4)
+ stw 6,0(3)
+ stw 7,4(3)
+ /* Return original dst pointer. */
+ ld 3,-16(1)
+ blr
+ .align 4
+4: bf 29,2b
+ lwz 6,0(4)
+ stw 6,0(3)
+6:
+ bf 30,5f
+ lhz 7,4(4)
+ sth 7,4(3)
+ bf 31,0f
+ lbz 8,6(4)
+ stb 8,6(3)
+ ld 3,-16(1)
+ blr
+ .align 4
+5:
+ bf 31,0f
+ lbz 6,4(4)
+ stb 6,4(3)
+ .align 4
+0:
+ /* Return original dst pointer. */
+ ld 3,-16(1)
blr
.align 4
@@ -173,27 +319,29 @@ EALIGN (BP_SYM (memcpy), 5, 0)
not. Use aligned doubleword loads from the source, shifted to realign
the data, to allow aligned destination stores. */
subf 5,10,12
- andi. 0,11,1
+ andi. 0,9,1
+ cmpldi cr6,11,0
sldi 10,10,3
+ mr 11,9
mr 4,3
ld 6,0(5)
ld 7,8(5)
subfic 9,10,64
beq 2f
sld 0,6,10
- addi 11,11,-1
+ cmpldi 11,1
mr 6,7
addi 4,4,-8
- cmpldi 11,0
+ addi 11,11,-1
b 1f
2: addi 5,5,8
.align 4
0: sld 0,6,10
srd 8,7,9
- addi 11,11,-2
+ cmpldi 11,2
ld 6,8(5)
or 0,0,8
- cmpldi 11,0
+ addi 11,11,-2
std 0,0(4)
sld 0,7,10
1: srd 8,6,9
@@ -204,8 +352,15 @@ EALIGN (BP_SYM (memcpy), 5, 0)
addi 5,5,16
addi 4,4,16
b 0b
+ .align 4
8:
std 0,8(4)
- b .L8
+ rldicr 0,31,0,60
+ mtcrf 0x01,31
+ bne cr6,.L9 /* If the tail is 0 bytes we are done! */
+ /* Return original dst pointer. */
+ ld 31,-8(1)
+ ld 3,-16(1)
+ blr
END_GEN_TB (BP_SYM (memcpy),TB_TOCLESS)
libc_hidden_builtin_def (memcpy)