Optimize long-word additions in SHA implementation

5 files changed, 61 insertions, 14 deletions

diff --git a/ChangeLog b/ChangeLog
index 802b5692d8..210bded935 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+2011-07-02  Ulrich Drepper  <drepper@gmail.com>
+
+	* crypt/sha512.h (struct sha512_ctx): Add union to access total also
+	as 128-bit value.
+	* crypt/sha512.c (sha512_process_block): Perform total addition using
+	128-bit if possible.
+	(__sha512_finish_ctx): Likewise.
+	* crypt/sha256.h (struct sha256_ctx): Add union to access total also
+	as 64-bit value.
+	* crypt/sha256.c (SWAP64): Define.
+	(sha256_process_block): Perform total addition using 64-bit if
+	possible.
+	(__sha256_finish_ctx): Likewise.
+
 2011-07-01  Ulrich Drepper  <drepper@gmail.com>
 
 	* nscd/pwdcache.c (cache_addpw): Cleanup.  Add branch prediction.
diff --git a/crypt/sha256.c b/crypt/sha256.c
index 941612e17b..0ca3355a05 100644
--- a/crypt/sha256.c
+++ b/crypt/sha256.c
@@ -1,6 +1,6 @@
 /* Functions to compute SHA256 message digest of files or memory blocks.
    according to the definition of SHA256 in FIPS 180-2.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2011 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -35,12 +35,23 @@
 # ifdef _LIBC
 #  include <byteswap.h>
 #  define SWAP(n) bswap_32 (n)
+#  define SWAP64(n) bswap_64 (n)
 # else
 #  define SWAP(n) \
     (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24))
+#  define SWAP64(n) \
+  (((n) << 56)					\
+   | (((n) & 0xff00) << 40)			\
+   | (((n) & 0xff0000) << 24)			\
+   | (((n) & 0xff000000) << 8)			\
+   | (((n) >> 8) & 0xff000000)			\
+   | (((n) >> 24) & 0xff0000)			\
+   | (((n) >> 40) & 0xff00)			\
+   | ((n) >> 56))
 # endif
 #else
 # define SWAP(n) (n)
+# define SWAP64(n) (n)
 #endif
 
 
@@ -89,10 +100,8 @@ sha256_process_block (const void *buffer, size_t len, struct sha256_ctx *ctx)
 
   /* First increment the byte count.  FIPS 180-2 specifies the possible
      length of the file up to 2^64 bits.  Here we only compute the
-     number of bytes.  Do a double word increment.  */
-  ctx->total[0] += len;
-  if (ctx->total[0] < len)
-    ++ctx->total[1];
+     number of bytes.  */
+  ctx->total64 += len;
 
   /* Process all bytes in the buffer with 64 bytes in each round of
      the loop.  */
@@ -186,7 +195,7 @@ __sha256_init_ctx (ctx)
   ctx->H[6] = 0x1f83d9ab;
   ctx->H[7] = 0x5be0cd19;
 
-  ctx->total[0] = ctx->total[1] = 0;
+  ctx->total64 = 0;
   ctx->buflen = 0;
 }
 
@@ -206,17 +215,19 @@ __sha256_finish_ctx (ctx, resbuf)
   size_t pad;
 
   /* Now count remaining bytes.  */
-  ctx->total[0] += bytes;
-  if (ctx->total[0] < bytes)
-    ++ctx->total[1];
+  ctx->total64 += bytes;
 
   pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes;
   memcpy (&ctx->buffer[bytes], fillbuf, pad);
 
   /* Put the 64-bit file length in *bits* at the end of the buffer.  */
+#ifdef _STRING_ARCH_unaligned
+  *(uint64_t *)  &ctx->buffer[bytes + pad] = SWAP64 (ctx->total64 << 3);
+#else
   *(uint32_t *) &ctx->buffer[bytes + pad + 4] = SWAP (ctx->total[0] << 3);
   *(uint32_t *) &ctx->buffer[bytes + pad] = SWAP ((ctx->total[1] << 3) |
 						  (ctx->total[0] >> 29));
+#endif
 
   /* Process last bytes.  */
   sha256_process_block (ctx->buffer, bytes + pad + 8, ctx);
diff --git a/crypt/sha256.h b/crypt/sha256.h
index be8b0772cb..fcf61485de 100644
--- a/crypt/sha256.h
+++ b/crypt/sha256.h
@@ -1,6 +1,6 @@
 /* Declaration of functions and data types used for SHA256 sum computing
    library functions.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2011 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -31,7 +31,11 @@ struct sha256_ctx
 {
   uint32_t H[8];
 
-  uint32_t total[2];
+  union
+  {
+    uint64_t total64;
+    uint32_t total[2];
+  };
   uint32_t buflen;
   char buffer[128] __attribute__ ((__aligned__ (__alignof__ (uint32_t))));
 };
diff --git a/crypt/sha512.c b/crypt/sha512.c
index 02127476fe..16b4877551 100644
--- a/crypt/sha512.c
+++ b/crypt/sha512.c
@@ -1,6 +1,6 @@
 /* Functions to compute SHA512 message digest of files or memory blocks.
    according to the definition of SHA512 in FIPS 180-2.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2011 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -121,9 +121,13 @@ sha512_process_block (const void *buffer, size_t len, struct sha512_ctx *ctx)
   /* First increment the byte count.  FIPS 180-2 specifies the possible
      length of the file up to 2^128 bits.  Here we only compute the
      number of bytes.  Do a double word increment.  */
+#ifdef USE_TOTAL128
+  ctx->total128 += len;
+#else
   ctx->total[0] += len;
   if (ctx->total[0] < len)
     ++ctx->total[1];
+#endif
 
   /* Process all bytes in the buffer with 128 bytes in each round of
      the loop.  */
@@ -237,9 +241,13 @@ __sha512_finish_ctx (ctx, resbuf)
   size_t pad;
 
   /* Now count remaining bytes.  */
+#ifdef USE_TOTAL128
+  ctx->total128 += bytes;
+#else
   ctx->total[0] += bytes;
   if (ctx->total[0] < bytes)
     ++ctx->total[1];
+#endif
 
   pad = bytes >= 112 ? 128 + 112 - bytes : 112 - bytes;
   memcpy (&ctx->buffer[bytes], fillbuf, pad);
diff --git a/crypt/sha512.h b/crypt/sha512.h
index 9952c5f862..90e55dccb2 100644
--- a/crypt/sha512.h
+++ b/crypt/sha512.h
@@ -1,6 +1,6 @@
 /* Declaration of functions and data types used for SHA512 sum computing
    library functions.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2011 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -24,6 +24,9 @@
 #include <limits.h>
 #include <stdint.h>
 #include <stdio.h>
+#ifdef _LIBC
+# include <bits/wordsize.h>
+#endif
 
 
 /* Structure to save state of computation between the single steps.  */
@@ -31,7 +34,14 @@ struct sha512_ctx
 {
   uint64_t H[8];
 
-  uint64_t total[2];
+  union
+  {
+#if defined __GNUC__ && __WORDSIZE == 64
+# define USE_TOTAL128
+    unsigned int total128 __attribute__ ((__mode__ (TI)));
+#endif
+    uint64_t total[2];
+  };
   uint64_t buflen;
   char buffer[256] __attribute__ ((__aligned__ (__alignof__ (uint64_t))));
 };