X-Git-Url: http://www.wagner.pp.ru/gitweb/?a=blobdiff_plain;ds=sidebyside;f=gosthash2012.c;h=91c2c9462518681df33e7536dba0b06e24dc73de;hb=HEAD;hp=3dacc3531955e6f12c43b59ea7277cb27518ea1b;hpb=b2e96b3086cc3033ddc31cc981b601ff5b9b5f08;p=openssl-gost%2Fengine.git

diff --git a/gosthash2012.c b/gosthash2012.c
index 3dacc35..91c2c94 100644
--- a/gosthash2012.c
+++ b/gosthash2012.c
@@ -9,6 +9,13 @@
  */
 
 #include "gosthash2012.h"
+#if defined(__x86_64__) || defined(__e2k__)
+# ifdef _MSC_VER
+#  include <intrin.h>
+# else
+#  include <x86intrin.h>
+# endif
+#endif
 
 #if defined(_WIN32) || defined(_WINDOWS)
 # define INLINE __inline
@@ -48,60 +55,63 @@ void init_gost2012_hash_ctx(gost2012_hash_ctx * CTX,
 
 static INLINE void pad(gost2012_hash_ctx * CTX)
 {
-    memset(&(CTX->buffer[CTX->bufsize]), 0, sizeof(CTX->buffer) - CTX->bufsize);
-    CTX->buffer[CTX->bufsize] = 1;
+    memset(&(CTX->buffer.B[CTX->bufsize]), 0, sizeof(CTX->buffer) - CTX->bufsize);
+    CTX->buffer.B[CTX->bufsize] = 1;
 
 }
 
-static INLINE void add512(const union uint512_u *x,
-                          const union uint512_u *y, union uint512_u *r)
+static INLINE void add512(union uint512_u * RESTRICT x,
+                          const union uint512_u * RESTRICT y)
 {
 #ifndef __GOST3411_BIG_ENDIAN__
-    unsigned int CF;
+    unsigned int CF = 0;
     unsigned int i;
 
-    CF = 0;
+# ifdef HAVE_ADDCARRY_U64
+    for (i = 0; i < 8; i++)
+        CF = _addcarry_u64(CF, x->QWORD[i] , y->QWORD[i], &(x->QWORD[i]));
+# else
     for (i = 0; i < 8; i++) {
-	const unsigned long long left = x->QWORD[i];
-	unsigned long long sum;
-
-	sum = left + y->QWORD[i] + CF;
-	/*
-	 * (sum == left): is noop, because it's possible only
-	 * when `left' is added with `0 + 0' or with `ULLONG_MAX + 1',
-	 * in that case `CF' (carry) retain previous value, which is correct,
-	 * because when `left + 0 + 0' there was no overflow (thus no carry),
-	 * and when `left + ULLONG_MAX + 1' value is wrapped back to
-	 * itself with overflow, thus creating carry.
-	 *
-	 * (sum != left):
-	 * if `sum' is not wrapped (sum > left) there should not be carry,
-	 * if `sum' is wrapped (sum < left) there should be carry.
-	 */
-	if (sum != left)
-	    CF = (sum < left);
-	r->QWORD[i] = sum;
+        const unsigned long long left = x->QWORD[i];
+        unsigned long long sum;
+
+        sum = left + y->QWORD[i] + CF;
+        /*
+         * (sum == left): is noop, because it's possible only
+         * when `left' is added with `0 + 0' or with `ULLONG_MAX + 1',
+         * in that case `CF' (carry) retain previous value, which is correct,
+         * because when `left + 0 + 0' there was no overflow (thus no carry),
+         * and when `left + ULLONG_MAX + 1' value is wrapped back to
+         * itself with overflow, thus creating carry.
+         *
+         * (sum != left):
+         * if `sum' is not wrapped (sum > left) there should not be carry,
+         * if `sum' is wrapped (sum < left) there should be carry.
+         */
+        if (sum != left)
+            CF = (sum < left);
+        x->QWORD[i] = sum;
     }
-#else
-    const unsigned char *xp, *yp;
-    unsigned char *rp;
+# endif /* !__x86_64__ */
+#else /* __GOST3411_BIG_ENDIAN__ */
+    const unsigned char *yp;
+    unsigned char *xp;
     unsigned int i;
     int buf;
 
-    xp = (const unsigned char *)&x[0];
+    xp = (unsigned char *)&x[0];
     yp = (const unsigned char *)&y[0];
-    rp = (unsigned char *)&r[0];
 
     buf = 0;
     for (i = 0; i < 64; i++) {
         buf = xp[i] + yp[i] + (buf >> 8);
-        rp[i] = (unsigned char)buf & 0xFF;
+        xp[i] = (unsigned char)buf & 0xFF;
     }
-#endif
+#endif /* __GOST3411_BIG_ENDIAN__ */
 }
 
-static void g(union uint512_u *h, const union uint512_u *N,
-              const unsigned char *m)
+static void g(union uint512_u *h, const union uint512_u * RESTRICT N,
+              const union uint512_u * RESTRICT m)
 {
 #ifdef __GOST3411_HAS_SSE2__
     __m128i xmm0, xmm2, xmm4, xmm6; /* XMMR0-quadruple */
@@ -111,7 +121,7 @@ static void g(union uint512_u *h, const union uint512_u *N,
     LOAD(N, xmm0, xmm2, xmm4, xmm6);
     XLPS128M(h, xmm0, xmm2, xmm4, xmm6);
 
-    LOAD(m, xmm1, xmm3, xmm5, xmm7);
+    ULOAD(m, xmm1, xmm3, xmm5, xmm7);
     XLPS128R(xmm0, xmm2, xmm4, xmm6, xmm1, xmm3, xmm5, xmm7);
 
     for (i = 0; i < 11; i++)
@@ -121,12 +131,15 @@ static void g(union uint512_u *h, const union uint512_u *N,
     X128R(xmm0, xmm2, xmm4, xmm6, xmm1, xmm3, xmm5, xmm7);
 
     X128M(h, xmm0, xmm2, xmm4, xmm6);
-    X128M(m, xmm0, xmm2, xmm4, xmm6);
-
-    UNLOAD(h, xmm0, xmm2, xmm4, xmm6);
+    ULOAD(m, xmm1, xmm3, xmm5, xmm7);
+    X128R(xmm0, xmm2, xmm4, xmm6, xmm1, xmm3, xmm5, xmm7);
 
+    STORE(h, xmm0, xmm2, xmm4, xmm6);
+# ifndef __i386__
     /* Restore the Floating-point status on the CPU */
+    /* This is only required on MMX, but EXTRACT32 is using MMX */
     _mm_empty();
+# endif
 #else
     union uint512_u Ki, data;
     unsigned int i;
@@ -145,47 +158,34 @@ static void g(union uint512_u *h, const union uint512_u *N,
     /* E() done */
 
     X((&data), h, (&data));
-    X((&data), ((const union uint512_u *)&m[0]), h);
+    X((&data), m, h);
 #endif
 }
 
-static INLINE void stage2(gost2012_hash_ctx * CTX, const unsigned char *data)
+static INLINE void stage2(gost2012_hash_ctx * CTX, const union uint512_u *data)
 {
-    union uint512_u m;
+    g(&(CTX->h), &(CTX->N), data);
 
-    memcpy(&m, data, sizeof(m));
-    g(&(CTX->h), &(CTX->N), (const unsigned char *)&m);
-
-    add512(&(CTX->N), &buffer512, &(CTX->N));
-    add512(&(CTX->Sigma), &m, &(CTX->Sigma));
+    add512(&(CTX->N), &buffer512);
+    add512(&(CTX->Sigma), data);
 }
 
 static INLINE void stage3(gost2012_hash_ctx * CTX)
 {
-    ALIGN(16) union uint512_u buf;
-
-    memset(&buf, 0x00, sizeof buf);
-    memcpy(&buf, &(CTX->buffer), CTX->bufsize);
-    memcpy(&(CTX->buffer), &buf, sizeof(uint512_u));
+    pad(CTX);
+    g(&(CTX->h), &(CTX->N), &(CTX->buffer));
+    add512(&(CTX->Sigma), &CTX->buffer);
 
-    memset(&buf, 0x00, sizeof buf);
+    memset(&(CTX->buffer.B[0]), 0, sizeof(uint512_u));
 #ifndef __GOST3411_BIG_ENDIAN__
-    buf.QWORD[0] = CTX->bufsize << 3;
+    CTX->buffer.QWORD[0] = CTX->bufsize << 3;
 #else
-    buf.QWORD[0] = BSWAP64(CTX->bufsize << 3);
+    CTX->buffer.QWORD[0] = BSWAP64(CTX->bufsize << 3);
 #endif
+    add512(&(CTX->N), &(CTX->buffer));
 
-    pad(CTX);
-
-    g(&(CTX->h), &(CTX->N), (const unsigned char *)&(CTX->buffer));
-
-    add512(&(CTX->N), &buf, &(CTX->N));
-    add512(&(CTX->Sigma), (const union uint512_u *)&CTX->buffer[0],
-           &(CTX->Sigma));
-
-    g(&(CTX->h), &buffer0, (const unsigned char *)&(CTX->N));
-
-    g(&(CTX->h), &buffer0, (const unsigned char *)&(CTX->Sigma));
+    g(&(CTX->h), &buffer0, &(CTX->N));
+    g(&(CTX->h), &buffer0, &(CTX->Sigma));
 }
 
 /*
@@ -195,32 +195,34 @@ static INLINE void stage3(gost2012_hash_ctx * CTX)
 void gost2012_hash_block(gost2012_hash_ctx * CTX,
                          const unsigned char *data, size_t len)
 {
-    size_t chunksize;
-
-    while (len > 63 && CTX->bufsize == 0) {
-        stage2(CTX, data);
-
-        data += 64;
-        len -= 64;
+    register size_t bufsize = CTX->bufsize;
+
+    if (bufsize == 0) {
+        while (len >= 64) {
+            memcpy(&CTX->buffer.B[0], data, 64);
+            stage2(CTX, &(CTX->buffer));
+            data += 64;
+            len -= 64;
+        }
     }
 
     while (len) {
-        chunksize = 64 - CTX->bufsize;
+        register size_t chunksize = 64 - bufsize;
         if (chunksize > len)
             chunksize = len;
 
-        memcpy(&CTX->buffer[CTX->bufsize], data, chunksize);
+        memcpy(&CTX->buffer.B[bufsize], data, chunksize);
 
-        CTX->bufsize += chunksize;
+        bufsize += chunksize;
         len -= chunksize;
         data += chunksize;
 
-        if (CTX->bufsize == 64) {
-            stage2(CTX, CTX->buffer);
-
-            CTX->bufsize = 0;
+        if (bufsize == 64) {
+            stage2(CTX, &(CTX->buffer) );
+            bufsize = 0;
         }
     }
+    CTX->bufsize = bufsize;
 }
 
 /*