Bugfix - carry bit overflow

[openssl-gost/engine.git] / gost_ec_keyx.c

diff --git a/gost_ec_keyx.c b/gost_ec_keyx.c
index 929c7861765e4288906159f9cdc93c9daaf545fb..30d8e5847f8f30ed04061979903b520350b76ec3 100644 (file)
--- a/gost_ec_keyx.c
+++ b/gost_ec_keyx.c
@@ -8,6 +8,7 @@
  *          Requires OpenSSL 0.9.9 for compilation                    *
  **********************************************************************/
 #include <openssl/evp.h>
+#include <openssl/err.h>
 #include <openssl/rand.h>
 #include <string.h>
 #include <openssl/objects.h>
@@ -21,29 +22,28 @@ static int VKO_compute_key(unsigned char *shared_key, size_t shared_key_size,
                            const EC_POINT *pub_key, EC_KEY *priv_key,
                            const unsigned char *ukm, int dgst_nid)
 {
-    unsigned char *databuf = NULL, *hashbuf = NULL;
+    unsigned char *databuf = NULL;
     BIGNUM *UKM = NULL, *p = NULL, *order = NULL, *X = NULL, *Y = NULL;
     const BIGNUM *key = EC_KEY_get0_private_key(priv_key);
     EC_POINT *pnt = EC_POINT_new(EC_KEY_get0_group(priv_key));
-    int i;
     BN_CTX *ctx = BN_CTX_new();
-    EVP_MD_CTX mdctx;
-    const EVP_MD *md;
+    EVP_MD_CTX *mdctx = NULL;
+    const EVP_MD *md = NULL;
     int effective_dgst_nid = (dgst_nid == NID_id_GostR3411_2012_512) ?
         NID_id_GostR3411_2012_256 : dgst_nid;
     int buf_len = (dgst_nid == NID_id_GostR3411_2012_512) ? 128 : 64,
         half_len = buf_len >> 1;
+               int ret = 0;
 
     if (!ctx) {
-        GOSTerr(GOST_F_VKO_COMPUTE_KEY, GOST_R_NO_MEMORY);
+        GOSTerr(GOST_F_VKO_COMPUTE_KEY, ERR_R_MALLOC_FAILURE);
         return 0;
     }
     BN_CTX_start(ctx);
 
-    databuf = OPENSSL_malloc(buf_len);
-    hashbuf = OPENSSL_malloc(buf_len);
-    if (!databuf || !hashbuf) {
-        GOSTerr(GOST_F_VKO_COMPUTE_KEY, GOST_R_MALLOC_FAILURE);
+    databuf = OPENSSL_zalloc(buf_len);
+    if (!databuf) {
+        GOSTerr(GOST_F_VKO_COMPUTE_KEY, ERR_R_MALLOC_FAILURE);
         goto err;
     }
 
@@ -60,7 +60,11 @@ static int VKO_compute_key(unsigned char *shared_key, size_t shared_key_size,
     Y = BN_CTX_get(ctx);
     EC_GROUP_get_order(EC_KEY_get0_group(priv_key), order, ctx);
     BN_mod_mul(p, key, UKM, order, ctx);
-    EC_POINT_mul(EC_KEY_get0_group(priv_key), pnt, NULL, pub_key, p, ctx);
+    if (!EC_POINT_mul
+        (EC_KEY_get0_group(priv_key), pnt, NULL, pub_key, p, ctx)) {
+        GOSTerr(GOST_F_VKO_COMPUTE_KEY, GOST_R_ERROR_POINT_MUL);
+        goto err;
+    }
     EC_POINT_get_affine_coordinates_GFp(EC_KEY_get0_group(priv_key),
                                         pnt, X, Y, ctx);
     /*
@@ -69,25 +73,31 @@ static int VKO_compute_key(unsigned char *shared_key, size_t shared_key_size,
     store_bignum(Y, databuf, half_len);
     store_bignum(X, databuf + half_len, half_len);
     /* And reverse byte order of whole buffer */
-    for (i = 0; i < buf_len; i++) {
-        hashbuf[buf_len - 1 - i] = databuf[i];
+               BUF_reverse(databuf, NULL, buf_len);
+
+    mdctx = EVP_MD_CTX_new();
+    if (!mdctx) {
+        GOSTerr(GOST_F_VKO_COMPUTE_KEY, ERR_R_MALLOC_FAILURE);
+        goto err;
     }
-    EVP_MD_CTX_init(&mdctx);
-    EVP_DigestInit_ex(&mdctx, md, NULL);
-    EVP_DigestUpdate(&mdctx, hashbuf, buf_len);
-    EVP_DigestFinal_ex(&mdctx, shared_key, NULL);
-    EVP_MD_CTX_cleanup(&mdctx);
+    EVP_MD_CTX_init(mdctx);
+    EVP_DigestInit_ex(mdctx, md, NULL);
+    EVP_DigestUpdate(mdctx, databuf, buf_len);
+    EVP_DigestFinal_ex(mdctx, shared_key, NULL);
+               ret = 32;
+
  err:
     BN_free(UKM);
     BN_CTX_end(ctx);
     BN_CTX_free(ctx);
+
     EC_POINT_free(pnt);
-    if (databuf)
-        OPENSSL_free(databuf);
-    if (hashbuf)
-        OPENSSL_free(hashbuf);
 
-    return 32;
+    EVP_MD_CTX_free(mdctx);
+
+    OPENSSL_free(databuf);
+
+    return ret;
 }
 
 /*
@@ -113,7 +123,7 @@ int pkey_gost_ec_derive(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keylen)
 
     if (key == NULL) {
         *keylen = 32;
-        return 32;
+        return 1;
     }
 
     EVP_PKEY_get_default_digest_nid(my_key, &dgst_nid);
@@ -126,10 +136,6 @@ int pkey_gost_ec_derive(EVP_PKEY_CTX *ctx, unsigned char *key, size_t *keylen)
     return (*keylen) ? 1 : 0;
 }
 
-/*
- * EVP_PKEY_METHOD callback encrypt
- * Implementation of GOST2001 key transport, cryptocom variation
- */
 /*
  * Generates ephemeral key based on pubk algorithm computes shared key using
  * VKO and returns filled up GOST_KEY_TRANSPORT structure
@@ -163,8 +169,7 @@ int pkey_GOST_ECcp_encrypt(EVP_PKEY_CTX *pctx, unsigned char *out,
     } else if (out) {
 
         if (RAND_bytes(ukm, 8) <= 0) {
-            GOSTerr(GOST_F_PKEY_GOST_ECCP_ENCRYPT,
-                    GOST_R_RANDOM_GENERATOR_FAILURE);
+            GOSTerr(GOST_F_PKEY_GOST_ECCP_ENCRYPT, GOST_R_RNG_ERROR);
             return 0;
         }
     }
@@ -180,9 +185,11 @@ int pkey_GOST_ECcp_encrypt(EVP_PKEY_CTX *pctx, unsigned char *out,
         key_is_ephemeral = 1;
         if (out) {
             sec_key = EVP_PKEY_new();
-            EVP_PKEY_assign(sec_key, EVP_PKEY_base_id(pubk), EC_KEY_new());
-            EVP_PKEY_copy_parameters(sec_key, pubk);
-            if (!gost_ec_keygen(EVP_PKEY_get0(sec_key))) {
+            if (! EVP_PKEY_assign(sec_key, EVP_PKEY_base_id(pubk), EC_KEY_new()) ||
+                                 ! EVP_PKEY_copy_parameters(sec_key, pubk) ||
+                ! gost_ec_keygen(EVP_PKEY_get0(sec_key))) {
+            GOSTerr(GOST_F_PKEY_GOST_ECCP_ENCRYPT,
+                    GOST_R_ERROR_COMPUTING_SHARED_KEY);
                 goto err;
             }
         }