Mixed-mode pkey sign and verify must construct DigestInfo for PKCS #1.5.

PKCS #11 expects a DigestInfo rather than a raw digest when passing a
pre-computed digest for PKCS #1.5 signature or verification, so the
rpc_pkey signature and verification calls do too.  This requires
special case handling of RSA when the user passes a digest handle in
mixed mode.  Annoying, but PKCS #1.5 is weird enoug that there's no
way to avoid some kind of special case handling, this approach has the
advantage of not requiring us to parse and reconstruct the ASN.1, and
is probably what PKCS #11 has trained software to expect in any case.

3 files changed, 50 insertions, 12 deletions

diff --git a/hal_internal.h b/hal_internal.h
index 1c06494..5e08c4e 100644
--- a/hal_internal.h
+++ b/hal_internal.h
@@ -244,6 +244,18 @@ extern const hal_rpc_pkey_dispatch_t hal_rpc_local_pkey_dispatch, hal_rpc_remote
 #define HAL_PKEY_HANDLE_PROXIMATE_FLAG  (1 << 31)
 
 /*
+ * Mostly used by the local_pkey code, but the mixed_pkey code needs
+ * it to pad hashes for RSA PKCS #1.5 signatures.  This may indicate
+ * that we need a slightly more general internal API here, but not
+ * worth worrying about as long as we can treat RSA as a special case
+ * and just pass the plain hash for everything else.
+ */
+
+extern hal_error_t hal_rpc_pkey_pkcs1_construct_digestinfo(const hal_hash_handle_t handle,
+                                                           uint8_t *digest_info, size_t *digest_info_len,
+                                                           const size_t digest_info_max);
+
+/*
  * Keystore API.
  *
  * The original design for this subsystem used two separate tables,
diff --git a/rpc_client.c b/rpc_client.c
index a517528..a952a6e 100644
--- a/rpc_client.c
+++ b/rpc_client.c
@@ -833,17 +833,30 @@ static hal_error_t pkey_mixed_sign(const hal_session_handle_t session,
                                              signature, signature_len, signature_max);
 
   hal_digest_algorithm_t alg;
+  hal_key_type_t pkey_type;
   size_t digest_len;
   hal_error_t err;
 
   if ((err = hal_rpc_hash_get_algorithm(hash, &alg))           != HAL_OK ||
-      (err = hal_rpc_hash_get_digest_length(alg, &digest_len)) != HAL_OK)
+      (err = hal_rpc_hash_get_digest_length(alg, &digest_len)) != HAL_OK ||
+      (err = hal_rpc_pkey_get_key_type(pkey, &pkey_type))      != HAL_OK)
     return err;
 
-  uint8_t digest[digest_len];
+  uint8_t digest[digest_len > signature_max ? digest_len : signature_max];
 
-  if ((err = hal_rpc_hash_finalize(hash, digest, digest_len)) != HAL_OK)
-    return err;
+  switch (pkey_type) {
+
+  case HAL_KEY_TYPE_RSA_PRIVATE:
+  case HAL_KEY_TYPE_RSA_PUBLIC:
+    if ((err = hal_rpc_pkey_pkcs1_construct_digestinfo(hash, digest, &digest_len, sizeof(digest))) != HAL_OK)
+      return err;
+    break;
+
+  default:
+    if ((err = hal_rpc_hash_finalize(hash, digest, digest_len)) != HAL_OK)
+      return err;
+
+  }
 
   return mixed_handle_dispatch(pkey)->sign(session, pkey, hal_hash_handle_none, digest, digest_len,
                                            signature, signature_len, signature_max);
@@ -860,17 +873,30 @@ static hal_error_t pkey_mixed_verify(const hal_session_handle_t session,
                                                signature, signature_len);
 
   hal_digest_algorithm_t alg;
+  hal_key_type_t pkey_type;
   size_t digest_len;
   hal_error_t err;
 
   if ((err = hal_rpc_hash_get_algorithm(hash, &alg))           != HAL_OK ||
-      (err = hal_rpc_hash_get_digest_length(alg, &digest_len)) != HAL_OK)
+      (err = hal_rpc_hash_get_digest_length(alg, &digest_len)) != HAL_OK ||
+      (err = hal_rpc_pkey_get_key_type(pkey, &pkey_type))      != HAL_OK)
     return err;
 
-  uint8_t digest[digest_len];
+  uint8_t digest[digest_len > signature_len ? digest_len : signature_len];
 
-  if ((err = hal_rpc_hash_finalize(hash, digest, digest_len)) != HAL_OK)
-    return err;
+  switch (pkey_type) {
+
+  case HAL_KEY_TYPE_RSA_PRIVATE:
+  case HAL_KEY_TYPE_RSA_PUBLIC:
+    if ((err = hal_rpc_pkey_pkcs1_construct_digestinfo(hash, digest, &digest_len, sizeof(digest))) != HAL_OK)
+      return err;
+    break;
+
+  default:
+    if ((err = hal_rpc_hash_finalize(hash, digest, digest_len)) != HAL_OK)
+      return err;
+
+  }
 
   return mixed_handle_dispatch(pkey)->verify(session, pkey, hal_hash_handle_none,
                                              digest, digest_len, signature, signature_len);
diff --git a/rpc_pkey.c b/rpc_pkey.c
index 6b548d5..d6efbe7 100644
--- a/rpc_pkey.c
+++ b/rpc_pkey.c
@@ -142,8 +142,8 @@ static inline pkey_slot_t *find_handle(const hal_pkey_handle_t handle)
  * basic) ASN.1 encoding, which we perform inline.
  */
 
-static hal_error_t pkcs1_construct_digestinfo(const hal_hash_handle_t handle,
-                                              uint8_t *digest_info, size_t *digest_info_len, const size_t digest_info_max)
+hal_error_t hal_rpc_pkey_pkcs1_construct_digestinfo(const hal_hash_handle_t handle,
+                                                    uint8_t *digest_info, size_t *digest_info_len, const size_t digest_info_max)
 {
   assert(digest_info != NULL && digest_info_len != NULL);
 
@@ -623,7 +623,7 @@ static hal_error_t pkey_local_sign_rsa(uint8_t *keybuf, const size_t keybuf_len,
     return HAL_ERROR_RESULT_TOO_LONG;
 
   if (input == NULL) {
-    if ((err = pkcs1_construct_digestinfo(hash, signature, &input_len, *signature_len)) != HAL_OK)
+    if ((err = hal_rpc_pkey_pkcs1_construct_digestinfo(hash, signature, &input_len, *signature_len)) != HAL_OK)
       return err;
     input = signature;
   }
@@ -751,7 +751,7 @@ static hal_error_t pkey_local_verify_rsa(uint8_t *keybuf, const size_t keybuf_le
     return err;
 
   if (input == NULL) {
-    if ((err = pkcs1_construct_digestinfo(hash, expected, &input_len, sizeof(expected))) != HAL_OK)
+    if ((err = hal_rpc_pkey_pkcs1_construct_digestinfo(hash, expected, &input_len, sizeof(expected))) != HAL_OK)
       return err;
     input = expected;
   }