Small cleanups in RPC code, e.g. to support null arguments.

- Add support for null pointer arguments in RPCs for get_digest_algorithm_id
  and get_public_key. This is years overdue, and would have obviated the need
  for get_public_key_len as a separate RPC.

- Refactor pkey_local_get_public_key_len in terms of pkey_local_get_public_key.

- Add more parameter sanity checks to rpc_api.c.

- Add a len_max parameter to hal_xdr_decode_variable_opaque, rather than
  having len be an in/out parameter. This brings xdr slightly more in line
  with the rest of the code base (again after literal years), and slightly
  simplifies several calls in rpc_client.c.

1 files changed, 12 insertions, 4 deletions

diff --git a/rpc_server.c b/rpc_server.c
index b363f23..9598413 100644
--- a/rpc_server.c
+++ b/rpc_server.c
@@ -170,10 +170,16 @@ static hal_error_t hash_get_digest_algorithm_id(const uint8_t **iptr, const uint
     if (nargs(1) + pad(len_max) > (uint32_t)(olimit - *optr))
         return HAL_ERROR_RPC_PACKET_OVERFLOW;
 
-    /* get the data directly into the output buffer */
-    if ((err = hal_rpc_hash_get_digest_algorithm_id(alg, *optr + nargs(1), &len, (size_t)len_max)) == HAL_OK) {
-        check(hal_xdr_encode_int(optr, olimit, len));
-        *optr += pad(len);
+    if (len_max == 0) {
+        if ((err = hal_rpc_hash_get_digest_algorithm_id(alg, NULL, &len, 0)) == HAL_OK)
+            check(hal_xdr_encode_int(optr, olimit, len));
+    }
+    else {
+        /* get the data directly into the output buffer */
+        if ((err = hal_rpc_hash_get_digest_algorithm_id(alg, *optr + nargs(1), &len, (size_t)len_max)) == HAL_OK) {
+            check(hal_xdr_encode_int(optr, olimit, len));
+            *optr += pad(len);
+        }
     }
 
     return err;
@@ -209,6 +215,8 @@ static hal_error_t hash_initialize(const uint8_t **iptr, const uint8_t * const i
     check(hal_xdr_decode_int(iptr, ilimit, &session.handle));
     check(hal_xdr_decode_int(iptr, ilimit, &alg));
     check(hal_xdr_decode_variable_opaque_ptr(iptr, ilimit, &key, &key_len));
+    if (key_len == 0)
+        key = NULL;
 
     check(hal_rpc_hash_initialize(client, session, &hash, (hal_digest_algorithm_t)alg, key, (size_t)key_len));