将OpenSSL RSA加密转换为GnuTLS

Question

出于许可原因，我正在尝试将以下代码从OpenSSL转换为GnuTLS：

  BIO *bioKey = BIO_new(BIO_s_mem());
  if (!bioKey)
  {
    DEBUG_ERROR("failed to allocate bioKey");
    spice_disconnect_channel(channel);
    return false;
  }

  BIO_write(bioKey, reply.pub_key, SPICE_TICKET_PUBKEY_BYTES);
  EVP_PKEY *rsaKey = d2i_PUBKEY_bio(bioKey, NULL);
  RSA *rsa = EVP_PKEY_get1_RSA(rsaKey);

  char enc[RSA_size(rsa)];
  if (RSA_public_encrypt(
        strlen(spice.password) + 1,
        (uint8_t*)spice.password,
        (uint8_t*)enc,
        rsa,
        RSA_PKCS1_OAEP_PADDING
  ) <= 0)
  {
    DEBUG_ERROR("rsa public encrypt failed");
    spice_disconnect_channel(channel);
    EVP_PKEY_free(rsaKey);
    BIO_free(bioKey);
    return false;
  }

  ssize_t rsaSize = RSA_size(rsa);
  EVP_PKEY_free(rsaKey);
  BIO_free(bioKey);

到目前为止，我提出了以下内容，但似乎输出的格式不正确(RSA_PKCS1_OEAP_PADDING)

  const gnutls_datum_t pubData =
  {
    .data = (void *)reply.pub_key,
    .size = SPICE_TICKET_PUBKEY_BYTES
  };

  gnutls_pubkey_t pubkey;
  if (gnutls_pubkey_init(&pubkey) < 0)
  {
    spice_disconnect_channel(channel);
    DEBUG_ERROR("gnutls_pubkey_init failed");
    return false;
  }

  if (gnutls_pubkey_import(pubkey, &pubData, GNUTLS_X509_FMT_DER) < 0)
  {
    gnutls_pubkey_deinit(pubkey);
    spice_disconnect_channel(channel);
    DEBUG_ERROR("gnutls_pubkey_import failed");
    return false;
  }

  const gnutls_datum_t input =
  {
    .data = (void *)spice.password,
    .size = strlen(spice.password) + 1
  };

  gnutls_datum_t out;
  if (gnutls_pubkey_encrypt_data(pubkey, 0, &input, &out) < 0)
  {
    gnutls_pubkey_deinit(pubkey);
    spice_disconnect_channel(channel);
    DEBUG_ERROR("gnutls_pubkey_encrypt_data failed");
    return false;
  }

  const char        *enc     = (char *)out.data;
  const unsigned int rsaSize = out.size;

我不是加密或这些库的专家，所以请仁慈。

Answer 1

GnuTLS根本不支持ES-OEAP。gnutls_pubkey_encrypt_data会生成PKCS#1填充的数据，因此无法使用。

解决方案是完全避免GnuTLS，并使用nettle和libgmp手动执行加密。我的解决方案基于FreeTDS中的示例：

http://www.freetds.org/reference/a00351_source.html

这实现了OEAP填充功能，并使用GMP执行RSA加密。