Make crypto.c compile/link with OpenSSL 3

- Initialize libssl using non-deprecated APIs

OpenSSL 3 deprecated `SSL_library_init` and `SSL_load_error_strings` in
favor of `OPENSSL_init_ssl`. Use `OPENSSL_init_ssl` when dealing with
OpenSSL 1.1 and newer to unbreak the build with OpenSSL 3.

- Move MD5 APIs to EVP_MD APIs

OpenSSL 3 deprecated all of the `MD5_`* APIs. Move to the equivalent
`EVP_MD`* APIs so the code doesn't need to be pinned down to 1.1
compatible APIs and uplifted at a later date.

Co-authored-by: Pierre Pronchery <[email protected]>
Co-authored-by: Ed Maste <[email protected]>
Signed-off-by: Enji Cooper <[email protected]>
Sponsored by:	The FreeBSD Foundation

Author: 3 people

crypto.c

@@ -33,13 +33,19 @@
  * SUCH DAMAGE.
  */
 
+#include <openssl/opensslv.h>
+#if (OPENSSL_VERSION_NUMBER >= 0x300000L)
+#define	IS_OPENSSL3	1
+#endif
+
 #include <openssl/x509.h>
 #include <openssl/md5.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #include <openssl/pem.h>
 #include <openssl/rand.h>
 
+#include <assert.h>
 #include <strings.h>
 #include <string.h>
 #include <syslog.h>
@@ -115,8 +121,16 @@ smtp_init_crypto(int fd, int feature, struct smtp_features* features)
 
 	/* XXX clean up on error/close */
 	/* Init SSL library */
+#if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+	error = OPENSSL_init_ssl(0, NULL);
+	if (error != 1) {
+		syslog(LOG_WARNING, "remote delivery deferred: SSL init failed: %s", ssl_errstr());
+		return (1);
+	}
+#else
 	SSL_library_init();
 	SSL_load_error_strings();
+#endif
 
 	// Allow any possible version
 #if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
@@ -225,7 +239,12 @@ void
 hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
     unsigned char* digest)
 {
-        MD5_CTX context;
+#ifdef	IS_OPENSSL3
+	EVP_MD		*md;
+	EVP_MD_CTX	*context;
+#else
+	MD5_CTX		context;
+#endif
         unsigned char k_ipad[65];    /* inner padding -
                                       * key XORd with ipad
                                       */
@@ -234,15 +253,23 @@ hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
                                       */
         unsigned char tk[16];
         int i;
-        /* if key is longer than 64 bytes reset it to key=MD5(key) */
-        if (key_len > 64) {
-
-                MD5_CTX      tctx;
 
-                MD5_Init(&tctx);
-                MD5_Update(&tctx, key, key_len);
-                MD5_Final(tk, &tctx);
+#ifdef	IS_OPENSSL3
+	context = EVP_MD_CTX_new();
+	assert(context != NULL);
+#endif
 
+        /* if key is longer than 64 bytes reset it to key=MD5(key) */
+        if (key_len > 64) {
+#ifdef	IS_OPENSSL3
+		EVP_DigestInit_ex(context, md, NULL);
+		EVP_DigestUpdate(context, key, key_len);
+		EVP_DigestFinal_ex(context, tk, NULL);
+#else
+                MD5_Init(&context);
+                MD5_Update(&context, key, key_len);
+                MD5_Final(tk, &context);
+#endif
                 key = tk;
                 key_len = 16;
         }
@@ -270,14 +297,43 @@ hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
                 k_ipad[i] ^= 0x36;
                 k_opad[i] ^= 0x5c;
         }
+
+#ifdef	IS_OPENSSL3
+        /**
+         * Perform inner MD5.
+         */
+
+	/* Init context for first pass. */
+	EVP_DigestInit_ex(context, md, NULL);
+	/* Start with inner pad. */
+	EVP_DigestUpdate(context, k_ipad, 64);
+	/* Update with text of datagram. */
+	EVP_DigestUpdate(context, text, text_len);
+	/* Finish up first pass. */
+	EVP_DigestFinal_ex(context, digest, NULL);
+
+	/**
+         * Perform outer MD5.
+         */
+
+	/* Re-init context for second pass. */
+	EVP_DigestInit_ex(context, md, NULL);
+	/* Start with outer pad. */
+	EVP_DigestUpdate(context, k_opad, 64);
+	/* Update with results of first hash. */
+	EVP_DigestUpdate(context, digest, 16);
+	/* Finish up second pass. */
+	EVP_DigestFinal_ex(context, digest, NULL);
+#else
         /*
          * perform inner MD5
          */
         MD5_Init(&context);                   /* init context for 1st
                                               * pass */
         MD5_Update(&context, k_ipad, 64);     /* start with inner pad */
         MD5_Update(&context, text, text_len); /* then text of datagram */
-        MD5_Final(digest, &context);          /* finish up 1st pass */
+
+	MD5_Final(digest, &context);          /* finish up 1st pass */
         /*
          * perform outer MD5
          */
@@ -287,6 +343,7 @@ hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
         MD5_Update(&context, digest, 16);     /* then results of 1st
                                               * hash */
         MD5_Final(digest, &context);          /* finish up 2nd pass */
+#endif
 }
 
 /*