path: root/dev-db/mariadb/files/mariadb-10.6.8-openssl3.patch

https://bugs.gentoo.org/849593
https://github.com/MariaDB/server/commit/f0fa40efada3f46d76b735fa61e9f20cc9ce18fc
https://github.com/MariaDB/server/commit/987d16a0b43c163264ab37d3e01795577f97d83b
https://github.com/MariaDB/server/commit/78412ab028509f07a801c9bf1f3792ad77bcfac6
https://github.com/MariaDB/server/commit/babb803222d4a5d5a3256720c59d056cc8ba7dc3

From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 8 Nov 2021 18:48:19 +0100
Subject: [PATCH] MDEV-25785 Add support for OpenSSL 3.0

Summary of changes

- MD_CTX_SIZE is increased

- EVP_CIPHER_CTX_buf_noconst(ctx) does not work anymore, points
  to nobody knows where. The assumption made previously was that
  (since the function does not seem to be documented)
  was that it points to the last partial source block.
  Add own partial block buffer for NOPAD encryption instead

- SECLEVEL in CipherString in openssl.cnf
  had been downgraded to 0, from 1, to make TLSv1.0 and TLSv1.1 possible
   (according to https://github.com/openssl/openssl/blob/openssl-3.0.0/NEWS.md
   even though the manual for SSL_CTX_get_security_level claims that it
   should not be necessary)

- Workaround Ssl_cipher_list issue, it now returns TLSv1.3 ciphers,
  in addition to what was set in --ssl-cipher

- ctx_buf buffer now must be aligned to 16 bytes with openssl(
  previously with WolfSSL only), ot crashes will happen

- updated aes-t , to be better debuggable
  using function, rather than a huge multiline macro
  added test that does "nopad" encryption piece-wise, to test
  replacement of EVP_CIPHER_CTX_buf_noconst

part of MDEV-28133
--- a/cmake/ssl.cmake
+++ b/cmake/ssl.cmake
@@ -139,6 +139,13 @@ MACRO (MYSQL_CHECK_SSL)
       SET(SSL_INTERNAL_INCLUDE_DIRS "")
       SET(SSL_DEFINES "-DHAVE_OPENSSL")
 
+      # Silence "deprecated in OpenSSL 3.0"
+      IF((NOT OPENSSL_VERSION) # 3.0 not determined by older cmake
+         OR NOT(OPENSSL_VERSION VERSION_LESS "3.0.0"))
+        SET(SSL_DEFINES "${SSL_DEFINES} -DOPENSSL_API_COMPAT=0x10100000L")
+        SET(CMAKE_REQUIRED_DEFINITIONS -DOPENSSL_API_COMPAT=0x10100000L)
+      ENDIF()
+
       SET(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
       SET(CMAKE_REQUIRED_LIBRARIES ${SSL_LIBRARIES})
       SET(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
@@ -152,6 +159,7 @@ MACRO (MYSQL_CHECK_SSL)
                           HAVE_X509_check_host)
       SET(CMAKE_REQUIRED_INCLUDES)
       SET(CMAKE_REQUIRED_LIBRARIES)
+      SET(CMAKE_REQUIRED_DEFINITIONS)
     ELSE()
       IF(WITH_SSL STREQUAL "system")
         MESSAGE(FATAL_ERROR "Cannot find appropriate system libraries for SSL. Use WITH_SSL=bundled to enable SSL support")
--- a/include/mysql/service_my_crypt.h
+++ b/include/mysql/service_my_crypt.h
@@ -45,7 +45,7 @@ extern "C" {
 /* The max key length of all supported algorithms */
 #define MY_AES_MAX_KEY_LENGTH 32
 
-#define MY_AES_CTX_SIZE 656
+#define MY_AES_CTX_SIZE 672
 
 enum my_aes_mode {
     MY_AES_ECB, MY_AES_CBC
--- a/include/ssl_compat.h
+++ b/include/ssl_compat.h
@@ -24,7 +24,7 @@
 #define SSL_LIBRARY OpenSSL_version(OPENSSL_VERSION)
 #define ERR_remove_state(X) ERR_clear_error()
 #define EVP_CIPHER_CTX_SIZE 176
-#define EVP_MD_CTX_SIZE 48
+#define EVP_MD_CTX_SIZE 72
 #undef EVP_MD_CTX_init
 #define EVP_MD_CTX_init(X) do { memset((X), 0, EVP_MD_CTX_SIZE); EVP_MD_CTX_reset(X); } while(0)
 #undef EVP_CIPHER_CTX_init
@@ -77,7 +77,6 @@
 #define DH_set0_pqg(D,P,Q,G)            ((D)->p= (P), (D)->g= (G))
 #endif
 
-#define EVP_CIPHER_CTX_buf_noconst(ctx) ((ctx)->buf)
 #define EVP_CIPHER_CTX_encrypting(ctx)  ((ctx)->encrypt)
 #define EVP_CIPHER_CTX_SIZE             sizeof(EVP_CIPHER_CTX)
 
--- a/mysql-test/lib/openssl.cnf
+++ b/mysql-test/lib/openssl.cnf
@@ -9,4 +9,4 @@ ssl_conf = ssl_section
 system_default = system_default_section
 
 [system_default_section]
-CipherString = ALL:@SECLEVEL=1
+CipherString = ALL:@SECLEVEL=0
--- a/mysql-test/main/ssl_cipher.result
+++ b/mysql-test/main/ssl_cipher.result
@@ -61,8 +61,8 @@ connect  ssl_con,localhost,root,,,,,SSL;
 SHOW STATUS LIKE 'Ssl_cipher';
 Variable_name	Value
 Ssl_cipher	AES128-SHA
-SHOW STATUS LIKE 'Ssl_cipher_list';
-Variable_name	Value
-Ssl_cipher_list	AES128-SHA
+SELECT VARIABLE_VALUE like '%AES128-SHA%' FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='Ssl_cipher_list';
+VARIABLE_VALUE like '%AES128-SHA%'
+1
 disconnect ssl_con;
 connection default;
--- a/mysql-test/main/ssl_cipher.test
+++ b/mysql-test/main/ssl_cipher.test
@@ -98,6 +98,6 @@ let $restart_parameters=--ssl-cipher=AES128-SHA;
 source include/restart_mysqld.inc;
 connect (ssl_con,localhost,root,,,,,SSL);
 SHOW STATUS LIKE 'Ssl_cipher';
-SHOW STATUS LIKE 'Ssl_cipher_list';
+SELECT VARIABLE_VALUE like '%AES128-SHA%' FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='Ssl_cipher_list';
 disconnect ssl_con;
 connection default;
--- a/mysys_ssl/my_crypt.cc
+++ b/mysys_ssl/my_crypt.cc
@@ -29,11 +29,7 @@
 #include <ssl_compat.h>
 #include <cstdint>
 
-#ifdef HAVE_WOLFSSL
 #define CTX_ALIGN 16
-#else
-#define CTX_ALIGN 0
-#endif
 
 class MyCTX
 {
@@ -100,8 +96,9 @@ class MyCTX_nopad : public MyCTX
 {
 public:
   const uchar *key;
-  uint klen, buf_len;
+  uint klen, source_tail_len;
   uchar oiv[MY_AES_BLOCK_SIZE];
+  uchar source_tail[MY_AES_BLOCK_SIZE];
 
   MyCTX_nopad() : MyCTX() { }
   ~MyCTX_nopad() { }
@@ -112,7 +109,7 @@ class MyCTX_nopad : public MyCTX
     compile_time_assert(MY_AES_CTX_SIZE >= sizeof(MyCTX_nopad));
     this->key= key;
     this->klen= klen;
-    this->buf_len= 0;
+    this->source_tail_len= 0;
     if (ivlen)
       memcpy(oiv, iv, ivlen);
     DBUG_ASSERT(ivlen == 0 || ivlen == sizeof(oiv));
@@ -123,26 +120,41 @@ class MyCTX_nopad : public MyCTX
     return res;
   }
 
+  /** Update last partial source block, stored in source_tail array. */
+  void update_source_tail(const uchar* src, uint slen)
+  {
+    if (!slen)
+      return;
+    uint new_tail_len= (source_tail_len + slen) % MY_AES_BLOCK_SIZE;
+    if (new_tail_len)
+    {
+      if (slen + source_tail_len < MY_AES_BLOCK_SIZE)
+      {
+        memcpy(source_tail + source_tail_len, src, slen);
+      }
+      else
+      {
+        DBUG_ASSERT(slen > new_tail_len);
+        memcpy(source_tail, src + slen - new_tail_len, new_tail_len);
+      }
+    }
+    source_tail_len= new_tail_len;
+  }
+
   int update(const uchar *src, uint slen, uchar *dst, uint *dlen)
   {
-    buf_len+= slen;
+    update_source_tail(src, slen);
     return MyCTX::update(src, slen, dst, dlen);
   }
 
   int finish(uchar *dst, uint *dlen)
   {
-    buf_len %= MY_AES_BLOCK_SIZE;
-    if (buf_len)
+    if (source_tail_len)
     {
-      uchar *buf= EVP_CIPHER_CTX_buf_noconst(ctx);
       /*
         Not much we can do, block ciphers cannot encrypt data that aren't
         a multiple of the block length. At least not without padding.
         Let's do something CTR-like for the last partial block.
-
-        NOTE this assumes that there are only buf_len bytes in the buf.
-        If OpenSSL will change that, we'll need to change the implementation
-        of this class too.
       */
       uchar mask[MY_AES_BLOCK_SIZE];
       uint mlen;
@@ -154,10 +166,10 @@ class MyCTX_nopad : public MyCTX
         return rc;
       DBUG_ASSERT(mlen == sizeof(mask));
 
-      for (uint i=0; i < buf_len; i++)
-        dst[i]= buf[i] ^ mask[i];
+      for (uint i=0; i < source_tail_len; i++)
+        dst[i]= source_tail[i] ^ mask[i];
     }
-    *dlen= buf_len;
+    *dlen= source_tail_len;
     return MY_AES_OK;
   }
 };
--- a/unittest/mysys/aes-t.c
+++ b/unittest/mysys/aes-t.c
@@ -21,27 +21,96 @@
 #include <string.h>
 #include <ctype.h>
 
-#define DO_TEST(mode, nopad, slen, fill, dlen, hash)                    \
-  SKIP_BLOCK_IF(mode == 0xDEADBEAF, nopad ? 4 : 5, #mode " not supported")     \
-  {                                                                     \
-    memset(src, fill, src_len= slen);                                   \
-    ok(my_aes_crypt(mode, nopad | ENCRYPTION_FLAG_ENCRYPT,              \
-                    src, src_len, dst, &dst_len,                        \
-                    key, sizeof(key), iv, sizeof(iv)) == MY_AES_OK,     \
-      "encrypt " #mode " %u %s", src_len, nopad ? "nopad" : "pad");     \
-    if (!nopad)                                                         \
-      ok (dst_len == my_aes_get_size(mode, src_len), "my_aes_get_size");\
-    my_md5(md5, (char*)dst, dst_len);                                   \
-    ok(dst_len == dlen && memcmp(md5, hash, sizeof(md5)) == 0, "md5");  \
-    ok(my_aes_crypt(mode, nopad | ENCRYPTION_FLAG_DECRYPT,              \
-                    dst, dst_len, ddst, &ddst_len,                      \
-                    key, sizeof(key), iv, sizeof(iv)) == MY_AES_OK,     \
-       "decrypt " #mode " %u", dst_len);                                \
-    ok(ddst_len == src_len && memcmp(src, ddst, src_len) == 0, "memcmp"); \
+
+/** Test streaming encryption, bytewise update.*/
+static int aes_crypt_bytewise(enum my_aes_mode mode, int flags, const unsigned char *src,
+                 unsigned int slen, unsigned char *dst, unsigned int *dlen,
+                 const unsigned char *key, unsigned int klen,
+                 const unsigned char *iv, unsigned int ivlen)
+{
+  /* Allocate context on odd address on stack, in order to
+   catch misalignment errors.*/
+  void *ctx= (char *)alloca(MY_AES_CTX_SIZE+1)+1;
+
+  int res1, res2;
+  uint d1= 0, d2;
+  uint i;
+
+  if ((res1= my_aes_crypt_init(ctx, mode, flags, key, klen, iv, ivlen)))
+    return res1;
+  for (i= 0; i < slen; i++)
+  {
+    uint tmp_d1=0;
+    res1= my_aes_crypt_update(ctx, src+i,1, dst, &tmp_d1);
+    if (res1)
+      return res1;
+    d1+= tmp_d1;
+    dst+= tmp_d1;
+  }
+  res2= my_aes_crypt_finish(ctx, dst, &d2);
+  *dlen= d1 + d2;
+  return res1 ? res1 : res2;
+}
+
+
+#ifndef HAVE_EncryptAes128Ctr
+const uint MY_AES_CTR=0xDEADBEAF;
+#endif
+#ifndef HAVE_EncryptAes128Gcm
+const uint MY_AES_GCM=0xDEADBEAF;
+#endif
+
+#define MY_AES_UNSUPPORTED(x)  (x == 0xDEADBEAF)
+
+static void do_test(uint mode, const char *mode_str, int nopad, uint slen,
+                    char fill, size_t dlen, const char *hash)
+{
+  uchar key[16]= {1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6};
+  uchar iv[16]= {2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7};
+  uchar src[1000], dst[1100], dst2[1100], ddst[1000];
+  uchar md5[MY_MD5_HASH_SIZE];
+  uint src_len, dst_len, dst_len2, ddst_len;
+  int result;
+
+  if (MY_AES_UNSUPPORTED(mode))
+  {
+    skip(nopad?7:6, "%s not supported", mode_str);
+    return;
+  }
+  memset(src, fill, src_len= slen);
+  result= my_aes_crypt(mode, nopad | ENCRYPTION_FLAG_ENCRYPT, src, src_len,
+                       dst, &dst_len, key, sizeof(key), iv, sizeof(iv));
+  ok(result == MY_AES_OK, "encrypt %s %u %s", mode_str, src_len,
+     nopad ? "nopad" : "pad");
+
+  if (nopad)
+  {
+    result= aes_crypt_bytewise(mode, nopad | ENCRYPTION_FLAG_ENCRYPT, src,
+                                src_len, dst2, &dst_len2, key, sizeof(key),
+                                iv, sizeof(iv));
+    ok(result == MY_AES_OK, "encrypt bytewise %s %u", mode_str, src_len);
+    /* Compare with non-bytewise encryption result*/
+    ok(dst_len == dst_len2 && memcmp(dst, dst2, dst_len) == 0,
+       "memcmp bytewise  %s %u", mode_str, src_len);
+  }
+  else
+  {
+    int dst_len_real= my_aes_get_size(mode, src_len);
+    ok(dst_len_real= dst_len, "my_aes_get_size");
   }
+  my_md5(md5, (char *) dst, dst_len);
+  ok(dst_len == dlen, "md5 len");
+  ok(memcmp(md5, hash, sizeof(md5)) == 0, "md5");
+  result= my_aes_crypt(mode, nopad | ENCRYPTION_FLAG_DECRYPT,
+                       dst, dst_len, ddst, &ddst_len, key, sizeof(key), iv,
+                       sizeof(iv));
+
+  ok(result == MY_AES_OK, "decrypt %s %u", mode_str, dst_len);
+  ok(ddst_len == src_len && memcmp(src, ddst, src_len) == 0, "memcmp");
+}
 
-#define DO_TEST_P(M,S,F,D,H) DO_TEST(M,0,S,F,D,H)
-#define DO_TEST_N(M,S,F,D,H) DO_TEST(M,ENCRYPTION_FLAG_NOPAD,S,F,D,H)
+#define DO_TEST_P(M, S, F, D, H) do_test(M, #M, 0, S, F, D, H)
+#define DO_TEST_N(M, S, F, D, H) do_test(M, #M, ENCRYPTION_FLAG_NOPAD, S, F, D, H)
 
 /* useful macro for debugging */
 #define PRINT_MD5()                                     \
@@ -53,25 +122,15 @@
     printf("\"\n");                                     \
   } while(0);
 
-#ifndef HAVE_EncryptAes128Ctr
-const uint MY_AES_CTR=0xDEADBEAF;
-#endif
-#ifndef HAVE_EncryptAes128Gcm
-const uint MY_AES_GCM=0xDEADBEAF;
-#endif
 
 int
 main(int argc __attribute__((unused)),char *argv[])
 {
-  uchar key[16]= {1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6};
-  uchar iv[16]=  {2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7};
-  uchar src[1000], dst[1100], ddst[1000];
-  uchar md5[MY_MD5_HASH_SIZE];
-  uint src_len, dst_len, ddst_len;
 
   MY_INIT(argv[0]);
 
-  plan(87);
+  plan(122);
+
   DO_TEST_P(MY_AES_ECB, 200, '.', 208, "\xd8\x73\x8e\x3a\xbc\x66\x99\x13\x7f\x90\x23\x52\xee\x97\x6f\x9a");
   DO_TEST_P(MY_AES_ECB, 128, '?', 144, "\x19\x58\x33\x85\x4c\xaa\x7f\x06\xd1\xb2\xec\xd7\xb7\x6a\xa9\x5b");
   DO_TEST_P(MY_AES_CBC, 159, '%', 160, "\x4b\x03\x18\x3d\xf1\xa7\xcd\xa1\x46\xb3\xc6\x8a\x92\xc0\x0f\xc9");

From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Fri, 4 Feb 2022 14:52:03 +0100
Subject: [PATCH] Revert "don't build with OpenSSL 3.0, it doesn't work before
 MDEV-25785"

This reverts commit c9beef43154a199bfcd9f71049c011a2ed77ca74, because
we have OpenSSL 3.0 support here.

part of MDEV-28133
--- a/cmake/ssl.cmake
+++ b/cmake/ssl.cmake
@@ -118,7 +118,7 @@ MACRO (MYSQL_CHECK_SSL)
     ENDIF()
     FIND_PACKAGE(OpenSSL)
     SET_PACKAGE_PROPERTIES(OpenSSL PROPERTIES TYPE RECOMMENDED)
-    IF(OPENSSL_FOUND AND OPENSSL_VERSION AND OPENSSL_VERSION VERSION_LESS "3.0.0")
+    IF(OPENSSL_FOUND)
       SET(OPENSSL_LIBRARY ${OPENSSL_SSL_LIBRARY})
       INCLUDE(CheckSymbolExists)
       SET(SSL_SOURCES "")

From: Honza Horak <hhorak@redhat.com>
Date: Tue, 8 Feb 2022 16:39:10 +0100
Subject: [PATCH] MDEV-27778 md5 in FIPS crashes with OpenSSL 3.0.0

OpenSSL 3.0.0+ does not support EVP_MD_CTX_FLAG_NON_FIPS_ALLOW any longer.
In OpenSSL 1.1.1 the non FIPS allowed flag is context specific, while
in 3.0.0+ it is a different EVP_MD provider.

Fixes #2010

part of MDEV-28133
--- a/mysys_ssl/my_md5.cc
+++ b/mysys_ssl/my_md5.cc
@@ -52,12 +52,23 @@ static void md5_result(EVP_MD_CTX *context, uchar digest[MD5_HASH_SIZE])
 
 static void md5_init(EVP_MD_CTX *context)
 {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+  EVP_MD *md5;
+  EVP_MD_CTX_init(context);
+  /* Ok to ignore FIPS: MD5 is not used for crypto here */
+  /* In OpenSSL 3.0.0+ it is a different EVP_MD provider */
+  md5 = EVP_MD_fetch(NULL, "MD5", "fips=no");
+  EVP_DigestInit_ex(context, md5, NULL);
+  EVP_MD_free(md5);
+#else
   EVP_MD_CTX_init(context);
 #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
   /* Ok to ignore FIPS: MD5 is not used for crypto here */
+  /* In OpenSSL 1.1.1 the non FIPS allowed flag is context specific */
   EVP_MD_CTX_set_flags(context, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
 #endif
   EVP_DigestInit_ex(context, EVP_md5(), NULL);
+#endif
 }
 
 static void md5_input(EVP_MD_CTX *context, const uchar *buf, unsigned len)

From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 23 May 2022 14:38:56 +0200
Subject: [PATCH] MDEV-28648 main.ssl_timeout fails with OpenSSL 3.0.3

Depending on OpenSSL version, and at least in 3.0.3, the client-side socket
timeout is reported as generic error (SSL_ERROR_SYSCALL), losing further
details (both errno and GetLastError() return 0). This results in client
reporting "Unknown OpenSSL error" 2026, instead of another generic
"Lost connection to server during query" 2013

Adjusted test case.
--- a/mysql-test/main/ssl_timeout.result
+++ b/mysql-test/main/ssl_timeout.result
@@ -5,6 +5,6 @@ SELECT (VARIABLE_VALUE <> '') AS have_ssl FROM INFORMATION_SCHEMA.SESSION_STATUS
 have_ssl
 1
 SELECT SLEEP(600);
-ERROR HY000: Lost connection to server during query
+Got one of the listed errors
 connection default;
 disconnect ssl_con;
--- a/mysql-test/main/ssl_timeout.test
+++ b/mysql-test/main/ssl_timeout.test
@@ -10,7 +10,7 @@ connect (ssl_con,localhost,root,,,,,SSL read_timeout=5);
 SELECT (VARIABLE_VALUE <> '') AS have_ssl FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='Ssl_cipher';
 
 # --error CR_SERVER_LOST
---error 2013
+--error 2013,2026
 SELECT SLEEP(600);
 
 connection default;