Handle overflows in FFI integer conversions

kornelski · kornelski · commit 06ca1fd7461d · 2026-02-04T16:37:34.000Z
diff --git a/boring/src/aes.rs b/boring/src/aes.rs
@@ -38,7 +38,7 @@
 //! ```
 //!
 use crate::ffi;
-use libc::{c_int, c_uint};
+use libc::c_int;
 use openssl_macros::corresponds;
 use std::mem::MaybeUninit;
 use std::ptr;
@@ -64,7 +64,7 @@ impl AesKey {
             let mut aes_key = MaybeUninit::uninit();
             let r = ffi::AES_set_encrypt_key(
                 key.as_ptr(),
-                key.len() as c_uint * 8,
+                (key.len() * 8).try_into().map_err(|_| KeyError(()))?,
                 aes_key.as_mut_ptr(),
             );
             if r == 0 {
@@ -88,7 +88,7 @@ impl AesKey {
             let mut aes_key = MaybeUninit::uninit();
             let r = ffi::AES_set_decrypt_key(
                 key.as_ptr(),
-                key.len() as c_uint * 8,
+                (key.len() * 8).try_into().map_err(|_| KeyError(()))?,
                 aes_key.as_mut_ptr(),
             );
 
diff --git a/boring/src/bio.rs b/boring/src/bio.rs
@@ -1,11 +1,12 @@
-use crate::ffi;
-use crate::ffi::BIO_new_mem_buf;
 use std::marker::PhantomData;
 use std::ptr;
 use std::slice;
 
 use crate::cvt_p;
 use crate::error::ErrorStack;
+use crate::ffi;
+use crate::ffi::BIO_new_mem_buf;
+use crate::try_int;
 
 pub struct MemBioSlice<'a>(*mut ffi::BIO, PhantomData<&'a [u8]>);
 
@@ -19,15 +20,9 @@ impl Drop for MemBioSlice<'_> {
 
 impl<'a> MemBioSlice<'a> {
     pub fn new(buf: &'a [u8]) -> Result<MemBioSlice<'a>, ErrorStack> {
-        #[cfg(not(feature = "legacy-compat-deprecated"))]
-        type BufLen = isize;
-        #[cfg(feature = "legacy-compat-deprecated")]
-        type BufLen = libc::c_int;
-
         ffi::init();
 
-        assert!(buf.len() <= BufLen::MAX as usize);
-        let bio = unsafe { cvt_p(BIO_new_mem_buf(buf.as_ptr().cast(), buf.len() as BufLen))? };
+        let bio = unsafe { cvt_p(BIO_new_mem_buf(buf.as_ptr().cast(), try_int(buf.len())?))? };
 
         Ok(MemBioSlice(bio, PhantomData))
     }
@@ -63,7 +58,7 @@ impl MemBio {
         unsafe {
             let mut ptr = ptr::null_mut();
             let len = ffi::BIO_get_mem_data(self.0, &mut ptr);
-            if ptr.is_null() {
+            if ptr.is_null() || len < 0 {
                 return &[];
             }
             slice::from_raw_parts(ptr.cast_const().cast(), len as usize)
diff --git a/boring/src/bn.rs b/boring/src/bn.rs
@@ -390,7 +390,7 @@ impl BigNumRef {
         unsafe {
             cvt(ffi::BN_generate_prime_ex(
                 self.as_ptr(),
-                bits as c_int,
+                c_int::from(bits),
                 c_int::from(safe),
                 add.map(|n| n.as_ptr()).unwrap_or(ptr::null_mut()),
                 rem.map(|n| n.as_ptr()).unwrap_or(ptr::null_mut()),
diff --git a/boring/src/dsa.rs b/boring/src/dsa.rs
@@ -5,7 +5,6 @@
 //! using the private key that can be validated with the public key but not be generated
 //! without the private key.
 
-use crate::ffi;
 use foreign_types::{ForeignType, ForeignTypeRef};
 use libc::c_uint;
 use openssl_macros::corresponds;
@@ -15,7 +14,9 @@ use std::ptr;
 
 use crate::bn::{BigNum, BigNumRef};
 use crate::error::ErrorStack;
+use crate::ffi;
 use crate::pkey::{HasParams, HasPrivate, HasPublic, Private, Public};
+use crate::try_int;
 use crate::{cvt, cvt_p};
 
 generic_foreign_type_and_impl_send_sync! {
@@ -195,7 +196,7 @@ impl Dsa<Private> {
             let dsa = Dsa::from_ptr(cvt_p(ffi::DSA_new())?);
             cvt(ffi::DSA_generate_parameters_ex(
                 dsa.0,
-                bits as c_uint,
+                c_uint::from(bits),
                 ptr::null(),
                 0,
                 ptr::null_mut(),
diff --git a/boring/src/ec.rs b/boring/src/ec.rs
@@ -15,7 +15,6 @@
 //! [`EcGroup`]: struct.EcGroup.html
 //! [`Nid`]: ../nid/struct.Nid.html
 //! [Eliptic Curve Cryptography]: https://wiki.openssl.org/index.php/Elliptic_Curve_Cryptography
-use crate::ffi;
 use foreign_types::{ForeignType, ForeignTypeRef};
 use libc::c_int;
 use openssl_macros::corresponds;
@@ -24,8 +23,10 @@ use std::ptr;
 
 use crate::bn::{BigNumContextRef, BigNumRef};
 use crate::error::ErrorStack;
+use crate::ffi;
 use crate::nid::Nid;
 use crate::pkey::{HasParams, HasPrivate, HasPublic, Params, Private, Public};
+use crate::try_int;
 use crate::{cvt, cvt_n, cvt_p, init};
 
 /// Compressed or Uncompressed conversion
diff --git a/boring/src/hash.rs b/boring/src/hash.rs
@@ -1,6 +1,4 @@
-use crate::ffi;
 use openssl_macros::corresponds;
-use std::convert::TryInto;
 use std::ffi::c_uint;
 use std::fmt;
 use std::io;
@@ -9,8 +7,10 @@ use std::ops::{Deref, DerefMut};
 use std::ptr;
 
 use crate::error::ErrorStack;
+use crate::ffi;
 use crate::ffi::{EVP_MD_CTX_free, EVP_MD_CTX_new};
 use crate::nid::Nid;
+use crate::try_int;
 use crate::{cvt, cvt_p};
 
 #[derive(Copy, Clone, PartialEq, Eq)]
@@ -210,7 +210,7 @@ impl Hasher {
             self.init()?;
         }
         unsafe {
-            let mut len = ffi::EVP_MAX_MD_SIZE.try_into().unwrap();
+            let mut len = try_int(ffi::EVP_MAX_MD_SIZE)?;
             let mut buf = [0; ffi::EVP_MAX_MD_SIZE as usize];
             cvt(ffi::EVP_DigestFinal_ex(
                 self.ctx,
@@ -220,7 +220,7 @@ impl Hasher {
             self.state = Finalized;
             Ok(DigestBytes {
                 buf,
-                len: len as usize,
+                len: try_int(len)?,
             })
         }
     }
diff --git a/boring/src/lib.rs b/boring/src/lib.rs
@@ -201,6 +201,15 @@ fn cvt_n(r: c_int) -> Result<c_int, ErrorStack> {
     }
 }
 
+fn try_int<F, T>(from: F) -> Result<T, ErrorStack>
+where
+    F: TryInto<T> + Send + Sync + Copy + 'static,
+    T: Send + Sync + Copy + 'static,
+{
+    from.try_into()
+        .map_err(|_| ErrorStack::internal_error_str("int overflow"))
+}
+
 unsafe extern "C" fn free_data_box<T>(
     _parent: *mut c_void,
     ptr: *mut c_void,
diff --git a/boring/src/macros.rs b/boring/src/macros.rs
@@ -60,12 +60,11 @@ macro_rules! private_key_to_pem {
         ) -> Result<Vec<u8>, crate::error::ErrorStack> {
             unsafe {
                 let bio = crate::bio::MemBio::new()?;
-                assert!(passphrase.len() <= ::libc::c_int::MAX as usize);
                 cvt($f(bio.as_ptr(),
                         self.as_ptr(),
                         cipher.as_ptr(),
                         passphrase.as_ptr() as *const _ as *mut _,
-                        passphrase.len() as ::libc::c_int,
+                        try_int(passphrase.len())?,
                         None,
                         ptr::null_mut()))?;
                 Ok(bio.get_buf().to_owned())
diff --git a/boring/src/pkcs5.rs b/boring/src/pkcs5.rs
@@ -1,11 +1,11 @@
 use crate::ffi;
-use libc::{c_int, c_uint};
+use std::ffi::c_int;
 use std::ptr;
 
 use crate::error::ErrorStack;
 use crate::hash::MessageDigest;
 use crate::symm::Cipher;
-use crate::{cvt, cvt_nz};
+use crate::{cvt, cvt_nz, try_int};
 
 #[derive(Clone, Eq, PartialEq, Hash, Debug)]
 pub struct KeyIvPair {
@@ -90,17 +90,14 @@ pub fn pbkdf2_hmac(
     key: &mut [u8],
 ) -> Result<(), ErrorStack> {
     unsafe {
-        assert!(pass.len() <= c_int::MAX as usize);
-        assert!(salt.len() <= c_int::MAX as usize);
-        assert!(key.len() <= c_int::MAX as usize);
-
         ffi::init();
+
         cvt(ffi::PKCS5_PBKDF2_HMAC(
             pass.as_ptr().cast(),
             pass.len(),
             salt.as_ptr(),
             salt.len(),
-            iter as c_uint,
+            try_int(iter)?,
             hash.as_ptr(),
             key.len(),
             key.as_mut_ptr(),
diff --git a/boring/src/pkey.rs b/boring/src/pkey.rs
@@ -40,7 +40,6 @@
 //! println!("{:?}", str::from_utf8(pub_key.as_slice()).unwrap());
 //! ```
 
-use crate::ffi;
 use foreign_types::{ForeignType, ForeignTypeRef};
 use libc::{c_int, c_long};
 use openssl_macros::corresponds;
@@ -54,7 +53,9 @@ use crate::dh::Dh;
 use crate::dsa::Dsa;
 use crate::ec::EcKey;
 use crate::error::ErrorStack;
+use crate::ffi;
 use crate::rsa::Rsa;
+use crate::try_int;
 use crate::util::{invoke_passwd_cb, CallbackState};
 use crate::{cvt, cvt_0i, cvt_p};
 
diff --git a/boring/src/rsa.rs b/boring/src/rsa.rs
@@ -23,7 +23,6 @@
 //! let mut buf = vec![0; rsa.size() as usize];
 //! let encrypted_len = rsa.public_encrypt(data, &mut buf, Padding::PKCS1).unwrap();
 //! ```
-use crate::ffi;
 use foreign_types::{ForeignType, ForeignTypeRef};
 use libc::c_int;
 use openssl_macros::corresponds;
@@ -33,7 +32,9 @@ use std::ptr;
 
 use crate::bn::{BigNum, BigNumRef};
 use crate::error::ErrorStack;
+use crate::ffi;
 use crate::pkey::{HasPrivate, HasPublic, Private, Public};
+use crate::try_int;
 use crate::{cvt, cvt_n, cvt_p};
 
 pub const EVP_PKEY_OP_SIGN: c_int = 1 << 3;
diff --git a/boring/src/ssl/bio.rs b/boring/src/ssl/bio.rs
@@ -101,8 +101,12 @@ unsafe fn state<'a, S: 'a>(bio: *mut BIO) -> &'a mut StreamState<S> {
 unsafe extern "C" fn bwrite<S: Write>(bio: *mut BIO, buf: *const c_char, len: c_int) -> c_int {
     BIO_clear_retry_flags(bio);
 
+    let Ok(len) = usize::try_from(len) else {
+        return -1;
+    };
+
     let state = state::<S>(bio);
-    let buf = slice::from_raw_parts(buf.cast(), len as usize);
+    let buf = slice::from_raw_parts(buf.cast(), len);
 
     match catch_unwind(AssertUnwindSafe(|| state.stream.write(buf))) {
         Ok(Ok(len)) => len as c_int,
@@ -123,8 +127,12 @@ unsafe extern "C" fn bwrite<S: Write>(bio: *mut BIO, buf: *const c_char, len: c_
 unsafe extern "C" fn bread<S: Read>(bio: *mut BIO, buf: *mut c_char, len: c_int) -> c_int {
     BIO_clear_retry_flags(bio);
 
+    let Ok(len) = usize::try_from(len) else {
+        return -1;
+    };
+
     let state = state::<S>(bio);
-    let buf = slice::from_raw_parts_mut(buf.cast(), len as usize);
+    let buf = slice::from_raw_parts_mut(buf.cast(), len);
 
     match catch_unwind(AssertUnwindSafe(|| state.stream.read(buf))) {
         Ok(Ok(len)) => len as c_int,
diff --git a/boring/src/ssl/mod.rs b/boring/src/ssl/mod.rs
diff --git a/boring/src/symm.rs b/boring/src/symm.rs
diff --git a/boring/src/x509/mod.rs b/boring/src/x509/mod.rs
