feat(dapi): add background health check for DAPI nodes

packages/rs-dapi-client/Cargo.toml

@@ -43,6 +43,7 @@ dapi-grpc = { path = "../dapi-grpc", features = [
   "client",
 ], default-features = false }
 futures = { version = "0.3.28" }
+tokio-util = { version = "0.7.12", default-features = false }
 http = { version = "1.1.0", default-features = false }
 http-serde = { version = "2.1", optional = true }
 

packages/rs-dapi-client/src/address_list.rs

@@ -11,7 +11,8 @@ use std::str::FromStr;
 use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
-const DEFAULT_BASE_BAN_PERIOD: Duration = Duration::from_secs(60);
+// Base ban period in seconds. Ban duration will increase exponentially with each subsequent ban.
+pub(crate) const DEFAULT_BASE_BAN_PERIOD: Duration = Duration::from_secs(60);
 
 /// DAPI address.
 #[derive(Debug, Clone, Eq)]
@@ -85,11 +86,20 @@ impl AddressStatus {
         self.ban_count += 1;
     }
 
-    /// Check if [Address] is banned.
+    /// Check if [Address] has a ban record (has been banned at least once and not yet unbanned).
+    ///
+    /// Note: This checks `ban_count > 0`, not whether the ban is currently active.
+    /// An address with an expired `banned_until` but non-zero `ban_count` will still
+    /// return `true` here. Use [`AddressList::get_live_address`] for temporal ban checking.
     pub fn is_banned(&self) -> bool {
         self.ban_count > 0
     }
 
+    /// Returns the number of times this address has been banned.
+    pub fn ban_count(&self) -> usize {
+        self.ban_count
+    }
+
     /// Clears ban record.
     pub fn unban(&mut self) {
         self.ban_count = 0;
@@ -155,6 +165,18 @@ impl AddressList {
         true
     }
 
+    /// Atomically resets the ban for an address: clears the ban history and applies
+    /// a fresh base-period ban. This avoids the TOCTOU gap that would exist with
+    /// separate `unban()` + `ban()` calls where a concurrent reader could observe
+    /// an unbanned state between the two operations.
+    pub fn reset_ban(&self, address: &Address) {
+        let mut guard = self.addresses.write().unwrap();
+        if let Some(status) = guard.get_mut(address) {
+            status.ban_count = 1;
+            status.banned_until = Some(chrono::Utc::now() + self.base_ban_period);
+        }
+    }
+
     /// Clears address' ban record
     /// Returns false if the address is not in the list.
     pub fn unban(&self, address: &Address) -> bool {
@@ -266,6 +288,42 @@ impl AddressList {
             .collect()
     }
 
+    /// Returns ALL addresses (both banned and unbanned).
+    pub fn get_all_addresses(&self) -> Vec<Address> {
+        let guard = self.addresses.read().unwrap();
+        guard.keys().cloned().collect()
+    }
+
+    /// Returns the earliest `banned_until` timestamp that is still in the future.
+    /// Returns `None` if no addresses are currently banned with a future expiry.
+    pub fn get_next_ban_expiry(&self) -> Option<chrono::DateTime<Utc>> {
+        let guard = self.addresses.read().unwrap();
+        let now = chrono::Utc::now();
+        guard
+            .values()
+            .filter_map(|status| status.banned_until)
+            .filter(|banned_until| *banned_until > now)
+            .min()
+    }
+
+    /// Returns addresses whose ban has expired but still have a ban record (ban_count > 0).
+    /// These are candidates for re-probing before being made available again.
+    pub fn get_expired_ban_addresses(&self) -> Vec<Address> {
+        let guard = self.addresses.read().unwrap();
+        let now = chrono::Utc::now();
+        guard
+            .iter()
+            .filter(|(_, status)| {
+                status.ban_count > 0
+                    && status
+                        .banned_until
+                        .map(|banned_until| banned_until <= now)
+                        .unwrap_or(true)
+            })
+            .map(|(addr, _)| addr.clone())
+            .collect()
+    }
+
     /// Get number of all addresses, both banned and not banned.
     pub fn len(&self) -> usize {
         self.addresses.read().unwrap().len()
@@ -592,4 +650,112 @@ mod tests {
         let list = AddressList::default();
         assert!(list.is_empty());
     }
+
+    #[test]
+    fn test_get_all_addresses_returns_both_banned_and_unbanned() {
+        let mut list = AddressList::new();
+        let addr1: Address = "http://127.0.0.1:3000".parse().unwrap();
+        let addr2: Address = "http://127.0.0.1:3001".parse().unwrap();
+        list.add(addr1.clone());
+        list.add(addr2.clone());
+        list.ban(&addr2);
+
+        let all = list.get_all_addresses();
+        assert_eq!(all.len(), 2);
+        assert!(all.contains(&addr1));
+        assert!(all.contains(&addr2));
+    }
+
+    #[test]
+    fn test_get_next_ban_expiry_returns_earliest() {
+        let mut list = AddressList::new();
+        let addr1: Address = "http://127.0.0.1:3000".parse().unwrap();
+        let addr2: Address = "http://127.0.0.1:3001".parse().unwrap();
+        list.add(addr1.clone());
+        list.add(addr2.clone());
+
+        list.ban(&addr1);
+        list.ban(&addr2);
+        list.ban(&addr2);
+
+        let expiry = list.get_next_ban_expiry();
+        assert!(expiry.is_some());
+    }
+
+    #[test]
+    fn test_get_next_ban_expiry_none_when_no_bans() {
+        let mut list = AddressList::new();
+        list.add("http://127.0.0.1:3000".parse().unwrap());
+        assert!(list.get_next_ban_expiry().is_none());
+    }
+
+    #[test]
+    fn test_get_expired_ban_addresses() {
+        let mut list = AddressList::with_settings(Duration::from_millis(1));
+        let addr1: Address = "http://127.0.0.1:3000".parse().unwrap();
+        let addr2: Address = "http://127.0.0.1:3001".parse().unwrap();
+        list.add(addr1.clone());
+        list.add(addr2.clone());
+
+        list.ban(&addr1);
+        std::thread::sleep(Duration::from_millis(50));
+
+        let expired = list.get_expired_ban_addresses();
+        assert!(expired.contains(&addr1));
+        assert!(!expired.contains(&addr2));
+    }
+
+    #[test]
+    fn test_reset_ban_clears_history_and_applies_base_period_ban() {
+        let mut list = AddressList::with_settings(Duration::from_secs(60));
+        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
+        list.add(addr.clone());
+
+        // Ban twice to build up ban_count
+        list.ban(&addr);
+        list.ban(&addr);
+        assert_eq!(
+            list.addresses.read().unwrap().get(&addr).unwrap().ban_count,
+            2
+        );
+
+        // reset_ban should set ban_count=1 and apply a fresh base-period ban
+        list.reset_ban(&addr);
+        let guard = list.addresses.read().unwrap();
+        let status = guard.get(&addr).unwrap();
+        assert_eq!(status.ban_count, 1, "reset_ban must set ban_count to 1");
+        assert!(
+            status.banned_until.is_some(),
+            "reset_ban must set banned_until"
+        );
+        // The address must be actively banned (not visible via get_live_address)
+        drop(guard);
+        assert!(
+            list.get_live_address().is_none(),
+            "reset_ban must keep address banned"
+        );
+    }
+
+    #[test]
+    fn test_reset_ban_on_nonexistent_address_is_noop() {
+        let list = AddressList::new();
+        let addr: Address = "http://127.0.0.1:3000".parse().unwrap();
+        // Must not panic
+        list.reset_ban(&addr);
+    }
+
+    #[test]
+    fn test_address_status_ban_count() {
+        let mut status = AddressStatus::default();
+        assert_eq!(status.ban_count(), 0);
+
+        status.ban(&Duration::from_secs(60));
+        assert_eq!(status.ban_count(), 1);
+
+        status.ban(&Duration::from_secs(60));
+        assert_eq!(status.ban_count(), 2);
+
+        status.unban();
+        assert_eq!(status.ban_count(), 0);
+    }
 }

packages/rs-dapi-client/src/dapi_client.rs

@@ -155,6 +155,16 @@ impl DapiClient {
     pub fn get_live_addresses(&self) -> Vec<crate::Address> {
         self.address_list.get_live_addresses()
     }
+
+    /// Returns a reference to the connection pool.
+    pub fn connection_pool(&self) -> &ConnectionPool {
+        &self.pool
+    }
+
+    /// Returns the client's request settings.
+    pub fn settings(&self) -> &RequestSettings {
+        &self.settings
+    }
 }
 
 /// Ban address in case of retryable error or unban it