taskito 0.6.0__tar.gz → 0.7.0__tar.gz

{taskito-0.6.0 → taskito-0.7.0}/Cargo.lock

@@ -19,9 +19,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
 [[package]]
 name = "arc-swap"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
+checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
 dependencies = [
  "rustversion",
 ]
@@ -551,9 +551,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "js-sys"
@@ -1201,10 +1201,11 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "taskito-async"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
+ "log",
  "pyo3",
  "taskito-core",
  "tokio",
@@ -1212,7 +1213,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-core"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1236,10 +1237,11 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
+ "log",
  "pyo3",
  "serde_json",
  "taskito-async",
@@ -1786,18 +1788,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.42"
+version = "0.8.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
+checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.42"
+version = "0.8.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
+checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
 dependencies = [
  "proc-macro2",
  "quote",

{taskito-0.6.0 → taskito-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.6.0
+Version: 0.7.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 
 [features]

taskito-0.7.0/crates/taskito-core/src/resilience/circuit_breaker.rs

@@ -0,0 +1,313 @@
+use crate::error::Result;
+use crate::job::now_millis;
+use crate::storage::models::CircuitBreakerRow;
+use crate::storage::{Storage, StorageBackend};
+
+/// Circuit breaker states.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(i32)]
+pub enum CircuitState {
+    Closed = 0,
+    Open = 1,
+    HalfOpen = 2,
+}
+
+impl CircuitState {
+    pub fn from_i32(v: i32) -> Self {
+        match v {
+            1 => Self::Open,
+            2 => Self::HalfOpen,
+            _ => Self::Closed,
+        }
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Closed => "closed",
+            Self::Open => "open",
+            Self::HalfOpen => "half_open",
+        }
+    }
+}
+
+/// Configuration for a task's circuit breaker.
+#[derive(Debug, Clone)]
+pub struct CircuitBreakerConfig {
+    pub threshold: i32,
+    pub window_ms: i64,
+    pub cooldown_ms: i64,
+    /// Number of probe requests allowed in HalfOpen state (default: 5).
+    pub half_open_max_probes: i32,
+    /// Required success rate (0.0–1.0) to close from HalfOpen (default: 0.8 = 80%).
+    pub half_open_success_rate: f64,
+}
+
+/// Circuit breaker manager backed by SQLite.
+pub struct CircuitBreaker {
+    storage: StorageBackend,
+}
+
+impl CircuitBreaker {
+    pub fn new(storage: StorageBackend) -> Self {
+        Self { storage }
+    }
+
+    /// Check if a task is allowed to execute. Returns true if allowed.
+    /// Transitions Open -> HalfOpen after cooldown.
+    pub fn allow(&self, task_name: &str) -> Result<bool> {
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(true), // No breaker configured = always allow
+        };
+
+        let now = now_millis();
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::Closed => Ok(true),
+            CircuitState::Open => {
+                // Check if cooldown has elapsed
+                let opened = row.opened_at.unwrap_or(0);
+                if now.saturating_sub(opened) >= row.cooldown_ms {
+                    // Transition to half-open: reset probe counters
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::HalfOpen as i32,
+                        half_open_at: Some(now),
+                        half_open_probe_count: 0,
+                        half_open_success_count: 0,
+                        half_open_failure_count: 0,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                    Ok(true)
+                } else {
+                    Ok(false) // Still in cooldown
+                }
+            }
+            CircuitState::HalfOpen => {
+                // Allow up to max_probes concurrent probes
+                if row.half_open_probe_count < row.half_open_max_probes {
+                    let updated = CircuitBreakerRow {
+                        half_open_probe_count: row.half_open_probe_count + 1,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                    Ok(true)
+                } else {
+                    // Check for timeout: if probes haven't completed within cooldown, re-open
+                    let half_open_since = row.half_open_at.unwrap_or(0);
+                    if now.saturating_sub(half_open_since) >= row.cooldown_ms {
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Open as i32,
+                            opened_at: Some(now),
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    }
+                    Ok(false)
+                }
+            }
+        }
+    }
+
+    /// Record a task success. In HalfOpen, tracks probes and closes when
+    /// the success rate threshold is met.
+    pub fn record_success(&self, task_name: &str) -> Result<()> {
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(()),
+        };
+
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::Closed if row.failure_count == 0 => Ok(()),
+            CircuitState::HalfOpen => {
+                let successes = row.half_open_success_count + 1;
+                let total = successes + row.half_open_failure_count;
+
+                if total >= row.half_open_max_probes {
+                    let rate = successes as f64 / total as f64;
+                    if rate >= row.half_open_success_rate {
+                        // Threshold met — close the circuit
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Closed as i32,
+                            failure_count: 0,
+                            opened_at: None,
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    } else {
+                        // Threshold not met — re-open
+                        let now = now_millis();
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Open as i32,
+                            opened_at: Some(now),
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    }
+                } else {
+                    // Still collecting samples
+                    let updated = CircuitBreakerRow {
+                        half_open_success_count: successes,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+                Ok(())
+            }
+            _ => {
+                // Closed with failures or Open — reset to clean Closed
+                let updated = CircuitBreakerRow {
+                    state: CircuitState::Closed as i32,
+                    failure_count: 0,
+                    opened_at: None,
+                    half_open_at: None,
+                    half_open_probe_count: 0,
+                    half_open_success_count: 0,
+                    half_open_failure_count: 0,
+                    ..row
+                };
+                self.storage.upsert_circuit_breaker(&updated)?;
+                Ok(())
+            }
+        }
+    }
+
+    /// Record a task failure. May trip the breaker open.
+    pub fn record_failure(&self, task_name: &str) -> Result<()> {
+        let now = now_millis();
+
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(()),
+        };
+
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::HalfOpen => {
+                let failures = row.half_open_failure_count + 1;
+                let total = row.half_open_success_count + failures;
+
+                // Check if the success rate can still be met
+                let remaining = row.half_open_max_probes - total;
+                let best_possible_rate = if row.half_open_max_probes > 0 {
+                    (row.half_open_success_count + remaining) as f64
+                        / row.half_open_max_probes as f64
+                } else {
+                    0.0
+                };
+
+                if total >= row.half_open_max_probes
+                    || best_possible_rate < row.half_open_success_rate
+                {
+                    // Either all samples collected and failed, or impossible to meet threshold
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::Open as i32,
+                        failure_count: row.failure_count.saturating_add(1),
+                        last_failure_at: Some(now),
+                        opened_at: Some(now),
+                        half_open_at: None,
+                        half_open_probe_count: 0,
+                        half_open_success_count: 0,
+                        half_open_failure_count: 0,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                } else {
+                    // Still collecting samples
+                    let updated = CircuitBreakerRow {
+                        failure_count: row.failure_count.saturating_add(1),
+                        last_failure_at: Some(now),
+                        half_open_failure_count: failures,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+            }
+            CircuitState::Closed => {
+                // Reset count if outside the window
+                let count = if let Some(last) = row.last_failure_at {
+                    if now.saturating_sub(last) > row.window_ms {
+                        1 // Window expired, start fresh
+                    } else {
+                        row.failure_count.saturating_add(1)
+                    }
+                } else {
+                    1
+                };
+
+                if count >= row.threshold {
+                    // Trip to open
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::Open as i32,
+                        failure_count: count,
+                        last_failure_at: Some(now),
+                        opened_at: Some(now),
+                        half_open_at: None,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                } else {
+                    let updated = CircuitBreakerRow {
+                        failure_count: count,
+                        last_failure_at: Some(now),
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+            }
+            CircuitState::Open => {
+                // Already open, just update failure count
+                let updated = CircuitBreakerRow {
+                    failure_count: row.failure_count.saturating_add(1),
+                    last_failure_at: Some(now),
+                    ..row
+                };
+                self.storage.upsert_circuit_breaker(&updated)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Register a circuit breaker for a task (idempotent).
+    pub fn register(&self, task_name: &str, config: &CircuitBreakerConfig) -> Result<()> {
+        if self.storage.get_circuit_breaker(task_name)?.is_some() {
+            return Ok(());
+        }
+
+        let row = CircuitBreakerRow {
+            task_name: task_name.to_string(),
+            state: CircuitState::Closed as i32,
+            failure_count: 0,
+            last_failure_at: None,
+            opened_at: None,
+            half_open_at: None,
+            threshold: config.threshold,
+            window_ms: config.window_ms,
+            cooldown_ms: config.cooldown_ms,
+            half_open_max_probes: config.half_open_max_probes,
+            half_open_success_rate: config.half_open_success_rate,
+            half_open_probe_count: 0,
+            half_open_success_count: 0,
+            half_open_failure_count: 0,
+        };
+        self.storage.upsert_circuit_breaker(&row)?;
+        Ok(())
+    }
+}

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/scheduler/mod.rs

@@ -79,6 +79,7 @@ pub enum ResultOutcome {
     Retry {
         job_id: String,
         task_name: String,
+        queue: String,
         error: String,
         retry_count: i32,
         timed_out: bool,
@@ -87,11 +88,16 @@ pub enum ResultOutcome {
     DeadLettered {
         job_id: String,
         task_name: String,
+        queue: String,
         error: String,
         timed_out: bool,
     },
     /// Task was cancelled during execution.
-    Cancelled { job_id: String, task_name: String },
+    Cancelled {
+        job_id: String,
+        task_name: String,
+        queue: String,
+    },
 }
 
 /// Per-task configuration for retry, rate limiting, and circuit breaker.
@@ -452,6 +458,8 @@ mod tests {
             threshold: 1,
             window_ms: 60_000,
             cooldown_ms: 300_000,
+            half_open_max_probes: 5,
+            half_open_success_rate: 0.8,
         };
         scheduler.register_task(
             "cb_task".to_string(),

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/scheduler/result_handler.rs

@@ -71,6 +71,14 @@ impl Scheduler {
                     log::error!("circuit breaker error for {task_name}: {e}");
                 }
 
+                // Look up the job to get the queue name for middleware context
+                let queue = self
+                    .storage
+                    .get_job(&job_id)?
+                    .as_ref()
+                    .map(|j| j.queue.clone())
+                    .unwrap_or_default();
+
                 // If should_retry is false (exception filtering), skip straight to DLQ
                 if !should_retry {
                     match self.storage.get_job(&job_id)? {
@@ -80,6 +88,7 @@ impl Scheduler {
                     return Ok(ResultOutcome::DeadLettered {
                         job_id,
                         task_name,
+                        queue,
                         error,
                         timed_out,
                     });
@@ -103,6 +112,7 @@ impl Scheduler {
                     Ok(ResultOutcome::Retry {
                         job_id,
                         task_name,
+                        queue,
                         error,
                         retry_count,
                         timed_out,
@@ -116,6 +126,7 @@ impl Scheduler {
                     Ok(ResultOutcome::DeadLettered {
                         job_id,
                         task_name,
+                        queue,
                         error,
                         timed_out,
                     })
@@ -140,7 +151,17 @@ impl Scheduler {
                 {
                     error!("failed to record metric for cancelled job {job_id}: {e}");
                 }
-                Ok(ResultOutcome::Cancelled { job_id, task_name })
+                let queue = self
+                    .storage
+                    .get_job(&job_id)?
+                    .as_ref()
+                    .map(|j| j.queue.clone())
+                    .unwrap_or_default();
+                Ok(ResultOutcome::Cancelled {
+                    job_id,
+                    task_name,
+                    queue,
+                })
             }
         }
     }

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/storage/models.rs

@@ -272,6 +272,24 @@ pub struct CircuitBreakerRow {
     pub threshold: i32,
     pub window_ms: i64,
     pub cooldown_ms: i64,
+    #[serde(default = "default_max_probes")]
+    pub half_open_max_probes: i32,
+    #[serde(default = "default_success_rate")]
+    pub half_open_success_rate: f64,
+    #[serde(default)]
+    pub half_open_probe_count: i32,
+    #[serde(default)]
+    pub half_open_success_count: i32,
+    #[serde(default)]
+    pub half_open_failure_count: i32,
+}
+
+fn default_max_probes() -> i32 {
+    5
+}
+
+fn default_success_rate() -> f64 {
+    0.8
 }
 
 // ── Workers ──────────────────────────────────────────────────────

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/storage/postgres/mod.rs

@@ -319,7 +319,12 @@ impl PostgresStorage {
                 half_open_at   BIGINT,
                 threshold      INTEGER NOT NULL DEFAULT 5,
                 window_ms      BIGINT NOT NULL DEFAULT 60000,
-                cooldown_ms    BIGINT NOT NULL DEFAULT 300000
+                cooldown_ms    BIGINT NOT NULL DEFAULT 300000,
+                half_open_max_probes   INTEGER NOT NULL DEFAULT 5,
+                half_open_success_rate DOUBLE PRECISION NOT NULL DEFAULT 0.8,
+                half_open_probe_count  INTEGER NOT NULL DEFAULT 0,
+                half_open_success_count INTEGER NOT NULL DEFAULT 0,
+                half_open_failure_count INTEGER NOT NULL DEFAULT 0
             )",
         )
         .execute(&mut conn)?;
@@ -407,6 +412,28 @@ impl PostgresStorage {
             "ALTER TABLE jobs ADD COLUMN IF NOT EXISTS namespace TEXT",
         );
 
+        // Migration: add sample-based half-open probes to circuit breakers
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE circuit_breakers ADD COLUMN IF NOT EXISTS half_open_max_probes INTEGER NOT NULL DEFAULT 5",
+        );
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE circuit_breakers ADD COLUMN IF NOT EXISTS half_open_success_rate DOUBLE PRECISION NOT NULL DEFAULT 0.8",
+        );
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE circuit_breakers ADD COLUMN IF NOT EXISTS half_open_probe_count INTEGER NOT NULL DEFAULT 0",
+        );
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE circuit_breakers ADD COLUMN IF NOT EXISTS half_open_success_count INTEGER NOT NULL DEFAULT 0",
+        );
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE circuit_breakers ADD COLUMN IF NOT EXISTS half_open_failure_count INTEGER NOT NULL DEFAULT 0",
+        );
+
         // ── Distributed Locks ─────────────────────────────
         diesel::sql_query(
             "CREATE TABLE IF NOT EXISTS distributed_locks (

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/storage/redis_backend/locks.rs

@@ -127,20 +127,32 @@ impl RedisStorage {
         let mut conn = self.conn()?;
         let pattern = self.key(&["lock", "*"]);
 
-        let keys: Vec<String> = redis::cmd("KEYS")
-            .arg(&pattern)
-            .query(&mut conn)
-            .map_err(map_err)?;
-
         let mut count = 0u64;
-        for key in keys {
-            let expires_at: Option<i64> = conn.hget(&key, "expires_at").map_err(map_err)?;
-            if let Some(exp) = expires_at {
-                if exp <= now {
-                    conn.del::<_, ()>(&key).map_err(map_err)?;
-                    count += 1;
+        let mut cursor: u64 = 0;
+        loop {
+            let (next_cursor, keys): (u64, Vec<String>) = redis::cmd("SCAN")
+                .arg(cursor)
+                .arg("MATCH")
+                .arg(&pattern)
+                .arg("COUNT")
+                .arg(100)
+                .query(&mut conn)
+                .map_err(map_err)?;
+
+            for key in keys {
+                let expires_at: Option<i64> = conn.hget(&key, "expires_at").map_err(map_err)?;
+                if let Some(exp) = expires_at {
+                    if exp <= now {
+                        conn.del::<_, ()>(&key).map_err(map_err)?;
+                        count += 1;
+                    }
                 }
             }
+
+            cursor = next_cursor;
+            if cursor == 0 {
+                break;
+            }
         }
 
         Ok(count)
@@ -151,11 +163,14 @@ impl RedisStorage {
         let now = now_millis();
         let ckey = self.key(&["exec_claim", job_id]);
 
-        // NX: set only if not exists
+        // NX: set only if not exists. PX: auto-expire after 24 hours so
+        // orphaned claims from dead workers don't block re-execution forever.
         let result: bool = redis::cmd("SET")
             .arg(&ckey)
             .arg(format!("{worker_id}:{now}"))
             .arg("NX")
+            .arg("PX")
+            .arg(86_400_000i64) // 24 hours in milliseconds
             .query(&mut conn)
             .map_err(map_err)?;
 

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/storage/schema.rs

@@ -134,6 +134,11 @@ diesel::table! {
         threshold -> Integer,
         window_ms -> BigInt,
         cooldown_ms -> BigInt,
+        half_open_max_probes -> Integer,
+        half_open_success_rate -> Double,
+        half_open_probe_count -> Integer,
+        half_open_success_count -> Integer,
+        half_open_failure_count -> Integer,
     }
 }
 

{taskito-0.6.0 → taskito-0.7.0}/crates/taskito-core/src/storage/sqlite/dead_letter.rs

@@ -54,8 +54,8 @@ impl SqliteStorage {
         // Cascade cancel dependents — log warning on failure since the DLQ
         // transaction already committed and we can't roll it back.
         if let Err(e) = self.cascade_cancel(&job_id, "dependency failed") {
-            eprintln!(
-                "[taskito] WARNING: cascade_cancel failed for job {}: {}. Dependent jobs may be left pending.",
+            log::warn!(
+                "[taskito] cascade_cancel failed for job {}: {}. Dependent jobs may be left pending.",
                 job_id, e
             );
         }