taskito 0.6.0__tar.gz → 0.8.0__tar.gz

{taskito-0.6.0 → taskito-0.8.0}/Cargo.lock

@@ -19,9 +19,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
 [[package]]
 name = "arc-swap"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
+checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
 dependencies = [
  "rustversion",
 ]
@@ -551,9 +551,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "js-sys"
@@ -1201,10 +1201,11 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "taskito-async"
-version = "0.6.0"
+version = "0.8.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
+ "log",
  "pyo3",
  "taskito-core",
  "tokio",
@@ -1212,7 +1213,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-core"
-version = "0.6.0"
+version = "0.8.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1236,10 +1237,11 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.6.0"
+version = "0.8.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
+ "log",
  "pyo3",
  "serde_json",
  "taskito-async",
@@ -1786,18 +1788,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.42"
+version = "0.8.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
+checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.42"
+version = "0.8.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
+checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
 dependencies = [
  "proc-macro2",
  "quote",

{taskito-0.6.0 → taskito-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.6.0
+Version: 0.8.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{taskito-0.6.0 → taskito-0.8.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.6.0"
+version = "0.8.0"
 edition = "2021"
 
 [features]

taskito-0.8.0/crates/taskito-core/src/resilience/circuit_breaker.rs

@@ -0,0 +1,313 @@
+use crate::error::Result;
+use crate::job::now_millis;
+use crate::storage::models::CircuitBreakerRow;
+use crate::storage::{Storage, StorageBackend};
+
+/// Circuit breaker states.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(i32)]
+pub enum CircuitState {
+    Closed = 0,
+    Open = 1,
+    HalfOpen = 2,
+}
+
+impl CircuitState {
+    pub fn from_i32(v: i32) -> Self {
+        match v {
+            1 => Self::Open,
+            2 => Self::HalfOpen,
+            _ => Self::Closed,
+        }
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Closed => "closed",
+            Self::Open => "open",
+            Self::HalfOpen => "half_open",
+        }
+    }
+}
+
+/// Configuration for a task's circuit breaker.
+#[derive(Debug, Clone)]
+pub struct CircuitBreakerConfig {
+    pub threshold: i32,
+    pub window_ms: i64,
+    pub cooldown_ms: i64,
+    /// Number of probe requests allowed in HalfOpen state (default: 5).
+    pub half_open_max_probes: i32,
+    /// Required success rate (0.0–1.0) to close from HalfOpen (default: 0.8 = 80%).
+    pub half_open_success_rate: f64,
+}
+
+/// Circuit breaker manager backed by SQLite.
+pub struct CircuitBreaker {
+    storage: StorageBackend,
+}
+
+impl CircuitBreaker {
+    pub fn new(storage: StorageBackend) -> Self {
+        Self { storage }
+    }
+
+    /// Check if a task is allowed to execute. Returns true if allowed.
+    /// Transitions Open -> HalfOpen after cooldown.
+    pub fn allow(&self, task_name: &str) -> Result<bool> {
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(true), // No breaker configured = always allow
+        };
+
+        let now = now_millis();
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::Closed => Ok(true),
+            CircuitState::Open => {
+                // Check if cooldown has elapsed
+                let opened = row.opened_at.unwrap_or(0);
+                if now.saturating_sub(opened) >= row.cooldown_ms {
+                    // Transition to half-open: reset probe counters
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::HalfOpen as i32,
+                        half_open_at: Some(now),
+                        half_open_probe_count: 0,
+                        half_open_success_count: 0,
+                        half_open_failure_count: 0,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                    Ok(true)
+                } else {
+                    Ok(false) // Still in cooldown
+                }
+            }
+            CircuitState::HalfOpen => {
+                // Allow up to max_probes concurrent probes
+                if row.half_open_probe_count < row.half_open_max_probes {
+                    let updated = CircuitBreakerRow {
+                        half_open_probe_count: row.half_open_probe_count + 1,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                    Ok(true)
+                } else {
+                    // Check for timeout: if probes haven't completed within cooldown, re-open
+                    let half_open_since = row.half_open_at.unwrap_or(0);
+                    if now.saturating_sub(half_open_since) >= row.cooldown_ms {
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Open as i32,
+                            opened_at: Some(now),
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    }
+                    Ok(false)
+                }
+            }
+        }
+    }
+
+    /// Record a task success. In HalfOpen, tracks probes and closes when
+    /// the success rate threshold is met.
+    pub fn record_success(&self, task_name: &str) -> Result<()> {
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(()),
+        };
+
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::Closed if row.failure_count == 0 => Ok(()),
+            CircuitState::HalfOpen => {
+                let successes = row.half_open_success_count + 1;
+                let total = successes + row.half_open_failure_count;
+
+                if total >= row.half_open_max_probes {
+                    let rate = successes as f64 / total as f64;
+                    if rate >= row.half_open_success_rate {
+                        // Threshold met — close the circuit
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Closed as i32,
+                            failure_count: 0,
+                            opened_at: None,
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    } else {
+                        // Threshold not met — re-open
+                        let now = now_millis();
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Open as i32,
+                            opened_at: Some(now),
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    }
+                } else {
+                    // Still collecting samples
+                    let updated = CircuitBreakerRow {
+                        half_open_success_count: successes,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+                Ok(())
+            }
+            _ => {
+                // Closed with failures or Open — reset to clean Closed
+                let updated = CircuitBreakerRow {
+                    state: CircuitState::Closed as i32,
+                    failure_count: 0,
+                    opened_at: None,
+                    half_open_at: None,
+                    half_open_probe_count: 0,
+                    half_open_success_count: 0,
+                    half_open_failure_count: 0,
+                    ..row
+                };
+                self.storage.upsert_circuit_breaker(&updated)?;
+                Ok(())
+            }
+        }
+    }
+
+    /// Record a task failure. May trip the breaker open.
+    pub fn record_failure(&self, task_name: &str) -> Result<()> {
+        let now = now_millis();
+
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(()),
+        };
+
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::HalfOpen => {
+                let failures = row.half_open_failure_count + 1;
+                let total = row.half_open_success_count + failures;
+
+                // Check if the success rate can still be met
+                let remaining = row.half_open_max_probes - total;
+                let best_possible_rate = if row.half_open_max_probes > 0 {
+                    (row.half_open_success_count + remaining) as f64
+                        / row.half_open_max_probes as f64
+                } else {
+                    0.0
+                };
+
+                if total >= row.half_open_max_probes
+                    || best_possible_rate < row.half_open_success_rate
+                {
+                    // Either all samples collected and failed, or impossible to meet threshold
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::Open as i32,
+                        failure_count: row.failure_count.saturating_add(1),
+                        last_failure_at: Some(now),
+                        opened_at: Some(now),
+                        half_open_at: None,
+                        half_open_probe_count: 0,
+                        half_open_success_count: 0,
+                        half_open_failure_count: 0,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                } else {
+                    // Still collecting samples
+                    let updated = CircuitBreakerRow {
+                        failure_count: row.failure_count.saturating_add(1),
+                        last_failure_at: Some(now),
+                        half_open_failure_count: failures,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+            }
+            CircuitState::Closed => {
+                // Reset count if outside the window
+                let count = if let Some(last) = row.last_failure_at {
+                    if now.saturating_sub(last) > row.window_ms {
+                        1 // Window expired, start fresh
+                    } else {
+                        row.failure_count.saturating_add(1)
+                    }
+                } else {
+                    1
+                };
+
+                if count >= row.threshold {
+                    // Trip to open
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::Open as i32,
+                        failure_count: count,
+                        last_failure_at: Some(now),
+                        opened_at: Some(now),
+                        half_open_at: None,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                } else {
+                    let updated = CircuitBreakerRow {
+                        failure_count: count,
+                        last_failure_at: Some(now),
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+            }
+            CircuitState::Open => {
+                // Already open, just update failure count
+                let updated = CircuitBreakerRow {
+                    failure_count: row.failure_count.saturating_add(1),
+                    last_failure_at: Some(now),
+                    ..row
+                };
+                self.storage.upsert_circuit_breaker(&updated)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Register a circuit breaker for a task (idempotent).
+    pub fn register(&self, task_name: &str, config: &CircuitBreakerConfig) -> Result<()> {
+        if self.storage.get_circuit_breaker(task_name)?.is_some() {
+            return Ok(());
+        }
+
+        let row = CircuitBreakerRow {
+            task_name: task_name.to_string(),
+            state: CircuitState::Closed as i32,
+            failure_count: 0,
+            last_failure_at: None,
+            opened_at: None,
+            half_open_at: None,
+            threshold: config.threshold,
+            window_ms: config.window_ms,
+            cooldown_ms: config.cooldown_ms,
+            half_open_max_probes: config.half_open_max_probes,
+            half_open_success_rate: config.half_open_success_rate,
+            half_open_probe_count: 0,
+            half_open_success_count: 0,
+            half_open_failure_count: 0,
+        };
+        self.storage.upsert_circuit_breaker(&row)?;
+        Ok(())
+    }
+}

{taskito-0.6.0 → taskito-0.8.0}/crates/taskito-core/src/scheduler/maintenance.rs

@@ -99,7 +99,7 @@ impl Scheduler {
                 depends_on: vec![],
                 expires_at: None,
                 result_ttl_ms: None,
-                namespace: None,
+                namespace: self.namespace.clone(),
             };
 
             if let Err(e) = self.storage.enqueue_unique(new_job) {

{taskito-0.6.0 → taskito-0.8.0}/crates/taskito-core/src/scheduler/mod.rs

@@ -79,6 +79,7 @@ pub enum ResultOutcome {
     Retry {
         job_id: String,
         task_name: String,
+        queue: String,
         error: String,
         retry_count: i32,
         timed_out: bool,
@@ -87,11 +88,16 @@ pub enum ResultOutcome {
     DeadLettered {
         job_id: String,
         task_name: String,
+        queue: String,
         error: String,
         timed_out: bool,
     },
     /// Task was cancelled during execution.
-    Cancelled { job_id: String, task_name: String },
+    Cancelled {
+        job_id: String,
+        task_name: String,
+        queue: String,
+    },
 }
 
 /// Per-task configuration for retry, rate limiting, and circuit breaker.
@@ -122,6 +128,7 @@ pub struct Scheduler {
     config: SchedulerConfig,
     shutdown: Arc<Notify>,
     paused_cache: Mutex<(HashSet<String>, Instant)>,
+    namespace: Option<String>,
 }
 
 /// Counters for tick-based scheduling of periodic maintenance tasks.
@@ -133,7 +140,12 @@ struct TickCounters {
 }
 
 impl Scheduler {
-    pub fn new(storage: StorageBackend, queues: Vec<String>, config: SchedulerConfig) -> Self {
+    pub fn new(
+        storage: StorageBackend,
+        queues: Vec<String>,
+        config: SchedulerConfig,
+        namespace: Option<String>,
+    ) -> Self {
         let rate_limiter = RateLimiter::new(storage.clone());
         let dlq = DeadLetterQueue::new(storage.clone());
         let circuit_breaker = CircuitBreaker::new(storage.clone());
@@ -149,6 +161,7 @@ impl Scheduler {
             config,
             shutdown: Arc::new(Notify::new()),
             paused_cache: Mutex::new((HashSet::new(), Instant::now())),
+            namespace,
         }
     }
 
@@ -243,6 +256,7 @@ mod tests {
             storage,
             vec!["default".to_string()],
             SchedulerConfig::default(),
+            None,
         )
     }
 
@@ -439,7 +453,7 @@ mod tests {
         // The second job should be back in pending with a future scheduled_at
         let jobs = scheduler
             .storage
-            .list_jobs(Some(JobStatus::Pending as i32), None, None, 10, 0)
+            .list_jobs(Some(JobStatus::Pending as i32), None, None, 10, 0, None)
             .unwrap();
         assert_eq!(jobs.len(), 1);
         assert!(jobs[0].scheduled_at > now_millis());
@@ -452,6 +466,8 @@ mod tests {
             threshold: 1,
             window_ms: 60_000,
             cooldown_ms: 300_000,
+            half_open_max_probes: 5,
+            half_open_success_rate: 0.8,
         };
         scheduler.register_task(
             "cb_task".to_string(),
@@ -478,7 +494,7 @@ mod tests {
         // Job should be rescheduled
         let jobs = scheduler
             .storage
-            .list_jobs(Some(JobStatus::Pending as i32), None, None, 10, 0)
+            .list_jobs(Some(JobStatus::Pending as i32), None, None, 10, 0, None)
             .unwrap();
         assert_eq!(jobs.len(), 1);
         assert!(jobs[0].scheduled_at > now_millis());
@@ -554,6 +570,7 @@ mod tests {
                 Some("periodic_task"),
                 10,
                 0,
+                None,
             )
             .unwrap();
         assert_eq!(jobs.len(), 1);
@@ -567,13 +584,13 @@ mod tests {
             result_ttl_ms: Some(1), // 1ms TTL
             ..SchedulerConfig::default()
         };
-        let scheduler = Scheduler::new(storage, vec!["default".to_string()], config);
+        let scheduler = Scheduler::new(storage, vec!["default".to_string()], config, None);
 
         // Enqueue, dequeue, and complete a job
         let job = scheduler.storage.enqueue(make_job("cleanup_task")).unwrap();
         scheduler
             .storage
-            .dequeue("default", now_millis() + 1000)
+            .dequeue("default", now_millis() + 1000, None)
             .unwrap();
         scheduler.storage.complete(&job.id, Some(vec![1])).unwrap();
 
@@ -597,7 +614,7 @@ mod tests {
             cleanup_interval: 1,
             ..SchedulerConfig::default()
         };
-        let scheduler = Scheduler::new(storage, vec!["default".to_string()], config);
+        let scheduler = Scheduler::new(storage, vec!["default".to_string()], config, None);
 
         scheduler.storage.enqueue(make_job("tick_task")).unwrap();
 

{taskito-0.6.0 → taskito-0.8.0}/crates/taskito-core/src/scheduler/poller.rs

@@ -51,7 +51,10 @@ impl Scheduler {
             return Ok(false);
         }
 
-        let job = match self.storage.dequeue_from(&active_queues, now)? {
+        let job = match self
+            .storage
+            .dequeue_from(&active_queues, now, self.namespace.as_deref())?
+        {
             Some(j) => j,
             None => return Ok(false),
         };

{taskito-0.6.0 → taskito-0.8.0}/crates/taskito-core/src/scheduler/result_handler.rs

@@ -71,6 +71,14 @@ impl Scheduler {
                     log::error!("circuit breaker error for {task_name}: {e}");
                 }
 
+                // Look up the job to get the queue name for middleware context
+                let queue = self
+                    .storage
+                    .get_job(&job_id)?
+                    .as_ref()
+                    .map(|j| j.queue.clone())
+                    .unwrap_or_default();
+
                 // If should_retry is false (exception filtering), skip straight to DLQ
                 if !should_retry {
                     match self.storage.get_job(&job_id)? {
@@ -80,6 +88,7 @@ impl Scheduler {
                     return Ok(ResultOutcome::DeadLettered {
                         job_id,
                         task_name,
+                        queue,
                         error,
                         timed_out,
                     });
@@ -103,6 +112,7 @@ impl Scheduler {
                     Ok(ResultOutcome::Retry {
                         job_id,
                         task_name,
+                        queue,
                         error,
                         retry_count,
                         timed_out,
@@ -116,6 +126,7 @@ impl Scheduler {
                     Ok(ResultOutcome::DeadLettered {
                         job_id,
                         task_name,
+                        queue,
                         error,
                         timed_out,
                     })
@@ -140,7 +151,17 @@ impl Scheduler {
                 {
                     error!("failed to record metric for cancelled job {job_id}: {e}");
                 }
-                Ok(ResultOutcome::Cancelled { job_id, task_name })
+                let queue = self
+                    .storage
+                    .get_job(&job_id)?
+                    .as_ref()
+                    .map(|j| j.queue.clone())
+                    .unwrap_or_default();
+                Ok(ResultOutcome::Cancelled {
+                    job_id,
+                    task_name,
+                    queue,
+                })
             }
         }
     }