taskito 0.5.0__tar.gz → 0.7.0__tar.gz

{taskito-0.5.0 → taskito-0.7.0}/Cargo.lock

@@ -19,9 +19,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
 [[package]]
 name = "arc-swap"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5"
+checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
 dependencies = [
  "rustversion",
 ]
@@ -69,9 +69,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cc"
-version = "1.2.56"
+version = "1.2.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
 dependencies = [
  "find-msvc-tools",
  "shlex",
@@ -551,9 +551,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "js-sys"
@@ -1201,10 +1201,11 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "taskito-async"
-version = "0.5.0"
+version = "0.7.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
+ "log",
  "pyo3",
  "taskito-core",
  "tokio",
@@ -1212,7 +1213,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-core"
-version = "0.5.0"
+version = "0.7.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1236,11 +1237,13 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.5.0"
+version = "0.7.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
+ "log",
  "pyo3",
+ "serde_json",
  "taskito-async",
  "taskito-core",
  "tokio",
@@ -1785,18 +1788,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.42"
+version = "0.8.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
+checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.42"
+version = "0.8.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
+checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
 dependencies = [
  "proc-macro2",
  "quote",

{taskito-0.5.0 → taskito-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.5.0
+Version: 0.7.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{taskito-0.5.0 → taskito-0.7.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.5.0"
+version = "0.7.0"
 edition = "2021"
 
 [features]

taskito-0.7.0/crates/taskito-core/src/resilience/circuit_breaker.rs

@@ -0,0 +1,313 @@
+use crate::error::Result;
+use crate::job::now_millis;
+use crate::storage::models::CircuitBreakerRow;
+use crate::storage::{Storage, StorageBackend};
+
+/// Circuit breaker states.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(i32)]
+pub enum CircuitState {
+    Closed = 0,
+    Open = 1,
+    HalfOpen = 2,
+}
+
+impl CircuitState {
+    pub fn from_i32(v: i32) -> Self {
+        match v {
+            1 => Self::Open,
+            2 => Self::HalfOpen,
+            _ => Self::Closed,
+        }
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Closed => "closed",
+            Self::Open => "open",
+            Self::HalfOpen => "half_open",
+        }
+    }
+}
+
+/// Configuration for a task's circuit breaker.
+#[derive(Debug, Clone)]
+pub struct CircuitBreakerConfig {
+    pub threshold: i32,
+    pub window_ms: i64,
+    pub cooldown_ms: i64,
+    /// Number of probe requests allowed in HalfOpen state (default: 5).
+    pub half_open_max_probes: i32,
+    /// Required success rate (0.0–1.0) to close from HalfOpen (default: 0.8 = 80%).
+    pub half_open_success_rate: f64,
+}
+
+/// Circuit breaker manager backed by SQLite.
+pub struct CircuitBreaker {
+    storage: StorageBackend,
+}
+
+impl CircuitBreaker {
+    pub fn new(storage: StorageBackend) -> Self {
+        Self { storage }
+    }
+
+    /// Check if a task is allowed to execute. Returns true if allowed.
+    /// Transitions Open -> HalfOpen after cooldown.
+    pub fn allow(&self, task_name: &str) -> Result<bool> {
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(true), // No breaker configured = always allow
+        };
+
+        let now = now_millis();
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::Closed => Ok(true),
+            CircuitState::Open => {
+                // Check if cooldown has elapsed
+                let opened = row.opened_at.unwrap_or(0);
+                if now.saturating_sub(opened) >= row.cooldown_ms {
+                    // Transition to half-open: reset probe counters
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::HalfOpen as i32,
+                        half_open_at: Some(now),
+                        half_open_probe_count: 0,
+                        half_open_success_count: 0,
+                        half_open_failure_count: 0,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                    Ok(true)
+                } else {
+                    Ok(false) // Still in cooldown
+                }
+            }
+            CircuitState::HalfOpen => {
+                // Allow up to max_probes concurrent probes
+                if row.half_open_probe_count < row.half_open_max_probes {
+                    let updated = CircuitBreakerRow {
+                        half_open_probe_count: row.half_open_probe_count + 1,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                    Ok(true)
+                } else {
+                    // Check for timeout: if probes haven't completed within cooldown, re-open
+                    let half_open_since = row.half_open_at.unwrap_or(0);
+                    if now.saturating_sub(half_open_since) >= row.cooldown_ms {
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Open as i32,
+                            opened_at: Some(now),
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    }
+                    Ok(false)
+                }
+            }
+        }
+    }
+
+    /// Record a task success. In HalfOpen, tracks probes and closes when
+    /// the success rate threshold is met.
+    pub fn record_success(&self, task_name: &str) -> Result<()> {
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(()),
+        };
+
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::Closed if row.failure_count == 0 => Ok(()),
+            CircuitState::HalfOpen => {
+                let successes = row.half_open_success_count + 1;
+                let total = successes + row.half_open_failure_count;
+
+                if total >= row.half_open_max_probes {
+                    let rate = successes as f64 / total as f64;
+                    if rate >= row.half_open_success_rate {
+                        // Threshold met — close the circuit
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Closed as i32,
+                            failure_count: 0,
+                            opened_at: None,
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    } else {
+                        // Threshold not met — re-open
+                        let now = now_millis();
+                        let updated = CircuitBreakerRow {
+                            state: CircuitState::Open as i32,
+                            opened_at: Some(now),
+                            half_open_at: None,
+                            half_open_probe_count: 0,
+                            half_open_success_count: 0,
+                            half_open_failure_count: 0,
+                            ..row
+                        };
+                        self.storage.upsert_circuit_breaker(&updated)?;
+                    }
+                } else {
+                    // Still collecting samples
+                    let updated = CircuitBreakerRow {
+                        half_open_success_count: successes,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+                Ok(())
+            }
+            _ => {
+                // Closed with failures or Open — reset to clean Closed
+                let updated = CircuitBreakerRow {
+                    state: CircuitState::Closed as i32,
+                    failure_count: 0,
+                    opened_at: None,
+                    half_open_at: None,
+                    half_open_probe_count: 0,
+                    half_open_success_count: 0,
+                    half_open_failure_count: 0,
+                    ..row
+                };
+                self.storage.upsert_circuit_breaker(&updated)?;
+                Ok(())
+            }
+        }
+    }
+
+    /// Record a task failure. May trip the breaker open.
+    pub fn record_failure(&self, task_name: &str) -> Result<()> {
+        let now = now_millis();
+
+        let row = match self.storage.get_circuit_breaker(task_name)? {
+            Some(r) => r,
+            None => return Ok(()),
+        };
+
+        let state = CircuitState::from_i32(row.state);
+
+        match state {
+            CircuitState::HalfOpen => {
+                let failures = row.half_open_failure_count + 1;
+                let total = row.half_open_success_count + failures;
+
+                // Check if the success rate can still be met
+                let remaining = row.half_open_max_probes - total;
+                let best_possible_rate = if row.half_open_max_probes > 0 {
+                    (row.half_open_success_count + remaining) as f64
+                        / row.half_open_max_probes as f64
+                } else {
+                    0.0
+                };
+
+                if total >= row.half_open_max_probes
+                    || best_possible_rate < row.half_open_success_rate
+                {
+                    // Either all samples collected and failed, or impossible to meet threshold
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::Open as i32,
+                        failure_count: row.failure_count.saturating_add(1),
+                        last_failure_at: Some(now),
+                        opened_at: Some(now),
+                        half_open_at: None,
+                        half_open_probe_count: 0,
+                        half_open_success_count: 0,
+                        half_open_failure_count: 0,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                } else {
+                    // Still collecting samples
+                    let updated = CircuitBreakerRow {
+                        failure_count: row.failure_count.saturating_add(1),
+                        last_failure_at: Some(now),
+                        half_open_failure_count: failures,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+            }
+            CircuitState::Closed => {
+                // Reset count if outside the window
+                let count = if let Some(last) = row.last_failure_at {
+                    if now.saturating_sub(last) > row.window_ms {
+                        1 // Window expired, start fresh
+                    } else {
+                        row.failure_count.saturating_add(1)
+                    }
+                } else {
+                    1
+                };
+
+                if count >= row.threshold {
+                    // Trip to open
+                    let updated = CircuitBreakerRow {
+                        state: CircuitState::Open as i32,
+                        failure_count: count,
+                        last_failure_at: Some(now),
+                        opened_at: Some(now),
+                        half_open_at: None,
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                } else {
+                    let updated = CircuitBreakerRow {
+                        failure_count: count,
+                        last_failure_at: Some(now),
+                        ..row
+                    };
+                    self.storage.upsert_circuit_breaker(&updated)?;
+                }
+            }
+            CircuitState::Open => {
+                // Already open, just update failure count
+                let updated = CircuitBreakerRow {
+                    failure_count: row.failure_count.saturating_add(1),
+                    last_failure_at: Some(now),
+                    ..row
+                };
+                self.storage.upsert_circuit_breaker(&updated)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Register a circuit breaker for a task (idempotent).
+    pub fn register(&self, task_name: &str, config: &CircuitBreakerConfig) -> Result<()> {
+        if self.storage.get_circuit_breaker(task_name)?.is_some() {
+            return Ok(());
+        }
+
+        let row = CircuitBreakerRow {
+            task_name: task_name.to_string(),
+            state: CircuitState::Closed as i32,
+            failure_count: 0,
+            last_failure_at: None,
+            opened_at: None,
+            half_open_at: None,
+            threshold: config.threshold,
+            window_ms: config.window_ms,
+            cooldown_ms: config.cooldown_ms,
+            half_open_max_probes: config.half_open_max_probes,
+            half_open_success_rate: config.half_open_success_rate,
+            half_open_probe_count: 0,
+            half_open_success_count: 0,
+            half_open_failure_count: 0,
+        };
+        self.storage.upsert_circuit_breaker(&row)?;
+        Ok(())
+    }
+}

{taskito-0.5.0 → taskito-0.7.0}/crates/taskito-core/src/scheduler/maintenance.rs

@@ -33,7 +33,7 @@ impl Scheduler {
 
         for job in stale_jobs {
             let error = format!("job timed out after {}ms", job.timeout_ms);
-            self.handle_result(JobResult::Failure {
+            let _ = self.handle_result(JobResult::Failure {
                 job_id: job.id.clone(),
                 error,
                 retry_count: job.retry_count,
@@ -41,6 +41,7 @@ impl Scheduler {
                 task_name: job.task_name.clone(),
                 wall_time_ns: 0,
                 should_retry: true,
+                timed_out: true,
             })?;
         }
 

{taskito-0.5.0 → taskito-0.7.0}/crates/taskito-core/src/scheduler/mod.rs

@@ -60,6 +60,7 @@ pub enum JobResult {
         task_name: String,
         wall_time_ns: i64,
         should_retry: bool,
+        timed_out: bool,
     },
     Cancelled {
         job_id: String,
@@ -68,12 +69,51 @@ pub enum JobResult {
     },
 }
 
+/// Outcome of processing a job result, returned to the caller for
+/// Python-side middleware hook dispatch.
+#[derive(Debug, Clone)]
+pub enum ResultOutcome {
+    /// Task completed successfully.
+    Success { job_id: String, task_name: String },
+    /// Task failed and will be retried.
+    Retry {
+        job_id: String,
+        task_name: String,
+        queue: String,
+        error: String,
+        retry_count: i32,
+        timed_out: bool,
+    },
+    /// Task exhausted retries and moved to the dead-letter queue.
+    DeadLettered {
+        job_id: String,
+        task_name: String,
+        queue: String,
+        error: String,
+        timed_out: bool,
+    },
+    /// Task was cancelled during execution.
+    Cancelled {
+        job_id: String,
+        task_name: String,
+        queue: String,
+    },
+}
+
 /// Per-task configuration for retry, rate limiting, and circuit breaker.
 #[derive(Debug, Clone)]
 pub struct TaskConfig {
     pub retry_policy: RetryPolicy,
     pub rate_limit: Option<crate::resilience::rate_limiter::RateLimitConfig>,
     pub circuit_breaker: Option<CircuitBreakerConfig>,
+    pub max_concurrent: Option<i32>,
+}
+
+/// Per-queue configuration for rate limiting and concurrency caps.
+#[derive(Debug, Clone)]
+pub struct QueueConfig {
+    pub rate_limit: Option<crate::resilience::rate_limiter::RateLimitConfig>,
+    pub max_concurrent: Option<i32>,
 }
 
 /// The central scheduler that coordinates job dispatch, retries, rate limiting, and circuit breakers.
@@ -83,6 +123,7 @@ pub struct Scheduler {
     dlq: DeadLetterQueue,
     circuit_breaker: CircuitBreaker,
     task_configs: HashMap<String, TaskConfig>,
+    queue_configs: HashMap<String, QueueConfig>,
     queues: Vec<String>,
     config: SchedulerConfig,
     shutdown: Arc<Notify>,
@@ -109,6 +150,7 @@ impl Scheduler {
             dlq,
             circuit_breaker,
             task_configs: HashMap::new(),
+            queue_configs: HashMap::new(),
             queues,
             config,
             shutdown: Arc::new(Notify::new()),
@@ -124,6 +166,10 @@ impl Scheduler {
         self.shutdown.clone()
     }
 
+    pub fn register_queue_config(&mut self, queue_name: String, config: QueueConfig) {
+        self.queue_configs.insert(queue_name, config);
+    }
+
     pub fn register_task(&mut self, task_name: String, config: TaskConfig) {
         if let Some(ref cb_config) = config.circuit_breaker {
             if let Err(e) = self.circuit_breaker.register(&task_name, cb_config) {
@@ -276,6 +322,7 @@ mod tests {
                 },
                 rate_limit: None,
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 
@@ -290,6 +337,7 @@ mod tests {
                 task_name: "retry_task".to_string(),
                 wall_time_ns: 500_000,
                 should_retry: true,
+                timed_out: false,
             })
             .unwrap();
 
@@ -312,6 +360,7 @@ mod tests {
                 task_name: "exhausted_task".to_string(),
                 wall_time_ns: 100,
                 should_retry: true,
+                timed_out: false,
             })
             .unwrap();
 
@@ -337,6 +386,7 @@ mod tests {
                 task_name: "no_retry_task".to_string(),
                 wall_time_ns: 100,
                 should_retry: false,
+                timed_out: false,
             })
             .unwrap();
 
@@ -373,6 +423,7 @@ mod tests {
                     refill_rate: 0.0,
                 }),
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 
@@ -407,6 +458,8 @@ mod tests {
             threshold: 1,
             window_ms: 60_000,
             cooldown_ms: 300_000,
+            half_open_max_probes: 5,
+            half_open_success_rate: 0.8,
         };
         scheduler.register_task(
             "cb_task".to_string(),
@@ -414,6 +467,7 @@ mod tests {
                 retry_policy: RetryPolicy::default(),
                 rate_limit: None,
                 circuit_breaker: Some(cb_config),
+                max_concurrent: None,
             },
         );
 
@@ -452,6 +506,7 @@ mod tests {
                 },
                 rate_limit: None,
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 

{taskito-0.5.0 → taskito-0.7.0}/crates/taskito-core/src/scheduler/poller.rs

@@ -14,6 +14,9 @@ const CIRCUIT_BREAKER_RETRY_DELAY_MS: i64 = 5000;
 /// Delay before re-scheduling a rate-limited job (ms).
 const RATE_LIMIT_RETRY_DELAY_MS: i64 = 1000;
 
+/// Delay before re-scheduling a concurrency-limited job (ms).
+const CONCURRENCY_RETRY_DELAY_MS: i64 = 500;
+
 impl Scheduler {
     pub(super) fn try_dispatch(&self, job_tx: &tokio::sync::mpsc::Sender<Job>) -> Result<bool> {
         let now = now_millis();
@@ -53,6 +56,26 @@ impl Scheduler {
             None => return Ok(false),
         };
 
+        // Check queue-level limits
+        if let Some(qcfg) = self.queue_configs.get(&job.queue) {
+            if let Some(ref rl_config) = qcfg.rate_limit {
+                let key = format!("queue:{}", job.queue);
+                if !self.rate_limiter.try_acquire(&key, rl_config)? {
+                    self.storage
+                        .retry(&job.id, now + RATE_LIMIT_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
+            if let Some(max_conc) = qcfg.max_concurrent {
+                let stats = self.storage.stats_by_queue(&job.queue)?;
+                if stats.running >= max_conc as i64 {
+                    self.storage
+                        .retry(&job.id, now + CONCURRENCY_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
+        }
+
         // Check circuit breaker for this task
         if let Some(config) = self.task_configs.get(&job.task_name) {
             if config.circuit_breaker.is_some() && !self.circuit_breaker.allow(&job.task_name)? {
@@ -68,6 +91,16 @@ impl Scheduler {
                     return Ok(false);
                 }
             }
+
+            // Check per-task concurrency limit
+            if let Some(max_conc) = config.max_concurrent {
+                let running = self.storage.count_running_by_task(&job.task_name)?;
+                if running >= max_conc as i64 {
+                    self.storage
+                        .retry(&job.id, now + CONCURRENCY_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
         }
 
         // Claim exactly-once execution