taskito 0.5.0__tar.gz → 0.6.0__tar.gz

{taskito-0.5.0 → taskito-0.6.0}/Cargo.lock

@@ -69,9 +69,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cc"
-version = "1.2.56"
+version = "1.2.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
 dependencies = [
  "find-msvc-tools",
  "shlex",
@@ -1201,7 +1201,7 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "taskito-async"
-version = "0.5.0"
+version = "0.6.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
@@ -1212,7 +1212,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-core"
-version = "0.5.0"
+version = "0.6.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1236,11 +1236,12 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.5.0"
+version = "0.6.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
  "pyo3",
+ "serde_json",
  "taskito-async",
  "taskito-core",
  "tokio",

{taskito-0.5.0 → taskito-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.5.0
+Version: 0.6.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 
 [features]

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/maintenance.rs

@@ -33,7 +33,7 @@ impl Scheduler {
 
         for job in stale_jobs {
             let error = format!("job timed out after {}ms", job.timeout_ms);
-            self.handle_result(JobResult::Failure {
+            let _ = self.handle_result(JobResult::Failure {
                 job_id: job.id.clone(),
                 error,
                 retry_count: job.retry_count,
@@ -41,6 +41,7 @@ impl Scheduler {
                 task_name: job.task_name.clone(),
                 wall_time_ns: 0,
                 should_retry: true,
+                timed_out: true,
             })?;
         }
 

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/mod.rs

@@ -60,6 +60,7 @@ pub enum JobResult {
         task_name: String,
         wall_time_ns: i64,
         should_retry: bool,
+        timed_out: bool,
     },
     Cancelled {
         job_id: String,
@@ -68,12 +69,45 @@ pub enum JobResult {
     },
 }
 
+/// Outcome of processing a job result, returned to the caller for
+/// Python-side middleware hook dispatch.
+#[derive(Debug, Clone)]
+pub enum ResultOutcome {
+    /// Task completed successfully.
+    Success { job_id: String, task_name: String },
+    /// Task failed and will be retried.
+    Retry {
+        job_id: String,
+        task_name: String,
+        error: String,
+        retry_count: i32,
+        timed_out: bool,
+    },
+    /// Task exhausted retries and moved to the dead-letter queue.
+    DeadLettered {
+        job_id: String,
+        task_name: String,
+        error: String,
+        timed_out: bool,
+    },
+    /// Task was cancelled during execution.
+    Cancelled { job_id: String, task_name: String },
+}
+
 /// Per-task configuration for retry, rate limiting, and circuit breaker.
 #[derive(Debug, Clone)]
 pub struct TaskConfig {
     pub retry_policy: RetryPolicy,
     pub rate_limit: Option<crate::resilience::rate_limiter::RateLimitConfig>,
     pub circuit_breaker: Option<CircuitBreakerConfig>,
+    pub max_concurrent: Option<i32>,
+}
+
+/// Per-queue configuration for rate limiting and concurrency caps.
+#[derive(Debug, Clone)]
+pub struct QueueConfig {
+    pub rate_limit: Option<crate::resilience::rate_limiter::RateLimitConfig>,
+    pub max_concurrent: Option<i32>,
 }
 
 /// The central scheduler that coordinates job dispatch, retries, rate limiting, and circuit breakers.
@@ -83,6 +117,7 @@ pub struct Scheduler {
     dlq: DeadLetterQueue,
     circuit_breaker: CircuitBreaker,
     task_configs: HashMap<String, TaskConfig>,
+    queue_configs: HashMap<String, QueueConfig>,
     queues: Vec<String>,
     config: SchedulerConfig,
     shutdown: Arc<Notify>,
@@ -109,6 +144,7 @@ impl Scheduler {
             dlq,
             circuit_breaker,
             task_configs: HashMap::new(),
+            queue_configs: HashMap::new(),
             queues,
             config,
             shutdown: Arc::new(Notify::new()),
@@ -124,6 +160,10 @@ impl Scheduler {
         self.shutdown.clone()
     }
 
+    pub fn register_queue_config(&mut self, queue_name: String, config: QueueConfig) {
+        self.queue_configs.insert(queue_name, config);
+    }
+
     pub fn register_task(&mut self, task_name: String, config: TaskConfig) {
         if let Some(ref cb_config) = config.circuit_breaker {
             if let Err(e) = self.circuit_breaker.register(&task_name, cb_config) {
@@ -276,6 +316,7 @@ mod tests {
                 },
                 rate_limit: None,
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 
@@ -290,6 +331,7 @@ mod tests {
                 task_name: "retry_task".to_string(),
                 wall_time_ns: 500_000,
                 should_retry: true,
+                timed_out: false,
             })
             .unwrap();
 
@@ -312,6 +354,7 @@ mod tests {
                 task_name: "exhausted_task".to_string(),
                 wall_time_ns: 100,
                 should_retry: true,
+                timed_out: false,
             })
             .unwrap();
 
@@ -337,6 +380,7 @@ mod tests {
                 task_name: "no_retry_task".to_string(),
                 wall_time_ns: 100,
                 should_retry: false,
+                timed_out: false,
             })
             .unwrap();
 
@@ -373,6 +417,7 @@ mod tests {
                     refill_rate: 0.0,
                 }),
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 
@@ -414,6 +459,7 @@ mod tests {
                 retry_policy: RetryPolicy::default(),
                 rate_limit: None,
                 circuit_breaker: Some(cb_config),
+                max_concurrent: None,
             },
         );
 
@@ -452,6 +498,7 @@ mod tests {
                 },
                 rate_limit: None,
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/poller.rs

@@ -14,6 +14,9 @@ const CIRCUIT_BREAKER_RETRY_DELAY_MS: i64 = 5000;
 /// Delay before re-scheduling a rate-limited job (ms).
 const RATE_LIMIT_RETRY_DELAY_MS: i64 = 1000;
 
+/// Delay before re-scheduling a concurrency-limited job (ms).
+const CONCURRENCY_RETRY_DELAY_MS: i64 = 500;
+
 impl Scheduler {
     pub(super) fn try_dispatch(&self, job_tx: &tokio::sync::mpsc::Sender<Job>) -> Result<bool> {
         let now = now_millis();
@@ -53,6 +56,26 @@ impl Scheduler {
             None => return Ok(false),
         };
 
+        // Check queue-level limits
+        if let Some(qcfg) = self.queue_configs.get(&job.queue) {
+            if let Some(ref rl_config) = qcfg.rate_limit {
+                let key = format!("queue:{}", job.queue);
+                if !self.rate_limiter.try_acquire(&key, rl_config)? {
+                    self.storage
+                        .retry(&job.id, now + RATE_LIMIT_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
+            if let Some(max_conc) = qcfg.max_concurrent {
+                let stats = self.storage.stats_by_queue(&job.queue)?;
+                if stats.running >= max_conc as i64 {
+                    self.storage
+                        .retry(&job.id, now + CONCURRENCY_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
+        }
+
         // Check circuit breaker for this task
         if let Some(config) = self.task_configs.get(&job.task_name) {
             if config.circuit_breaker.is_some() && !self.circuit_breaker.allow(&job.task_name)? {
@@ -68,6 +91,16 @@ impl Scheduler {
                     return Ok(false);
                 }
             }
+
+            // Check per-task concurrency limit
+            if let Some(max_conc) = config.max_concurrent {
+                let running = self.storage.count_running_by_task(&job.task_name)?;
+                if running >= max_conc as i64 {
+                    self.storage
+                        .retry(&job.id, now + CONCURRENCY_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
         }
 
         // Claim exactly-once execution

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/result_handler.rs

@@ -3,11 +3,14 @@ use log::{error, warn};
 use crate::error::Result;
 use crate::storage::Storage;
 
-use super::{JobResult, Scheduler};
+use super::{JobResult, ResultOutcome, Scheduler};
 
 impl Scheduler {
     /// Handle a completed or failed job result from a worker.
-    pub fn handle_result(&self, result: JobResult) -> Result<()> {
+    ///
+    /// Returns a [`ResultOutcome`] describing the action taken, so the
+    /// caller can dispatch Python-side middleware hooks and events.
+    pub fn handle_result(&self, result: JobResult) -> Result<ResultOutcome> {
         match result {
             JobResult::Success {
                 job_id,
@@ -32,6 +35,11 @@ impl Scheduler {
                 if let Err(e) = self.circuit_breaker.record_success(task_name) {
                     error!("circuit breaker error for {task_name}: {e}");
                 }
+
+                Ok(ResultOutcome::Success {
+                    job_id,
+                    task_name: task_name.clone(),
+                })
             }
             JobResult::Failure {
                 job_id,
@@ -41,6 +49,7 @@ impl Scheduler {
                 task_name,
                 wall_time_ns,
                 should_retry,
+                timed_out,
             } => {
                 // Clear execution claim so it can be retried
                 if let Err(e) = self.storage.complete_execution(&job_id) {
@@ -68,7 +77,12 @@ impl Scheduler {
                         Some(job) => self.dlq.move_to_dlq(&job, &error, None)?,
                         None => warn!("job {job_id} disappeared before DLQ move"),
                     }
-                    return Ok(());
+                    return Ok(ResultOutcome::DeadLettered {
+                        job_id,
+                        task_name,
+                        error,
+                        timed_out,
+                    });
                 }
 
                 let policy = self
@@ -86,12 +100,25 @@ impl Scheduler {
                 if retry_count < effective_max {
                     let next_at = policy.next_retry_at(retry_count);
                     self.storage.retry(&job_id, next_at)?;
+                    Ok(ResultOutcome::Retry {
+                        job_id,
+                        task_name,
+                        error,
+                        retry_count,
+                        timed_out,
+                    })
                 } else {
                     // Move to DLQ
                     match self.storage.get_job(&job_id)? {
                         Some(job) => self.dlq.move_to_dlq(&job, &error, None)?,
                         None => warn!("job {job_id} disappeared before DLQ move"),
                     }
+                    Ok(ResultOutcome::DeadLettered {
+                        job_id,
+                        task_name,
+                        error,
+                        timed_out,
+                    })
                 }
             }
             JobResult::Cancelled {
@@ -113,8 +140,8 @@ impl Scheduler {
                 {
                     error!("failed to record metric for cancelled job {job_id}: {e}");
                 }
+                Ok(ResultOutcome::Cancelled { job_id, task_name })
             }
         }
-        Ok(())
     }
 }

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/storage/diesel_common/jobs.rs

@@ -916,6 +916,19 @@ macro_rules! impl_diesel_job_ops {
                 Ok(affected as u64)
             }
 
+            /// Count running jobs for a specific task name (for per-task concurrency limiting).
+            pub fn count_running_by_task(&self, task_name: &str) -> Result<i64> {
+                let mut conn = self.conn()?;
+
+                let count: i64 = jobs::table
+                    .filter(jobs::task_name.eq(task_name))
+                    .filter(jobs::status.eq(JobStatus::Running as i32))
+                    .count()
+                    .get_result(&mut conn)?;
+
+                Ok(count)
+            }
+
             /// Purge job errors older than the given timestamp.
             pub fn purge_job_errors(&self, older_than_ms: i64) -> Result<u64> {
                 let mut conn = self.conn()?;

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/storage/mod.rs

@@ -444,6 +444,12 @@ macro_rules! impl_storage {
             ) -> $crate::error::Result<u64> {
                 self.purge_execution_claims(older_than_ms)
             }
+            fn count_running_by_task(
+                &self,
+                task_name: &str,
+            ) -> $crate::error::Result<i64> {
+                self.count_running_by_task(task_name)
+            }
             fn stats_by_queue(
                 &self,
                 queue_name: &str,
@@ -792,6 +798,9 @@ impl Storage for StorageBackend {
     fn purge_execution_claims(&self, older_than_ms: i64) -> Result<u64> {
         delegate!(self, purge_execution_claims, older_than_ms)
     }
+    fn count_running_by_task(&self, task_name: &str) -> Result<i64> {
+        delegate!(self, count_running_by_task, task_name)
+    }
     fn stats_by_queue(&self, queue_name: &str) -> Result<QueueStats> {
         delegate!(self, stats_by_queue, queue_name)
     }

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/storage/redis_backend/jobs.rs

@@ -725,6 +725,24 @@ impl RedisStorage {
         Ok(stats)
     }
 
+    /// Count running jobs for a specific task name (for per-task concurrency limiting).
+    pub fn count_running_by_task(&self, task_name: &str) -> Result<i64> {
+        let mut conn = self.conn()?;
+        let by_task_key = self.key(&["jobs", "by_task", task_name]);
+        let job_ids: Vec<String> = conn.smembers(&by_task_key).map_err(map_err)?;
+
+        let mut count: i64 = 0;
+        for id in &job_ids {
+            if let Some(job) = self.load_job(&mut conn, id)? {
+                if job.status == JobStatus::Running {
+                    count += 1;
+                }
+            }
+        }
+
+        Ok(count)
+    }
+
     pub fn stats_by_queue(&self, queue_name: &str) -> Result<QueueStats> {
         let mut conn = self.conn()?;
         let by_queue_key = self.key(&["jobs", "by_queue", queue_name]);

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/storage/sqlite/tests.rs

@@ -362,6 +362,31 @@ fn test_cascade_cancel_on_dlq() {
     assert!(b.error.unwrap().contains("dependency failed"));
 }
 
+#[test]
+fn test_count_running_by_task() {
+    let storage = test_storage();
+    storage.enqueue(make_job("task_a")).unwrap();
+    storage.enqueue(make_job("task_a")).unwrap();
+    storage.enqueue(make_job("task_b")).unwrap();
+
+    // No running jobs yet
+    assert_eq!(storage.count_running_by_task("task_a").unwrap(), 0);
+
+    let now = now_millis() + 1000;
+    // Dequeue one task_a (becomes running)
+    storage.dequeue("default", now).unwrap().unwrap();
+
+    assert_eq!(storage.count_running_by_task("task_a").unwrap(), 1);
+    assert_eq!(storage.count_running_by_task("task_b").unwrap(), 0);
+
+    // Dequeue second task_a
+    storage.dequeue("default", now).unwrap().unwrap();
+    assert_eq!(storage.count_running_by_task("task_a").unwrap(), 2);
+
+    // Nonexistent task should return 0
+    assert_eq!(storage.count_running_by_task("no_such_task").unwrap(), 0);
+}
+
 #[test]
 fn test_enqueue_rejects_missing_dependency() {
     let storage = test_storage();

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-core/src/storage/traits.rs

@@ -161,6 +161,10 @@ pub trait Storage: Send + Sync + Clone {
     fn complete_execution(&self, job_id: &str) -> Result<()>;
     fn purge_execution_claims(&self, older_than_ms: i64) -> Result<u64>;
 
+    // ── Per-task concurrency ──────────────────────────────────────
+
+    fn count_running_by_task(&self, task_name: &str) -> Result<i64>;
+
     // ── Per-queue stats ──────────────────────────────────────────
 
     fn stats_by_queue(&self, queue_name: &str) -> Result<QueueStats>;

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-python"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 
 [features]
@@ -22,3 +22,4 @@ crossbeam-channel = { workspace = true }
 uuid = { workspace = true }
 async-trait = { workspace = true }
 taskito-async = { path = "../taskito-async", optional = true }
+serde_json = { workspace = true }

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-python/src/async_worker.rs

@@ -118,6 +118,7 @@ impl WorkerDispatcher for AsyncWorkerPool {
                                 task_name,
                                 wall_time_ns,
                                 should_retry,
+                                timed_out: false,
                             }
                         }
                     }

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-python/src/py_config.rs

@@ -26,13 +26,17 @@ pub struct PyTaskConfig {
     pub circuit_breaker_cooldown: Option<i64>,
     #[pyo3(get, set)]
     pub retry_delays: Option<Vec<f64>>,
+    #[pyo3(get, set)]
+    pub max_retry_delay: Option<i64>,
+    #[pyo3(get, set)]
+    pub max_concurrent: Option<i32>,
 }
 
 #[pymethods]
 #[allow(clippy::too_many_arguments)]
 impl PyTaskConfig {
     #[new]
-    #[pyo3(signature = (name, max_retries=3, retry_backoff=1.0, timeout=300, priority=0, rate_limit=None, queue="default".to_string(), circuit_breaker_threshold=None, circuit_breaker_window=None, circuit_breaker_cooldown=None, retry_delays=None))]
+    #[pyo3(signature = (name, max_retries=3, retry_backoff=1.0, timeout=300, priority=0, rate_limit=None, queue="default".to_string(), circuit_breaker_threshold=None, circuit_breaker_window=None, circuit_breaker_cooldown=None, retry_delays=None, max_retry_delay=None, max_concurrent=None))]
     pub fn new(
         name: String,
         max_retries: i32,
@@ -45,6 +49,8 @@ impl PyTaskConfig {
         circuit_breaker_window: Option<i64>,
         circuit_breaker_cooldown: Option<i64>,
         retry_delays: Option<Vec<f64>>,
+        max_retry_delay: Option<i64>,
+        max_concurrent: Option<i32>,
     ) -> Self {
         Self {
             name,
@@ -58,6 +64,8 @@ impl PyTaskConfig {
             circuit_breaker_window,
             circuit_breaker_cooldown,
             retry_delays,
+            max_retry_delay,
+            max_concurrent,
         }
     }
 }

{taskito-0.5.0 → taskito-0.6.0}/crates/taskito-python/src/py_queue/mod.rs

@@ -33,6 +33,9 @@ pub struct PyQueue {
     pub(crate) default_priority: i32,
     pub(crate) shutdown_flag: Arc<AtomicBool>,
     pub(crate) result_ttl_ms: Option<i64>,
+    pub(crate) scheduler_poll_interval_ms: u64,
+    pub(crate) scheduler_reap_interval: u32,
+    pub(crate) scheduler_cleanup_interval: u32,
 }
 
 #[pymethods]
@@ -43,7 +46,8 @@ pub struct PyQueue {
 )]
 impl PyQueue {
     #[new]
-    #[pyo3(signature = (db_path=".taskito/taskito.db", workers=0, default_retry=3, default_timeout=300, default_priority=0, result_ttl=None, backend="sqlite", db_url=None, schema="taskito", pool_size=None))]
+    #[pyo3(signature = (db_path=".taskito/taskito.db", workers=0, default_retry=3, default_timeout=300, default_priority=0, result_ttl=None, backend="sqlite", db_url=None, schema="taskito", pool_size=None, scheduler_poll_interval_ms=50, scheduler_reap_interval=100, scheduler_cleanup_interval=1200))]
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         db_path: &str,
         workers: usize,
@@ -55,6 +59,9 @@ impl PyQueue {
         db_url: Option<&str>,
         schema: &str,
         pool_size: Option<u32>,
+        scheduler_poll_interval_ms: u64,
+        scheduler_reap_interval: u32,
+        scheduler_cleanup_interval: u32,
     ) -> PyResult<Self> {
         let storage = match backend {
             "sqlite" => {
@@ -117,6 +124,9 @@ impl PyQueue {
             default_priority,
             shutdown_flag: Arc::new(AtomicBool::new(false)),
             result_ttl_ms: result_ttl.map(|s| s * 1000),
+            scheduler_poll_interval_ms,
+            scheduler_reap_interval,
+            scheduler_cleanup_interval,
         })
     }