taskito 0.9.0__tar.gz → 0.10.0__tar.gz

{taskito-0.9.0 → taskito-0.10.0}/Cargo.lock

@@ -1217,7 +1217,7 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "taskito-async"
-version = "0.9.0"
+version = "0.10.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
@@ -1229,7 +1229,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-core"
-version = "0.9.0"
+version = "0.10.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1253,7 +1253,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.9.0"
+version = "0.10.0"
 dependencies = [
  "async-trait",
  "base64",

{taskito-0.9.0 → taskito-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.9.0
+Version: 0.10.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -49,17 +49,17 @@ License-File: LICENSE
 Summary: Rust-powered task queue for Python. No broker required.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-Project-URL: Changelog, https://github.com/pratyush618/taskito/blob/master/docs/changelog.md
+Project-URL: Changelog, https://github.com/ByteVeda/taskito/blob/master/docs/changelog.md
 Project-URL: Documentation, https://taskito.grigori.in
-Project-URL: Homepage, https://github.com/pratyush618/taskito
-Project-URL: Issues, https://github.com/pratyush618/taskito/issues
-Project-URL: Repository, https://github.com/pratyush618/taskito
+Project-URL: Homepage, https://github.com/ByteVeda/taskito
+Project-URL: Issues, https://github.com/ByteVeda/taskito/issues
+Project-URL: Repository, https://github.com/ByteVeda/taskito
 
 # taskito
 
 [![PyPI version](https://img.shields.io/pypi/v/taskito.svg)](https://pypi.org/project/taskito/)
 [![Python versions](https://img.shields.io/pypi/pyversions/taskito.svg)](https://pypi.org/project/taskito/)
-[![License](https://img.shields.io/pypi/l/taskito.svg)](https://github.com/pratyush618/taskito/blob/master/LICENSE)
+[![License](https://img.shields.io/pypi/l/taskito.svg)](https://github.com/ByteVeda/taskito/blob/master/LICENSE)
 
 A Rust-powered task queue for Python. No broker required — just SQLite or Postgres.
 

{taskito-0.9.0 → taskito-0.10.0}/README.md

@@ -2,7 +2,7 @@
 
 [![PyPI version](https://img.shields.io/pypi/v/taskito.svg)](https://pypi.org/project/taskito/)
 [![Python versions](https://img.shields.io/pypi/pyversions/taskito.svg)](https://pypi.org/project/taskito/)
-[![License](https://img.shields.io/pypi/l/taskito.svg)](https://github.com/pratyush618/taskito/blob/master/LICENSE)
+[![License](https://img.shields.io/pypi/l/taskito.svg)](https://github.com/ByteVeda/taskito/blob/master/LICENSE)
 
 A Rust-powered task queue for Python. No broker required — just SQLite or Postgres.
 

{taskito-0.9.0 → taskito-0.10.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.9.0"
+version = "0.10.0"
 edition = "2021"
 
 [features]

{taskito-0.9.0 → taskito-0.10.0}/crates/taskito-core/src/scheduler/mod.rs

@@ -22,6 +22,9 @@ pub use crate::job::Job;
 pub struct SchedulerConfig {
     /// Interval between scheduler poll cycles.
     pub poll_interval: Duration,
+    /// Priority boost per second of wait time. `None` disables aging.
+    /// Example: `Some(1)` boosts priority by 1 per second of wait.
+    pub aging_factor: Option<i64>,
     /// Reap stale jobs every N iterations.
     pub reap_interval: u32,
     /// Check periodic tasks every N iterations.
@@ -36,6 +39,7 @@ impl Default for SchedulerConfig {
     fn default() -> Self {
         Self {
             poll_interval: Duration::from_millis(50),
+            aging_factor: None,
             reap_interval: 100,
             periodic_check_interval: 60,
             cleanup_interval: 1200,
@@ -116,6 +120,34 @@ pub struct QueueConfig {
     pub max_concurrent: Option<i32>,
 }
 
+/// In-memory cache of average task execution duration.
+/// Updated on every `handle_result()` — no DB queries needed.
+struct TaskDurationCache {
+    durations: HashMap<String, (u64, i64)>, // (count, sum_ns)
+}
+
+impl TaskDurationCache {
+    fn new() -> Self {
+        Self {
+            durations: HashMap::new(),
+        }
+    }
+
+    fn record(&mut self, task_name: &str, wall_time_ns: i64) {
+        let entry = self.durations.entry(task_name.to_string()).or_default();
+        entry.0 += 1;
+        entry.1 = entry.1.saturating_add(wall_time_ns);
+    }
+
+    #[allow(dead_code)]
+    fn avg_ns(&self, task_name: &str) -> Option<i64> {
+        self.durations
+            .get(task_name)
+            .filter(|(count, _)| *count > 0)
+            .map(|(count, sum)| sum / *count as i64)
+    }
+}
+
 /// The central scheduler that coordinates job dispatch, retries, rate limiting, and circuit breakers.
 pub struct Scheduler {
     storage: StorageBackend,
@@ -129,6 +161,7 @@ pub struct Scheduler {
     shutdown: Arc<Notify>,
     paused_cache: Mutex<(HashSet<String>, Instant)>,
     namespace: Option<String>,
+    duration_cache: Mutex<TaskDurationCache>,
 }
 
 /// Counters for tick-based scheduling of periodic maintenance tasks.
@@ -162,6 +195,7 @@ impl Scheduler {
             shutdown: Arc::new(Notify::new()),
             paused_cache: Mutex::new((HashSet::new(), Instant::now())),
             namespace,
+            duration_cache: Mutex::new(TaskDurationCache::new()),
         }
     }
 
@@ -192,24 +226,44 @@ impl Scheduler {
 
     /// Run the scheduler loop. Polls for ready jobs and dispatches them
     /// to the worker pool via the provided channel.
+    ///
+    /// Uses adaptive polling: starts at `poll_interval`, backs off
+    /// exponentially (up to 1s) when no jobs are found, resets immediately
+    /// when a job is dispatched.
     pub async fn run(&self, job_tx: tokio::sync::mpsc::Sender<Job>) {
         let mut counters = TickCounters::default();
+        let base_interval = self.config.poll_interval;
+        let max_interval = Duration::from_millis(200);
+        let mut current_interval = base_interval;
 
         loop {
             tokio::select! {
                 _ = self.shutdown.notified() => break,
-                _ = tokio::time::sleep(self.config.poll_interval) => {}
+                _ = tokio::time::sleep(current_interval) => {}
             }
 
-            self.tick(&job_tx, &mut counters);
+            let had_work = self.tick(&job_tx, &mut counters);
+            if had_work {
+                current_interval = base_interval;
+            } else {
+                current_interval = (current_interval * 2).min(max_interval);
+            }
         }
     }
 
     /// Execute one iteration of the scheduler loop.
-    fn tick(&self, job_tx: &tokio::sync::mpsc::Sender<Job>, counters: &mut TickCounters) {
-        if let Err(e) = self.try_dispatch(job_tx) {
-            error!("scheduler error: {e}");
-        }
+    /// Returns true if any work was done (job dispatched or periodic task enqueued),
+    /// which resets the adaptive poll interval.
+    fn tick(&self, job_tx: &tokio::sync::mpsc::Sender<Job>, counters: &mut TickCounters) -> bool {
+        let dispatched = match self.try_dispatch(job_tx) {
+            Ok(d) => d,
+            Err(e) => {
+                error!("scheduler error: {e}");
+                false
+            }
+        };
+
+        let mut had_maintenance = false;
 
         counters.reap += 1;
         counters.periodic += 1;
@@ -225,8 +279,13 @@ impl Scheduler {
             .periodic
             .is_multiple_of(self.config.periodic_check_interval)
         {
-            if let Err(e) = self.check_periodic() {
-                error!("periodic check error: {e}");
+            match self.check_periodic() {
+                Ok(()) => {
+                    // Periodic tasks may have been enqueued — reset polling
+                    // so the next tick picks them up quickly.
+                    had_maintenance = true;
+                }
+                Err(e) => error!("periodic check error: {e}"),
             }
         }
 
@@ -238,6 +297,8 @@ impl Scheduler {
                 error!("auto-cleanup error: {e}");
             }
         }
+
+        dispatched || had_maintenance
     }
 }
 

{taskito-0.9.0 → taskito-0.10.0}/crates/taskito-core/src/scheduler/result_handler.rs

@@ -36,6 +36,11 @@ impl Scheduler {
                     error!("circuit breaker error for {task_name}: {e}");
                 }
 
+                // Update in-memory duration cache for smart scheduling
+                if let Ok(mut cache) = self.duration_cache.lock() {
+                    cache.record(task_name, wall_time_ns);
+                }
+
                 Ok(ResultOutcome::Success {
                     job_id,
                     task_name: task_name.clone(),
@@ -71,6 +76,11 @@ impl Scheduler {
                     log::error!("circuit breaker error for {task_name}: {e}");
                 }
 
+                // Update in-memory duration cache for smart scheduling
+                if let Ok(mut cache) = self.duration_cache.lock() {
+                    cache.record(&task_name, wall_time_ns);
+                }
+
                 // Look up the job to get the queue name for middleware context
                 let queue = self
                     .storage

{taskito-0.9.0 → taskito-0.10.0}/crates/taskito-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-python"
-version = "0.9.0"
+version = "0.10.0"
 edition = "2021"
 
 [features]

taskito-0.10.0/crates/taskito-python/src/prefork/dispatch.rs

@@ -0,0 +1,65 @@
+//! Job dispatch strategies for distributing work across child processes.
+
+/// Selects the child with the fewest in-flight jobs (least-loaded).
+/// Falls back to round-robin if all children have equal load.
+pub fn least_loaded(in_flight_counts: &[u32]) -> usize {
+    in_flight_counts
+        .iter()
+        .enumerate()
+        .min_by_key(|(_, &count)| count)
+        .map(|(idx, _)| idx)
+        .unwrap_or(0)
+}
+
+/// Selects the child with the lowest weighted load.
+///
+/// Score = in_flight_count * avg_task_duration_ns. A worker with 1 slow job
+/// scores higher than one with 3 fast jobs, enabling better load distribution
+/// across heterogeneous workloads.
+///
+/// Falls back to `least_loaded` when `avg_duration_ns` is 0.
+#[allow(dead_code)]
+pub fn weighted_least_loaded(in_flight_counts: &[u32], avg_duration_ns: i64) -> usize {
+    if avg_duration_ns <= 0 {
+        return least_loaded(in_flight_counts);
+    }
+    in_flight_counts
+        .iter()
+        .enumerate()
+        .min_by_key(|(_, &count)| count as i64 * avg_duration_ns)
+        .map(|(idx, _)| idx)
+        .unwrap_or(0)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_least_loaded_picks_idle() {
+        assert_eq!(least_loaded(&[3, 0, 2]), 1);
+    }
+
+    #[test]
+    fn test_least_loaded_picks_first_on_tie() {
+        assert_eq!(least_loaded(&[1, 1, 1]), 0);
+    }
+
+    #[test]
+    fn test_least_loaded_single() {
+        assert_eq!(least_loaded(&[5]), 0);
+    }
+
+    #[test]
+    fn test_weighted_picks_lowest_score() {
+        // Worker 0: 2 in-flight * 100ns = 200
+        // Worker 1: 1 in-flight * 100ns = 100 ← pick this
+        // Worker 2: 3 in-flight * 100ns = 300
+        assert_eq!(weighted_least_loaded(&[2, 1, 3], 100), 1);
+    }
+
+    #[test]
+    fn test_weighted_falls_back_on_zero_duration() {
+        assert_eq!(weighted_least_loaded(&[3, 0, 2], 0), 1); // same as least_loaded
+    }
+}

{taskito-0.9.0 → taskito-0.10.0}/py_src/taskito/__init__.py

@@ -99,4 +99,4 @@ try:
 
     __version__ = _get_version("taskito")
 except PackageNotFoundError:
-    __version__ = "0.9.0"
+    __version__ = "0.10.0"