taskito 0.3.0__tar.gz → 0.5.0__tar.gz

{taskito-0.3.0 → taskito-0.5.0}/Cargo.lock

@@ -26,6 +26,17 @@ dependencies = [
  "rustversion",
 ]
 
+[[package]]
+name = "async-trait"
+version = "0.1.89"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.5.0"
@@ -183,9 +194,9 @@ dependencies = [
 
 [[package]]
 name = "diesel"
-version = "2.3.6"
+version = "2.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b6c2fc184a6fb6ebcf5f9a5e3bbfa84d8fd268cdfcce4ed508979a6259494d"
+checksum = "f4ae09a41a4b89f94ec1e053623da8340d996bc32c6517d325a9daad9b239358"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -520,6 +531,15 @@ dependencies = [
  "rustversion",
 ]
 
+[[package]]
+name = "inventory"
+version = "0.3.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -673,9 +693,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.21.3"
+version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "openssl-src"
@@ -688,9 +708,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.111"
+version = "0.9.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
+checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
 dependencies = [
  "cc",
  "libc",
@@ -835,6 +855,7 @@ checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884"
 dependencies = [
  "cfg-if",
  "indoc",
+ "inventory",
  "libc",
  "memoffset",
  "once_cell",
@@ -1178,10 +1199,22 @@ version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
+[[package]]
+name = "taskito-async"
+version = "0.5.0"
+dependencies = [
+ "async-trait",
+ "crossbeam-channel",
+ "pyo3",
+ "taskito-core",
+ "tokio",
+]
+
 [[package]]
 name = "taskito-core"
-version = "0.3.0"
+version = "0.5.0"
 dependencies = [
+ "async-trait",
  "chrono",
  "chrono-tz",
  "cron",
@@ -1203,10 +1236,12 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.3.0"
+version = "0.5.0"
 dependencies = [
+ "async-trait",
  "crossbeam-channel",
  "pyo3",
+ "taskito-async",
  "taskito-core",
  "tokio",
  "uuid",
@@ -1214,9 +1249,9 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.26.0"
+version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
  "getrandom 0.4.2",
@@ -1750,18 +1785,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.40"
+version = "0.8.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
+checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.40"
+version = "0.8.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
+checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
 dependencies = [
  "proc-macro2",
  "quote",

{taskito-0.3.0 → taskito-0.5.0}/Cargo.toml

@@ -19,4 +19,5 @@ chrono-tz = "0.10"
 log = "0.4"
 redis = { version = "0.27", features = ["script"] }
 openssl-sys = { version = "0.9", features = ["vendored"] }
-pyo3 = { version = "0.22", features = ["extension-module"] }
+pyo3 = { version = "0.22", features = ["multiple-pymethods"] }
+async-trait = "0.1"

{taskito-0.3.0 → taskito-0.5.0}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.3.0
+Version: 0.5.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -14,6 +13,7 @@ Classifier: Programming Language :: Rust
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Distributed Computing
 Requires-Dist: cloudpickle>=3.0
+Requires-Dist: boto3>=1.20 ; extra == 'aws'
 Requires-Dist: pytest>=7.0 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21 ; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
@@ -25,17 +25,20 @@ Requires-Dist: cryptography ; extra == 'encryption'
 Requires-Dist: fastapi>=0.100.0 ; extra == 'fastapi'
 Requires-Dist: pydantic>=2.0 ; extra == 'fastapi'
 Requires-Dist: flask>=2.0 ; extra == 'flask'
+Requires-Dist: google-cloud-storage>=2.0 ; extra == 'gcs'
 Requires-Dist: msgpack ; extra == 'msgpack'
 Requires-Dist: opentelemetry-api ; extra == 'otel'
 Requires-Dist: opentelemetry-sdk ; extra == 'otel'
 Requires-Dist: prometheus-client ; extra == 'prometheus'
 Requires-Dist: sentry-sdk ; extra == 'sentry'
+Provides-Extra: aws
 Provides-Extra: dev
 Provides-Extra: django
 Provides-Extra: docs
 Provides-Extra: encryption
 Provides-Extra: fastapi
 Provides-Extra: flask
+Provides-Extra: gcs
 Provides-Extra: msgpack
 Provides-Extra: otel
 Provides-Extra: postgres
@@ -44,7 +47,7 @@ Provides-Extra: redis
 Provides-Extra: sentry
 License-File: LICENSE
 Summary: Rust-powered task queue for Python. No broker required.
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Changelog, https://github.com/pratyush618/taskito/blob/master/docs/changelog.md
 Project-URL: Documentation, https://taskito.grigori.in

{taskito-0.3.0 → taskito-0.5.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.3.0"
+version = "0.5.0"
 edition = "2021"
 
 [features]
@@ -24,6 +24,7 @@ chrono = { workspace = true }
 cron = { workspace = true }
 chrono-tz = { workspace = true }
 log = { workspace = true }
+async-trait = { workspace = true }
 redis = { workspace = true, optional = true }
 
 [dev-dependencies]

{taskito-0.3.0 → taskito-0.5.0}/crates/taskito-core/src/error.rs

@@ -35,6 +35,9 @@ pub enum QueueError {
     #[error("dependency not found: {0}")]
     DependencyNotFound(String),
 
+    #[error("lock not acquired: {0}")]
+    LockNotAcquired(String),
+
     #[error("{0}")]
     Other(String),
 }

{taskito-0.3.0 → taskito-0.5.0}/crates/taskito-core/src/lib.rs

@@ -4,6 +4,7 @@ pub mod periodic;
 pub mod resilience;
 pub mod scheduler;
 pub mod storage;
+pub mod worker;
 
 // Primary public API — the types most consumers need.
 pub use error::QueueError;

taskito-0.5.0/crates/taskito-core/src/scheduler/maintenance.rs

@@ -0,0 +1,139 @@
+use log::{error, info, warn};
+
+use crate::error::Result;
+use crate::job::{now_millis, NewJob};
+use crate::periodic::{next_cron_time, next_cron_time_tz};
+use crate::storage::Storage;
+
+use super::{JobResult, Scheduler};
+
+/// Default max retries for periodic tasks.
+const PERIODIC_DEFAULT_MAX_RETRIES: i32 = 3;
+
+/// Default timeout for periodic tasks (ms).
+const PERIODIC_DEFAULT_TIMEOUT_MS: i64 = 300_000;
+
+impl Scheduler {
+    pub(super) fn reap_stale(&self) -> Result<()> {
+        let now = now_millis();
+        // Expire pending jobs that passed their TTL
+        if let Err(e) = self.storage.expire_pending_jobs(now) {
+            warn!("expire_pending_jobs error: {e}");
+        }
+        // Reap expired distributed locks
+        if let Err(e) = self.storage.reap_expired_locks(now) {
+            warn!("reap_expired_locks error: {e}");
+        }
+        // Purge stale execution claims (older than 1 hour)
+        let claim_cutoff = now.saturating_sub(3_600_000);
+        if let Err(e) = self.storage.purge_execution_claims(claim_cutoff) {
+            warn!("purge_execution_claims error: {e}");
+        }
+        let stale_jobs = self.storage.reap_stale_jobs(now)?;
+
+        for job in stale_jobs {
+            let error = format!("job timed out after {}ms", job.timeout_ms);
+            self.handle_result(JobResult::Failure {
+                job_id: job.id.clone(),
+                error,
+                retry_count: job.retry_count,
+                max_retries: job.max_retries,
+                task_name: job.task_name.clone(),
+                wall_time_ns: 0,
+                should_retry: true,
+            })?;
+        }
+
+        Ok(())
+    }
+
+    /// Purge old completed/dead jobs and their errors if result_ttl_ms is set.
+    pub(super) fn auto_cleanup(&self) -> Result<()> {
+        let ttl = match self.config.result_ttl_ms {
+            Some(t) => t,
+            None => return Ok(()),
+        };
+
+        let cutoff = now_millis().saturating_sub(ttl);
+
+        // Use per-job TTL aware cleanup
+        let completed = self.storage.purge_completed_with_ttl(cutoff)?;
+        let dead = self.storage.purge_dead(cutoff)?;
+        let errors = self.storage.purge_job_errors(cutoff)?;
+        let metrics = self.storage.purge_metrics(cutoff).unwrap_or_else(|e| {
+            warn!("purge_metrics failed: {e}");
+            0
+        });
+        let logs = self.storage.purge_task_logs(cutoff).unwrap_or_else(|e| {
+            warn!("purge_task_logs failed: {e}");
+            0
+        });
+
+        if completed + dead + errors + metrics + logs > 0 {
+            info!(
+                "auto-cleanup: purged {completed} completed, {dead} dead, {errors} errors, {metrics} metrics, {logs} logs"
+            );
+        }
+
+        Ok(())
+    }
+
+    /// Check for periodic tasks that are due and enqueue them.
+    pub(super) fn check_periodic(&self) -> Result<()> {
+        let now = now_millis();
+        let due_tasks = self.storage.get_due_periodic(now)?;
+
+        for task in due_tasks {
+            let unique_key = Some(format!("periodic:{}:{}", task.name, now));
+            let new_job = NewJob {
+                queue: task.queue.clone(),
+                task_name: task.task_name.clone(),
+                payload: Self::build_periodic_payload(&task.args),
+                priority: 0,
+                scheduled_at: now,
+                max_retries: PERIODIC_DEFAULT_MAX_RETRIES,
+                timeout_ms: PERIODIC_DEFAULT_TIMEOUT_MS,
+                unique_key,
+                metadata: None,
+                depends_on: vec![],
+                expires_at: None,
+                result_ttl_ms: None,
+                namespace: None,
+            };
+
+            if let Err(e) = self.storage.enqueue_unique(new_job) {
+                error!("failed to enqueue periodic task '{}': {e}", task.name);
+                continue;
+            }
+
+            let next_run = match if let Some(ref tz) = task.timezone {
+                next_cron_time_tz(&task.cron_expr, now, tz)
+            } else {
+                next_cron_time(&task.cron_expr, now)
+            } {
+                Ok(t) => t,
+                Err(e) => {
+                    error!("failed to compute next run for '{}': {e}", task.name);
+                    continue;
+                }
+            };
+
+            if let Err(e) = self
+                .storage
+                .update_periodic_schedule(&task.name, now, next_run)
+            {
+                error!("failed to update schedule for '{}': {e}", task.name);
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Build a cloudpickle-compatible payload from stored args.
+    fn build_periodic_payload(args: &Option<Vec<u8>>) -> Vec<u8> {
+        match args {
+            Some(blob) => blob.clone(),
+            None => Vec::new(),
+        }
+    }
+}