taskito 0.12.3__tar.gz → 0.13.0__tar.gz

{taskito-0.12.3 → taskito-0.13.0}/Cargo.lock

@@ -67,9 +67,9 @@ dependencies = [
 
 [[package]]
 name = "autocfg"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "base64"
@@ -94,9 +94,9 @@ checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
 
 [[package]]
 name = "bumpalo"
-version = "3.20.2"
+version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
 [[package]]
 name = "byteorder"
@@ -344,9 +344,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
 
 [[package]]
 name = "equivalent"
@@ -676,9 +676,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "js-sys"
-version = "0.3.98"
+version = "0.3.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
+checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
 dependencies = [
  "cfg-if",
  "futures-util",
@@ -783,9 +783,9 @@ dependencies = [
 
 [[package]]
 name = "num-conv"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
+checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
 
 [[package]]
 name = "num-integer"
@@ -1203,9 +1203,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
 
 [[package]]
 name = "rsqlite-vfs"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d"
+checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
 dependencies = [
  "hashbrown 0.16.1",
  "thiserror",
@@ -1289,9 +1289,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.149"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
  "itoa",
  "memchr",
@@ -1352,9 +1352,9 @@ dependencies = [
 
 [[package]]
 name = "sqlite-wasm-rs"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b2c760607300407ddeaee518acf28c795661b7108c75421303dbefb237d3a36"
+checksum = "cdd578e94101503d97e2b286bbf8db2135035ca24b2ce4cbf3f9e2fb2bbf1eee"
 dependencies = [
  "cc",
  "js-sys",
@@ -1404,7 +1404,7 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "taskito-async"
-version = "0.12.3"
+version = "0.13.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
@@ -1416,7 +1416,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-core"
-version = "0.12.3"
+version = "0.13.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1440,7 +1440,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.12.3"
+version = "0.13.0"
 dependencies = [
  "async-trait",
  "base64",
@@ -1460,11 +1460,12 @@ dependencies = [
 
 [[package]]
 name = "taskito-workflows"
-version = "0.12.3"
+version = "0.13.0"
 dependencies = [
  "dagron-core",
  "diesel",
  "log",
+ "redis",
  "serde",
  "serde_json",
  "taskito-core",
@@ -1658,9 +1659,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.121"
+version = "0.2.122"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
+checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -1671,9 +1672,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.121"
+version = "0.2.122"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
+checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1681,9 +1682,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.121"
+version = "0.2.122"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
+checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -1694,9 +1695,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.121"
+version = "0.2.122"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
+checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
 dependencies = [
  "unicode-ident",
 ]

{taskito-0.12.3 → taskito-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.12.3
+Version: 0.13.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.12.3"
+version = "0.13.0"
 edition = "2021"
 
 [features]

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-core/src/storage/redis_backend/mod.rs

@@ -64,6 +64,14 @@ impl RedisStorage {
             .get_connection()
             .map_err(|e| QueueError::Other(format!("Redis connection error: {e}")))
     }
+
+    /// The key prefix used for every Redis key this storage writes.
+    ///
+    /// Exposed so adjacent stores (e.g. the workflow store) can namespace
+    /// their own keys under the same prefix without re-parsing the URL.
+    pub fn prefix(&self) -> &str {
+        &self.prefix
+    }
 }
 
 fn map_err(e: redis::RedisError) -> QueueError {

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-python/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "taskito-python"
-version = "0.12.3"
+version = "0.13.0"
 edition = "2021"
 
 [features]
 default = []
 extension-module = ["pyo3/extension-module"]
-postgres = ["taskito-core/postgres"]
-redis = ["taskito-core/redis"]
+postgres = ["taskito-core/postgres", "taskito-workflows?/postgres"]
+redis = ["taskito-core/redis", "taskito-workflows?/redis"]
 native-async = ["dep:taskito-async"]
 workflows = ["dep:taskito-workflows"]
 

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-python/src/py_job.rs

@@ -44,6 +44,7 @@ pub struct PyJob {
     pub namespace: Option<String>,
 
     status_val: i32,
+    payload_bytes: Vec<u8>,
     result_bytes: Option<Vec<u8>>,
 }
 
@@ -56,6 +57,13 @@ impl PyJob {
             .as_str()
     }
 
+    /// The serialized ``(args, kwargs)`` payload that was enqueued for this
+    /// job. Use the queue's per-task serializer to decode.
+    #[getter]
+    pub fn payload_bytes(&self) -> &[u8] {
+        &self.payload_bytes
+    }
+
     #[getter]
     pub fn result_bytes(&self) -> Option<&[u8]> {
         self.result_bytes.as_deref()
@@ -93,6 +101,7 @@ impl From<Job> for PyJob {
             notes: job.notes,
             namespace: job.namespace,
             status_val: job.status as i32,
+            payload_bytes: job.payload,
             result_bytes: job.result,
         }
     }

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-python/src/py_queue/workflow_ops/fan_out.rs

@@ -65,7 +65,12 @@ impl PyQueue {
                 return Ok(Vec::new());
             }
 
+            // Enqueue all child jobs first, then atomically batch-insert the
+            // matching workflow_nodes. The batch insert wraps every row in one
+            // transaction, so a crash partway through node creation no longer
+            // leaves the run with half-tracked children.
             let mut child_job_ids = Vec::with_capacity(child_names.len());
+            let mut wf_nodes = Vec::with_capacity(child_names.len());
             for (child_name, payload) in child_names.iter().zip(child_payloads.into_iter()) {
                 let new_job = NewJob {
                     queue: queue_owned.clone(),
@@ -86,7 +91,7 @@ impl PyQueue {
                 let job = self.storage.enqueue(new_job)?;
                 child_job_ids.push(job.id.clone());
 
-                let wf_node = WorkflowNode {
+                wf_nodes.push(WorkflowNode {
                     id: uuid::Uuid::now_v7().to_string(),
                     run_id: run_id_owned.clone(),
                     node_name: child_name.clone(),
@@ -98,10 +103,14 @@ impl PyQueue {
                     started_at: None,
                     completed_at: None,
                     error: None,
-                };
-                wf_storage.create_workflow_node(&wf_node)?;
+                    compensation_job_id: None,
+                    compensation_started_at: None,
+                    compensation_completed_at: None,
+                    compensation_error: None,
+                });
             }
 
+            wf_storage.create_workflow_nodes_batch(&wf_nodes)?;
             wf_storage.set_workflow_node_fan_out_count(&run_id_owned, &parent_name_owned, count)?;
             Ok(child_job_ids)
         });

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-python/src/py_queue/workflow_ops/lifecycle.rs

@@ -119,6 +119,10 @@ impl PyQueue {
                     started_at: None,
                     completed_at: Some(now),
                     error: None,
+                    compensation_job_id: None,
+                    compensation_started_at: None,
+                    compensation_completed_at: None,
+                    compensation_error: None,
                 };
                 wf_storage
                     .create_workflow_node(&wf_node)
@@ -140,6 +144,10 @@ impl PyQueue {
                     started_at: None,
                     completed_at: None,
                     error: None,
+                    compensation_job_id: None,
+                    compensation_started_at: None,
+                    compensation_completed_at: None,
+                    compensation_error: None,
                 };
                 wf_storage
                     .create_workflow_node(&wf_node)
@@ -208,6 +216,10 @@ impl PyQueue {
                 started_at: None,
                 completed_at: None,
                 error: None,
+                compensation_job_id: None,
+                compensation_started_at: None,
+                compensation_completed_at: None,
+                compensation_error: None,
             };
             wf_storage
                 .create_workflow_node(&wf_node)

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-python/src/py_queue/workflow_ops/mod.rs

@@ -11,6 +11,7 @@ mod gates;
 mod lifecycle;
 mod nodes;
 mod queries;
+mod saga;
 
 use std::collections::HashMap;
 
@@ -19,6 +20,10 @@ use pyo3::prelude::*;
 
 use taskito_core::error::Result as CoreResult;
 use taskito_core::storage::{Storage, StorageBackend};
+#[cfg(feature = "postgres")]
+use taskito_workflows::WorkflowPostgresStorage;
+#[cfg(feature = "redis")]
+use taskito_workflows::WorkflowRedisStorage;
 use taskito_workflows::{
     StepMetadata, WorkflowNode, WorkflowNodeStatus, WorkflowSqliteStorage, WorkflowState,
     WorkflowStorage, WorkflowStorageBackend,
@@ -41,17 +46,13 @@ pub(super) fn workflow_storage(queue: &PyQueue) -> PyResult<WorkflowStorageBacke
             .map(WorkflowStorageBackend::Sqlite)
             .map_err(|e| PyRuntimeError::new_err(e.to_string()))?,
         #[cfg(feature = "postgres")]
-        StorageBackend::Postgres(_) => {
-            return Err(PyRuntimeError::new_err(
-                "workflows are currently only supported on the SQLite backend",
-            ))
-        }
+        StorageBackend::Postgres(s) => WorkflowPostgresStorage::new(s.clone())
+            .map(WorkflowStorageBackend::Postgres)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?,
         #[cfg(feature = "redis")]
-        StorageBackend::Redis(_) => {
-            return Err(PyRuntimeError::new_err(
-                "workflows are currently only supported on the SQLite backend",
-            ))
-        }
+        StorageBackend::Redis(s) => WorkflowRedisStorage::new(s.clone())
+            .map(WorkflowStorageBackend::Redis)
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))?,
     };
     // If another thread raced us to initialize, our value is ignored — either
     // handle is equivalent because the underlying pool is shared.
@@ -211,6 +212,10 @@ mod tests {
             started_at: None,
             completed_at: None,
             error: None,
+            compensation_job_id: None,
+            compensation_started_at: None,
+            compensation_completed_at: None,
+            compensation_error: None,
         };
         wf_storage.create_workflow_node(&node).unwrap();
         node

taskito-0.13.0/crates/taskito-python/src/py_queue/workflow_ops/saga.rs

@@ -0,0 +1,124 @@
+//! PyO3 entry points for the saga orchestrator (Python-side).
+//!
+//! The orchestrator lives in `py_src/taskito/workflows/saga/orchestrator.py`
+//! and drives reverse-order compensation. It needs to:
+//!
+//! - Transition the workflow run's state to `Compensating`, then to one
+//!   of `Compensated` / `CompensationFailed` when the saga finishes.
+//! - Mark individual nodes as `Compensating` (with the comp job id),
+//!   `Compensated`, or `CompensationFailed`.
+//!
+//! Each of those operations maps directly to a `WorkflowStorage` trait
+//! method added in the saga-foundations PR; this file exposes them as
+//! Python-callable `PyQueue` methods.
+
+use pyo3::exceptions::PyRuntimeError;
+use pyo3::prelude::*;
+
+use taskito_workflows::{WorkflowState, WorkflowStorage};
+
+use crate::py_queue::workflow_ops::workflow_storage;
+use crate::py_queue::PyQueue;
+
+#[pymethods]
+impl PyQueue {
+    /// Set a workflow run's state to `Compensating`.
+    pub fn set_workflow_run_compensating(&self, py: Python<'_>, run_id: &str) -> PyResult<()> {
+        let wf = workflow_storage(self)?;
+        let rid = run_id.to_string();
+        py.allow_threads(|| wf.update_workflow_run_state(&rid, WorkflowState::Compensating, None))
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+
+    /// Set a workflow run's state to `Compensated` and stamp `completed_at`.
+    pub fn set_workflow_run_compensated(
+        &self,
+        py: Python<'_>,
+        run_id: &str,
+        completed_at: i64,
+    ) -> PyResult<()> {
+        let wf = workflow_storage(self)?;
+        let rid = run_id.to_string();
+        py.allow_threads(|| {
+            wf.update_workflow_run_state(&rid, WorkflowState::Compensated, None)?;
+            wf.set_workflow_run_completed(&rid, completed_at)?;
+            Ok::<_, taskito_core::error::QueueError>(())
+        })
+        .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+
+    /// Set a workflow run's state to `CompensationFailed` and stamp
+    /// `completed_at`. Optionally records an error string on the run.
+    #[pyo3(signature = (run_id, completed_at, error=None))]
+    pub fn set_workflow_run_compensation_failed(
+        &self,
+        py: Python<'_>,
+        run_id: &str,
+        completed_at: i64,
+        error: Option<String>,
+    ) -> PyResult<()> {
+        let wf = workflow_storage(self)?;
+        let rid = run_id.to_string();
+        py.allow_threads(|| {
+            wf.update_workflow_run_state(
+                &rid,
+                WorkflowState::CompensationFailed,
+                error.as_deref(),
+            )?;
+            wf.set_workflow_run_completed(&rid, completed_at)?;
+            Ok::<_, taskito_core::error::QueueError>(())
+        })
+        .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+
+    /// Mark a node as `Compensating`, recording the compensation job id
+    /// and start timestamp.
+    pub fn set_workflow_node_compensation_job(
+        &self,
+        py: Python<'_>,
+        run_id: &str,
+        node_name: &str,
+        compensation_job_id: &str,
+        started_at: i64,
+    ) -> PyResult<()> {
+        let wf = workflow_storage(self)?;
+        let rid = run_id.to_string();
+        let nn = node_name.to_string();
+        let cjid = compensation_job_id.to_string();
+        py.allow_threads(|| wf.set_workflow_node_compensation_job(&rid, &nn, &cjid, started_at))
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+
+    /// Mark a node as `Compensated`.
+    pub fn set_workflow_node_compensated(
+        &self,
+        py: Python<'_>,
+        run_id: &str,
+        node_name: &str,
+        completed_at: i64,
+    ) -> PyResult<()> {
+        let wf = workflow_storage(self)?;
+        let rid = run_id.to_string();
+        let nn = node_name.to_string();
+        py.allow_threads(|| wf.set_workflow_node_compensated(&rid, &nn, completed_at))
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+
+    /// Mark a node as `CompensationFailed`, recording the error and
+    /// completion timestamp.
+    pub fn set_workflow_node_compensation_failed(
+        &self,
+        py: Python<'_>,
+        run_id: &str,
+        node_name: &str,
+        error: &str,
+        completed_at: i64,
+    ) -> PyResult<()> {
+        let wf = workflow_storage(self)?;
+        let rid = run_id.to_string();
+        let nn = node_name.to_string();
+        let err = error.to_string();
+        py.allow_threads(|| wf.set_workflow_node_compensation_failed(&rid, &nn, &err, completed_at))
+            .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+    }
+}

{taskito-0.12.3 → taskito-0.13.0}/crates/taskito-workflows/Cargo.toml

@@ -1,8 +1,13 @@
 [package]
 name = "taskito-workflows"
-version = "0.12.3"
+version = "0.13.0"
 edition = "2021"
 
+[features]
+default = []
+postgres = ["taskito-core/postgres", "diesel/postgres"]
+redis = ["taskito-core/redis", "dep:redis"]
+
 [dependencies]
 taskito-core = { path = "../taskito-core" }
 dagron-core = { workspace = true }
@@ -11,3 +16,8 @@ serde_json = { workspace = true }
 uuid = { workspace = true }
 diesel = { workspace = true }
 log = { workspace = true }
+redis = { workspace = true, optional = true }
+
+[dev-dependencies]
+uuid = { workspace = true }
+serde_json = { workspace = true }