taskito 0.19.0__tar.gz → 0.20.0__tar.gz

{taskito-0.19.0 → taskito-0.20.0}/Cargo.lock

@@ -214,9 +214,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.15"
+version = "0.5.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
+checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -298,9 +298,9 @@ dependencies = [
 
 [[package]]
 name = "diesel"
-version = "2.3.10"
+version = "2.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29fe29a87fb84c631ffb3ba21798c4b1f3a964701ba78f0dce4bf8668562ec88"
+checksum = "e54d1f576cd3a3460f212a4615fd12ce1b6303c095b79a44449ffbe627753dc1"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -1334,6 +1334,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "sea-query"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d190cfb3bcceb8a8d7d04dee5a0c77f60c7627979cdcb47fdcb8934f009badf"
+
 [[package]]
 name = "semver"
 version = "1.0.27"
@@ -1481,7 +1487,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "taskito-core"
-version = "0.19.0"
+version = "0.20.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1495,6 +1501,7 @@ dependencies = [
  "pq-sys",
  "rand 0.10.2",
  "redis",
+ "sea-query",
  "serde",
  "serde_json",
  "tempfile",
@@ -1505,7 +1512,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-java"
-version = "0.19.0"
+version = "0.20.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
@@ -1524,7 +1531,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-mesh"
-version = "0.19.0"
+version = "0.20.0"
 dependencies = [
  "base64",
  "bincode",
@@ -1539,7 +1546,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-node"
-version = "0.19.0"
+version = "0.20.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
@@ -1558,7 +1565,7 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.19.0"
+version = "0.20.0"
 dependencies = [
  "async-trait",
  "base64",
@@ -1578,12 +1585,13 @@ dependencies = [
 
 [[package]]
 name = "taskito-workflows"
-version = "0.19.0"
+version = "0.20.0"
 dependencies = [
  "dagron-core",
  "diesel",
  "log",
  "redis",
+ "sea-query",
  "serde",
  "serde_json",
  "taskito-core",
@@ -1597,7 +1605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.1",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -1750,9 +1758,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
 
 [[package]]
 name = "uuid"
-version = "1.23.4"
+version = "1.23.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
+checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a"
 dependencies = [
  "getrandom 0.4.1",
  "js-sys",

{taskito-0.19.0 → taskito-0.20.0}/Cargo.toml

@@ -22,6 +22,11 @@ openssl-sys = { version = "0.9", features = ["vendored"] }
 pyo3 = { version = "0.29", features = ["multiple-pymethods"] }
 async-trait = "0.1"
 dagron-core = { git = "https://github.com/ByteVeda/dagron.git", rev = "d1b61aaf2ed2d516b9a239f089a55b143cb05f65" }
+# Programmatic DDL builder — renders dialect-correct schema SQL executed through
+# the Diesel connection. `default-features = false` drops the mysql backend and
+# derive macros we don't use; `backend-postgres` is enabled per-crate under the
+# `postgres` cargo feature.
+sea-query = { version = "1", default-features = false, features = ["backend-sqlite"] }
 
 # Release cdylibs ship inside SDK packages; strip + thin-LTO keep them small.
 [profile.release]

{taskito-0.19.0 → taskito-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.19.0
+Version: 0.20.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -13,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Rust
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Distributed Computing
+Requires-Dist: cbor2>=5.6
 Requires-Dist: cloudpickle>=3.0
 Requires-Dist: msgpack>=1.0
 Requires-Dist: tzdata ; platform_system == 'Windows'

{taskito-0.19.0 → taskito-0.20.0}/crates/taskito-core/BINDING_CONTRACT.md

@@ -17,8 +17,53 @@ depending on it). Keep Python/Node/Java specifics in the shell.
 them. Each shell serializes args/kwargs at enqueue and deserializes them in the
 worker — using whatever serializer it wants. The Python shell defaults to cloudpickle
 (Python-only). **Cross-language constraint:** a job enqueued by one language and run
-by another requires both to agree on a neutral serializer (JSON or msgpack);
-cloudpickle is Python-internal only.
+by another requires both sides to use the wire envelope below.
+
+## Wire envelope (cross-SDK payloads)
+A wire payload is one tag byte followed by the codec body. The tag records which
+codec produced the body, so any shell can dispatch a decoder (or reject clearly)
+without out-of-band configuration:
+
+| Tag    | Body        | Cross-SDK | Notes |
+|--------|-------------|-----------|-------|
+| `0x00` | native      | **never** — reject with a clear error | Language-native codec (e.g. pickle). Same-language producer/consumer only. |
+| `0x01` | msgpack     | optional  | Legacy tagged format; shells MAY read it, SHOULD NOT write it cross-SDK. |
+| `0x02` | CBOR (RFC 8949) | **default** | The cross-SDK wire format. |
+| `0x03` | reserved    | —         | Tagged JSON (not yet specified). |
+| `0x04+`| reserved    | —         | Future (protobuf, …). |
+
+Untagged payloads predate the envelope and are same-SDK legacy; a shell MUST NOT
+assume any tag discipline unless the task is configured with a tagged serializer
+on both sides. (Sniffing is unsafe: raw msgpack/CBOR bodies can begin with any
+byte value.)
+
+**Why CBOR over JSON**: integers survive — JS `Number` is exact only to 2^53−1
+while other languages carry 64-bit/unbounded ints; CBOR bignums round-trip them
+losslessly. IANA tags also round-trip datetimes (tag 0/1), decimals (tag 4), and
+byte strings without a hand-rolled registry. Mature codecs exist everywhere
+(`cbor2`, `cbor-x`, `jackson-dataformat-cbor`, `fxamacker/cbor`, …).
+
+**Call body** (`Job.payload`): a 2-element CBOR array `[args, kwargs]` — `args`
+an array, `kwargs` a map (empty map when the language has no keyword arguments).
+Job-scoped extras belong in the `metadata`/`notes` columns, not the payload.
+Convention for cross-SDK tasks: prefer a single object argument
+(`args = [ {…} ]`, `kwargs = {}`) — it maps cleanly onto every language's
+handler-binding model.
+
+**Result body** (`Job.result`): a bare CBOR value (no array wrapper).
+
+**Cross-SDK rules**:
+- A shell reading tag `0x00` on a payload it did not produce MUST fail with an
+  error naming the tag, not a generic decode error.
+- Producer and consumer of a task MUST be configured with the same wire
+  serializer; the tag is a self-check, not a negotiation mechanism.
+- Delivery-side semantics (retries, DLQ, acks) are unaffected — the envelope is
+  purely a payload contract.
+
+**Test vectors** (hex, `0x02`-tagged CBOR):
+- call `f(1, "a")`, no kwargs → `02 82 82 01 61 61 a0` — `[ [1, "a"], {} ]`
+- result `true` → `02 f5`
+- big int `2^53` → `02 1b 00 20 00 00 00 00 00 00`
 
 ## Dispatch call sequence
 1. Shell constructs `Storage` (SQLite default; `postgres`/`redis` features) — `storage/traits.rs`.
@@ -51,6 +96,36 @@ Scheduler.handle_result ─▶ ResultOutcome ─▶ shell emits events / middlew
 Channels: inbound `tokio::sync::mpsc::Receiver<Job>` (async); outbound
 `crossbeam_channel::Sender<JobResult>` (sync, cloneable).
 
+## Task errors (structured, cross-SDK)
+When a task raises, the shell reports the failure as a **canonical JSON object**
+serialized into `JobResult::Failure.error` (and thus into `jobs.error`,
+`job_errors.error`, `dead_letter.error` — the storage layer never interprets it):
+
+```json
+{"errtype": "ValueError", "message": "bad value 42", "traceback": ["...frame...", "..."]}
+```
+
+- `errtype` — the exception's class name, as idiomatic per language (qualified
+  where the language has a notion of it). Required.
+- `message` — the human-readable message, verbatim (keeps `error_like`
+  substring filters useful). Required, may be empty.
+- `traceback` — array of strings, best-effort per shell; `[]` when the
+  language/runtime can't provide frames. Required key.
+
+**Fallback rule (readers)**: an error string that does not parse as a JSON
+object with a `message` key is a plain legacy/system string and MUST be
+surfaced as-is. Core-generated maintenance errors (timeouts, worker-death
+recovery, expiry, cancellation) remain plain strings by design.
+
+**Retry semantics**: `retry_on`/`dont_retry_on`-style filtering matches on the
+live exception object before formatting — the stored string never drives retry
+decisions.
+
+**Test vector** (assert byte-exact in each shell's formatter):
+input errtype `BoomError`, message `it broke`, traceback `["frame1", "frame2"]` →
+`{"errtype":"BoomError","message":"it broke","traceback":["frame1","frame2"]}`
+(JSON with those three keys in that order, no extra whitespace).
+
 ## Types the shell produces / consumes
 - **`Job`** — `job.rs`. Fields incl. `id`, `queue`, `task_name`, `payload: Vec<u8>` (opaque),
   `status`, `priority`, `retry_count`, `max_retries`, `timeout_ms`, `unique_key`,

{taskito-0.19.0 → taskito-0.20.0}/crates/taskito-core/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "taskito-core"
-version = "0.19.0"
+version = "0.20.0"
 edition = "2021"
 
 [features]
 default = []
-postgres = ["diesel/postgres", "pq-sys", "openssl-sys"]
+postgres = ["diesel/postgres", "pq-sys", "openssl-sys", "sea-query/backend-postgres"]
 redis = ["dep:redis"]
 # Event-driven scheduler wakeups. OFF by default — the shipped wheel keeps
 # polling. Opt-in for source builds that want lower dispatch latency.
@@ -28,6 +28,7 @@ cron = { workspace = true }
 chrono-tz = { workspace = true }
 log = { workspace = true }
 async-trait = { workspace = true }
+sea-query = { workspace = true }
 redis = { workspace = true, optional = true }
 
 [dev-dependencies]

taskito-0.20.0/crates/taskito-core/build.rs

@@ -0,0 +1,96 @@
+//! Path-based migration auto-discovery.
+//!
+//! Scans the crate's `migrations/` directory at build time and generates the
+//! module declarations plus an ordered `all()` registry into `OUT_DIR`, which
+//! `migrations/mod.rs` pulls in with `include!`. Modelled on Alembic's "read
+//! every file in the versions/ directory" discovery: dropping a new
+//! `mXXXX_*.rs` file into `migrations/` registers it automatically — no manual
+//! `mod` line and no hand-maintained list.
+//!
+//! Identity is the migration's own `version()` (never the filename); the
+//! migrator sorts by it at runtime, so this file only needs to *discover* the
+//! migrations, not order them. Each file must contain exactly one
+//! `impl Migration for <Struct>` — the struct name is read from that line.
+
+use std::fs;
+use std::path::{Path, PathBuf};
+
+fn main() {
+    let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");
+    let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR");
+    let migrations_dir = Path::new(&manifest_dir).join("migrations");
+
+    // Re-run when a migration file is added, removed, or edited. The directory
+    // line catches add/remove (its mtime changes); the per-file lines catch
+    // edits to existing migrations.
+    println!("cargo:rerun-if-changed={}", migrations_dir.display());
+
+    let mut files = migration_files(&migrations_dir);
+    files.sort();
+
+    let mut module_decls = String::new();
+    let mut registry_entries = String::new();
+    for path in &files {
+        println!("cargo:rerun-if-changed={}", path.display());
+        let module = path
+            .file_stem()
+            .and_then(|s| s.to_str())
+            .expect("migration file stem");
+        let struct_name = struct_name(path);
+        module_decls.push_str(&format!(
+            "#[path = {path:?}]\nmod {module};\n",
+            path = path.to_string_lossy(),
+        ));
+        registry_entries.push_str(&format!("        Box::new({module}::{struct_name}),\n"));
+    }
+
+    let generated = format!(
+        "// @generated by build.rs — do not edit. One entry per migrations/mXXXX_*.rs file.\n\
+         {module_decls}\n\
+         /// Every discovered migration. The migrator sorts these by `version()`.\n\
+         pub fn all() -> Vec<Box<dyn Migration>> {{\n    vec![\n{registry_entries}    ]\n}}\n"
+    );
+    fs::write(
+        Path::new(&out_dir).join("migrations_generated.rs"),
+        generated,
+    )
+    .expect("write generated migration registry");
+}
+
+/// The `mXXXX_*.rs` files in the migrations directory (excludes `mod.rs`).
+fn migration_files(dir: &Path) -> Vec<PathBuf> {
+    fs::read_dir(dir)
+        .unwrap_or_else(|e| panic!("read migrations dir {}: {e}", dir.display()))
+        .filter_map(|entry| entry.ok().map(|e| e.path()))
+        .filter(|path| {
+            path.extension().is_some_and(|ext| ext == "rs")
+                && path
+                    .file_stem()
+                    .and_then(|s| s.to_str())
+                    .is_some_and(|stem| stem.starts_with('m') && stem != "mod")
+        })
+        .collect()
+}
+
+/// Read the struct name from the file's `impl Migration for <Struct>` line.
+fn struct_name(path: &Path) -> String {
+    let content = fs::read_to_string(path)
+        .unwrap_or_else(|e| panic!("read migration {}: {e}", path.display()));
+    const MARKER: &str = "impl Migration for ";
+    let start = content.find(MARKER).unwrap_or_else(|| {
+        panic!(
+            "migration {} has no `impl Migration for <Struct>`",
+            path.display()
+        )
+    }) + MARKER.len();
+    let name: String = content[start..]
+        .chars()
+        .take_while(|c| c.is_alphanumeric() || *c == '_')
+        .collect();
+    assert!(
+        !name.is_empty(),
+        "migration {} has a malformed `impl Migration for` line",
+        path.display()
+    );
+    name
+}