cairnq 0.2.0__tar.gz

cairnq-0.2.0/.gitignore

@@ -0,0 +1,25 @@
+# Python
+__pycache__/
+*.pyc
+.venv/
+.pytest_cache/
+*.egg-info/
+dist/
+build/
+
+# Node
+node_modules/
+dist/
+*.tsbuildinfo
+
+# Vendored protocol (generated by scripts/vendor-protocol.mjs at publish time).
+# py lands beside the package; node lands in dist/_protocol (already ignored above).
+cairnq-py/cairnq/_protocol/
+
+# SQLite working files
+*.db
+*.db-wal
+*.db-shm
+
+# Editor / OS
+.DS_Store

cairnq-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jannchie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cairnq-0.2.0/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.4
+Name: cairnq
+Version: 0.2.0
+Summary: SQLite-first, cross-language, storage-centered durable task runtime
+Project-URL: Homepage, https://github.com/Jannchie/cairnq
+Project-URL: Repository, https://github.com/Jannchie/cairnq
+Project-URL: Issues, https://github.com/Jannchie/cairnq/issues
+Author-email: Jannchie <jannchie@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: aiosqlite,cross-language,durable,job-queue,sqlite,task-queue,task-runtime,worker
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Requires-Dist: aiosqlite>=0.19
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Provides-Extra: postgres
+Requires-Dist: asyncpg>=0.29; extra == 'postgres'
+Description-Content-Type: text/markdown
+
+# cairnq (Python)
+
+SQLite-first, cross-language, storage-centered durable task runtime. The Python
+SDK. API and worker processes coordinate only through a shared SQLite file.
+
+```python
+from cairnq import CairnQ, Worker
+
+# Worker side — a handler always receives (ctx, payload).
+worker = Worker.sqlite("tasks.db")
+
+@worker.task                          # registered under the function name, "create_summary"
+async def create_summary(ctx, payload):
+    await ctx.progress(0.2, "reading")
+    return {"summary": await llm.summarize(payload["text"])}
+
+worker.serve()                        # blocking entry point; Ctrl-C closes cleanly
+
+# API side (in your server) — submit returns immediately.
+tasks = CairnQ.sqlite("tasks.db")
+task = await tasks.submit("create_summary", {"text": text}, key=f"summary:{aid}")
+```
+
+`@worker.task` defaults the task name to the function's name. Pass a string for a
+dotted/namespaced name: `@worker.task("summary.create")`.
+
+Synchronous call (submit + wait):
+
+```python
+from cairnq import TaskFailed, TaskTimeout
+
+try:
+    result = await tasks.call("create_summary", {"text": text}, wait_timeout_ms=10_000)
+except TaskFailed as e:
+    log(e.code, e.message, e.retryable)   # envelope fields, no e.error["code"] digging
+except TaskTimeout as e:
+    ...  # e.task_id keeps running
+```
+
+Inspect a task by id/key without memorizing status strings:
+
+```python
+task = await tasks.get_by_key(key)
+if task and task.succeeded:        # also .failed / .canceled / .running / .queued / .is_terminal
+    use(task.result)
+```
+
+Optionally define a task once and share the symbol across both ends — the name
+lives in one place (no string drift), and `call()` is typed as the task's result:
+
+```python
+from cairnq import TaskDef
+
+summarize = TaskDef[dict, dict]("summarize")
+
+@worker.task(summarize)            # registered under summarize.name
+async def handle(ctx, payload): ...
+
+result = await tasks.call(summarize, {"text": text})
+```
+
+Opt-in: every API still accepts a plain name string (cross-language callers use it).
+
+## Running it in production
+
+```python
+worker = Worker.sqlite(
+    "tasks.db",
+    concurrency=4,
+    retry_backoff_ms=1_000,   # doubles per attempt, capped by retry_backoff_max_ms (30s); 0 disables
+    on_error=lambda exc, info: log.warning("worker survived %s: %s", info, exc),
+)
+
+# Nothing else deletes rows. Sweep terminal tasks on a schedule.
+await tasks.purge(older_than_ms=7 * 24 * 3600_000)
+```
+
+A handler that does real side effects should bail out when it loses its lease —
+the task is already running on another worker and nothing it writes is recorded:
+
+```python
+@worker.task("long.job")
+async def long_job(ctx, payload):
+    for chunk in chunks:
+        if ctx.lost_lease or await ctx.canceled():
+            return
+        await process(chunk)
+```
+
+## Multi-host
+
+Same code, Postgres instead of the file — `CairnQ.postgres(dsn)` /
+`Worker.postgres(dsn)`. Install with `pip install cairnq[postgres]`.
+
+The protocol (schema + canonical SQL) lives in `../cairnq-protocol` and is shared
+verbatim with the TypeScript SDK. See `../cairnq-protocol/PROTOCOL.md`.

cairnq-0.2.0/README.md

@@ -0,0 +1,96 @@
+# cairnq (Python)
+
+SQLite-first, cross-language, storage-centered durable task runtime. The Python
+SDK. API and worker processes coordinate only through a shared SQLite file.
+
+```python
+from cairnq import CairnQ, Worker
+
+# Worker side — a handler always receives (ctx, payload).
+worker = Worker.sqlite("tasks.db")
+
+@worker.task                          # registered under the function name, "create_summary"
+async def create_summary(ctx, payload):
+    await ctx.progress(0.2, "reading")
+    return {"summary": await llm.summarize(payload["text"])}
+
+worker.serve()                        # blocking entry point; Ctrl-C closes cleanly
+
+# API side (in your server) — submit returns immediately.
+tasks = CairnQ.sqlite("tasks.db")
+task = await tasks.submit("create_summary", {"text": text}, key=f"summary:{aid}")
+```
+
+`@worker.task` defaults the task name to the function's name. Pass a string for a
+dotted/namespaced name: `@worker.task("summary.create")`.
+
+Synchronous call (submit + wait):
+
+```python
+from cairnq import TaskFailed, TaskTimeout
+
+try:
+    result = await tasks.call("create_summary", {"text": text}, wait_timeout_ms=10_000)
+except TaskFailed as e:
+    log(e.code, e.message, e.retryable)   # envelope fields, no e.error["code"] digging
+except TaskTimeout as e:
+    ...  # e.task_id keeps running
+```
+
+Inspect a task by id/key without memorizing status strings:
+
+```python
+task = await tasks.get_by_key(key)
+if task and task.succeeded:        # also .failed / .canceled / .running / .queued / .is_terminal
+    use(task.result)
+```
+
+Optionally define a task once and share the symbol across both ends — the name
+lives in one place (no string drift), and `call()` is typed as the task's result:
+
+```python
+from cairnq import TaskDef
+
+summarize = TaskDef[dict, dict]("summarize")
+
+@worker.task(summarize)            # registered under summarize.name
+async def handle(ctx, payload): ...
+
+result = await tasks.call(summarize, {"text": text})
+```
+
+Opt-in: every API still accepts a plain name string (cross-language callers use it).
+
+## Running it in production
+
+```python
+worker = Worker.sqlite(
+    "tasks.db",
+    concurrency=4,
+    retry_backoff_ms=1_000,   # doubles per attempt, capped by retry_backoff_max_ms (30s); 0 disables
+    on_error=lambda exc, info: log.warning("worker survived %s: %s", info, exc),
+)
+
+# Nothing else deletes rows. Sweep terminal tasks on a schedule.
+await tasks.purge(older_than_ms=7 * 24 * 3600_000)
+```
+
+A handler that does real side effects should bail out when it loses its lease —
+the task is already running on another worker and nothing it writes is recorded:
+
+```python
+@worker.task("long.job")
+async def long_job(ctx, payload):
+    for chunk in chunks:
+        if ctx.lost_lease or await ctx.canceled():
+            return
+        await process(chunk)
+```
+
+## Multi-host
+
+Same code, Postgres instead of the file — `CairnQ.postgres(dsn)` /
+`Worker.postgres(dsn)`. Install with `pip install cairnq[postgres]`.
+
+The protocol (schema + canonical SQL) lives in `../cairnq-protocol` and is shared
+verbatim with the TypeScript SDK. See `../cairnq-protocol/PROTOCOL.md`.

cairnq-0.2.0/bench/run.py

@@ -0,0 +1,156 @@
+"""CairnQ micro-benchmark (Python SDK).
+
+    uv run python bench/run.py               # SQLite on a temp file
+    uv run python bench/run.py postgres      # against CAIRNQ_TEST_PG_DSN (use a throwaway DB)
+
+Workload sizes, poll settings and row labels MUST match cairnq-node/bench/run.ts
+— the two benches exist to be compared against each other.
+
+Reports client-op throughput (submit/get/cancel/purge), worker drain throughput,
+and end-to-end call() latency — the latency rows are the ones that show the
+polling floor: with the default intervals a call can't finish faster than the
+worker's claim poll plus wait()'s read poll.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import tempfile
+from time import perf_counter
+
+from cairnq import CairnQ, PostgresStore, SQLiteStore, TaskDef, TaskStore, Worker
+
+N_SUBMIT = 1000
+N_GET = 2000
+N_DRAIN = 500
+N_CALL = 40
+
+noop = TaskDef("bench.noop")
+rows: list[list[str]] = []
+
+
+def ops_per_sec(n: int, ms: float) -> int:
+    return round(n / ms * 1000)
+
+
+def pct(sorted_ms: list[float], q: float) -> float:
+    return sorted_ms[min(len(sorted_ms) - 1, int(q * len(sorted_ms)))]
+
+
+async def timed(label, n, fn):
+    t0 = perf_counter()
+    out = await fn()
+    rows.append([label, f"{ops_per_sec(n, (perf_counter() - t0) * 1000)} ops/s"])
+    return out
+
+
+def push_latency(label: str, lat: list[float]) -> None:
+    rows.append([label, f"p50 {pct(lat, 0.5):.0f} ms", f"p95 {pct(lat, 0.95):.0f} ms"])
+
+
+async def call_latencies(tasks: CairnQ, **call_kwargs) -> list[float]:
+    lat = []
+    for _ in range(N_CALL):
+        t0 = perf_counter()
+        await tasks.call(noop, {}, wait_timeout_ms=15_000, **call_kwargs)
+        lat.append((perf_counter() - t0) * 1000)
+    return sorted(lat)
+
+
+async def purge_all(tasks: CairnQ) -> None:
+    """Sweep every terminal task, honoring purge's bounded-batch contract."""
+    while len(await tasks.purge(older_than_ms=0, limit=1_000)) == 1_000:
+        pass
+
+
+async def drain_settled(tasks: CairnQ) -> None:
+    """The drain counter fires on the last handler's return; its complete write —
+    and the handful still in flight — land just after. Wait them out."""
+    while await tasks.list(status="running", limit=1):
+        await asyncio.sleep(0.005)
+
+
+async def run(backend: str, tasks: CairnQ, worker_store: TaskStore) -> None:
+    await tasks.connect()
+
+    # ---- client ops, no worker running. "bench.unclaimed" is a name no worker
+    # registers, so these rows sit queued until the cancel pass below.
+    async def submit_all() -> list[str]:
+        return [(await tasks.submit("bench.unclaimed", {"i": i})).id for i in range(N_SUBMIT)]
+
+    ids = await timed("submit", N_SUBMIT, submit_all)
+    n_ids = len(ids)
+
+    async def get_all() -> None:
+        for i in range(N_GET):
+            await tasks.get(ids[i % n_ids])
+
+    await timed("get", N_GET, get_all)
+
+    async def cancel_all() -> None:
+        for task_id in ids:
+            await tasks.cancel(task_id)
+
+    await timed("cancel", N_SUBMIT, cancel_all)
+    await timed("purge", N_SUBMIT, lambda: purge_all(tasks))
+
+    # ---- drain throughput + call latency at the default poll intervals. The
+    # worker store connects up front so one-time setup stays out of the clock;
+    # both workers share it (they never run at the same time).
+    await asyncio.gather(*(tasks.submit(noop, {"i": i}) for i in range(N_DRAIN)))
+    done = 0
+    all_done = asyncio.Event()
+
+    def counting_noop(ctx, payload):
+        nonlocal done
+        done += 1
+        if done == N_DRAIN:
+            all_done.set()
+        return {}
+
+    w = Worker(worker_store, ["default"], concurrency=8)
+    w.task(noop)(counting_noop)
+    await worker_store.connect()
+    t0 = perf_counter()
+    async with w.background():
+        await all_done.wait()
+        await drain_settled(tasks)
+        drain_ms = (perf_counter() - t0) * 1000
+        lat = await call_latencies(tasks)
+    rows.append([f"drain {N_DRAIN} (conc 8)", f"{ops_per_sec(N_DRAIN, drain_ms)} tasks/s"])
+    push_latency("call e2e (default polls)", lat)
+
+    # ---- call latency with the poll intervals tuned down.
+    w2 = Worker(worker_store, ["default"], concurrency=8, poll_interval_ms=25)
+    w2.task(noop)(lambda ctx, payload: {})
+    async with w2.background():
+        lat2 = await call_latencies(tasks, poll_ms=10)
+    push_latency("call e2e (poll 25ms, wait 10ms)", lat2)
+
+    if backend == "postgres":
+        await purge_all(tasks)  # shared DB — leave it clean
+    await tasks.close()
+    await worker_store.close()
+
+    print(f"cairnq-py bench  backend={backend}")
+    for row in rows:
+        print(f"  {row[0]:<32}{'  '.join(row[1:])}")
+
+
+async def main() -> None:
+    backend = sys.argv[1] if len(sys.argv) > 1 else "sqlite"
+    if backend == "postgres":
+        dsn = os.environ.get("CAIRNQ_TEST_PG_DSN")
+        if not dsn:
+            raise SystemExit("set CAIRNQ_TEST_PG_DSN (point it at a throwaway database)")
+        await run(backend, CairnQ.postgres(dsn), PostgresStore(dsn))
+    else:
+        with tempfile.TemporaryDirectory(prefix="cairnq-") as tmp:
+            path = os.path.join(tmp, "tasks.db")
+            await run(backend, CairnQ.sqlite(path), SQLiteStore(path))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

cairnq-0.2.0/cairnq/__init__.py

@@ -0,0 +1,53 @@
+"""CairnQ — SQLite-first, cross-language, storage-centered durable task runtime."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
+from .client import CairnQ
+from .context import TaskContext
+from .errors import (
+    AlreadyExists,
+    CairnQError,
+    LostLease,
+    ProtocolVersionMismatch,
+    SerializationError,
+    TaskCanceled,
+    TaskError,
+    TaskFailed,
+    TaskTimeout,
+)
+from .models import STATUSES, Task, TaskDef, TaskStatus
+from .store import Conflict, PostgresStore, SQLiteStore, TaskStore
+from .worker import Worker
+
+# Read from the installed package metadata rather than repeated here: the version
+# lives in pyproject.toml alone, so a release bump can't leave this behind.
+try:
+    __version__ = _pkg_version("cairnq")
+except PackageNotFoundError:  # source tree without an install
+    __version__ = "0.0.0.dev0"
+
+__all__ = [
+    "CairnQ",
+    "Worker",
+    "TaskContext",
+    "Task",
+    "TaskDef",
+    "TaskStatus",
+    "STATUSES",
+    "Conflict",
+    "TaskStore",
+    "SQLiteStore",
+    "PostgresStore",
+    "CairnQError",
+    "AlreadyExists",
+    "TaskTimeout",
+    "TaskFailed",
+    "TaskCanceled",
+    "TaskError",
+    "LostLease",
+    "ProtocolVersionMismatch",
+    "SerializationError",
+]

cairnq-0.2.0/cairnq/_ids.py

@@ -0,0 +1,29 @@
+"""ULID-style id generation. Format pinned by PROTOCOL.md and asserted by
+conformance: `<prefix>_` + 26-char Crockford base32 (48-bit ms timestamp +
+80-bit randomness), lexicographically time-sortable. Must match the TS SDK."""
+
+from __future__ import annotations
+
+import os
+import time
+
+_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+
+
+def new_ulid(ts_ms: int | None = None) -> str:
+    if ts_ms is None:
+        ts_ms = int(time.time() * 1000)
+    value = (ts_ms << 80) | int.from_bytes(os.urandom(10), "big")
+    chars = []
+    for _ in range(26):
+        chars.append(_CROCKFORD[value & 0x1F])
+        value >>= 5
+    return "".join(reversed(chars))
+
+
+def new_id(prefix: str = "task") -> str:
+    return f"{prefix}_{new_ulid()}"
+
+
+def now_ms() -> int:
+    return int(time.time() * 1000)

cairnq-0.2.0/cairnq/_protocol/migrations/postgres/0001_init.sql

@@ -0,0 +1,75 @@
+-- CairnQ canonical schema (Postgres dialect) — protocol_version 1
+-- Single source of truth for the task table on Postgres. Idempotent; safe to
+-- re-run. Time is stored as bigint epoch milliseconds (`*_ms`), generated by the
+-- DB clock inside each statement (NOT supplied by the SDK — see sql/postgres/*).
+-- JSON columns are jsonb. Status is a CHECK constraint (not a PG enum) so the
+-- conformance status-set check matches across dialects. Ordered migrations are
+-- canonical; there is no separate schema.sql.
+
+create table if not exists cairnq_tasks (
+    id text primary key,
+
+    name text not null,
+    queue text not null default 'default',
+
+    status text not null check (
+        status in ('queued', 'running', 'succeeded', 'failed', 'canceled')
+    ),
+
+    payload jsonb not null,
+    result jsonb,
+    error jsonb,
+    metadata jsonb not null default '{}'::jsonb,
+
+    progress double precision,
+    message text,
+
+    attempt integer not null default 0,
+    max_attempts integer not null default 3,
+
+    priority integer not null default 0,
+
+    worker_id text,
+    lease_until_ms bigint,
+
+    run_at_ms bigint not null,
+
+    cancel_requested_at_ms bigint,
+
+    parent_id text,
+    root_id text,
+    correlation_id text,
+
+    created_at_ms bigint not null,
+    updated_at_ms bigint not null,
+    completed_at_ms bigint
+);
+
+-- Serves the claim query: WHERE queue=? AND status='queued' ORDER BY priority
+-- desc, created_at_ms asc (run_at_ms applied as a residual filter).
+create index if not exists cairnq_tasks_claim_idx
+    on cairnq_tasks (queue, status, priority desc, created_at_ms);
+create index if not exists cairnq_tasks_status_idx on cairnq_tasks (status);
+create index if not exists cairnq_tasks_name_idx on cairnq_tasks (name);
+create index if not exists cairnq_tasks_root_idx on cairnq_tasks (root_id);
+create index if not exists cairnq_tasks_correlation_idx on cairnq_tasks (correlation_id);
+
+-- key = business-stable pointer to the *current* task for that key.
+-- task_id = one concrete execution. Kept separate (not a unique constraint on
+-- tasks) so reuse / reject / replace are natural.
+create table if not exists cairnq_task_keys (
+    key text primary key,
+    task_id text not null references cairnq_tasks(id) on delete cascade,
+    created_at_ms bigint not null,
+    updated_at_ms bigint not null
+);
+
+create table if not exists cairnq_meta (
+    key text primary key,
+    value text not null
+);
+
+insert into cairnq_meta (key, value) values ('protocol_version', '1')
+    on conflict (key) do nothing;
+insert into cairnq_meta (key, value) values ('schema_version', '1')
+    on conflict (key) do nothing;

cairnq-0.2.0/cairnq/_protocol/migrations/postgres/0002_purge_index.sql

@@ -0,0 +1,6 @@
+-- Serves purge.sql: scan terminal tasks in completion order. Without it the
+-- retention sweep is a full table scan of exactly the rows that accumulate most.
+create index if not exists cairnq_tasks_completed_idx
+    on cairnq_tasks (completed_at_ms);
+
+update cairnq_meta set value = '2' where key = 'schema_version';

cairnq-0.2.0/cairnq/_protocol/migrations/postgres/0003_notify.sql

@@ -0,0 +1,38 @@
+-- Push-based wakeups (Postgres only). A row trigger emits:
+--   cairnq_queued  (payload: queue name)  when a task becomes claimable-soon:
+--                                         inserted queued, or requeued by a
+--                                         retryable fail / retry / recovery;
+--   cairnq_done    (payload: task id)     when a task reaches a terminal status.
+-- The trigger lives in the database, not in the SDKs, so every writer — either
+-- SDK, any version, even hand-run SQL — wakes listeners. See PROTOCOL.md
+-- ("Push wakeups") for the contract; in short, notifications only cut a poll
+-- sleep short and are never required for correctness. Additive:
+-- protocol_version stays 1.
+--
+-- Trigger guards, hottest write first:
+--   - WHEN keeps claim (-> 'running', the most frequent status write) from
+--     entering plpgsql at all;
+--   - UPDATE OF status keeps heartbeat/progress from firing the trigger;
+--   - the IS DISTINCT FROM checks skip a SET that rewrites the same value
+--     (e.g. cancel.sql on an already-running task).
+
+create or replace function cairnq_notify() returns trigger as $$
+begin
+    if new.status = 'queued'
+       and (tg_op = 'INSERT' or old.status is distinct from new.status) then
+        perform pg_notify('cairnq_queued', new.queue);
+    elsif tg_op = 'UPDATE'
+          and new.status in ('succeeded', 'failed', 'canceled')
+          and old.status not in ('succeeded', 'failed', 'canceled') then
+        perform pg_notify('cairnq_done', new.id);
+    end if;
+    return null;
+end;
+$$ language plpgsql;
+
+drop trigger if exists cairnq_tasks_notify on cairnq_tasks;
+create trigger cairnq_tasks_notify
+after insert or update of status on cairnq_tasks
+for each row
+when (new.status is distinct from 'running')
+execute function cairnq_notify();