ardiq 0.1.1__tar.gz

ardiq-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 17tayyy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ardiq-0.1.1/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: ardiq
+Version: 0.1.1
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Classifier: Framework :: AsyncIO
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Operating System :: POSIX :: Linux
+Requires-Dist: msgpack>=1.1.2
+Requires-Dist: typer>=0.26.0
+License-File: LICENSE
+Summary: A fast distributed task queue with a Rust core and a Python API, backed by Redis streams.
+Keywords: task-queue,redis,rust,async,asyncio,distributed,jobs,worker
+Author-email: 17tayyy <oscarfdst@proton.me>
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/17tayyy/ardiq
+
+# ArdiQ
+
+A fast distributed task queue with a **Rust core** and a clean **Python API**, backed by Redis streams.
+
+ArdiQ runs the worker loop and all Redis I/O in Rust (via [PyO3](https://pyo3.rs) + [tokio](https://tokio.rs)); you write tasks in plain Python. The two meet at a single async callback, with the GIL held only for the microseconds it takes to start a task and read its result — so a single process handles high concurrency.
+
+> **Early stage.** The engine is solid — priorities, retries with backoff, delayed/scheduled tasks, crash recovery, result storage — but the ergonomics layer is still growing.
+
+## Features
+
+- 🦀 **Rust core** — the loop and Redis I/O run on tokio, off the GIL
+- **Priority queues** — higher-priority tasks are consumed first
+- **Delayed & scheduled** tasks (`delay_ms` / `schedule_ms`)
+- **Automatic retries** with quadratic backoff, configurable per task
+- **Crash recovery** — in-flight tasks of a dead worker are reclaimed (`XAUTOCLAIM`)
+- **Results** with TTL, plus task **status** (`queued` / `running` / `complete` / `not_found`)
+- **Sync & async tasks** — blocking sync functions run in a thread pool
+- **CLI worker** (`ardiq run module:app`) and **burst mode** (drain the queue and exit)
+
+## Installation
+
+ArdiQ isn't on PyPI yet. Build it from source — you'll need [Rust](https://rustup.rs) and [uv](https://docs.astral.sh/uv/):
+
+```console
+$ git clone https://github.com/17tayyy/ardiq
+$ cd ardiq
+$ uv sync            # builds the Rust extension and installs dependencies
+```
+
+You also need a Redis server. For local development:
+
+```console
+$ docker compose up -d
+```
+
+## Quickstart
+
+Define an app and some tasks (`example.py`):
+
+```python
+from ardiq import Ardiq
+
+app = Ardiq(redis_url="redis://localhost:6379", queue_name="example")
+
+
+@app.task()
+async def add(a: int, b: int) -> int:
+    return a + b
+
+
+@app.task(max_retries=3)
+def slow_double(x: int) -> int:   # sync task — runs in a thread
+    return x * 2
+```
+
+Start a worker:
+
+```console
+$ ardiq run example:app
+```
+
+Enqueue tasks from anywhere and read their results:
+
+```python
+import asyncio
+from example import add
+
+
+async def main():
+    job = await add.enqueue(2, 3)        # returns a Job handle
+    print(job.id)
+    print(await job.status())            # 'queued' | 'running' | 'complete'
+    print(await job.result(timeout=5))   # waits → TaskResult(success=True, value=5, tries=1)
+
+
+asyncio.run(main())
+```
+
+Or run the whole thing in one process with `python example.py`, which enqueues a
+few tasks and processes them in burst mode.
+
+## Configuration
+
+`Ardiq(...)` accepts:
+
+| Option | Default | Description |
+|---|---|---|
+| `redis_url` | `redis://localhost:6379` | Redis connection URL |
+| `queue_name` | `"default"` | Logical queue (key namespace) |
+| `priorities` | `["default"]` | Priority names, **lowest-first** |
+| `concurrency` | `16` | Max tasks running at once |
+| `prefetch` | `concurrency * 2` | Max tasks held in memory (drives backpressure) |
+| `idle_timeout_ms` | `60000` | When an unrenewed in-flight task may be reclaimed |
+| `result_ttl_ms` | `300000` | How long results live (`0` drops, negative keeps forever) |
+| `burst` | `False` | Exit once the queue drains |
+| `serializer` / `deserializer` | msgpack | Wire codec; pass `pickle.dumps`/`pickle.loads` to send datetimes/objects |
+
+`@app.task(...)` accepts `name`, `max_retries` (default 3), `backoff_ms`, `timeout` (seconds), and `priority`.
+Use `task.options(delay_ms=…, schedule_ms=…, priority=…, task_id=…).enqueue(...)` for one-off overrides.
+
+## Development
+
+```console
+$ docker compose up -d      # Redis on localhost:6379
+$ uv run pytest             # test suite (needs Redis)
+$ uv run ruff check .       # lint
+$ uv run ty check ardiq tests   # type-check
+```
+
+After changing the Rust core, rebuild with `uv sync --reinstall-package ardiq`.
+
+## License
+
+[MIT](LICENSE)
+

ardiq-0.1.1/README.md

@@ -0,0 +1,114 @@
+# ArdiQ
+
+A fast distributed task queue with a **Rust core** and a clean **Python API**, backed by Redis streams.
+
+ArdiQ runs the worker loop and all Redis I/O in Rust (via [PyO3](https://pyo3.rs) + [tokio](https://tokio.rs)); you write tasks in plain Python. The two meet at a single async callback, with the GIL held only for the microseconds it takes to start a task and read its result — so a single process handles high concurrency.
+
+> **Early stage.** The engine is solid — priorities, retries with backoff, delayed/scheduled tasks, crash recovery, result storage — but the ergonomics layer is still growing.
+
+## Features
+
+- 🦀 **Rust core** — the loop and Redis I/O run on tokio, off the GIL
+- **Priority queues** — higher-priority tasks are consumed first
+- **Delayed & scheduled** tasks (`delay_ms` / `schedule_ms`)
+- **Automatic retries** with quadratic backoff, configurable per task
+- **Crash recovery** — in-flight tasks of a dead worker are reclaimed (`XAUTOCLAIM`)
+- **Results** with TTL, plus task **status** (`queued` / `running` / `complete` / `not_found`)
+- **Sync & async tasks** — blocking sync functions run in a thread pool
+- **CLI worker** (`ardiq run module:app`) and **burst mode** (drain the queue and exit)
+
+## Installation
+
+ArdiQ isn't on PyPI yet. Build it from source — you'll need [Rust](https://rustup.rs) and [uv](https://docs.astral.sh/uv/):
+
+```console
+$ git clone https://github.com/17tayyy/ardiq
+$ cd ardiq
+$ uv sync            # builds the Rust extension and installs dependencies
+```
+
+You also need a Redis server. For local development:
+
+```console
+$ docker compose up -d
+```
+
+## Quickstart
+
+Define an app and some tasks (`example.py`):
+
+```python
+from ardiq import Ardiq
+
+app = Ardiq(redis_url="redis://localhost:6379", queue_name="example")
+
+
+@app.task()
+async def add(a: int, b: int) -> int:
+    return a + b
+
+
+@app.task(max_retries=3)
+def slow_double(x: int) -> int:   # sync task — runs in a thread
+    return x * 2
+```
+
+Start a worker:
+
+```console
+$ ardiq run example:app
+```
+
+Enqueue tasks from anywhere and read their results:
+
+```python
+import asyncio
+from example import add
+
+
+async def main():
+    job = await add.enqueue(2, 3)        # returns a Job handle
+    print(job.id)
+    print(await job.status())            # 'queued' | 'running' | 'complete'
+    print(await job.result(timeout=5))   # waits → TaskResult(success=True, value=5, tries=1)
+
+
+asyncio.run(main())
+```
+
+Or run the whole thing in one process with `python example.py`, which enqueues a
+few tasks and processes them in burst mode.
+
+## Configuration
+
+`Ardiq(...)` accepts:
+
+| Option | Default | Description |
+|---|---|---|
+| `redis_url` | `redis://localhost:6379` | Redis connection URL |
+| `queue_name` | `"default"` | Logical queue (key namespace) |
+| `priorities` | `["default"]` | Priority names, **lowest-first** |
+| `concurrency` | `16` | Max tasks running at once |
+| `prefetch` | `concurrency * 2` | Max tasks held in memory (drives backpressure) |
+| `idle_timeout_ms` | `60000` | When an unrenewed in-flight task may be reclaimed |
+| `result_ttl_ms` | `300000` | How long results live (`0` drops, negative keeps forever) |
+| `burst` | `False` | Exit once the queue drains |
+| `serializer` / `deserializer` | msgpack | Wire codec; pass `pickle.dumps`/`pickle.loads` to send datetimes/objects |
+
+`@app.task(...)` accepts `name`, `max_retries` (default 3), `backoff_ms`, `timeout` (seconds), and `priority`.
+Use `task.options(delay_ms=…, schedule_ms=…, priority=…, task_id=…).enqueue(...)` for one-off overrides.
+
+## Development
+
+```console
+$ docker compose up -d      # Redis on localhost:6379
+$ uv run pytest             # test suite (needs Redis)
+$ uv run ruff check .       # lint
+$ uv run ty check ardiq tests   # type-check
+```
+
+After changing the Rust core, rebuild with `uv sync --reinstall-package ardiq`.
+
+## License
+
+[MIT](LICENSE)

ardiq-0.1.1/ardiq/__init__.py

@@ -0,0 +1,346 @@
+"""ArdiQ Python API: the Ardiq app, the @task decorator, and task handles.
+
+The Rust core owns the loop and Redis I/O; an `Ardiq` app owns its task registry,
+its wire codec, and the executor the core calls back into for every task.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, NamedTuple
+
+import msgpack
+
+from ardiq._core import ArdiqCore
+
+__all__ = ["Ardiq", "Job", "Task", "TaskInfo", "TaskResult"]
+
+# Outcome codes for the Rust core's executor protocol.
+SUCCESS, FAILURE, RETRY = 0, 1, 2
+DEFAULT_MAX_RETRIES = 3
+
+
+def _now_ms() -> int:
+    return int(time.time() * 1000)
+
+
+def _default_dumps(obj: Any) -> bytes:
+    return msgpack.packb(obj)
+
+
+def _default_loads(data: bytes) -> Any:
+    return msgpack.unpackb(data, raw=False)
+
+
+@dataclass(slots=True)
+class _Registered:
+    fn: Callable[..., Any]
+    max_retries: int
+    backoff_ms: int
+    is_async: bool
+    timeout: float | None  # seconds; None = no timeout
+
+
+class TaskResult(NamedTuple):
+    """A decoded result envelope. On failure, `value` holds the error repr.
+
+    Times are epoch ms: `enqueue_time` when enqueued, `start`/`finish` around
+    execution.
+    """
+
+    success: bool
+    value: Any
+    tries: int
+    enqueue_time: int = 0
+    start: int = 0
+    finish: int = 0
+
+    @property
+    def duration_ms(self) -> int:
+        return self.finish - self.start
+
+
+class TaskInfo(NamedTuple):
+    """Snapshot of an unfinished task (queued, scheduled, or running)."""
+
+    task_id: str
+    fn_name: str
+    args: tuple
+    kwargs: dict
+    enqueue_time: int
+    tries: int
+    status: str
+    scheduled_at: int | None = None  # epoch ms if waiting in the delayed queue
+
+
+@dataclass(frozen=True, slots=True)
+class Job:
+    """Handle to an enqueued task."""
+
+    app: Ardiq
+    id: str
+
+    async def result(self, timeout: float | None = None) -> TaskResult | None:
+        return await self.app.result(self.id, timeout=timeout)
+
+    async def status(self) -> str:
+        return await self.app.status(self.id)
+
+    async def info(self) -> TaskInfo | None:
+        return await self.app.info(self.id)
+
+
+class Task:
+    """A registered task. Call it to run inline, or `.enqueue` to dispatch."""
+
+    def __init__(
+        self, app: Ardiq, name: str, fn: Callable[..., Any], priority: str | None
+    ):
+        self.app = app
+        self.name = name
+        self.fn = fn
+        self.priority = priority
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        return self.fn(*args, **kwargs)
+
+    async def enqueue(self, *args: Any, **kwargs: Any) -> Job:
+        return await self.app._enqueue(self, args, kwargs)
+
+    def options(
+        self,
+        *,
+        task_id: str | None = None,
+        priority: str | None = None,
+        delay_ms: int = 0,
+        schedule_ms: int = 0,
+        expire_ms: int = 0,
+    ) -> _BoundTask:
+        return _BoundTask(self, task_id, priority, delay_ms, schedule_ms, expire_ms)
+
+
+@dataclass(frozen=True, slots=True)
+class _BoundTask:
+    """A task plus enqueue options, kept off `enqueue(*args, **kwargs)`."""
+
+    task: Task
+    task_id: str | None
+    priority: str | None
+    delay_ms: int
+    schedule_ms: int
+    expire_ms: int
+
+    async def enqueue(self, *args: Any, **kwargs: Any) -> Job:
+        return await self.task.app._enqueue(
+            self.task,
+            args,
+            kwargs,
+            task_id=self.task_id,
+            priority=self.priority,
+            delay_ms=self.delay_ms,
+            schedule_ms=self.schedule_ms,
+            expire_ms=self.expire_ms,
+        )
+
+
+class Ardiq:
+    """App: owns the core, its task registry, and its wire codec."""
+
+    def __init__(
+        self,
+        redis_url: str | None = None,
+        queue_name: str = "default",
+        priorities: list[str] | None = None,
+        *,
+        serializer: Callable[[Any], bytes] | None = None,
+        deserializer: Callable[[bytes], Any] | None = None,
+        **core_kwargs: Any,
+    ):
+        self._dumps = serializer or _default_dumps
+        self._loads = deserializer or _default_loads
+        self._registry: dict[str, _Registered] = {}
+        config = {
+            "redis_url": redis_url,
+            "queue_name": queue_name,
+            "priorities": priorities,
+            **core_kwargs,
+        }
+        self._core = ArdiqCore({k: v for k, v in config.items() if v is not None})
+
+    @property
+    def worker_id(self) -> str:
+        return self._core.worker_id
+
+    @property
+    def burst(self) -> bool:
+        return self._core.burst
+
+    @burst.setter
+    def burst(self, value: bool) -> None:
+        self._core.burst = value
+
+    @property
+    def tasks(self) -> list[str]:
+        """Names of the registered tasks."""
+        return list(self._registry)
+
+    def task(
+        self,
+        fn: Callable[..., Any] | None = None,
+        *,
+        name: str | None = None,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        backoff_ms: int = 0,
+        timeout: float | None = None,
+        priority: str | None = None,
+    ) -> Any:
+        def wrap(fn: Callable[..., Any]) -> Task:
+            task_name = name or getattr(fn, "__name__", None)
+            if task_name is None:
+                raise TypeError("@task needs an explicit name for this callable")
+            self._registry[task_name] = _Registered(
+                fn,
+                max_retries,
+                backoff_ms,
+                asyncio.iscoroutinefunction(fn),
+                timeout,
+            )
+            return Task(self, task_name, fn, priority)
+
+        return wrap(fn) if fn is not None else wrap
+
+    async def _enqueue(
+        self,
+        task: Task,
+        args: tuple,
+        kwargs: dict,
+        *,
+        task_id: str | None = None,
+        priority: str | None = None,
+        delay_ms: int = 0,
+        schedule_ms: int = 0,
+        expire_ms: int = 0,
+    ) -> Job:
+        job_id = task_id or uuid.uuid4().hex
+        payload = self._pack(task.name, args, kwargs)
+        await self._core.enqueue(
+            job_id, payload, priority or task.priority, delay_ms, schedule_ms, expire_ms
+        )
+        return Job(self, job_id)
+
+    def _pack(self, fn_name: str, args: tuple, kwargs: dict) -> bytes:
+        return self._dumps(
+            {"f": fn_name, "a": list(args), "k": kwargs, "t": _now_ms()}
+        )
+
+    def _envelope(
+        self, success: bool, result: Any, tries: int, enqueue_time: int, start: int
+    ) -> bytes:
+        return self._dumps(
+            {
+                "s": success,
+                "r": result,
+                "t": tries,
+                "et": enqueue_time,
+                "st": start,
+                "ft": _now_ms(),
+            }
+        )
+
+    def _unpack(self, raw: bytes | None) -> TaskResult | None:
+        if raw is None:
+            return None
+        env = self._loads(raw)
+        return TaskResult(
+            env["s"],
+            env["r"],
+            env["t"],
+            env.get("et", 0),
+            env.get("st", 0),
+            env.get("ft", 0),
+        )
+
+    async def _execute(
+        self, task_id: str, payload: bytes, tries: int
+    ) -> tuple[int, bytes, int]:
+        """The core's per-task callback. Returns (outcome, result_bytes, retry_ms)."""
+        data = self._loads(payload)
+        enqueue_time = int(data.get("t", 0))
+        start = _now_ms()
+        reg = self._registry.get(data["f"])
+        if reg is None:
+            env = self._envelope(
+                False, f"unknown task {data['f']!r}", tries, enqueue_time, start
+            )
+            return FAILURE, env, 0
+
+        try:
+            if reg.is_async:
+                coro = reg.fn(*data["a"], **data["k"])
+            else:
+                coro = asyncio.to_thread(reg.fn, *data["a"], **data["k"])
+            if reg.timeout is not None:
+                result = await asyncio.wait_for(coro, reg.timeout)
+            else:
+                result = await coro
+        except Exception as exc:
+            if isinstance(exc, TimeoutError) and reg.timeout is not None:
+                err = f"timed out after {reg.timeout}s"
+            else:
+                err = repr(exc)
+            if tries <= reg.max_retries:
+                return RETRY, b"", reg.backoff_ms  # 0 = core's default backoff
+            return FAILURE, self._envelope(False, err, tries, enqueue_time, start), 0
+
+        return SUCCESS, self._envelope(True, result, tries, enqueue_time, start), 0
+
+    async def run(self) -> None:
+        await self._core.run(self._execute)
+
+    def stop(self) -> None:
+        self._core.stop()
+
+    async def queue_size(self) -> int:
+        return await self._core.queue_size()
+
+    async def result(
+        self, task_id: str, timeout: float | None = None
+    ) -> TaskResult | None:
+        """Fetch a task's result. With `timeout` (seconds), wait for it to be
+        stored, raising `TimeoutError` if it isn't in time; without, return the
+        result now or `None` if it isn't ready."""
+        if timeout is None:
+            return self._unpack(await self._core.result(task_id))
+        deadline = time.monotonic() + timeout
+        while True:
+            raw = await self._core.result(task_id)
+            if raw is not None:
+                return self._unpack(raw)
+            if time.monotonic() >= deadline:
+                raise TimeoutError(f"no result for {task_id!r} within {timeout}s")
+            await asyncio.sleep(0.05)
+
+    async def status(self, task_id: str) -> str:
+        return await self._core.status(task_id)
+
+    async def info(self, task_id: str) -> TaskInfo | None:
+        """Metadata for an unfinished task, or `None` if it's finished/unknown
+        (use `result` for finished tasks)."""
+        payload, tries, scheduled_at = await self._core.task_info(task_id)
+        if payload is None:
+            return None
+        data = self._loads(payload)
+        return TaskInfo(
+            task_id=task_id,
+            fn_name=data["f"],
+            args=tuple(data["a"]),
+            kwargs=data["k"],
+            enqueue_time=int(data["t"]),
+            tries=tries,
+            status=await self.status(task_id),
+            scheduled_at=scheduled_at or None,
+        )

ardiq-0.1.1/ardiq/__main__.py

@@ -0,0 +1,4 @@
+from ardiq.cli import main
+
+if __name__ == "__main__":
+    main()

ardiq-0.1.1/ardiq/_core.pyi

@@ -0,0 +1,29 @@
+"""Type stubs for the Rust extension `ardiq._core`."""
+
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+Executor = Callable[[str, bytes, int], Awaitable[tuple[int, bytes, int]]]
+
+def init_logging(verbose: bool) -> None: ...
+
+class ArdiqCore:
+    def __init__(self, config: dict[str, Any]) -> None: ...
+    @property
+    def worker_id(self) -> str: ...
+    burst: bool
+    def enqueue(
+        self,
+        task_id: str,
+        payload: bytes,
+        priority: str | None = None,
+        delay_ms: int = 0,
+        schedule_ms: int = 0,
+        expire_ms: int = 0,
+    ) -> Awaitable[bool]: ...
+    def run(self, callback: Executor) -> Awaitable[None]: ...
+    def stop(self) -> None: ...
+    def queue_size(self) -> Awaitable[int]: ...
+    def result(self, task_id: str) -> Awaitable[bytes | None]: ...  # core: no timeout
+    def status(self, task_id: str) -> Awaitable[str]: ...
+    def task_info(self, task_id: str) -> Awaitable[tuple[bytes | None, int, int]]: ...