fronta 0.1.0__py3-none-any.whl

fronta/__init__.py

@@ -0,0 +1,58 @@
+"""Fronta: distributed task processing on PostgreSQL with sandboxed process execution."""
+
+from importlib.metadata import version
+
+from fronta.config import Settings
+from fronta.definitions import Context, ProcessTaskDefinition, TaskDefinition, process_task, task
+from fronta.errors import (
+    ConfigurationError,
+    FrontaError,
+    InputValidationError,
+    InvalidInput,
+    NonRetryableError,
+    NotCancellable,
+    PayloadTooLarge,
+    ProgressTooLarge,
+    ResultSerializationError,
+    SandboxError,
+    TaskNotFound,
+    UnknownTaskType,
+)
+from fronta.model import Backoff, Policy, Sandbox, State, TaskRow, TaskSummary, TaskTypeRow
+from fronta.runtime import close_pool, configure, open_pool
+from fronta.worker import Worker
+
+__version__: str = version("fronta")
+
+__all__ = [
+    "Backoff",
+    "ConfigurationError",
+    "Context",
+    "FrontaError",
+    "InputValidationError",
+    "InvalidInput",
+    "NonRetryableError",
+    "NotCancellable",
+    "PayloadTooLarge",
+    "Policy",
+    "ProcessTaskDefinition",
+    "ProgressTooLarge",
+    "ResultSerializationError",
+    "Sandbox",
+    "SandboxError",
+    "Settings",
+    "State",
+    "TaskDefinition",
+    "TaskNotFound",
+    "TaskRow",
+    "TaskSummary",
+    "TaskTypeRow",
+    "UnknownTaskType",
+    "Worker",
+    "__version__",
+    "close_pool",
+    "configure",
+    "open_pool",
+    "process_task",
+    "task",
+]

fronta/cli.py

@@ -0,0 +1,134 @@
+"""`fronta` command line: `db init`, `worker`, `server`."""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import logging
+import os
+import sys
+from pathlib import Path
+
+import click
+import psycopg
+from pydantic import ValidationError
+
+from fronta import store
+from fronta.config import Settings
+from fronta.errors import ConfigurationError
+from fronta.worker import Worker
+
+log = logging.getLogger(__name__)
+
+SERVER_EXTRA_MODULES = frozenset({"fastapi", "starlette", "uvicorn", "jinja2", "mcp", "jsonschema"})
+"""Top-level modules provided by the `server` extra."""
+
+
+def configure_logging() -> None:
+    """Our loggers follow `LOG_LEVEL_OURS` (default INFO), libraries `LOG_LEVEL_LIBS` (WARNING)."""
+    logging.basicConfig(
+        level=os.environ.get("LOG_LEVEL_LIBS", "WARNING").upper(),
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        stream=sys.stderr,
+    )
+    logging.getLogger("fronta").setLevel(os.environ.get("LOG_LEVEL_OURS", "INFO").upper())
+
+
+def load_object(target: str, expected: type) -> object:
+    """Import `module:attr` (the current directory is importable, as with uvicorn)."""
+    module_name, sep, attr = target.partition(":")
+    if not sep or not module_name or not attr:
+        msg = f"expected module:attr, got {target!r}"
+        raise click.BadParameter(msg)
+    cwd = str(Path.cwd())
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError as exc:
+        msg = f"cannot import {module_name!r}: {exc}"
+        raise click.BadParameter(msg) from exc
+    try:
+        obj = getattr(module, attr)
+    except AttributeError as exc:
+        msg = f"{module_name!r} has no attribute {attr!r}"
+        raise click.BadParameter(msg) from exc
+    if not isinstance(obj, expected):
+        msg = f"{target} is {type(obj).__name__}, expected {expected.__name__}"
+        raise click.BadParameter(msg)
+    return obj
+
+
+@click.group()
+@click.version_option(package_name="fronta")
+def main() -> None:
+    """Fronta: distributed task processing on PostgreSQL."""
+
+
+@main.group()
+def db() -> None:
+    """Database administration."""
+
+
+@db.command("init")
+@click.option("--dsn", envvar="FRONTA_DSN", required=True, help="PostgreSQL DSN (or FRONTA_DSN).")
+def db_init(dsn: str) -> None:
+    """Create the `fronta` schema (idempotent)."""
+
+    async def run() -> None:
+        async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as conn:
+            await store.init_schema(conn)
+
+    try:
+        asyncio.run(run())
+    except psycopg.Error as exc:
+        raise click.ClickException(f"database error: {exc}") from exc
+    click.echo("fronta schema is ready")
+
+
+@main.command()
+@click.argument("target")
+def worker(target: str) -> None:
+    """Run the worker TARGET (`module:attr`, a `fronta.Worker`) until SIGTERM/SIGINT."""
+    configure_logging()
+    instance = load_object(target, Worker)
+    assert isinstance(instance, Worker)  # noqa: S101  # narrowed by load_object
+    try:
+        settings = instance.settings  # FRONTA_* is read here, not when the module was imported
+    except ValidationError as exc:
+        raise click.ClickException(f"invalid settings: {exc}") from exc
+    log.info("worker target %s, concurrency %d", target, settings.concurrency)
+    try:
+        sys.exit(asyncio.run(instance.run()))
+    except psycopg.OperationalError as exc:  # could not reach the database at start
+        raise click.ClickException(f"database unavailable: {exc}") from exc
+
+
+@main.command()
+@click.option(
+    "--host", default=None, help="Bind address (default FRONTA_SERVER_HOST or 127.0.0.1)."
+)
+@click.option("--port", default=None, type=int, help="Port (default FRONTA_SERVER_PORT or 8000).")
+def server(host: str | None, port: int | None) -> None:
+    """Serve the REST API, the MCP endpoint and the dashboard (needs `fronta[server]`)."""
+    configure_logging()
+    try:
+        from fronta.server import serve  # noqa: PLC0415  # optional dependency, loaded on demand
+    except ImportError as exc:
+        if (exc.name or "").split(".")[0] not in SERVER_EXTRA_MODULES:
+            raise  # a genuine broken import, not a missing extra
+        msg = f"the server needs the optional dependencies: pip install 'fronta[server]' ({exc})"
+        raise click.ClickException(msg) from exc
+    overrides = {
+        k: v for k, v in {"server_host": host, "server_port": port}.items() if v is not None
+    }
+    try:
+        settings = Settings()  # type: ignore[call-arg]  # pydantic-settings reads FRONTA_DSN
+        if overrides:
+            settings = Settings(**{**settings.model_dump(), **overrides})  # re-validated
+    except ValidationError as exc:
+        raise click.ClickException(f"invalid settings: {exc}") from exc
+    try:
+        serve(settings)
+    except ConfigurationError as exc:
+        raise click.ClickException(str(exc)) from exc

fronta/codec.py

@@ -0,0 +1,111 @@
+"""JSON encoding with byte caps and error-metadata shaping.
+
+Caps count UTF-8 bytes of the compact JSON encoding (`separators=(",", ":")`, non-ASCII kept as is).
+"""
+
+from __future__ import annotations
+
+import json
+import traceback
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from fronta.model import JSON
+
+_JSON_KW: dict[str, Any] = {"separators": (",", ":"), "ensure_ascii": False, "allow_nan": False}
+
+
+class Unstorable(ValueError):
+    """The value cannot live in a `jsonb` column (NUL characters, NaN, infinity)."""
+
+
+class OverCap(ValueError):
+    """The encoding exceeds its byte cap."""
+
+
+def encode(value: JSON) -> str:
+    """Compact JSON.
+
+    Raises `Unstorable` for NaN/inf and for NUL characters (which `jsonb` rejects), and
+    `TypeError` for anything that is not JSON: unknown objects and non-string mapping keys
+    (`json.dumps` would silently turn `{1: ...}` into `{"1": ...}`).
+    """
+    _check(value)
+    try:
+        return json.dumps(value, **_JSON_KW)
+    except ValueError as exc:
+        raise Unstorable(str(exc)) from exc
+
+
+def _check(value: object) -> None:
+    if isinstance(value, str):
+        if "\x00" in value:
+            msg = "NUL characters cannot be stored in jsonb"
+            raise Unstorable(msg)
+    elif isinstance(value, dict):
+        for key, item in value.items():
+            if not isinstance(key, str):
+                msg = f"mapping keys must be strings, got {type(key).__name__}"
+                raise TypeError(msg)
+            _check(key)
+            _check(item)
+    elif isinstance(value, list | tuple):
+        for item in value:
+            _check(item)
+
+
+def sanitize(text: str) -> str:
+    """Make text storable: NUL is the one code point `jsonb` rejects."""
+    return text.replace("\x00", "\ufffd")
+
+
+def utf8_len(text: str) -> int:
+    return len(text.encode("utf-8"))
+
+
+def encode_capped(value: JSON, cap: int, what: str) -> str:
+    """Encode and enforce the cap. Raises `OverCap` with a message naming `what`."""
+    text = encode(value)
+    size = utf8_len(text)
+    if size > cap:
+        msg = f"{what} is {size} bytes, cap is {cap} bytes"
+        raise OverCap(msg)
+    return text
+
+
+def error_metadata(exc: BaseException, cap: int) -> dict[str, Any]:
+    """Structured error for a failed attempt: type, message, traceback — truncated to `cap`."""
+    tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
+    return truncate(
+        {"type": type(exc).__qualname__, "message": sanitize(str(exc)), "traceback": sanitize(tb)},
+        cap,
+        keep_tail=("traceback",),
+    )
+
+
+def truncate(error: dict[str, Any], cap: int, keep_tail: tuple[str, ...] = ()) -> dict[str, Any]:
+    """Shrink the longest string fields until the encoding fits `cap`; marks `truncated: true`.
+
+    Fields in `keep_tail` keep their end (the useful part of a traceback or a log stream); other
+    fields keep their start. Slicing happens on code points, so the output stays valid UTF-8.
+    Always terminates: a field that cannot shrink further is emptied, and when nothing is left
+    to cut only the type survives.
+    """
+    data = dict(error)
+    if utf8_len(encode(data)) <= cap:
+        return data
+    data["truncated"] = True
+    while utf8_len(encode(data)) > cap:
+        name, text = max(
+            ((k, v) for k, v in data.items() if isinstance(v, str) and v),
+            key=lambda kv: len(kv[1]),
+            default=(None, ""),
+        )
+        if name is None:
+            return {"type": str(error.get("type", ""))[:64], "truncated": True}
+        keep = len(text) // 2
+        if keep == 0:
+            data[name] = ""
+        else:
+            data[name] = text[-keep:] if name in keep_tail else text[:keep]
+    return data

fronta/config.py

@@ -0,0 +1,62 @@
+"""Typed configuration: parsed once from `FRONTA_*` environment variables, validated at startup."""
+
+from __future__ import annotations
+
+from pydantic import Field, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+KIB = 1024
+MIB = 1024 * KIB
+
+
+class Settings(BaseSettings):
+    """Runtime settings. Durations are seconds; caps are UTF-8 bytes of the JSON encoding."""
+
+    model_config = SettingsConfigDict(env_prefix="FRONTA_", extra="ignore", frozen=True)
+
+    dsn: str = Field(min_length=1, description="PostgreSQL connection string")
+
+    # Liveness
+    lease_s: float = Field(30.0, gt=0)
+    heartbeat_s: float = Field(10.0, gt=0)
+    reaper_interval_s: float = Field(15.0, gt=0)
+    poll_interval_s: float = Field(5.0, gt=0)
+    grace_s: float = Field(30.0, ge=0)
+    kill_timeout_s: float = Field(5.0, gt=0)
+
+    # Worker
+    concurrency: int = Field(10, ge=1)
+    pool_size: int = Field(4, ge=1)
+    connect_timeout_s: float = Field(10.0, ge=1)
+    statement_timeout_s: float = Field(30.0, ge=0.001)
+
+    # Retention
+    retention_s: float = Field(7 * 86400.0, ge=0)
+    purge_interval_s: float = Field(600.0, gt=0)
+    purge_batch: int = Field(1000, ge=1)
+
+    # Caps
+    payload_cap: int = Field(MIB, ge=KIB)
+    result_cap: int = Field(MIB, ge=KIB)
+    progress_cap: int = Field(64 * KIB, ge=64)
+    error_cap: int = Field(64 * KIB, ge=KIB)
+    list_page_size: int = Field(50, ge=1)
+    list_page_max: int = Field(200, ge=1)
+
+    # Server
+    server_host: str = "127.0.0.1"
+    server_port: int = Field(8000, ge=1, le=65535)
+    server_token: str | None = Field(None, min_length=1)  # an empty secret is a misconfiguration
+
+    # Sandbox
+    bwrap_path: str = "bwrap"
+
+    @model_validator(mode="after")
+    def _check_relations(self) -> Settings:
+        if self.heartbeat_s >= self.lease_s:
+            msg = f"heartbeat_s ({self.heartbeat_s}) must be shorter than lease_s ({self.lease_s})"
+            raise ValueError(msg)
+        if self.list_page_size > self.list_page_max:
+            msg = "list_page_size must not exceed list_page_max"
+            raise ValueError(msg)
+        return self

fronta/definitions.py

@@ -0,0 +1,229 @@
+"""Task definitions: the `task` / `process_task` decorators, `Sandbox`, and `enqueue`."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable, Mapping
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Any, Protocol
+
+from pydantic import BaseModel
+
+from fronta import codec, runtime, store
+from fronta.errors import PayloadTooLarge
+from fronta.model import Backoff, Executor, NewTask, Policy, Sandbox, TaskTypeSpec
+
+if TYPE_CHECKING:
+    import asyncio
+    import logging
+
+    from fronta.model import JSON
+
+
+class Context[StateT](Protocol):
+    """What a handler receives. Implemented by the worker."""
+
+    task_id: int
+    attempt: int
+    state: StateT
+    log: logging.LoggerAdapter[logging.Logger]
+    cancelled: asyncio.Event
+
+    async def progress(self, value: JSON) -> None:
+        """Store progress (any JSON value up to the progress cap; over-cap raises)."""
+
+    async def enqueue[I: BaseModel](  # noqa: PLR0913  # public signature fixed by SPEC.md
+        self,
+        task: TaskDefinition[I, Any],
+        input: I,
+        *,
+        priority: int = 0,
+        run_at: datetime | None = None,
+        key: str | None = None,
+        concurrency_key: str | None = None,
+    ) -> int:
+        """Enqueue another task, immediately and independently of this task's outcome."""
+
+
+type Handler[InputT: BaseModel, OutputT] = Callable[[Context[Any], InputT], Awaitable[OutputT]]
+
+
+def _seconds(value: float | timedelta) -> float:
+    return value.total_seconds() if isinstance(value, timedelta) else float(value)
+
+
+class TaskDefinition[InputT: BaseModel, OutputT]:
+    """An asyncio task type: name, models, policy and the handler."""
+
+    executor = Executor.ASYNCIO
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        input_model: type[InputT],
+        output_model: type[BaseModel] | None,
+        policy: Policy,
+        handler: Handler[InputT, OutputT] | None,
+    ) -> None:
+        store.check_name(name)
+        self.name = name
+        self.input_model = input_model
+        self.output_model = output_model
+        self.policy = policy
+        self.handler = handler
+
+    def __repr__(self) -> str:
+        return f"<{type(self).__name__} {self.name!r}>"
+
+    @property
+    def spec(self) -> TaskTypeSpec:
+        return TaskTypeSpec(
+            name=self.name,
+            executor=self.executor,
+            input_schema=self.input_model.model_json_schema(mode="validation"),
+            output_schema=(
+                None
+                if self.output_model is None
+                else self.output_model.model_json_schema(mode="serialization")
+            ),
+            policy=self.policy,
+        )
+
+    def encode_input(self, input: InputT | Mapping[str, Any], cap: int) -> str:
+        """Validate against the input model and encode; enforce the payload cap."""
+        model = (
+            input if isinstance(input, self.input_model) else self.input_model.model_validate(input)
+        )
+        value = model.model_dump(mode="json")
+        if not isinstance(value, dict):
+            msg = f"input of {self.name!r} must serialize to a JSON object"
+            raise TypeError(msg)
+        try:
+            return codec.encode_capped(value, cap, "payload")
+        except codec.OverCap as exc:
+            raise PayloadTooLarge(str(exc)) from exc
+
+    async def enqueue(  # noqa: PLR0913  # public signature fixed by SPEC.md section 3
+        self,
+        input: InputT | Mapping[str, Any],
+        *,
+        conn: store.Conn | None = None,
+        priority: int = 0,
+        run_at: datetime | None = None,
+        key: str | None = None,
+        concurrency_key: str | None = None,
+    ) -> int:
+        """Enqueue and return the task id (or the id of the active task with the same key).
+
+        With `conn` the insert joins the caller's transaction (never committed by Fronta);
+        without it the process-global pool is used and the insert is committed here.
+        """
+        settings = runtime.get_settings()
+        new_task = NewTask(
+            type=self.name,
+            input_json=self.encode_input(input, settings.payload_cap),
+            policy=self.policy,
+            priority=priority,
+            run_at=run_at,
+            key=key,
+            concurrency_key=concurrency_key,
+        )
+        deadline = settings.statement_timeout_s
+        if conn is not None:
+            return await store.enqueue(conn, new_task, deadline_s=deadline)
+        pool = await runtime.get_pool()
+        async with pool.connection() as own_conn, own_conn.transaction():
+            return await store.enqueue(own_conn, new_task, deadline_s=deadline)
+
+
+class ProcessTaskDefinition[InputT: BaseModel](TaskDefinition[InputT, dict[str, Any]]):
+    """A sandboxed executable: input on stdin, result `{exit_code, stdout, stderr, truncated}`."""
+
+    executor = Executor.PROCESS
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        argv: tuple[str, ...],
+        input_model: type[InputT],
+        policy: Policy,
+        sandbox: Sandbox,
+    ) -> None:
+        if not argv:
+            msg = "argv must not be empty"
+            raise ValueError(msg)
+        super().__init__(
+            name, input_model=input_model, output_model=None, policy=policy, handler=None
+        )
+        self.argv = argv
+        self.sandbox = sandbox
+
+
+def _policy(
+    max_attempts: int,
+    attempt_timeout: float | timedelta,
+    backoff: Backoff,
+    max_concurrency: int | None,
+    max_concurrency_per_key: int | None,
+) -> Policy:
+    return Policy(
+        max_attempts=max_attempts,
+        attempt_timeout_s=_seconds(attempt_timeout),
+        backoff=backoff,
+        max_concurrency=max_concurrency,
+        max_concurrency_per_key=max_concurrency_per_key,
+    )
+
+
+def task[InputT: BaseModel, OutputT](  # noqa: PLR0913  # public signature fixed by SPEC.md
+    name: str,
+    *,
+    input: type[InputT],
+    output: type[BaseModel] | None = None,
+    max_attempts: int = 3,
+    attempt_timeout: float | timedelta = 3600.0,
+    backoff: Backoff | None = None,
+    max_concurrency: int | None = None,
+    max_concurrency_per_key: int | None = None,
+) -> Callable[[Handler[InputT, OutputT]], TaskDefinition[InputT, OutputT]]:
+    """Declare an asyncio task type. Decorates `async def handler(ctx, input) -> output`."""
+    policy = _policy(
+        max_attempts,
+        attempt_timeout,
+        backoff or Backoff(),
+        max_concurrency,
+        max_concurrency_per_key,
+    )
+
+    def decorate(handler: Handler[InputT, OutputT]) -> TaskDefinition[InputT, OutputT]:
+        return TaskDefinition(
+            name, input_model=input, output_model=output, policy=policy, handler=handler
+        )
+
+    return decorate
+
+
+def process_task[InputT: BaseModel](  # noqa: PLR0913  # public signature fixed by SPEC.md
+    name: str,
+    argv: tuple[str, ...] | list[str],
+    *,
+    input: type[InputT],
+    sandbox: Sandbox | None = None,
+    max_attempts: int = 3,
+    attempt_timeout: float | timedelta = 3600.0,
+    backoff: Backoff | None = None,
+    max_concurrency: int | None = None,
+    max_concurrency_per_key: int | None = None,
+) -> ProcessTaskDefinition[InputT]:
+    """Declare a sandboxed process task type. `argv[0]` is resolved inside the sandbox."""
+    policy = _policy(
+        max_attempts,
+        attempt_timeout,
+        backoff or Backoff(),
+        max_concurrency,
+        max_concurrency_per_key,
+    )
+    return ProcessTaskDefinition(
+        name, argv=tuple(argv), input_model=input, policy=policy, sandbox=sandbox or Sandbox()
+    )

fronta/errors.py

@@ -0,0 +1,53 @@
+"""Domain exceptions.
+
+They live in one bottom-layer module because they form the public API surface (`fronta.X`) and are
+raised and handled across layers (codec -> executors -> worker/server).
+"""
+
+
+class FrontaError(Exception):
+    """Base class of every Fronta error."""
+
+
+class ConfigurationError(FrontaError):
+    """Invalid settings or definition parameters."""
+
+
+class PayloadTooLarge(FrontaError):
+    """Enqueue input exceeds the payload cap (UTF-8 bytes of the JSON encoding)."""
+
+
+class ProgressTooLarge(FrontaError):
+    """`ctx.progress()` value exceeds the progress cap."""
+
+
+class InputValidationError(FrontaError):
+    """Stored input does not match the input model at claim time. Fails the task without retry."""
+
+
+class ResultSerializationError(FrontaError):
+    """Result is not JSON, not finite, over the cap, or violates the output model. No retry."""
+
+
+class NonRetryableError(FrontaError):
+    """Raised by a handler to fail the task without retry."""
+
+
+class UnknownTaskType(FrontaError):
+    """The task type has not been published by any worker."""
+
+
+class TaskNotFound(FrontaError):
+    """No task with this id."""
+
+
+class NotCancellable(FrontaError):
+    """The task is already terminal."""
+
+
+class InvalidInput(FrontaError):
+    """Server-side input does not match the published JSON schema."""
+
+
+class SandboxError(FrontaError):
+    """The sandbox cannot be set up on this host (probe failed) or a spawn failed."""