payload-documents-worker-builder 0.1.0__tar.gz

payload_documents_worker_builder-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.env

payload_documents_worker_builder-0.1.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: payload-documents-worker-builder
+Version: 0.1.0
+Summary: Parametrizable taskiq-based worker for Payload CMS — FastAPI kick app + Postgres LISTEN/NOTIFY broker + LlamaParse parse-document task.
+Project-URL: Homepage, https://github.com/Zetesis-Labs/PayloadAgents
+Project-URL: Repository, https://github.com/Zetesis-Labs/PayloadAgents
+Project-URL: Issues, https://github.com/Zetesis-Labs/PayloadAgents/issues
+Author: Zetesis Labs
+License: MIT
+Keywords: llamaparse,payload,postgres,taskiq,worker
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: fastapi>=0.115
+Requires-Dist: httpx>=0.28
+Requires-Dist: pydantic>=2.9
+Requires-Dist: structlog>=24.1
+Requires-Dist: taskiq-redis>=1.1.1
+Requires-Dist: taskiq>=0.11.18
+Requires-Dist: uvicorn[standard]>=0.34
+Description-Content-Type: text/markdown
+
+# payload-documents-worker-builder
+
+Parametrizable [taskiq](https://taskiq-python.github.io/) worker for Payload CMS, ready to drop into a workspace.
+
+Provides:
+
+- `RuntimeConfig` — pydantic config (one place for everything: DB DSN, Payload base URL, internal secret, LlamaCloud API key).
+- `create_broker(config)` — Redis stream broker (via `taskiq-redis`) with smart retry middleware.
+- `create_app(config)` — bundles broker + tasks + a small FastAPI HTTP "kicker" that exposes `POST /tasks/parse-document` so the Next.js side can enqueue parses without speaking the taskiq protocol directly.
+- `parse_document_task` — built-in task that uploads a Payload document to LlamaParse, polls until done, and writes `parsed_text` + `parse_status` back via Payload REST.
+
+## Usage
+
+```python
+from payload_documents_worker_builder import RuntimeConfig, create_app
+from pydantic import SecretStr
+
+config = RuntimeConfig(
+    app_name="my-worker",
+    redis_url="redis://redis:6379",
+    payload_url="http://app:3000",
+    payload_service_token=SecretStr("..."),  # Payload API key with write access
+    llama_cloud_api_key=SecretStr("..."),
+    internal_secret=SecretStr("dev"),
+    documents_collection_slug="documents",
+)
+
+app, broker = create_app(config)
+```
+
+Run two processes side by side:
+
+```bash
+uvicorn my_worker.main:app --host 0.0.0.0 --port 8001  # HTTP kicker
+taskiq worker my_worker.main:broker                     # task consumer
+```
+
+## Architecture
+
+```
+   Next.js (Payload)               payload-documents-worker (uvicorn)        payload-documents-worker (taskiq)
+  ─────────────────────────       ────────────────────────────    ────────────────────────────
+  POST /api/documents/{id}/parse  POST /tasks/parse-document      consume `parse_document` task
+   ├ stamps parse_status='queued'  ├ broker.kiq()                  ├ download file from Payload
+   └ HTTP→ kicker                  └ returns 202                   ├ upload to LlamaCloud
+                                                                   ├ poll status
+                                                                   └ PATCH parsed_text/status
+                                          │                                  │
+                                          └─────────── Redis Stream ─────────┘
+```
+
+## Public API
+
+```python
+from payload_documents_worker_builder import (
+    create_app,
+    create_broker,
+    RuntimeConfig,
+    LlamaParseClient,
+    PayloadClient,
+    parse_document_task,
+)
+```

payload_documents_worker_builder-0.1.0/README.md

@@ -0,0 +1,63 @@
+# payload-documents-worker-builder
+
+Parametrizable [taskiq](https://taskiq-python.github.io/) worker for Payload CMS, ready to drop into a workspace.
+
+Provides:
+
+- `RuntimeConfig` — pydantic config (one place for everything: DB DSN, Payload base URL, internal secret, LlamaCloud API key).
+- `create_broker(config)` — Redis stream broker (via `taskiq-redis`) with smart retry middleware.
+- `create_app(config)` — bundles broker + tasks + a small FastAPI HTTP "kicker" that exposes `POST /tasks/parse-document` so the Next.js side can enqueue parses without speaking the taskiq protocol directly.
+- `parse_document_task` — built-in task that uploads a Payload document to LlamaParse, polls until done, and writes `parsed_text` + `parse_status` back via Payload REST.
+
+## Usage
+
+```python
+from payload_documents_worker_builder import RuntimeConfig, create_app
+from pydantic import SecretStr
+
+config = RuntimeConfig(
+    app_name="my-worker",
+    redis_url="redis://redis:6379",
+    payload_url="http://app:3000",
+    payload_service_token=SecretStr("..."),  # Payload API key with write access
+    llama_cloud_api_key=SecretStr("..."),
+    internal_secret=SecretStr("dev"),
+    documents_collection_slug="documents",
+)
+
+app, broker = create_app(config)
+```
+
+Run two processes side by side:
+
+```bash
+uvicorn my_worker.main:app --host 0.0.0.0 --port 8001  # HTTP kicker
+taskiq worker my_worker.main:broker                     # task consumer
+```
+
+## Architecture
+
+```
+   Next.js (Payload)               payload-documents-worker (uvicorn)        payload-documents-worker (taskiq)
+  ─────────────────────────       ────────────────────────────    ────────────────────────────
+  POST /api/documents/{id}/parse  POST /tasks/parse-document      consume `parse_document` task
+   ├ stamps parse_status='queued'  ├ broker.kiq()                  ├ download file from Payload
+   └ HTTP→ kicker                  └ returns 202                   ├ upload to LlamaCloud
+                                                                   ├ poll status
+                                                                   └ PATCH parsed_text/status
+                                          │                                  │
+                                          └─────────── Redis Stream ─────────┘
+```
+
+## Public API
+
+```python
+from payload_documents_worker_builder import (
+    create_app,
+    create_broker,
+    RuntimeConfig,
+    LlamaParseClient,
+    PayloadClient,
+    parse_document_task,
+)
+```

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/__init__.py

@@ -0,0 +1,39 @@
+"""Public API for `payload-documents-worker-builder`.
+
+Mirrors the shape of `agno_agent_builder.__init__`: a single import surface
+that exposes the factory, the config, the clients and the tasks. Consumers
+should never reach into submodules.
+"""
+
+from payload_documents_worker_builder.app import WorkerApp, create_app
+from payload_documents_worker_builder.broker import create_broker
+from payload_documents_worker_builder.clients import (
+    LlamaParseClient,
+    LlamaParseError,
+    LlamaParseJob,
+    LlamaParseStatus,
+    PayloadClient,
+    PayloadError,
+)
+from payload_documents_worker_builder.config import RuntimeConfig
+from payload_documents_worker_builder.tasks import (
+    PARSE_DOCUMENT_TASK_NAME,
+    register_parse_document_task,
+)
+
+__all__ = [
+    "PARSE_DOCUMENT_TASK_NAME",
+    "LlamaParseClient",
+    "LlamaParseError",
+    "LlamaParseJob",
+    "LlamaParseStatus",
+    "PayloadClient",
+    "PayloadError",
+    "RuntimeConfig",
+    "WorkerApp",
+    "create_app",
+    "create_broker",
+    "register_parse_document_task",
+]
+
+__version__ = "0.1.0"

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/app.py

@@ -0,0 +1,49 @@
+"""Top-level factory.
+
+Consumers call ``create_app(config)`` and get back two ready-to-run handles:
+
+* ``broker`` — the taskiq broker. Pass it to the taskiq CLI:
+  ``taskiq worker my_worker.main:broker``.
+* ``app`` — the FastAPI HTTP kicker. Run with uvicorn:
+  ``uvicorn my_worker.main:app``.
+
+Both share the same ``RuntimeConfig`` so logs, retries, and credentials line
+up across processes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+
+from fastapi import FastAPI
+from taskiq import AsyncBroker
+
+from payload_documents_worker_builder.broker import create_broker
+from payload_documents_worker_builder.config import RuntimeConfig
+from payload_documents_worker_builder.http import create_http_app
+from payload_documents_worker_builder.lifecycle import configure_logging
+from payload_documents_worker_builder.tasks import register_parse_document_task
+
+
+@dataclass(slots=True, frozen=True)
+class WorkerApp:
+    """Bundle returned by :func:`create_app`. Exposed as a dataclass so consumers
+    can ``app, broker = create_app(config)`` (`__iter__` below) or address
+    fields by name explicitly."""
+
+    app: FastAPI
+    broker: AsyncBroker
+
+    def __iter__(self) -> Iterator[FastAPI | AsyncBroker]:
+        yield self.app
+        yield self.broker
+
+
+def create_app(config: RuntimeConfig) -> WorkerApp:
+    """Build the broker, register built-in tasks, and wrap a FastAPI kicker."""
+    configure_logging(config)
+    broker = create_broker(config)
+    register_parse_document_task(broker, config)
+    http_app = create_http_app(broker, config)
+    return WorkerApp(app=http_app, broker=broker)

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/broker.py

@@ -0,0 +1,19 @@
+"""Broker factory.
+
+Returns a configured `RedisStreamBroker` with `SmartRetryMiddleware`. Mirrors
+the pattern from nixon's `nixon_worker_core.broker_factory.create_broker`:
+one place that owns broker config so consumers never instantiate
+`RedisStreamBroker` directly.
+"""
+
+from __future__ import annotations
+
+from taskiq import AsyncBroker, SmartRetryMiddleware
+from taskiq_redis import RedisStreamBroker
+
+from payload_documents_worker_builder.config import RuntimeConfig
+
+
+def create_broker(config: RuntimeConfig) -> AsyncBroker:
+    """Build the broker the consumer's `main.py` should expose to taskiq."""
+    return RedisStreamBroker(url=config.redis_url).with_middlewares(SmartRetryMiddleware())

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/clients/__init__.py

@@ -0,0 +1,18 @@
+"""HTTP clients used by built-in tasks (Payload + LlamaParse)."""
+
+from payload_documents_worker_builder.clients.llama_parse import (
+    LlamaParseClient,
+    LlamaParseError,
+    LlamaParseJob,
+    LlamaParseStatus,
+)
+from payload_documents_worker_builder.clients.payload import PayloadClient, PayloadError
+
+__all__ = [
+    "LlamaParseClient",
+    "LlamaParseError",
+    "LlamaParseJob",
+    "LlamaParseStatus",
+    "PayloadClient",
+    "PayloadError",
+]

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/clients/_errors.py

@@ -0,0 +1,28 @@
+"""Shared error-handling helpers for HTTP clients in this package."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import httpx
+
+_DETAIL_TRUNCATE = 500
+
+
+def make_raise_for_status(
+    exc_cls: type[Exception], prefix: str
+) -> Callable[[httpx.Response, str], None]:
+    """Return a `_raise_for_status(response, op)` bound to the given exception class + prefix.
+
+    Each client (Payload, LlamaParse, …) wraps non-2xx responses in its own
+    exception type but the rest of the logic is identical: format
+    `<prefix> <op> failed: HTTP <code> — <body>` and raise.
+    """
+
+    def _raise_for_status(response: httpx.Response, op: str) -> None:
+        if response.is_success:
+            return
+        detail = response.text[:_DETAIL_TRUNCATE]
+        raise exc_cls(f"{prefix} {op} failed: HTTP {response.status_code} — {detail}")
+
+    return _raise_for_status

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/clients/llama_parse.py

@@ -0,0 +1,136 @@
+"""Minimal async LlamaParse client.
+
+Ported from `packages/payload-documents/src/llama-parse/client.ts`. Only the
+endpoints the parse-document task needs:
+
+* ``POST /api/parsing/upload``       — kick off a parse job
+* ``GET  /api/parsing/job/{id}``     — poll status
+* ``GET  /api/parsing/job/{id}/result/markdown`` — fetch parsed markdown
+
+Use as an async context manager so the underlying ``httpx.AsyncClient`` (and
+its connection pool) is shared across all calls within one task instead of a
+new TLS handshake per request::
+
+    async with LlamaParseClient(api_key=...) as client:
+        job = await client.upload(...)
+        ...
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from types import TracebackType
+from typing import Any, Literal, Self
+
+import httpx
+
+from ._errors import make_raise_for_status
+
+LlamaParseStatus = Literal["PENDING", "SUCCESS", "ERROR", "CANCELLED"]
+DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai"
+
+
+class LlamaParseError(Exception):
+    """Wraps any non-2xx response or transport failure with a helpful message."""
+
+
+_raise_for_status = make_raise_for_status(LlamaParseError, "LlamaParse")
+
+
+@dataclass(slots=True)
+class LlamaParseJob:
+    id: str
+    status: LlamaParseStatus
+    error: str | None = None
+
+
+class LlamaParseClient:
+    """Tiny httpx-backed client. Use via ``async with`` to share the httpx pool."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = 60.0,
+    ) -> None:
+        if not api_key:
+            raise LlamaParseError("LlamaParse API key is required")
+        self._base_url = base_url.rstrip("/")
+        self._headers = {"Authorization": f"Bearer {api_key}"}
+        self._timeout = timeout
+        self._http: httpx.AsyncClient | None = None
+
+    async def __aenter__(self) -> Self:
+        self._http = httpx.AsyncClient(timeout=self._timeout)
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if self._http is not None:
+            await self._http.aclose()
+            self._http = None
+
+    async def upload(
+        self,
+        *,
+        file_bytes: bytes,
+        filename: str,
+        language: str | None = None,
+        parsing_instruction: str | None = None,
+        mode: str | None = None,
+    ) -> LlamaParseJob:
+        data: dict[str, Any] = {}
+        if language is not None:
+            data["language"] = language
+        if parsing_instruction is not None:
+            data["parsing_instruction"] = parsing_instruction
+        if mode is not None:
+            data["parse_mode"] = mode
+
+        response = await self._client().post(
+            f"{self._base_url}/api/parsing/upload",
+            headers=self._headers,
+            files={"file": (filename, file_bytes)},
+            data=data,
+        )
+        _raise_for_status(response, "upload")
+        return _parse_job(response.json())
+
+    async def status(self, job_id: str) -> LlamaParseJob:
+        response = await self._client().get(
+            f"{self._base_url}/api/parsing/job/{job_id}",
+            headers=self._headers,
+        )
+        _raise_for_status(response, "status")
+        return _parse_job(response.json())
+
+    async def fetch_markdown(self, job_id: str) -> str:
+        response = await self._client().get(
+            f"{self._base_url}/api/parsing/job/{job_id}/result/markdown",
+            headers=self._headers,
+        )
+        _raise_for_status(response, "fetch_markdown")
+        markdown = response.json().get("markdown")
+        if not isinstance(markdown, str):
+            raise LlamaParseError(f"LlamaParse returned no markdown for job {job_id}")
+        return markdown
+
+    def _client(self) -> httpx.AsyncClient:
+        if self._http is None:
+            raise LlamaParseError(
+                "LlamaParseClient must be used inside `async with` (httpx pool not initialised)"
+            )
+        return self._http
+
+
+def _parse_job(payload: dict[str, Any]) -> LlamaParseJob:
+    return LlamaParseJob(
+        id=payload["id"],
+        status=payload.get("status", "PENDING"),
+        error=payload.get("error"),
+    )

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/clients/payload.py

@@ -0,0 +1,126 @@
+"""Tiny Payload CMS REST client used by built-in tasks.
+
+The worker only needs:
+
+* fetch the document context (parser knobs + filename)
+* fetch the binary attached to that document
+* stamp parse results back (parsed_text / parse_status / parse_error / ...)
+
+All Payload-side calls go through dedicated internal endpoints exposed by the
+``payload-documents`` plugin and authenticated with the shared
+``X-Internal-Secret`` header. The endpoints use Payload's local API with
+``overrideAccess: true`` server-side and the binary endpoint defers to a
+host-provided resolver for the actual storage read, so the plugin stays
+storage-agnostic and host apps can keep the documents collection's access
+control honestly locked down (multi-tenant filters, admin-only writes, etc.)
+without poking a service-account bypass into it.
+
+Use as an async context manager so the underlying ``httpx.AsyncClient`` (and
+its connection pool) is shared across all calls within one task instead of a
+new TLS handshake per request::
+
+    async with PayloadClient(base_url=..., internal_secret=...) as client:
+        ctx = await client.fetch_parse_context(slug, doc_id)
+        ...
+"""
+
+from __future__ import annotations
+
+from types import TracebackType
+from typing import Self
+
+import httpx
+
+from ._errors import make_raise_for_status
+from .types import ParseContext, ParseResultUpdate
+
+INTERNAL_SECRET_HEADER = "X-Internal-Secret"  # noqa: S105 — header name, not a secret value
+
+
+class PayloadError(Exception):
+    """Surfaced when Payload returns a non-2xx response."""
+
+
+_raise_for_status = make_raise_for_status(PayloadError, "Payload")
+
+
+class PayloadClient:
+    """Async REST client. Use via ``async with`` to share the httpx pool."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        internal_secret: str,
+        timeout: float = 60.0,
+    ) -> None:
+        if not base_url:
+            raise PayloadError("Payload base URL is required")
+        if not internal_secret:
+            raise PayloadError("Internal secret is required for plugin endpoints")
+        self._base_url = base_url.rstrip("/")
+        self._headers = {INTERNAL_SECRET_HEADER: internal_secret}
+        self._timeout = timeout
+        self._http: httpx.AsyncClient | None = None
+
+    async def __aenter__(self) -> Self:
+        self._http = httpx.AsyncClient(timeout=self._timeout)
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if self._http is not None:
+            await self._http.aclose()
+            self._http = None
+
+    async def fetch_parse_context(self, collection: str, doc_id: str | int) -> ParseContext:
+        """GET the plugin's internal read endpoint.
+
+        Returns a projection containing only the fields the worker needs to
+        drive the parse: ``id, url, filename, mimeType, language,
+        parsing_instruction, mode``.
+        """
+        path = self._endpoint(collection, doc_id, "parse-context")
+        response = await self._client().get(path, headers=self._headers)
+        _raise_for_status(response, f"GET {path}")
+        return response.json()
+
+    async def fetch_parse_file(self, collection: str, doc_id: str | int) -> bytes:
+        """GET the plugin's internal binary endpoint.
+
+        The plugin defers the actual storage read to a host-provided resolver
+        (S3/R2 GetObject, local fs, ...) and streams the result back.
+        """
+        path = self._endpoint(collection, doc_id, "parse-file")
+        response = await self._client().get(path, headers=self._headers)
+        _raise_for_status(response, f"GET {path}")
+        return response.content
+
+    async def submit_parse_result(
+        self,
+        collection: str,
+        doc_id: str | int,
+        data: ParseResultUpdate,
+    ) -> None:
+        """POST to the plugin's internal write endpoint.
+
+        Body is whitelisted server-side; only the parse_* fields are accepted
+        regardless of what we send.
+        """
+        path = self._endpoint(collection, doc_id, "parse-result")
+        response = await self._client().post(path, headers=self._headers, json=dict(data))
+        _raise_for_status(response, f"POST {path}")
+
+    def _client(self) -> httpx.AsyncClient:
+        if self._http is None:
+            raise PayloadError(
+                "PayloadClient must be used inside `async with` (httpx pool not initialised)"
+            )
+        return self._http
+
+    def _endpoint(self, collection: str, doc_id: str | int, op: str) -> str:
+        return f"{self._base_url}/api/{collection}/{doc_id}/{op}"

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/clients/types.py

@@ -0,0 +1,44 @@
+"""Typed dicts for the JSON contracts the Payload plugin endpoints expose.
+
+Mirrors the TypeScript types in `packages/payload-documents/src/plugin/types.ts`
+and `endpoints/parse-{context,result}-endpoint.ts`. Worth duplicating because
+the alternative — `dict[str, Any]` everywhere — drops type info at every
+boundary.
+"""
+
+from __future__ import annotations
+
+from typing import Literal, NotRequired, TypedDict
+
+ParseStatus = Literal["idle", "pending", "processing", "done", "error"]
+
+
+class ParseContext(TypedDict):
+    """Response shape of `GET /api/<collection>/<id>/parse-context`.
+
+    Field set is hard-coded server-side (see `parse-context-endpoint.ts`); the
+    plugin only returns what the worker needs to drive the LlamaParse upload.
+    """
+
+    id: str | int
+    url: NotRequired[str | None]
+    filename: NotRequired[str | None]
+    mimeType: NotRequired[str | None]
+    language: NotRequired[str | None]
+    parsing_instruction: NotRequired[str | None]
+    mode: NotRequired[str | None]
+
+
+class ParseResultUpdate(TypedDict, total=False):
+    """Request body for `POST /api/<collection>/<id>/parse-result`.
+
+    Fields are whitelisted server-side (see `parse-result-endpoint.ts`); any
+    keys outside this set are silently dropped, but typing them here means the
+    caller catches typos at lint/check time.
+    """
+
+    parsed_text: str | None
+    parse_status: ParseStatus
+    parse_error: str | None
+    parse_job_id: str | None
+    parsed_at: str | None

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/config.py

@@ -0,0 +1,67 @@
+"""Runtime configuration consumed by the worker library.
+
+Mirrors the shape of `agno_agent_builder.RuntimeConfig`: one pydantic model
+populated by the consumer, no env loading inside the lib so multi-tenant
+deploys can build several `RuntimeConfig` instances from a single env file.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field, HttpUrl, SecretStr
+
+
+class RuntimeConfig(BaseModel):
+    """All knobs the consumer needs to fill in to run a worker.
+
+    The attribute split mirrors the surface area of the upstream
+    `agno-agent-builder` so consumers using both libraries see the same
+    shape twice (just with `database_url`/`payload_url`/`llama_cloud_api_key`
+    instead of an `agent_source`).
+    """
+
+    app_name: str = Field(
+        description="FastAPI title and structlog identity. Shows up in logs and /health.",
+    )
+
+    # ── Broker ─────────────────────────────────────────────────────────────
+    redis_url: str = Field(
+        description="Redis connection URL used by taskiq-redis as the broker (e.g. redis://redis:6379).",
+    )
+
+    # ── Payload CMS ────────────────────────────────────────────────────────
+    payload_url: HttpUrl = Field(
+        description="Base URL for the Payload REST API (e.g. http://app:3000).",
+    )
+    documents_collection_slug: str = Field(
+        default="documents",
+        description="Payload collection slug for documents. Must expose the `parse_*` fields shipped by `@zetesis/payload-documents`.",
+    )
+
+    # ── LlamaParse ─────────────────────────────────────────────────────────
+    llama_cloud_api_key: SecretStr = Field(
+        description="LlamaCloud API key used to upload + poll parsing jobs.",
+    )
+    llama_parse_base_url: HttpUrl = Field(
+        default=HttpUrl("https://api.cloud.llamaindex.ai"),
+        description="Override only if you point to a self-hosted LlamaCloud-compatible service.",
+    )
+    llama_parse_poll_interval_s: float = Field(
+        default=5.0,
+        description="Seconds between successive LlamaCloud status polls.",
+    )
+    llama_parse_poll_timeout_s: float = Field(
+        default=600.0,
+        description="Hard cap on how long a single parse task waits before failing.",
+    )
+
+    # ── Internal HTTP kicker ──────────────────────────────────────────────
+    internal_secret: SecretStr = Field(
+        description="Shared secret required by every `POST /tasks/*` request (X-Internal-Secret header).",
+    )
+    public_paths: tuple[str, ...] = Field(
+        default=("/health", "/ready", "/docs", "/openapi.json"),
+        description="Paths the InternalAuthMiddleware lets through without the secret.",
+    )
+
+    # ── Logging ───────────────────────────────────────────────────────────
+    log_level: str = Field(default="INFO")

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/http.py

@@ -0,0 +1,87 @@
+"""HTTP kicker.
+
+Tiny FastAPI app the consumer process exposes so the Next.js side (or any
+HTTP client) can enqueue a task without speaking the taskiq Redis protocol.
+
+* ``GET /health`` and ``GET /ready`` — Kubernetes / Compose probes (no auth).
+* ``POST /tasks/parse-document`` — body ``{"document_id": "<id>"}``, gated by
+  the ``X-Internal-Secret`` header (matched against ``config.internal_secret``).
+"""
+
+from __future__ import annotations
+
+import hmac
+from typing import Any
+
+import structlog
+from fastapi import FastAPI, HTTPException, Request, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from taskiq import AsyncBroker
+
+from payload_documents_worker_builder.config import RuntimeConfig
+from payload_documents_worker_builder.tasks import PARSE_DOCUMENT_TASK_NAME
+
+logger = structlog.get_logger("payload_documents_worker_builder.http")
+
+
+class ParseDocumentRequest(BaseModel):
+    document_id: str
+
+
+def create_http_app(broker: AsyncBroker, config: RuntimeConfig) -> FastAPI:
+    """Build the FastAPI app the consumer hands to uvicorn."""
+    app = FastAPI(title=config.app_name)
+
+    @app.middleware("http")
+    async def _internal_auth(request: Request, call_next: Any) -> Any:
+        if request.url.path in config.public_paths:
+            return await call_next(request)
+        provided = request.headers.get("x-internal-secret", "")
+        expected = config.internal_secret.get_secret_value()
+        if not hmac.compare_digest(provided, expected):
+            return JSONResponse(
+                {"error": "Forbidden"},
+                status_code=status.HTTP_403_FORBIDDEN,
+            )
+        return await call_next(request)
+
+    @app.on_event("startup")
+    async def _startup() -> None:  # pyright: ignore[reportUnusedFunction]
+        if not broker.is_worker_process:
+            await broker.startup()
+            logger.info("Broker connected (kicker side)", url=config.redis_url)
+
+    @app.on_event("shutdown")
+    async def _shutdown() -> None:  # pyright: ignore[reportUnusedFunction]
+        if not broker.is_worker_process:
+            await broker.shutdown()
+
+    @app.get("/health")
+    async def health() -> dict[str, str]:  # pyright: ignore[reportUnusedFunction]
+        return {"status": "ok"}
+
+    @app.get("/ready")
+    async def ready() -> dict[str, str]:  # pyright: ignore[reportUnusedFunction]
+        return {"status": "ok"}
+
+    @app.post("/tasks/parse-document", status_code=status.HTTP_202_ACCEPTED)
+    async def kick_parse_document(  # pyright: ignore[reportUnusedFunction]
+        body: ParseDocumentRequest,
+    ) -> dict[str, str]:
+        if not body.document_id.strip():
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="document_id is required",
+            )
+        task = broker.find_task(PARSE_DOCUMENT_TASK_NAME)
+        if task is None:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Task {PARSE_DOCUMENT_TASK_NAME} is not registered",
+            )
+        await task.kiq(body.document_id)
+        logger.info("Enqueued parse-document task", document_id=body.document_id)
+        return {"status": "queued", "task": PARSE_DOCUMENT_TASK_NAME}
+
+    return app

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/lifecycle.py

@@ -0,0 +1,32 @@
+"""Logging + structured boot for the worker process.
+
+Mirrors the spirit of `agno_agent_builder.app.create_app` lifespan but kept
+much simpler — there's no registry/listener to bootstrap. We just configure
+structlog so taskiq + FastAPI logs share the same JSON sink.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import structlog
+
+from payload_documents_worker_builder.config import RuntimeConfig
+
+
+def configure_logging(config: RuntimeConfig) -> None:
+    """Idempotent structlog setup so both processes (uvicorn + taskiq) match."""
+    level = getattr(logging, config.log_level.upper(), logging.INFO)
+    logging.basicConfig(level=level, format="%(message)s")
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            structlog.processors.add_log_level,
+            structlog.processors.TimeStamper(fmt="iso"),
+            structlog.processors.StackInfoRenderer(),
+            structlog.processors.format_exc_info,
+            structlog.processors.JSONRenderer(),
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(level),
+        cache_logger_on_first_use=True,
+    )

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/tasks/__init__.py

@@ -0,0 +1,12 @@
+"""Built-in tasks.
+
+Currently exposes the LlamaParse parse-document task. Adding more tasks later
+is the same pattern: define them here and have `register_tasks` wire them.
+"""
+
+from payload_documents_worker_builder.tasks.parse_document import (
+    PARSE_DOCUMENT_TASK_NAME,
+    register_parse_document_task,
+)
+
+__all__ = ["PARSE_DOCUMENT_TASK_NAME", "register_parse_document_task"]

payload_documents_worker_builder-0.1.0/payload_documents_worker_builder/tasks/parse_document.py

@@ -0,0 +1,195 @@
+"""Parse-document task.
+
+Single responsibility: take a Payload document id, download its file, kick a
+LlamaParse upload, poll until the result is ready, write the parsed markdown
+back into Payload, and stamp `parse_status` accordingly.
+
+The task is registered against a broker via ``register_parse_document_task``
+so consumers can compose multiple workers on the same broker without us
+hard-coding the binding.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from datetime import UTC, datetime
+
+import httpx
+import structlog
+from taskiq import AsyncBroker
+
+from payload_documents_worker_builder.clients.llama_parse import (
+    LlamaParseClient,
+    LlamaParseError,
+    LlamaParseJob,
+)
+from payload_documents_worker_builder.clients.payload import PayloadClient, PayloadError
+from payload_documents_worker_builder.clients.types import ParseContext
+from payload_documents_worker_builder.config import RuntimeConfig
+
+PARSE_DOCUMENT_TASK_NAME = "documents.parse"
+DEFAULT_FILENAME = "upload.bin"
+
+logger = structlog.get_logger("payload_documents_worker_builder.parse_document")
+
+
+def register_parse_document_task(broker: AsyncBroker, config: RuntimeConfig) -> None:
+    """Bind the parse-document task to ``broker``.
+
+    The task is named ``documents.parse``. Kick it from any taskiq client
+    (or via the FastAPI HTTP kicker) with a single string arg: the Payload
+    document id (numeric ids serialise to string just fine).
+    """
+
+    decorator = broker.task(
+        task_name=PARSE_DOCUMENT_TASK_NAME,
+        retry_on_error=True,
+        max_retries=2,
+    )
+
+    async def parse_document(document_id: str) -> None:
+        await _run_parse_document(document_id, config)
+
+    decorator(parse_document)
+
+
+async def _run_parse_document(document_id: str, config: RuntimeConfig) -> None:
+    """Orchestrator: each phase is its own coroutine for unit-testability."""
+    log = logger.bind(document_id=document_id, collection=config.documents_collection_slug)
+    log.info("Parse document task started")
+
+    async with (
+        PayloadClient(
+            base_url=str(config.payload_url),
+            internal_secret=config.internal_secret.get_secret_value(),
+        ) as payload,
+        LlamaParseClient(
+            api_key=config.llama_cloud_api_key.get_secret_value(),
+            base_url=str(config.llama_parse_base_url),
+        ) as llama,
+    ):
+        try:
+            await _mark_processing(payload, config, document_id)
+            ctx, file_bytes = await _fetch_inputs(payload, config, document_id, log)
+            job = await _submit_to_llama(llama, ctx, file_bytes, log)
+            await _record_job_id(payload, config, document_id, job)
+            markdown = await _poll_until_done(llama, job.id, config, log)
+            await _writeback_success(payload, config, document_id, markdown, log)
+            log.info("Parse document task succeeded")
+        except (LlamaParseError, PayloadError) as exc:
+            log.exception("Parse document task failed")
+            await _stamp_error(payload, config, document_id, str(exc))
+            raise
+
+
+async def _mark_processing(payload: PayloadClient, config: RuntimeConfig, document_id: str) -> None:
+    await payload.submit_parse_result(
+        config.documents_collection_slug,
+        document_id,
+        {"parse_status": "processing", "parse_error": None},
+    )
+
+
+async def _fetch_inputs(
+    payload: PayloadClient,
+    config: RuntimeConfig,
+    document_id: str,
+    log: structlog.stdlib.BoundLogger,
+) -> tuple[ParseContext, bytes]:
+    ctx = await payload.fetch_parse_context(config.documents_collection_slug, document_id)
+    log.info("Downloading upload from Payload", filename=_resolve_filename(ctx))
+    file_bytes = await payload.fetch_parse_file(config.documents_collection_slug, document_id)
+    return ctx, file_bytes
+
+
+async def _submit_to_llama(
+    llama: LlamaParseClient,
+    ctx: ParseContext,
+    file_bytes: bytes,
+    log: structlog.stdlib.BoundLogger,
+) -> LlamaParseJob:
+    filename = _resolve_filename(ctx)
+    log.info("Uploading to LlamaParse", filename=filename, size=len(file_bytes))
+    job = await llama.upload(
+        file_bytes=file_bytes,
+        filename=filename,
+        language=ctx.get("language"),
+        parsing_instruction=ctx.get("parsing_instruction"),
+        mode=ctx.get("mode"),
+    )
+    log.info("LlamaParse job created", llama_job_id=job.id)
+    return job
+
+
+async def _record_job_id(
+    payload: PayloadClient, config: RuntimeConfig, document_id: str, job: LlamaParseJob
+) -> None:
+    await payload.submit_parse_result(
+        config.documents_collection_slug,
+        document_id,
+        {"parse_job_id": job.id},
+    )
+
+
+async def _writeback_success(
+    payload: PayloadClient,
+    config: RuntimeConfig,
+    document_id: str,
+    markdown: str,
+    log: structlog.stdlib.BoundLogger,
+) -> None:
+    log.info("Parse complete; writing back to Payload", chars=len(markdown))
+    await payload.submit_parse_result(
+        config.documents_collection_slug,
+        document_id,
+        {
+            "parsed_text": markdown,
+            "parse_status": "done",
+            "parse_error": None,
+            "parsed_at": _now_iso(),
+        },
+    )
+
+
+async def _poll_until_done(
+    client: LlamaParseClient,
+    job_id: str,
+    config: RuntimeConfig,
+    log: structlog.stdlib.BoundLogger,
+) -> str:
+    deadline = asyncio.get_event_loop().time() + config.llama_parse_poll_timeout_s
+    while asyncio.get_event_loop().time() <= deadline:
+        job = await client.status(job_id)
+        if job.status == "SUCCESS":
+            return await client.fetch_markdown(job_id)
+        if job.status in ("ERROR", "CANCELLED"):
+            raise LlamaParseError(
+                f"LlamaParse job {job_id} ended in {job.status}: {job.error or 'no detail'}"
+            )
+        log.debug("Polling LlamaParse", status=job.status)
+        await asyncio.sleep(config.llama_parse_poll_interval_s)
+    raise LlamaParseError(
+        f"LlamaParse job {job_id} timed out after {config.llama_parse_poll_timeout_s}s"
+    )
+
+
+def _resolve_filename(ctx: ParseContext) -> str:
+    filename = ctx.get("filename")
+    return filename if isinstance(filename, str) and filename else DEFAULT_FILENAME
+
+
+async def _stamp_error(
+    payload: PayloadClient, config: RuntimeConfig, document_id: str, message: str
+) -> None:
+    """Best-effort error stamp — never raises so we don't shadow the original exception."""
+    with contextlib.suppress(PayloadError, httpx.HTTPError):
+        await payload.submit_parse_result(
+            config.documents_collection_slug,
+            document_id,
+            {"parse_status": "error", "parse_error": message[:500]},
+        )
+
+
+def _now_iso() -> str:
+    return datetime.now(UTC).isoformat()

payload_documents_worker_builder-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[project]
+name = "payload-documents-worker-builder"
+version = "0.1.0"
+description = "Parametrizable taskiq-based worker for Payload CMS — FastAPI kick app + Postgres LISTEN/NOTIFY broker + LlamaParse parse-document task."
+requires-python = ">=3.12"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Zetesis Labs" }]
+keywords = ["payload", "worker", "taskiq", "llamaparse", "postgres"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Framework :: FastAPI",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+dependencies = [
+    "fastapi>=0.115",
+    "httpx>=0.28",
+    "pydantic>=2.9",
+    "structlog>=24.1",
+    "taskiq>=0.11.18",
+    "taskiq-redis>=1.1.1",
+    "uvicorn[standard]>=0.34",
+]
+
+[project.urls]
+Homepage = "https://github.com/Zetesis-Labs/PayloadAgents"
+Repository = "https://github.com/Zetesis-Labs/PayloadAgents"
+Issues = "https://github.com/Zetesis-Labs/PayloadAgents/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["payload_documents_worker_builder"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"payload_documents_worker_builder/py.typed" = "payload_documents_worker_builder/py.typed"