svc-infra 0.1.595__py3-none-any.whl → 0.1.597__py3-none-any.whl

svc_infra/db/outbox.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterable, List, Optional, Protocol
+
+
+@dataclass
+class OutboxMessage:
+    id: int
+    topic: str
+    payload: Dict[str, Any]
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    attempts: int = 0
+    processed_at: Optional[datetime] = None
+
+
+class OutboxStore(Protocol):
+    def enqueue(self, topic: str, payload: Dict[str, Any]) -> OutboxMessage:
+        pass
+
+    def fetch_next(self, *, topics: Optional[Iterable[str]] = None) -> Optional[OutboxMessage]:
+        """Return the next undispatched, unprocessed message (FIFO per-topic), or None.
+
+        Notes:
+        - Messages with attempts > 0 are considered "dispatched" to the job queue and won't be re-enqueued.
+        - Delivery retries are handled by the job queue worker, not by re-reading the outbox.
+        """
+        pass
+
+    def mark_processed(self, msg_id: int) -> None:
+        pass
+
+    def mark_failed(self, msg_id: int) -> None:
+        pass
+
+
+class InMemoryOutboxStore:
+    """Simple in-memory outbox for tests and local runs."""
+
+    def __init__(self):
+        self._seq = 0
+        self._messages: List[OutboxMessage] = []
+
+    def enqueue(self, topic: str, payload: Dict[str, Any]) -> OutboxMessage:
+        self._seq += 1
+        msg = OutboxMessage(id=self._seq, topic=topic, payload=dict(payload))
+        self._messages.append(msg)
+        return msg
+
+    def fetch_next(self, *, topics: Optional[Iterable[str]] = None) -> Optional[OutboxMessage]:
+        allowed = set(topics) if topics else None
+        for msg in self._messages:
+            if msg.processed_at is not None:
+                continue
+            # skip already dispatched messages (attempts>0)
+            if msg.attempts > 0:
+                continue
+            if allowed is not None and msg.topic not in allowed:
+                continue
+            return msg
+        return None
+
+    def mark_processed(self, msg_id: int) -> None:
+        for msg in self._messages:
+            if msg.id == msg_id:
+                msg.processed_at = datetime.now(timezone.utc)
+                return
+
+    def mark_failed(self, msg_id: int) -> None:
+        for msg in self._messages:
+            if msg.id == msg_id:
+                msg.attempts += 1
+                return
+
+
+class SqlOutboxStore:
+    """Skeleton for a SQL-backed outbox store.
+
+    Implementations should:
+    - INSERT on enqueue
+    - SELECT FOR UPDATE SKIP LOCKED (or equivalent) to fetch next
+    - UPDATE processed_at (and attempts on failure)
+    """
+
+    def __init__(self, session_factory):
+        self._session_factory = session_factory
+
+    # Placeholders to outline the API; not implemented here.
+    def enqueue(
+        self, topic: str, payload: Dict[str, Any]
+    ) -> OutboxMessage:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def fetch_next(
+        self, *, topics: Optional[Iterable[str]] = None
+    ) -> Optional[OutboxMessage]:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def mark_processed(self, msg_id: int) -> None:  # pragma: no cover - skeleton
+        raise NotImplementedError
+
+    def mark_failed(self, msg_id: int) -> None:  # pragma: no cover - skeleton
+        raise NotImplementedError

svc_infra/db/sql/versioning.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from sqlalchemy import Integer
+from sqlalchemy.orm import Mapped, mapped_column
+
+
+class Versioned:
+    """Mixin for optimistic locking with integer version.
+
+    - Initialize version=1 on insert (via default=1)
+    - Bump version in app code before commit to detect mismatches.
+    """
+
+    version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)

svc_infra/jobs/builtins/outbox_processor.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+from svc_infra.db.outbox import OutboxStore
+from svc_infra.jobs.queue import JobQueue
+
+
+def make_outbox_tick(
+    outbox: OutboxStore,
+    queue: JobQueue,
+    *,
+    topics: Optional[Iterable[str]] = None,
+    job_name_prefix: str = "outbox",
+):
+    """Return an async task function to move one outbox message into the job queue.
+
+    - It fetches at most one unprocessed message per tick to avoid starving others.
+    - The enqueued job name is f"{job_name_prefix}.{topic}" to allow routing.
+    - The job payload contains `outbox_id`, `topic`, and original `payload`.
+    """
+
+    dispatched: set[int] = set()
+
+    async def _tick():
+        # Outbox is sync; this wrapper is async for scheduler compatibility
+        msg = outbox.fetch_next(topics=topics)
+        if not msg:
+            return
+        if msg.id in dispatched:
+            return
+        job_name = f"{job_name_prefix}.{msg.topic}"
+        queue.enqueue(job_name, {"outbox_id": msg.id, "topic": msg.topic, "payload": msg.payload})
+        # mark as dispatched (bump attempts) so it won't be re-enqueued by fetch_next
+        outbox.mark_failed(msg.id)
+        dispatched.add(msg.id)
+
+    return _tick

svc_infra/jobs/builtins/webhook_delivery.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+from typing import Dict
+
+import httpx
+
+from svc_infra.db.inbox import InboxStore
+from svc_infra.db.outbox import OutboxStore
+from svc_infra.jobs.queue import Job
+
+
+def _compute_signature(secret: str, payload: Dict) -> str:
+    body = json.dumps(payload, separators=(",", ":")).encode()
+    return hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+
+
+def make_webhook_handler(
+    *,
+    outbox: OutboxStore,
+    inbox: InboxStore,
+    get_webhook_url_for_topic,
+    get_secret_for_topic,
+    header_name: str = "X-Signature",
+):
+    """Return an async job handler to deliver webhooks.
+
+    Expected job payload shape:
+    {"outbox_id": int, "topic": str, "payload": {...}}
+    """
+
+    async def _handler(job: Job) -> None:
+        data = job.payload or {}
+        outbox_id = data.get("outbox_id")
+        topic = data.get("topic")
+        payload = data.get("payload") or {}
+        if not outbox_id or not topic:
+            # Nothing we can do; ack to avoid poison loop
+            return
+        # dedupe by outbox_id via inbox
+        key = f"webhook:{outbox_id}"
+        if not inbox.mark_if_new(key, ttl_seconds=24 * 3600):
+            # already delivered
+            outbox.mark_processed(int(outbox_id))
+            return
+        url = get_webhook_url_for_topic(topic)
+        secret = get_secret_for_topic(topic)
+        sig = _compute_signature(secret, payload)
+        async with httpx.AsyncClient(timeout=10) as client:
+            resp = await client.post(url, json=payload, headers={header_name: sig})
+            if 200 <= resp.status_code < 300:
+                outbox.mark_processed(int(outbox_id))
+                return
+            # allow retry on non-2xx: raise to trigger fail/backoff
+            raise RuntimeError(f"webhook delivery failed: {resp.status_code}")
+
+    return _handler

svc_infra/jobs/easy.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import os
+
+from redis import Redis
+
+from .queue import InMemoryJobQueue, JobQueue
+from .redis_queue import RedisJobQueue
+from .scheduler import InMemoryScheduler
+
+
+class JobsConfig:
+    def __init__(self, driver: str | None = None):
+        # Future: support redis/sql drivers via extras
+        self.driver = driver or os.getenv("JOBS_DRIVER", "memory").lower()
+
+
+def easy_jobs(*, driver: str | None = None) -> tuple[JobQueue, InMemoryScheduler]:
+    """One-call wiring for jobs: returns (queue, scheduler).
+
+    Defaults to in-memory implementations for local/dev. ENV override via JOBS_DRIVER.
+    """
+    cfg = JobsConfig(driver=driver)
+    # Choose backend
+    if cfg.driver == "redis":
+        url = os.getenv("REDIS_URL", "redis://localhost:6379/0")
+        client = Redis.from_url(url)
+        queue = RedisJobQueue(client)
+    else:
+        queue = InMemoryJobQueue()
+    scheduler = InMemoryScheduler()
+    return queue, scheduler

svc_infra/jobs/loader.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+import os
+from typing import Awaitable, Callable
+
+from .scheduler import InMemoryScheduler
+
+
+def _resolve_target(path: str) -> Callable[[], Awaitable[None]]:
+    mod_name, func_name = path.split(":", 1)
+    mod = importlib.import_module(mod_name)
+    fn = getattr(mod, func_name)
+    if asyncio.iscoroutinefunction(fn):
+        return fn  # type: ignore[return-value]
+
+    # wrap sync into async
+    async def _wrapped():
+        fn()
+
+    return _wrapped
+
+
+def schedule_from_env(scheduler: InMemoryScheduler, env_var: str = "JOBS_SCHEDULE_JSON") -> None:
+    data = os.getenv(env_var)
+    if not data:
+        return
+    try:
+        tasks = json.loads(data)
+    except json.JSONDecodeError:
+        return
+    if not isinstance(tasks, list):
+        return
+    for t in tasks:
+        try:
+            name = t["name"]
+            interval = int(t.get("interval_seconds", 60))
+            target = t["target"]
+            fn = _resolve_target(target)
+            scheduler.add_task(name, interval, fn)
+        except Exception:
+            # ignore bad entries
+            continue

svc_infra/jobs/queue.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, Optional, Protocol
+
+
+@dataclass
+class Job:
+    id: str
+    name: str
+    payload: Dict[str, Any]
+    available_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    attempts: int = 0
+    max_attempts: int = 5
+    backoff_seconds: int = 60  # base backoff for retry
+    last_error: Optional[str] = None
+
+
+class JobQueue(Protocol):
+    def enqueue(self, name: str, payload: Dict[str, Any], *, delay_seconds: int = 0) -> Job:
+        pass
+
+    def reserve_next(self) -> Optional[Job]:
+        pass
+
+    def ack(self, job_id: str) -> None:
+        pass
+
+    def fail(self, job_id: str, *, error: str | None = None) -> None:
+        pass
+
+
+class InMemoryJobQueue:
+    """Simple in-memory queue for tests and local runs.
+
+    Single-threaded reserve/ack/fail semantics. Not suitable for production.
+    """
+
+    def __init__(self):
+        self._seq = 0
+        self._jobs: list[Job] = []
+
+    def _next_id(self) -> str:
+        self._seq += 1
+        return str(self._seq)
+
+    def enqueue(self, name: str, payload: Dict[str, Any], *, delay_seconds: int = 0) -> Job:
+        when = datetime.now(timezone.utc) + timedelta(seconds=delay_seconds)
+        job = Job(id=self._next_id(), name=name, payload=dict(payload), available_at=when)
+        self._jobs.append(job)
+        return job
+
+    def reserve_next(self) -> Optional[Job]:
+        now = datetime.now(timezone.utc)
+        for job in self._jobs:
+            if job.available_at <= now and job.attempts >= 0 and job.attempts < job.max_attempts:
+                job.attempts += 1
+                return job
+        return None
+
+    def ack(self, job_id: str) -> None:
+        self._jobs = [j for j in self._jobs if j.id != job_id]
+
+    def fail(self, job_id: str, *, error: str | None = None) -> None:
+        now = datetime.now(timezone.utc)
+        for job in self._jobs:
+            if job.id == job_id:
+                job.last_error = error
+                # Exponential backoff: base * attempts
+                delay = job.backoff_seconds * max(1, job.attempts)
+                job.available_at = now + timedelta(seconds=delay)
+                return

svc_infra/jobs/redis_queue.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import Dict, Optional
+
+from redis import Redis
+
+from .queue import Job, JobQueue
+
+
+class RedisJobQueue(JobQueue):
+    """Redis-backed job queue with visibility timeout and delayed retries.
+
+    Keys (with optional prefix):
+      - {p}:ready (LIST)        ready job ids
+      - {p}:processing (LIST)   in-flight job ids
+      - {p}:processing_vt (ZSET) id -> visible_at (epoch seconds)
+      - {p}:delayed (ZSET)      id -> available_at (epoch seconds)
+      - {p}:seq (STRING)        INCR for job ids
+      - {p}:job:{id} (HASH)     job fields (json payload)
+      - {p}:dlq (LIST)          dead-letter job ids
+    """
+
+    def __init__(self, client: Redis, *, prefix: str = "jobs", visibility_timeout: int = 60):
+        self._r = client
+        self._p = prefix
+        self._vt = visibility_timeout
+
+    # Key helpers
+    def _k(self, name: str) -> str:
+        return f"{self._p}:{name}"
+
+    def _job_key(self, job_id: str) -> str:
+        return f"{self._p}:job:{job_id}"
+
+    # Core ops
+    def enqueue(self, name: str, payload: Dict, *, delay_seconds: int = 0) -> Job:
+        now = datetime.now(timezone.utc)
+        job_id = str(self._r.incr(self._k("seq")))
+        job = Job(id=job_id, name=name, payload=dict(payload))
+        # Persist job
+        data = asdict(job)
+        data["payload"] = json.dumps(data["payload"])  # store payload as JSON string
+        # available_at stored as ISO format
+        data["available_at"] = job.available_at.isoformat()
+        self._r.hset(
+            self._job_key(job_id), mapping={k: str(v) for k, v in data.items() if v is not None}
+        )
+        if delay_seconds and delay_seconds > 0:
+            at = int(now.timestamp()) + int(delay_seconds)
+            self._r.zadd(self._k("delayed"), {job_id: at})
+        else:
+            # push to ready
+            self._r.lpush(self._k("ready"), job_id)
+        return job
+
+    def _move_due_delayed_to_ready(self) -> None:
+        now_ts = int(datetime.now(timezone.utc).timestamp())
+        ids = self._r.zrangebyscore(self._k("delayed"), "-inf", now_ts)
+        if not ids:
+            return
+        pipe = self._r.pipeline()
+        for jid in ids:
+            jid_s = jid.decode() if isinstance(jid, (bytes, bytearray)) else str(jid)
+            pipe.lpush(self._k("ready"), jid_s)
+            pipe.zrem(self._k("delayed"), jid_s)
+        pipe.execute()
+
+    def _requeue_timed_out_processing(self) -> None:
+        now_ts = int(datetime.now(timezone.utc).timestamp())
+        ids = self._r.zrangebyscore(self._k("processing_vt"), "-inf", now_ts)
+        if not ids:
+            return
+        pipe = self._r.pipeline()
+        for jid in ids:
+            jid_s = jid.decode() if isinstance(jid, (bytes, bytearray)) else str(jid)
+            pipe.lrem(self._k("processing"), 1, jid_s)
+            pipe.lpush(self._k("ready"), jid_s)
+            pipe.zrem(self._k("processing_vt"), jid_s)
+            # clear stale visibility timestamp so next reservation can set a fresh one
+            pipe.hdel(self._job_key(jid_s), "visible_at")
+        pipe.execute()
+
+    def reserve_next(self) -> Optional[Job]:
+        # opportunistically move due delayed jobs
+        self._move_due_delayed_to_ready()
+        # move timed-out processing jobs back to ready before reserving
+        self._requeue_timed_out_processing()
+        jid = self._r.rpoplpush(self._k("ready"), self._k("processing"))
+        if not jid:
+            return None
+        job_id = jid.decode() if isinstance(jid, (bytes, bytearray)) else str(jid)
+        key = self._job_key(job_id)
+        data = self._r.hgetall(key)
+        if not data:
+            # corrupted entry; ack and skip
+            self._r.lrem(self._k("processing"), 1, job_id)
+            return None
+
+        # Decode fields
+        def _get(field: str, default: Optional[str] = None) -> Optional[str]:
+            val = (
+                data.get(field.encode())
+                if isinstance(next(iter(data.keys())), bytes)
+                else data.get(field)
+            )
+            if val is None:
+                return default
+            return val.decode() if isinstance(val, (bytes, bytearray)) else str(val)
+
+        attempts = int(_get("attempts", "0")) + 1
+        max_attempts = int(_get("max_attempts", "5"))
+        backoff_seconds = int(_get("backoff_seconds", "60"))
+        name = _get("name", "") or ""
+        payload_json = _get("payload", "{}") or "{}"
+        try:
+            payload = json.loads(payload_json)
+        except Exception:  # pragma: no cover
+            payload = {}
+        available_at_str = _get("available_at")
+        available_at = (
+            datetime.fromisoformat(available_at_str)
+            if available_at_str
+            else datetime.now(timezone.utc)
+        )
+        # If exceeded max_attempts → DLQ and skip
+        if attempts > max_attempts:
+            self._r.lrem(self._k("processing"), 1, job_id)
+            self._r.lpush(self._k("dlq"), job_id)
+            return None
+        # Update attempts and visibility timeout
+        visible_at = int(datetime.now(timezone.utc).timestamp()) + int(self._vt)
+        pipe = self._r.pipeline()
+        pipe.hset(key, mapping={"attempts": attempts, "visible_at": visible_at})
+        pipe.zadd(self._k("processing_vt"), {job_id: visible_at})
+        pipe.execute()
+        return Job(
+            id=job_id,
+            name=name,
+            payload=payload,
+            available_at=available_at,
+            attempts=attempts,
+            max_attempts=max_attempts,
+            backoff_seconds=backoff_seconds,
+        )
+
+    def ack(self, job_id: str) -> None:
+        self._r.lrem(self._k("processing"), 1, job_id)
+        self._r.zrem(self._k("processing_vt"), job_id)
+        self._r.delete(self._job_key(job_id))
+
+    def fail(self, job_id: str, *, error: str | None = None) -> None:
+        key = self._job_key(job_id)
+        data = self._r.hgetall(key)
+        if not data:
+            # nothing to do
+            self._r.lrem(self._k("processing"), 1, job_id)
+            return
+
+        def _get(field: str, default: Optional[str] = None) -> Optional[str]:
+            val = (
+                data.get(field.encode())
+                if isinstance(next(iter(data.keys())), bytes)
+                else data.get(field)
+            )
+            if val is None:
+                return default
+            return val.decode() if isinstance(val, (bytes, bytearray)) else str(val)
+
+        attempts = int(_get("attempts", "0"))
+        max_attempts = int(_get("max_attempts", "5"))
+        backoff_seconds = int(_get("backoff_seconds", "60"))
+        now_ts = int(datetime.now(timezone.utc).timestamp())
+        # DLQ if at or beyond max_attempts
+        if attempts >= max_attempts:
+            self._r.lrem(self._k("processing"), 1, job_id)
+            self._r.zrem(self._k("processing_vt"), job_id)
+            self._r.lpush(self._k("dlq"), job_id)
+            return
+        delay = backoff_seconds * max(1, attempts)
+        available_at_ts = now_ts + delay
+        mapping = {
+            "last_error": error or "",
+            "available_at": datetime.fromtimestamp(available_at_ts, tz=timezone.utc).isoformat(),
+        }
+        self._r.hset(key, mapping=mapping)
+        self._r.lrem(self._k("processing"), 1, job_id)
+        self._r.zrem(self._k("processing_vt"), job_id)
+        self._r.zadd(self._k("delayed"), {job_id: available_at_ts})

svc_infra/jobs/scheduler.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Awaitable, Callable, Dict
+
+CronFunc = Callable[[], Awaitable[None]]
+
+
+@dataclass
+class ScheduledTask:
+    name: str
+    interval_seconds: int
+    func: CronFunc
+    next_run_at: datetime
+
+
+class InMemoryScheduler:
+    """Interval-based scheduler for simple periodic tasks (tests/local).
+
+    Not a full cron parser. Tracks next_run_at per task.
+    """
+
+    def __init__(self):
+        self._tasks: Dict[str, ScheduledTask] = {}
+
+    def add_task(self, name: str, interval_seconds: int, func: CronFunc) -> None:
+        now = datetime.now(timezone.utc)
+        self._tasks[name] = ScheduledTask(
+            name=name,
+            interval_seconds=interval_seconds,
+            func=func,
+            next_run_at=now + timedelta(seconds=interval_seconds),
+        )
+
+    async def tick(self) -> None:
+        now = datetime.now(timezone.utc)
+        for task in self._tasks.values():
+            if task.next_run_at <= now:
+                await task.func()
+                task.next_run_at = now + timedelta(seconds=task.interval_seconds)

svc_infra/jobs/worker.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import Awaitable, Callable
+
+from .queue import Job, JobQueue
+
+ProcessFunc = Callable[[Job], Awaitable[None]]
+
+
+async def process_one(queue: JobQueue, handler: ProcessFunc) -> bool:
+    """Reserve a job, process with handler, ack on success or fail with backoff.
+
+    Returns True if a job was processed (success or fail), False if no job was available.
+    """
+    job = queue.reserve_next()
+    if not job:
+        return False
+    try:
+        await handler(job)
+    except Exception as exc:  # pragma: no cover - exercise in tests by raising
+        queue.fail(job.id, error=str(exc))
+        return True
+    queue.ack(job.id)
+    return True

{svc_infra-0.1.595.dist-info → svc_infra-0.1.597.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: svc-infra
-Version: 0.1.595
+Version: 0.1.597
 Summary: Infrastructure for building and deploying prod-ready services
 License: MIT
 Keywords: fastapi,sqlalchemy,alembic,auth,infra,async,pydantic