svc-infra 0.1.595__py3-none-any.whl → 1.1.0__py3-none-any.whl

svc_infra/http/__init__.py

@@ -0,0 +1,13 @@
+from .client import (
+    get_default_timeout_seconds,
+    make_timeout,
+    new_async_httpx_client,
+    new_httpx_client,
+)
+
+__all__ = [
+    "get_default_timeout_seconds",
+    "new_httpx_client",
+    "new_async_httpx_client",
+    "make_timeout",
+]

svc_infra/http/client.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import os
+from contextvars import ContextVar
+from typing import Any
+
+import httpx
+
+from svc_infra.app.env import pick
+
+# Context var for request ID propagation across async boundaries
+_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+def set_request_id(request_id: str | None) -> None:
+    """Set the current request ID for propagation to outbound HTTP calls."""
+    _request_id_ctx.set(request_id)
+
+
+def get_request_id() -> str | None:
+    """Get the current request ID for propagation."""
+    return _request_id_ctx.get()
+
+
+def _merge_request_id_header(headers: dict[str, str] | None) -> dict[str, str]:
+    """Merge X-Request-Id header into headers dict if request ID is set."""
+    result = dict(headers) if headers else {}
+    request_id = get_request_id()
+    if request_id and "X-Request-Id" not in result:
+        result["X-Request-Id"] = request_id
+    return result
+
+
+def _parse_float_env(name: str, default: float) -> float:
+    raw = os.getenv(name)
+    if raw is None or raw == "":
+        return default
+    try:
+        return float(raw)
+    except ValueError:
+        return default
+
+
+def get_default_timeout_seconds() -> float:
+    """Return default outbound HTTP client timeout in seconds.
+
+    Env var: HTTP_CLIENT_TIMEOUT_SECONDS (float)
+    Defaults: 10.0 seconds for all envs unless overridden; tweakable via pick() if needed.
+    """
+    default = pick(prod=10.0, nonprod=10.0)
+    return _parse_float_env("HTTP_CLIENT_TIMEOUT_SECONDS", default)
+
+
+def make_timeout(seconds: float | None = None) -> httpx.Timeout:
+    s = seconds if seconds is not None else get_default_timeout_seconds()
+    # Apply same timeout for connect/read/write/pool for simplicity
+    return httpx.Timeout(timeout=s)
+
+
+def new_httpx_client(
+    *,
+    timeout_seconds: float | None = None,
+    headers: dict[str, str] | None = None,
+    base_url: str | None = None,
+    propagate_request_id: bool = True,
+    **kwargs: Any,
+) -> httpx.Client:
+    """Create a sync httpx Client with default timeout and optional headers/base_url.
+
+    Callers can override timeout_seconds; remaining kwargs are forwarded to httpx.Client.
+    If propagate_request_id=True (default), X-Request-Id header is added from context.
+    """
+    timeout = make_timeout(timeout_seconds)
+    merged_headers = _merge_request_id_header(headers) if propagate_request_id else headers
+    # httpx doesn't accept base_url=None; only pass if non-None
+    client_kwargs = {"timeout": timeout, "headers": merged_headers, **kwargs}
+    if base_url is not None:
+        client_kwargs["base_url"] = base_url
+    return httpx.Client(**client_kwargs)
+
+
+def new_async_httpx_client(
+    *,
+    timeout_seconds: float | None = None,
+    headers: dict[str, str] | None = None,
+    base_url: str | None = None,
+    propagate_request_id: bool = True,
+    **kwargs: Any,
+) -> httpx.AsyncClient:
+    """Create an async httpx AsyncClient with default timeout and optional headers/base_url.
+
+    Callers can override timeout_seconds; remaining kwargs are forwarded to httpx.AsyncClient.
+    If propagate_request_id=True (default), X-Request-Id header is added from context.
+    """
+    timeout = make_timeout(timeout_seconds)
+    merged_headers = _merge_request_id_header(headers) if propagate_request_id else headers
+    # httpx doesn't accept base_url=None; only pass if non-None
+    client_kwargs = {"timeout": timeout, "headers": merged_headers, **kwargs}
+    if base_url is not None:
+        client_kwargs["base_url"] = base_url
+    return httpx.AsyncClient(**client_kwargs)

svc_infra/jobs/__init__.py

@@ -0,0 +1,79 @@
+"""Background jobs module providing queue abstraction and worker utilities.
+
+This module provides a flexible background job system with multiple backends:
+
+- **InMemoryJobQueue**: Simple in-memory queue for tests and local development
+- **RedisJobQueue**: Production-ready Redis-backed queue with visibility timeout
+- **InMemoryScheduler**: Interval-based scheduler for periodic tasks
+
+Example:
+    from svc_infra.jobs import easy_jobs, Job
+
+    # Initialize queue and scheduler (auto-detects Redis or uses memory)
+    queue, scheduler = easy_jobs()
+
+    # Enqueue a job
+    job = queue.enqueue("send_email", {"to": "user@example.com"})
+    print(f"Enqueued job: {job.id}")
+
+    # Process jobs with a worker
+    from svc_infra.jobs import process_one
+
+    async def handler(job: Job):
+        if job.name == "send_email":
+            await send_email(job.payload["to"])
+
+    await process_one(queue, handler)
+
+Environment Variables:
+    JOBS_DRIVER: Backend driver ("memory" or "redis"), defaults to "memory"
+    REDIS_URL: Redis connection URL for redis driver
+    JOB_DEFAULT_TIMEOUT_SECONDS: Per-job execution timeout
+    JOBS_SCHEDULE_JSON: JSON array of scheduled task definitions
+
+See Also:
+    - docs/jobs.md for detailed documentation
+    - svc_infra.jobs.builtins for webhook delivery and outbox processing
+"""
+
+from __future__ import annotations
+
+# Easy setup function
+from .easy import easy_jobs
+
+# Loader for schedule configuration
+from .loader import schedule_from_env
+
+# Core queue abstractions
+from .queue import InMemoryJobQueue, Job, JobQueue
+
+# Redis-backed queue for production
+from .redis_queue import RedisJobQueue
+
+# Runner for long-lived workers
+from .runner import WorkerRunner
+
+# Scheduler for periodic tasks
+from .scheduler import InMemoryScheduler, ScheduledTask
+
+# Worker utilities
+from .worker import process_one
+
+__all__ = [
+    # Core types
+    "Job",
+    "JobQueue",
+    # Queue implementations
+    "InMemoryJobQueue",
+    "RedisJobQueue",
+    # Scheduler
+    "InMemoryScheduler",
+    "ScheduledTask",
+    # Easy setup
+    "easy_jobs",
+    # Worker utilities
+    "process_one",
+    "WorkerRunner",
+    # Configuration loader
+    "schedule_from_env",
+]

svc_infra/jobs/builtins/outbox_processor.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from svc_infra.db.outbox import OutboxStore
+from svc_infra.jobs.queue import JobQueue
+
+
+def make_outbox_tick(
+    outbox: OutboxStore,
+    queue: JobQueue,
+    *,
+    topics: Iterable[str] | None = None,
+    job_name_prefix: str = "outbox",
+):
+    """Return an async task function to move one outbox message into the job queue.
+
+    - It fetches at most one unprocessed message per tick to avoid starving others.
+    - The enqueued job name is f"{job_name_prefix}.{topic}" to allow routing.
+    - The job payload contains `outbox_id`, `topic`, and original `payload`.
+    """
+
+    dispatched: set[int] = set()
+
+    async def _tick():
+        # Outbox is sync; this wrapper is async for scheduler compatibility
+        msg = outbox.fetch_next(topics=topics)
+        if not msg:
+            return
+        if msg.id in dispatched:
+            return
+        job_name = f"{job_name_prefix}.{msg.topic}"
+        queue.enqueue(job_name, {"outbox_id": msg.id, "topic": msg.topic, "payload": msg.payload})
+        # mark as dispatched (bump attempts) so it won't be re-enqueued by fetch_next
+        outbox.mark_failed(msg.id)
+        dispatched.add(msg.id)
+
+    return _tick

svc_infra/jobs/builtins/webhook_delivery.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import os
+
+from svc_infra.db.inbox import InboxStore
+from svc_infra.db.outbox import OutboxStore
+from svc_infra.http import get_default_timeout_seconds, new_async_httpx_client
+from svc_infra.jobs.queue import Job
+from svc_infra.webhooks.encryption import decrypt_secret
+from svc_infra.webhooks.signing import sign
+
+
+def make_webhook_handler(
+    *,
+    outbox: OutboxStore,
+    inbox: InboxStore,
+    get_webhook_url_for_topic,
+    get_secret_for_topic,
+    header_name: str = "X-Signature",
+):
+    """Return an async job handler to deliver webhooks.
+
+    Expected job payload shape:
+    {"outbox_id": int, "topic": str, "payload": {...}}
+    """
+
+    async def _handler(job: Job) -> None:
+        data = job.payload or {}
+        outbox_id = data.get("outbox_id")
+        topic = data.get("topic")
+        payload = data.get("payload") or {}
+        if not outbox_id or not topic:
+            # Nothing we can do; ack to avoid poison loop
+            return
+        # dedupe marker key (marked after successful delivery)
+        key = f"webhook:{outbox_id}"
+        if inbox.is_marked(key):
+            # already delivered
+            outbox.mark_processed(int(outbox_id))
+            return
+        event = payload.get("event") if isinstance(payload, dict) else None
+        subscription = payload.get("subscription") if isinstance(payload, dict) else None
+        if event is not None and subscription is not None:
+            delivery_payload = event
+            url = subscription.get("url") or get_webhook_url_for_topic(topic)
+            # Decrypt secret (handles both encrypted and plaintext for backwards compat)
+            raw_secret = subscription.get("secret") or get_secret_for_topic(topic)
+            secret = decrypt_secret(raw_secret)
+            subscription_id = subscription.get("id")
+        else:
+            delivery_payload = payload
+            url = get_webhook_url_for_topic(topic)
+            secret = get_secret_for_topic(topic)
+            subscription_id = None
+        sig = sign(secret, delivery_payload)
+        headers = {
+            header_name: sig,
+            "X-Event-Id": str(outbox_id),
+            "X-Topic": str(topic),
+            "X-Attempt": str(job.attempts or 1),
+            "X-Signature-Alg": "hmac-sha256",
+            "X-Signature-Version": "v1",
+        }
+        if subscription_id:
+            headers["X-Webhook-Subscription"] = str(subscription_id)
+        # include event payload version if present
+        version = None
+        if isinstance(delivery_payload, dict):
+            version = delivery_payload.get("version")
+        if version is not None:
+            headers["X-Payload-Version"] = str(version)
+        # Derive timeout: dedicated WEBHOOK_DELIVERY_TIMEOUT_SECONDS or default HTTP client timeout
+        timeout_seconds = None
+        env_timeout = os.getenv("WEBHOOK_DELIVERY_TIMEOUT_SECONDS")
+        if env_timeout:
+            try:
+                timeout_seconds = float(env_timeout)
+            except ValueError:
+                timeout_seconds = get_default_timeout_seconds()
+        else:
+            timeout_seconds = get_default_timeout_seconds()
+
+        async with new_async_httpx_client(timeout_seconds=timeout_seconds) as client:
+            resp = await client.post(url, json=delivery_payload, headers=headers)
+            if 200 <= resp.status_code < 300:
+                # record delivery and mark processed
+                inbox.mark_if_new(key, ttl_seconds=24 * 3600)
+                outbox.mark_processed(int(outbox_id))
+                return
+            # allow retry on non-2xx: raise to trigger fail/backoff
+            raise RuntimeError(f"webhook delivery failed: {resp.status_code}")
+
+    return _handler

svc_infra/jobs/easy.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import os
+
+from redis import Redis
+
+from .queue import InMemoryJobQueue, JobQueue
+from .redis_queue import RedisJobQueue
+from .scheduler import InMemoryScheduler
+
+
+class JobsConfig:
+    def __init__(self, driver: str | None = None):
+        # Future: support redis/sql drivers via extras
+        self.driver = driver or os.getenv("JOBS_DRIVER", "memory").lower()
+
+
+def easy_jobs(*, driver: str | None = None) -> tuple[JobQueue, InMemoryScheduler]:
+    """One-call wiring for jobs: returns (queue, scheduler).
+
+    Defaults to in-memory implementations for local/dev. ENV override via JOBS_DRIVER.
+    """
+    cfg = JobsConfig(driver=driver)
+    # Choose backend
+    queue: JobQueue
+    if cfg.driver == "redis":
+        url = os.getenv("REDIS_URL", "redis://localhost:6379/0")
+        client = Redis.from_url(url)
+        queue = RedisJobQueue(client)
+    else:
+        queue = InMemoryJobQueue()
+    scheduler = InMemoryScheduler()
+    return queue, scheduler

svc_infra/jobs/loader.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+import logging
+import os
+from collections.abc import Awaitable, Callable
+from typing import cast
+
+from .scheduler import InMemoryScheduler
+
+logger = logging.getLogger(__name__)
+
+
+def _resolve_target(path: str) -> Callable[[], Awaitable[None]]:
+    mod_name, func_name = path.split(":", 1)
+    mod = importlib.import_module(mod_name)
+    fn = getattr(mod, func_name)
+    if asyncio.iscoroutinefunction(fn):
+        return cast("Callable[[], Awaitable[None]]", fn)
+
+    # wrap sync into async
+    async def _wrapped():
+        fn()
+
+    return _wrapped
+
+
+def schedule_from_env(scheduler: InMemoryScheduler, env_var: str = "JOBS_SCHEDULE_JSON") -> None:
+    data = os.getenv(env_var)
+    if not data:
+        return
+    try:
+        tasks = json.loads(data)
+    except json.JSONDecodeError:
+        return
+    if not isinstance(tasks, list):
+        return
+    for t in tasks:
+        try:
+            name = t["name"]
+            interval = int(t.get("interval_seconds", 60))
+            target = t["target"]
+            fn = _resolve_target(target)
+            scheduler.add_task(name, interval, fn)
+        except Exception as e:
+            logger.warning("Failed to load scheduled job entry %s: %s", t, e)
+            continue

svc_infra/jobs/queue.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import logging
+import os
+import warnings
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import Any, Protocol
+
+logger = logging.getLogger(__name__)
+
+_INMEMORY_WARNED = False
+
+
+def _check_inmemory_production_warning(class_name: str) -> None:
+    """Warn if in-memory store is used in production."""
+    global _INMEMORY_WARNED
+    if _INMEMORY_WARNED:
+        return
+    env = os.getenv("ENV", "development").lower()
+    if env in ("production", "staging", "prod"):
+        _INMEMORY_WARNED = True
+        msg = (
+            f"{class_name} is being used in {env} environment. "
+            "This is NOT suitable for production - data will be lost on restart. "
+            "Use RedisJobQueue instead."
+        )
+        warnings.warn(msg, RuntimeWarning, stacklevel=3)
+        logger.critical(msg)
+
+
+@dataclass
+class Job:
+    id: str
+    name: str
+    payload: dict[str, Any]
+    available_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    attempts: int = 0
+    max_attempts: int = 5
+    backoff_seconds: int = 60  # base backoff for retry
+    last_error: str | None = None
+
+
+class JobQueue(Protocol):
+    def enqueue(self, name: str, payload: dict[str, Any], *, delay_seconds: int = 0) -> Job:
+        pass
+
+    def reserve_next(self) -> Job | None:
+        pass
+
+    def ack(self, job_id: str) -> None:
+        pass
+
+    def fail(self, job_id: str, *, error: str | None = None) -> None:
+        pass
+
+
+class InMemoryJobQueue:
+    """Simple in-memory queue for tests and local runs.
+
+    Single-threaded reserve/ack/fail semantics. Not suitable for production.
+    """
+
+    def __init__(self):
+        _check_inmemory_production_warning("InMemoryJobQueue")
+        self._seq = 0
+        self._jobs: list[Job] = []
+
+    def _next_id(self) -> str:
+        self._seq += 1
+        return str(self._seq)
+
+    def enqueue(self, name: str, payload: dict[str, Any], *, delay_seconds: int = 0) -> Job:
+        when = datetime.now(UTC) + timedelta(seconds=delay_seconds)
+        job = Job(id=self._next_id(), name=name, payload=dict(payload), available_at=when)
+        self._jobs.append(job)
+        return job
+
+    def reserve_next(self) -> Job | None:
+        now = datetime.now(UTC)
+        for job in self._jobs:
+            if job.available_at <= now and job.attempts >= 0 and job.attempts < job.max_attempts:
+                job.attempts += 1
+                return job
+        return None
+
+    def ack(self, job_id: str) -> None:
+        self._jobs = [j for j in self._jobs if j.id != job_id]
+
+    def fail(self, job_id: str, *, error: str | None = None) -> None:
+        now = datetime.now(UTC)
+        for job in self._jobs:
+            if job.id == job_id:
+                job.last_error = error
+                # Exponential backoff: base * attempts
+                delay = job.backoff_seconds * max(1, job.attempts)
+                if delay > 0:
+                    # Add a tiny fudge so an immediate subsequent poll in ultra-fast
+                    # environments (like our acceptance API) doesn't re-reserve the job.
+                    # This keeps tests deterministic without impacting semantics.
+                    job.available_at = now + timedelta(seconds=delay, milliseconds=250)
+                else:
+                    # When backoff is explicitly zero (e.g., unit tests forcing
+                    # immediate retry), make the job available right away.
+                    job.available_at = now
+                return