py-app-runner 0.5.49.dev0__py3-none-any.whl

py_app_runner/queue/interface.py

@@ -0,0 +1,63 @@
+"""What both drivers answer to.
+
+A Protocol rather than a base class, so neither driver inherits anything it does not use and
+a test double does not have to subclass to stand in for one.
+
+The shapes are fixed to the database driver's vocabulary - `pending`, `delayed`, `reserved`
+- even where redis models the same idea differently. There is one status command and it
+prints one table, so the driver is what adapts.
+"""
+
+from typing import Any, Protocol, runtime_checkable
+
+from py_app_runner.queue.job import Job
+
+
+@runtime_checkable
+class QueueDriver(Protocol):
+    async def push(
+        self,
+        name: str,
+        payload: dict[str, Any] | None = None,
+        delay: int = 0,
+        queue: str = "default",
+        priority: int = 0,
+        unique: str | None = None,
+        max_attempts: int = 3,
+    ) -> int:
+        """Queue a job, returning its id - or the id of the job already holding `unique`."""
+        ...
+
+    async def reserve(self, queues: list[str], timeout: int, worker: str) -> Job | None:
+        """Claim the next due job, or None. `queues` is a precedence order, not a merged
+        sort. The claim lasts `timeout` seconds and must become claimable again after that
+        without anybody having to release it."""
+        ...
+
+    async def delete(self, job: Job) -> None:
+        """Done. Forget it."""
+        ...
+
+    async def release(self, job: Job, delay: int = 0, error: str = "") -> None:
+        """Put it back for another attempt. Must not change `attempts` - reserving already
+        counted this one."""
+        ...
+
+    async def fail(self, job: Job, error: str) -> None:
+        """Out of attempts. Keep it where a human will find it."""
+        ...
+
+    async def pending(self, queue: str | None = None) -> int:
+        """How many could be picked up right now: excludes jobs not yet due and jobs another
+        worker currently holds."""
+        ...
+
+    async def stats(self) -> list[dict[str, Any]]: ...
+
+    async def failed_count(self) -> int: ...
+
+    async def failed_rows(self, limit: int) -> list[dict[str, Any]]: ...
+
+    async def retry_failed(self, job_id: int | None, max_attempts: int) -> int: ...
+
+    async def forget_failed(self, job_id: int | None, before: str | None) -> int: ...

py_app_runner/queue/job.py

@@ -0,0 +1,46 @@
+"""The job as a handler sees it, and the errors the queue raises."""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+class QueueError(Exception):
+    """A queue that is configured or called wrongly. Never raised for a job that failed -
+    that is what the failed table is for."""
+
+
+@dataclass
+class Job:
+    id: int
+    queue: str
+    name: str
+    payload: dict[str, Any]
+    # The row exactly as it was written. Carried so a job moved to the failed table keeps
+    # byte-identical payload, rather than a re-encoding of a decoding of it.
+    payload_json: str
+    # Includes this attempt: reserving *is* the attempt, so the first run sees 1.
+    attempts: int
+    max_attempts: int
+    # Driver bookmark. Empty on the database driver, the stream entry id on redis.
+    handle: str = ""
+
+    _release_delay: int | None = field(default=None, repr=False)
+
+    def release(self, delay: int = 0) -> None:
+        """Put this job back for another attempt.
+
+        A deliberate "not now" - a rate limit upstream, a file that has not landed yet -
+        rather than a failure. It does not count against the attempt budget any differently
+        from a crash, because the claim already counted it.
+        """
+
+        self._release_delay = max(0, delay)
+
+    def was_released(self) -> bool:
+        return self._release_delay is not None
+
+    def release_delay(self) -> int:
+        return self._release_delay or 0
+
+    def is_last_attempt(self) -> bool:
+        return self.attempts >= self.max_attempts

py_app_runner/queue/worker.py

@@ -0,0 +1,221 @@
+"""The worker loop.
+
+Polls rather than blocks, so latency is bounded by `sleep` and nothing holds a connection
+open waiting. One reserve, one job, one completion per iteration.
+
+The per-job timeout is `asyncio.timeout()`, which genuinely cancels the task rather than
+asking it to stop at a point it may never reach - a signal-based timeout cannot interrupt a
+job blocked in a query, which is the case most worth interrupting. The visibility timeout
+stays as the backstop for a worker that dies outright rather than merely overrunning.
+"""
+
+import asyncio
+import logging
+import time
+import traceback
+from collections.abc import Callable
+from typing import Any
+
+from py_app_runner.queue.handler import resolve
+from py_app_runner.queue.job import Job
+
+Out = Callable[[str], None]
+
+_logger = logging.getLogger(__name__)
+
+# Five consecutive failures to reserve means the queue is unreadable, not that it is empty.
+# Exiting lets a supervisor restart the process rather than having it sit there logging.
+MAX_CONSECUTIVE_FAILURES = 5
+
+
+def backoff(attempt: int, steps: list[int] | int | None = None) -> int:
+    """Delay before the next attempt, given how many have already been made.
+
+    A list gives one delay per attempt and repeats the last entry forever, so a job that
+    keeps failing settles at a sensible interval rather than growing without bound.
+    """
+
+    if steps is None:
+        steps = [10, 60, 300]
+
+    if isinstance(steps, int):
+        return max(0, steps)
+
+    if not steps:
+        return 0
+
+    return max(0, steps[min(max(0, attempt - 1), len(steps) - 1)])
+
+
+class Worker:
+    def __init__(
+        self,
+        queue: Any,
+        handlers: dict[str, Any] | None = None,
+        backoff_steps: list[int] | int | None = None,
+        out: Out | None = None,
+        worker_id: str = "worker",
+    ) -> None:
+        self.queue = queue
+        self.handlers = handlers or {}
+        self.backoff_steps = backoff_steps
+        self.out = out or print
+        self.worker_id = worker_id
+        self.should_quit = False
+
+    def stop(self) -> None:
+        """Ask the loop to finish the job in hand and exit."""
+
+        self.should_quit = True
+
+    async def run(
+        self,
+        queues: list[str],
+        timeout: int = 300,
+        sleep: float = 1.0,
+        max_jobs: int = 0,
+        max_time: int = 0,
+        stop_when_empty: bool = False,
+    ) -> int:
+        """Returns a process exit code: 0 for every ordinary end, 1 only when the queue
+        could not be read repeatedly. A failed *job* never changes the exit code - that is
+        what the failed table is for."""
+
+        self.out(f"Worker {self.worker_id} watching {', '.join(queues)}")
+
+        started = time.monotonic()
+        done = 0
+        failures = 0
+
+        while True:
+            if self.should_quit:
+                self.out("Stopping: asked to shut down")
+                break
+
+            try:
+                job = await self.queue.reserve(queues, timeout, self.worker_id)
+                failures = 0
+            except Exception as e:
+                failures += 1
+                self.out(f"error: could not reserve a job: {e}")
+                if failures >= MAX_CONSECUTIVE_FAILURES:
+                    self.out(f"Stopping: the queue has been unreadable {MAX_CONSECUTIVE_FAILURES} times running")
+                    return 1
+
+                await self._rest(sleep)
+                continue
+
+            if job is None:
+                if stop_when_empty:
+                    self.out("Nothing left to do")
+                    break
+
+                reason = self._limit_reached(done, started, max_jobs, max_time)
+                if reason:
+                    self.out(f"Stopping: {reason}")
+                    break
+
+                await self._rest(sleep)
+                continue
+
+            await self._run_job(job, timeout)
+            done += 1
+
+            reason = self._limit_reached(done, started, max_jobs, max_time)
+            if reason:
+                self.out(f"Stopping: {reason}")
+                break
+
+        self.out(f"Ran {done} job(s)")
+        return 0
+
+    async def run_next(self, queues: list[str], timeout: int = 300) -> bool:
+        """Reserve and run exactly one job. The single-shot primitive, for tests - it
+        installs no signal handlers, applies no limits and does not catch reserve failures."""
+
+        job = await self.queue.reserve(queues, timeout, self.worker_id)
+        if job is None:
+            return False
+
+        await self._run_job(job, timeout)
+        return True
+
+    ###############
+    ### Running ###
+    ###############
+
+    async def _run_job(self, job: Job, timeout: int) -> None:
+        self.out(f"-> {job.name} #{job.id} (attempt {job.attempts}/{job.max_attempts})")
+        started = time.monotonic()
+
+        try:
+            # Resolution is inside the try on purpose: a handler that cannot be built fails
+            # the job through the same release/fail path as one that threw, rather than
+            # being a separate outcome nobody handles.
+            handler = resolve(job.name, self.handlers)
+            async with asyncio.timeout(timeout):
+                await handler(job.payload, job)
+        except asyncio.CancelledError:
+            # A cancellation that is not ours - the process is shutting down. Release the
+            # job so it is picked up promptly rather than waiting out its reservation, and
+            # let the cancellation continue.
+            await self._safe_release(job, 0, "Worker was cancelled mid-job")
+            raise
+        except Exception as e:
+            # QueueError included: a handler that cannot be resolved is a job that cannot
+            # run, and it goes through the same release/fail budget as any other failure.
+            await self._job_failed(job, e)
+            return
+
+        if job.was_released():
+            await self.queue.release(job, job.release_delay(), "")
+            self.out(f"   released, back in {job.release_delay()}s")
+            return
+
+        await self.queue.delete(job)
+        self.out(f"   done in {int((time.monotonic() - started) * 1000)}ms")
+
+    async def _job_failed(self, job: Job, error: BaseException) -> None:
+        detail = f"{type(error).__name__}: {error}\n{''.join(traceback.format_exception(error))}"
+
+        if job.is_last_attempt():
+            await self.queue.fail(job, detail)
+            self.out(f"   failed for good: {error}")
+            # Both, deliberately: process output goes to a supervisor log nobody reads,
+            # and the application's own logging is where the rest of its problems surface.
+            _logger.error("Queue job %s #%s failed permanently: %s", job.name, job.id, error)
+            return
+
+        delay = backoff(job.attempts, self.backoff_steps)
+        await self.queue.release(job, delay, detail)
+        self.out(f"   failed, retrying in {delay}s: {error}")
+
+    async def _safe_release(self, job: Job, delay: int, error: str) -> None:
+        try:
+            await self.queue.release(job, delay, error)
+        except Exception as e:
+            # Already unwinding; the reservation will expire on its own.
+            _logger.warning("Queue could not release job #%s during shutdown: %s", job.id, e)
+
+    ###############
+    ### Limits ####
+    ###############
+
+    def _limit_reached(self, done: int, started: float, max_jobs: int, max_time: int) -> str:
+        if max_jobs > 0 and done >= max_jobs:
+            return f"ran {done} jobs"
+
+        if max_time > 0 and (time.monotonic() - started) >= max_time:
+            return f"been going for {int(time.monotonic() - started)}s"
+
+        return ""
+
+    async def _rest(self, sleep: float) -> None:
+        """Sleep in short slices so a shutdown request is noticed promptly rather than after
+        a full poll interval."""
+
+        remaining = sleep
+        while remaining > 0 and not self.should_quit:
+            slice_length = min(0.25, remaining)
+            await asyncio.sleep(slice_length)
+            remaining -= slice_length

py_app_runner/registry.py

@@ -0,0 +1,54 @@
+from typing import Any
+
+
+class AppRegistry:
+    """
+    Singleton registry holding project-specific classes and configuration
+    that the framework needs. Must be configured before the runner starts.
+    """
+
+    _config: dict[str, Any] = {}
+    _users_model_cls: type | None = None
+    _api_keys_model_cls: type | None = None
+    _redis_channel: str = ""
+    _api_key_use_db: bool = False
+
+    @classmethod
+    def configure(
+        cls,
+        *,
+        config: dict[str, Any],
+        users_model: type,
+        api_keys_model: type,
+        redis_channel: str = "",
+        api_key_use_db: bool = False,
+    ) -> None:
+        cls._config = config
+        cls._users_model_cls = users_model
+        cls._api_keys_model_cls = api_keys_model
+        cls._redis_channel = redis_channel
+        cls._api_key_use_db = api_key_use_db
+
+    @classmethod
+    def config(cls) -> dict[str, Any]:
+        return cls._config
+
+    @classmethod
+    def users_model(cls) -> type:
+        if cls._users_model_cls is None:
+            raise RuntimeError("AppRegistry not configured: users_model is None")
+        return cls._users_model_cls
+
+    @classmethod
+    def api_keys_model(cls) -> type:
+        if cls._api_keys_model_cls is None:
+            raise RuntimeError("AppRegistry not configured: api_keys_model is None")
+        return cls._api_keys_model_cls
+
+    @classmethod
+    def redis_channel(cls) -> str:
+        return cls._redis_channel
+
+    @classmethod
+    def api_key_use_db(cls) -> bool:
+        return cls._api_key_use_db

py_app_runner/request_handler/auth_service.py

@@ -0,0 +1,123 @@
+import datetime as dt
+import uuid
+from typing import Literal, TypedDict, cast
+
+import jwt
+
+from py_app_runner.registry import AppRegistry
+from py_app_runner.utils import CustomJSONEncoder
+
+JwtType = Literal["user", "device", "impersonation"]
+
+
+class JwtPayload(TypedDict, total=False):
+    sub: str
+    typ: JwtType
+    iat: int
+    exp: int
+    jti: str
+    imp: str  # impersonator public_id (only for typ="impersonation")
+
+
+class AuthService:
+    def __init__(self, logger):
+        self.logger = logger
+
+    def _now(self) -> dt.datetime:
+        return dt.datetime.now(dt.UTC)
+
+    def create_access_jwt(
+        self,
+        *,
+        subject_public_id: str,
+        token_type: JwtType,
+        ttl_seconds: int = 15 * 60,
+    ) -> str:
+        """
+        Create short-lived access JWT.
+        'sub' MUST be the public_id UUID string of the user/device.
+        """
+        now = self._now()
+        payload: dict = {
+            "sub": str(subject_public_id),
+            "typ": token_type,
+            "iat": int(now.timestamp()),
+            "exp": int((now + dt.timedelta(seconds=ttl_seconds)).timestamp()),
+            "jti": str(uuid.uuid4()),
+        }
+        secret = AppRegistry.config()["jwt"]["secret"]
+        return jwt.encode(payload, secret, algorithm="HS256", json_encoder=CustomJSONEncoder)  # type: ignore
+
+    def create_impersonation_jwt(
+        self,
+        *,
+        target_public_id: str,
+        impersonator_public_id: str,
+        ttl_seconds: int = 30 * 60,
+    ) -> str:
+        """
+        Create an impersonation JWT.
+        sub = target user, imp = superadmin who initiated impersonation.
+        """
+        now = self._now()
+        payload: dict = {
+            "sub": str(target_public_id),
+            "typ": "impersonation",
+            "imp": str(impersonator_public_id),
+            "iat": int(now.timestamp()),
+            "exp": int((now + dt.timedelta(seconds=ttl_seconds)).timestamp()),
+            "jti": str(uuid.uuid4()),
+        }
+        secret = AppRegistry.config()["jwt"]["secret"]
+        return jwt.encode(payload, secret, algorithm="HS256", json_encoder=CustomJSONEncoder)  # type: ignore
+
+    def verify_access_jwt(self, token: str, *, expected_type: JwtType) -> JwtPayload | None:
+        """
+        Verify JWT signature + exp, enforce expected typ.
+        Returns payload dict if valid else None.
+        """
+        try:
+            payload = cast(
+                JwtPayload,
+                jwt.decode(
+                    token,
+                    AppRegistry.config()["jwt"]["secret"],
+                    algorithms=["HS256"],
+                    options={
+                        "require": ["exp", "iat", "sub", "typ"],
+                    },
+                    leeway=10,  # seconds clock skew tolerance
+                ),
+            )
+        except jwt.ExpiredSignatureError:
+            self.logger.debug("JWT expired")
+            return None
+        except jwt.InvalidTokenError:
+            self.logger.debug("JWT invalid")
+            return None
+
+        if payload.get("typ") != expected_type:
+            self.logger.debug("JWT type mismatch: expected=%s got=%s", expected_type, payload.get("typ"))
+            return None
+
+        # Validate UUID format for sub/jti (defensive)
+        try:
+            uuid.UUID(payload["sub"])
+            uuid.UUID(payload["jti"])
+        except Exception:
+            self.logger.debug("JWT payload UUID fields invalid")
+            return None
+
+        # For impersonation tokens, also validate the impersonator UUID
+        if payload.get("typ") == "impersonation":
+            imp = payload.get("imp")
+            if not imp:
+                self.logger.debug("Impersonation JWT missing imp claim")
+                return None
+            try:
+                uuid.UUID(imp)
+            except Exception:
+                self.logger.debug("Impersonation JWT imp field invalid")
+                return None
+
+        return payload