jasil 0.1.0__py3-none-any.whl

jasil/__init__.py

@@ -0,0 +1,48 @@
+"""Platform substrate: deployment profile, event envelope, and providers.
+
+Public (pure) surface — safe to import anywhere; pulls in no backends:
+    - ``DeploymentProfile`` — the deployment shape.
+    - ``Event`` / ``new_event`` — the event envelope.
+    - ``StateProvider`` / ``StorageProvider`` / ``EventBusProvider`` /
+      ``LockProvider`` / ``ClockProvider`` — the capability providers.
+
+The composition root (``Platform`` / ``build_platform``) lives in
+``jasil.container`` and is imported explicitly where the platform is
+built (startup, ``deps``). Keeping it out of this ``__init__`` means importing
+the pure profile/capabilities modules never drags the concrete backends in.
+"""
+
+from importlib.metadata import PackageNotFoundError as _PackageNotFoundError
+from importlib.metadata import version as _version
+
+from jasil.events import Event, new_event
+from jasil.profile import DeploymentProfile
+from jasil.providers import (
+    ClockProvider,
+    EventBusProvider,
+    LockProvider,
+    StateBackendUnavailableError,
+    StateProvider,
+    StorageProvider,
+    TieredFailureOutcome,
+)
+
+try:
+    __version__ = _version("jasil")
+except _PackageNotFoundError:  # pragma: no cover - running from an unbuilt source tree
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    # Package metadata
+    "__version__",
+    "ClockProvider",
+    "DeploymentProfile",
+    "Event",
+    "EventBusProvider",
+    "LockProvider",
+    "StateBackendUnavailableError",
+    "StateProvider",
+    "StorageProvider",
+    "TieredFailureOutcome",
+    "new_event",
+]

jasil/_core/__init__.py

@@ -0,0 +1,5 @@
+"""Vendored low-level helpers.
+
+Private to JASIL: nothing here is part of the public API, and these modules must
+not import from the substrate, the backends, or the jobs layer.
+"""

jasil/_core/dialects.py

@@ -0,0 +1,43 @@
+"""Database-dialect capability checks.
+
+JASIL's tables are portable across SQLite, PostgreSQL and MySQL, but the
+concurrency primitives are not uniformly available. Rather than testing
+``dialect.name == "postgresql"`` at each call site — which silently downgrades
+every other database — the checks live here.
+"""
+
+from typing import Any
+
+__all__ = ["supports_skip_locked"]
+
+
+def supports_skip_locked(bind: Any) -> bool:
+    """Whether ``bind`` supports ``SELECT ... FOR UPDATE SKIP LOCKED``.
+
+    Used by the claim, relay and prune queries so concurrent workers take
+    disjoint batches instead of contending. Where it is unavailable the caller
+    falls back to a plain select, which stays *correct* for a single worker but
+    offers no protection against two workers claiming the same row — so this
+    must not report a false positive.
+
+    Args:
+        bind: The session's bind (an ``Engine`` or ``Connection``), or ``None``.
+
+    Returns:
+        True when the clause is supported. SQLite always returns False: it has
+        no row-level locking, and a single writer makes the clause unnecessary.
+    """
+    if bind is None:
+        return False
+    dialect = bind.dialect
+    if dialect.name == "postgresql":
+        return True
+    if dialect.name == "mysql":
+        # Populated on first connect; treat "unknown" as unsupported.
+        version = getattr(dialect, "server_version_info", None)
+        if not version:
+            return False
+        if getattr(dialect, "is_mariadb", False):
+            return version >= (10, 6)
+        return version >= (8, 0, 1)
+    return False

jasil/_core/identity.py

@@ -0,0 +1,34 @@
+"""Process identity for competing-consumer coordination.
+
+A single helper so the durable-job worker and the Redis Streams consumer derive
+their identifier the same way. The value distinguishes competing consumers
+across replicas in logs, lease holders (``processing_jobs.locked_by``), and
+Redis pending-entry lists.
+"""
+
+import hashlib
+import os
+import socket
+
+from jasil._core.limits import MAX_WORKER_ID_LENGTH
+
+
+def process_identity() -> str:
+    """Return a stable per-process identifier that fits the lease column.
+
+    Uniqueness is the point: this value is what ``claim_jobs`` compares against
+    to decide whether *this* worker won a row, so two live processes sharing one
+    identity would run the same job twice. A long hostname therefore cannot
+    simply be clipped — the truncated form carries a digest of the full identity
+    instead, which stays distinct where a shared prefix would not.
+
+    Returns:
+        ``"{hostname}-{pid}"`` — unique per process, stable for its lifetime — or
+        a digest-suffixed form of it when that would exceed
+        :data:`~jasil._core.limits.MAX_WORKER_ID_LENGTH`.
+    """
+    identity = f"{socket.gethostname()}-{os.getpid()}"
+    if len(identity) <= MAX_WORKER_ID_LENGTH:
+        return identity
+    digest = hashlib.blake2s(identity.encode(), digest_size=8).hexdigest()
+    return f"{identity[: MAX_WORKER_ID_LENGTH - len(digest) - 1]}-{digest}"

jasil/_core/limits.py

@@ -0,0 +1,95 @@
+"""Bounds on the values JASIL persists.
+
+Two shapes of bound, because the values differ in kind:
+
+* :func:`check_length` **raises**. The envelope's ``event_id`` / ``event_type`` /
+  ``source`` and a durable ``subscriber_id`` are written by the producing code,
+  never by a user, so an over-long one is a bug to fix rather than a value to
+  clip — a truncated ``event_type`` routes to the wrong subscribers, or to none.
+* :func:`fit_length` **truncates**, leaving a marker. Failure text, the joined
+  subscriber list, and a worker's identity are derived and diagnostic, so
+  clipping them loses nothing that matters while refusing would lose the record
+  itself. The marker is what stops a reader assuming the value is complete.
+
+Either way the bound is applied where the value enters the system, so an
+over-long value is one clear outcome instead of a truncation error on PostgreSQL
+and MySQL, silent acceptance on SQLite, and — because the publish seam swallows
+delivery failures — a lost event on all three.
+
+**Every constant here is imported by the model declaring the column it bounds**,
+so a width and the code protecting it cannot drift apart. The one pair that was
+allowed to drift is what produced the bug recorded on
+:data:`MAX_HANDLER_NAME_LENGTH`.
+"""
+
+__all__ = [
+    "MAX_HANDLER_NAME_LENGTH",
+    "MAX_STORED_ERROR_LENGTH",
+    "MAX_WORKER_ID_LENGTH",
+    "check_length",
+    "fit_length",
+]
+
+#: Cap on the failure text written to ``event_log.error_message`` and
+#: ``processing_jobs.last_error``. Both are unbounded ``Text`` columns, so this
+#: exists to stop one pathological exception (a driver dumping a whole query, a
+#: deeply nested cause chain) from bloating every retry's row.
+MAX_STORED_ERROR_LENGTH = 4000
+
+#: Width of ``event_log.handler_name``, which holds the comma-joined list of every
+#: subscriber that ran for one event — so its length grows with the subscriber
+#: count and is unbounded from the writing layer's point of view. Overflowing it
+#: made PostgreSQL reject the whole UPDATE, and because event-log writes are
+#: deliberately best-effort (swallowed by the recorder so observability never
+#: breaks processing) the failure was silent: the handlers had already run, so
+#: the work completed while the row stayed ``published`` forever.
+MAX_HANDLER_NAME_LENGTH = 500
+
+#: Width of ``event_log.worker_id`` and ``processing_jobs.locked_by``, both of
+#: which hold a process identity. That value derives from the machine's hostname
+#: (see :func:`jasil._core.identity.process_identity`), so a deployment cannot shorten it
+#: and the library has to guarantee the fit itself.
+MAX_WORKER_ID_LENGTH = 100
+
+_TRUNCATION_MARKER = "..."
+
+
+def check_length(value: str, *, field: str, limit: int) -> None:
+    """Raise when ``value`` would not fit the ``limit``-character column for ``field``.
+
+    Args:
+        value: The identifier to check.
+        field: Its name, used in the error message.
+        limit: The character limit of the column it is stored in.
+
+    Raises:
+        ValueError: When ``value`` is longer than ``limit``. These identifiers are
+            written by the producing code, never by users, so an over-long one is
+            a bug to fix rather than a value to clip: a truncated ``event_type``
+            would route to the wrong subscribers, or to none at all.
+    """
+    if len(value) > limit:
+        raise ValueError(
+            f"{field} is {len(value)} characters, which exceeds the {limit}-character limit "
+            f"and would be rejected or truncated on write: {value[:limit]!r}..."
+        )
+
+
+def fit_length(value: str | None, limit: int) -> str | None:
+    """Clamp ``value`` to ``limit`` characters, marking it when it had to be cut.
+
+    For the derived, diagnostic values :func:`check_length` would be the wrong
+    tool for: refusing one discards the very record being written.
+
+    Args:
+        value: The text about to be stored, or ``None``.
+        limit: The character limit of the column it is stored in.
+
+    Returns:
+        ``value`` unchanged when it is ``None`` or already fits, otherwise
+        exactly ``limit`` characters ending in ``...``, so a reader can tell the
+        value was cut rather than assume it is complete.
+    """
+    if value is None or len(value) <= limit:
+        return value
+    return value[: limit - len(_TRUNCATION_MARKER)] + _TRUNCATION_MARKER

jasil/_core/network.py

@@ -0,0 +1,199 @@
+"""SSRF guard for operator-configured outbound hosts.
+
+Vendored from a host application's network helpers, trimmed to the outbound
+half. The inbound half (proxy-aware client-IP extraction, trusted-proxy
+resolution) is a web-framework concern and deliberately did not come across:
+JASIL serves no requests, and keeping it out is what lets this module stay free
+of ``fastapi``.
+
+The allowlist is passed in rather than read from a global, so the host owns the
+policy and the guard stays a pure function.
+"""
+
+import ipaddress
+import logging
+import re
+import socket
+from collections.abc import Sequence
+
+logger = logging.getLogger(__name__)
+
+# RFC 1123 hostname syntax: labels of 1-63 alphanumeric/hyphen characters,
+# separated by dots. Hyphens may not start or end a label.
+_HOSTNAME_RE = re.compile(
+    r"^(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?)"  # first label
+    r"(?:\.(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?))*$",  # more
+    re.IGNORECASE,
+)
+
+# Reasons a destination is refused. Phrased to read correctly after a host value.
+_UNRESOLVABLE = "hostname could not be resolved"
+_UNPARSEABLE = "resolves to an unparseable address"
+_NON_PUBLIC = "resolves to a non-public address"
+_NOT_AN_AUTHORITY = "is not a bare host[:port] authority"
+
+
+def _is_valid_hostname(value: str) -> bool:
+    """Return True when ``value`` is a syntactically valid RFC 1123 hostname.
+
+    Rejects values carrying URL schemes, ports, or other non-hostname characters
+    (e.g. ``caddy:8080`` or ``http://caddy``).
+
+    Args:
+        value: Candidate hostname to validate.
+
+    Returns:
+        True when ``value`` conforms to RFC 1123 hostname syntax.
+    """
+    return len(value) <= 253 and _HOSTNAME_RE.match(value) is not None
+
+
+def _is_private_or_reserved(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """Return True if ``addr`` belongs to any non-routable range.
+
+    Combines every "do not dial" predicate ``ipaddress`` exposes: private
+    (RFC1918, fc00::/7), loopback, link-local, multicast, unspecified, and
+    reserved. Any of these would let an attacker pivot to internal
+    infrastructure or a cloud metadata service.
+    """
+    return (
+        addr.is_private
+        or addr.is_loopback
+        or addr.is_link_local
+        or addr.is_multicast
+        or addr.is_unspecified
+        or addr.is_reserved
+    )
+
+
+def _split_allowlist(
+    allowed_hosts: Sequence[str],
+) -> tuple[frozenset[str], tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...]]:
+    """Classify allowlist entries into hostnames and IP networks."""
+    hosts: set[str] = set()
+    networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
+    for entry in allowed_hosts:
+        try:
+            networks.append(ipaddress.ip_network(entry, strict=False))
+        except ValueError:
+            hosts.add(entry.lower())
+    return frozenset(hosts), tuple(networks)
+
+
+def _address_rejection_reason(
+    hostname: str,
+    *,
+    allowed_hosts: Sequence[str],
+    purpose: str | None = None,
+) -> str | None:
+    """Return why ``hostname`` must not be dialed, or None when every address is safe.
+
+    Resolves every A/AAAA record and requires all of them to be public unicast. A
+    single private/loopback/link-local answer rejects the host — this defends
+    against DNS rebinding, where an attacker-controlled name returns a public IP
+    on the first lookup and a private IP on the next.
+
+    Args:
+        hostname: The hostname to resolve, without a port.
+        allowed_hosts: Hostnames and CIDRs exempt from the address denylist.
+        purpose: Optional short tag identifying the outbound call, used only for
+            audit logging.
+
+    Returns:
+        A reason string, or ``None`` when the host may be dialed.
+    """
+    try:
+        infos = socket.getaddrinfo(hostname, None)
+    except socket.gaierror:
+        return _UNRESOLVABLE
+
+    exempt_hostnames, exempt_networks = _split_allowlist(allowed_hosts)
+    for info in infos:
+        ip_text = info[4][0]
+        try:
+            addr = ipaddress.ip_address(ip_text)
+        except ValueError:
+            # Defensive: a resolver answer we cannot parse is treated as unsafe.
+            return _UNPARSEABLE
+        if not _is_private_or_reserved(addr):
+            continue
+        # Audit trail: every allowlisted private destination is logged so
+        # operators can review what the exception is being used for. A hostname
+        # entry is the broader of the two, and says so.
+        if hostname.lower() in exempt_hostnames:
+            logger.warning(
+                "SSRF allowlist hit: dialing private address %s for host %s (purpose=%s). That entry is a "
+                "hostname, so every address the name resolves to is exempt \u2014 including any it starts "
+                "returning later. Prefer a CIDR entry, which cannot widen.",
+                ip_text,
+                hostname,
+                purpose or "unspecified",
+            )
+        elif any(addr in network for network in exempt_networks):
+            logger.info(
+                "SSRF allowlist hit: dialing private address %s for host %s (purpose=%s)",
+                ip_text,
+                hostname,
+                purpose or "unspecified",
+            )
+        else:
+            return _NON_PUBLIC
+    return None
+
+
+def host_rejection_reason(
+    host: str | None,
+    *,
+    allowed_hosts: Sequence[str] = (),
+    purpose: str | None = None,
+) -> str | None:
+    """Return why an operator-configured ``host[:port]`` must not be dialed, or None.
+
+    Returns a reason rather than raising, because callers handed a host from
+    configuration must *degrade* (disable the feature) rather than fail.
+
+    The value must be a plain ``host[:port]``. One carrying a scheme, path, or
+    credentials would otherwise be interpolated into a URL by the caller and
+    silently redirect the request elsewhere — ``"evil.example.com/x"`` becomes
+    ``"https://evil.example.com/x/reverse"``, whose hostname check passes.
+
+    **This is a time-of-check guard, and callers use it once.** The geocoding
+    backend validates its host while the composition root assembles the platform,
+    then dials the resulting URL for the life of the process — so a name that
+    resolves publicly at startup and privately afterwards is not caught. What is
+    caught is the case that matters for an operator-supplied setting: a host that
+    was already internal when it was configured. Refusing redirects (which the
+    geocoding backend does) closes the other easy pivot. Full TOCTOU safety would
+    mean pinning the resolved public IP and dialing it with the original Host
+    header on every request.
+
+    Resolution is also unbounded: :func:`socket.getaddrinfo` takes no timeout, so
+    an unresponsive resolver stalls whoever called this. At startup that shows up
+    as a slow boot rather than a hung request.
+
+    Args:
+        host: The configured host authority, or ``None``.
+        allowed_hosts: Hostnames and CIDRs exempt from the address denylist, for
+            reaching a self-hosted service on a private network. An entry that is
+            a *hostname* exempts every address it resolves to — including a cloud
+            metadata endpoint — so prefer a CIDR where you can; a hostname
+            exemption is logged at ``WARNING`` each time it is taken.
+        purpose: Optional short tag identifying the outbound call, used only for
+            audit logging.
+
+    Returns:
+        A short human-readable reason, or ``None`` when the host may be dialed.
+    """
+    if host is None:
+        return _NOT_AN_AUTHORITY
+
+    hostname, separator, port = host.rpartition(":")
+    if not separator:
+        hostname = host
+    elif not (port.isdigit() and len(port) <= 5):
+        return _NOT_AN_AUTHORITY
+
+    if not _is_valid_hostname(hostname):
+        return _NOT_AN_AUTHORITY
+
+    return _address_rejection_reason(hostname, allowed_hosts=allowed_hosts, purpose=purpose)

jasil/_core/optional_deps.py

@@ -0,0 +1,30 @@
+"""Guard for features behind an optional extra.
+
+Turns a missing optional dependency into one clear message naming the extra to
+install, instead of a bare ``ModuleNotFoundError`` from somewhere deep in the
+call stack.
+"""
+
+from typing import Any
+
+__all__ = ["require"]
+
+
+def require(module: Any, *, package: str, extra: str, feature: str) -> Any:
+    """Return ``module``, or raise explaining which extra provides it.
+
+    Args:
+        module: The lazily imported module, or ``None`` when the import failed.
+        package: Distribution name, e.g. ``alembic``.
+        extra: The JASIL extra providing it, e.g. ``migrations``.
+        feature: Human-readable feature name for the message.
+
+    Raises:
+        ImportError: When ``module`` is ``None``.
+    """
+    if module is None:
+        raise ImportError(
+            f"{feature} requires the '{package}' package, which is not installed. "
+            f"Install it with: pip install 'jasil[{extra}]'"
+        )
+    return module

jasil/_core/pruning.py

@@ -0,0 +1,58 @@
+"""Bounded batch deletes shared by the substrate's prunable tables.
+
+Every table :mod:`jasil.retention` prunes (``event_log``, relayed
+``event_outbox`` rows, ``completed`` ``processing_jobs``) is deleted the
+same way, and only the model and the filter differ. The batching is what
+keeps each delete transaction short, so a prune pass never holds locks on
+a hot table long enough to block the relay or the worker.
+"""
+
+from typing import Any
+
+from sqlalchemy import ColumnExpressionArgument, delete, select
+from sqlalchemy.orm import Session
+
+from jasil._core.dialects import supports_skip_locked
+
+# Rows deleted per batch when pruning; bounded so each delete transaction
+# stays short.
+PRUNE_BATCH_SIZE = 1000
+# Safety cap on batches per pass so a pathological backlog cannot spin
+# forever; the next scheduled pass continues where this one left off.
+PRUNE_MAX_BATCHES = 10_000
+
+
+def bounded_delete(
+    model: type[Any],
+    *conditions: ColumnExpressionArgument[bool],
+    db: Session,
+    batch_size: int = PRUNE_BATCH_SIZE,
+) -> int:
+    """
+    Delete the rows matching ``conditions`` in bounded, committed batches.
+
+    Args:
+        model: Mapped class to delete from; must expose an ``id`` column.
+        *conditions: Filters selecting the rows that are safe to prune.
+        db: Active database session.
+        batch_size: Maximum rows deleted per batch.
+
+    Returns:
+        The total number of rows deleted.
+    """
+    total = 0
+    for _ in range(PRUNE_MAX_BATCHES):
+        id_stmt = select(model.id).where(*conditions).limit(batch_size)
+        if supports_skip_locked(db.bind):  # pragma: no cover - server-side locking, not exercised on SQLite
+            # Concurrent prunes step over each other's claimed page rather
+            # than blocking on it.
+            id_stmt = id_stmt.with_for_update(skip_locked=True)
+        ids = list(db.execute(id_stmt).scalars().all())
+        if not ids:
+            break
+        db.execute(delete(model).where(model.id.in_(ids)))
+        db.commit()
+        total += len(ids)
+        if len(ids) < batch_size:
+            break
+    return total