hugpy-control 0.2.0a0__py3-none-any.whl

hugpy_control/__init__.py

@@ -0,0 +1,49 @@
+"""hugpy-control: the job/settings/principal control plane of the Hugpy ecosystem.
+
+Submodules (all stdlib + ``hugpy_platform``; none import an engine, fleet or
+server package):
+
+    bus         in-process typed message bus + topic constants
+    jobs        JobStore — the one all-transport job store (chat, media, downloads)
+    shared      SqliteMirror — the cross-process mirror the JobStore rides on
+    settings    SettingsStore — namespaced runtime settings (F4)
+    principals  PrincipalStore — identities, tokens, capability checks (F2)
+    calllog     append-only call log + tail reader
+    job_schemas compat re-export of the download-job names
+
+Submodules are resolved lazily (PEP 562) so ``import hugpy_control`` stays
+side-effect free: the store singletons are only created when the module that
+owns them is first imported.
+"""
+from __future__ import annotations
+
+import importlib
+
+try:  # the installed distribution's version: the workspace tag/commit, never a literal
+    from importlib.metadata import version as _dist_version
+    __version__ = _dist_version("hugpy-control")
+except Exception:  # noqa: BLE001 — source tree without metadata
+    __version__ = "0.0.0+unknown"
+
+__all__ = [
+    "__version__",
+    "bus",
+    "calllog",
+    "job_schemas",
+    "jobs",
+    "principals",
+    "settings",
+    "shared",
+]
+
+_SUBMODULES = frozenset(n for n in __all__ if not n.startswith("__"))
+
+
+def __getattr__(name: str):
+    if name in _SUBMODULES:
+        return importlib.import_module(f"{__name__}.{name}")
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__():
+    return sorted(set(globals()) | _SUBMODULES)

hugpy_control/bus.py

@@ -0,0 +1,240 @@
+"""F1 — one typed, frozen, addressed message bus. Control rides the same bus.
+
+Scope (deliberate): this is the STATE/CONTROL plane, not the token data
+plane. Token streams stay on their SSE pipes — publishing every token through
+a fan-out bus would tax the hot path for nothing. What travels here:
+
+    job.created / job.status / job.done      lifecycle (published by the
+                                             JobStore adapter, see wire below)
+    control.cancel                           stop a generation, any transport
+    catalog.changed                          on-disk model inventory changed
+                                             (storage publishes, engine consumes)
+    control.restart / control.* (future)     worker ops, module updates
+    worker.* / keeper.* (future)             registries, health, keeper comms
+
+Envelope: BusMessage — frozen, addressed (principal / channel / target /
+job_id), serializable (to_dict/from_dict), so an HTTP or SSE relay can carry
+it across processes verbatim. In-process today (one bus per process, same
+one-gunicorn-process model as the JobStore); remote transports reach it
+through thin route adapters that publish/subscribe on their behalf.
+
+Delivery: each Subscription owns a bounded queue. Publishing never blocks —
+on overflow the OLDEST message is dropped and the drop is counted loudly on
+the subscription (a slow consumer must not stall cancels for everyone else).
+"""
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Iterator, Optional
+
+logger = logging.getLogger(__name__)
+
+# Topic conventions. Match exact ("control.cancel") or prefix-wildcard
+# ("job.*" matches job.created, job.status, ...; "*" matches everything).
+TOPIC_JOB_CREATED = "job.created"
+TOPIC_JOB_STATUS = "job.status"
+TOPIC_JOB_DONE = "job.done"          # any terminal state; payload carries which
+TOPIC_CONTROL_CANCEL = "control.cancel"
+# Catalog invalidation (partition protocol, see EXTRACTION_GUIDE §4): published by
+# hugpy_storage after a download/delete changes what is on disk; consumed by
+# hugpy_engine to drop its catalog/manifest caches. Payload convention:
+# {"model_key": str|None, "reason": "download"|"delete"|"prune"|"reconcile"|...}.
+TOPIC_CATALOG_CHANGED = "catalog.changed"
+
+
+@dataclass(frozen=True)
+class BusMessage:
+    topic: str
+    id: str = field(default_factory=lambda: uuid.uuid4().hex)
+    ts: float = field(default_factory=time.time)
+    source: Optional[str] = None      # transport/component that published
+    principal: Optional[str] = None   # who caused it (F2 threads through here)
+    channel: Optional[str] = None     # conversational context
+    target: Optional[str] = None      # addressed recipient; None = broadcast
+    job_id: Optional[str] = None
+    payload: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "topic": self.topic, "id": self.id, "ts": self.ts,
+            "source": self.source, "principal": self.principal,
+            "channel": self.channel, "target": self.target,
+            "job_id": self.job_id, "payload": dict(self.payload),
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "BusMessage":
+        return cls(
+            topic=str(d.get("topic") or ""),
+            id=str(d.get("id") or uuid.uuid4().hex),
+            ts=float(d.get("ts") or time.time()),
+            source=d.get("source"), principal=d.get("principal"),
+            channel=d.get("channel"), target=d.get("target"),
+            job_id=d.get("job_id"), payload=dict(d.get("payload") or {}),
+        )
+
+
+def _topic_matches(pattern: str, topic: str) -> bool:
+    if pattern == "*" or pattern == topic:
+        return True
+    if pattern.endswith(".*"):
+        return topic.startswith(pattern[:-1])
+    return False
+
+
+class Subscription:
+    """One consumer's mailbox. Iterate it (blocking) or get(timeout=...).
+    close() detaches it from the bus and unblocks any pending get."""
+
+    _SENTINEL = object()
+
+    def __init__(self, bus: "Bus", topics: tuple[str, ...],
+                 target: Optional[str], maxsize: int) -> None:
+        self._bus = bus
+        self.topics = topics
+        self.target = target
+        self._q: "queue.Queue[Any]" = queue.Queue(maxsize=maxsize)
+        self.dropped = 0
+        self.closed = False
+
+    def _matches(self, msg: BusMessage) -> bool:
+        if self.target is not None and msg.target not in (None, self.target):
+            return False
+        return any(_topic_matches(p, msg.topic) for p in self.topics)
+
+    def _offer(self, msg: BusMessage) -> None:
+        while True:
+            try:
+                self._q.put_nowait(msg)
+                return
+            except queue.Full:
+                try:
+                    self._q.get_nowait()
+                    self.dropped += 1
+                    if self.dropped in (1, 100) or self.dropped % 1000 == 0:
+                        logger.warning(
+                            "bus subscription %s dropped %d message(s) "
+                            "(slow consumer)", self.topics, self.dropped)
+                except queue.Empty:
+                    pass
+
+    def get(self, timeout: Optional[float] = None) -> Optional[BusMessage]:
+        try:
+            item = self._q.get(timeout=timeout)
+        except queue.Empty:
+            return None
+        return None if item is self._SENTINEL else item
+
+    def __iter__(self) -> Iterator[BusMessage]:
+        while not self.closed:
+            item = self._q.get()
+            if item is self._SENTINEL:
+                return
+            yield item
+
+    def close(self) -> None:
+        self.closed = True
+        self._bus._detach(self)
+        try:
+            self._q.put_nowait(self._SENTINEL)
+        except queue.Full:
+            pass
+
+
+class Bus:
+    def __init__(self) -> None:
+        self._subs: list[Subscription] = []
+        self._lock = threading.Lock()
+
+    def subscribe(self, *topics: str, target: Optional[str] = None,
+                  maxsize: int = 256) -> Subscription:
+        sub = Subscription(self, topics or ("*",), target, maxsize)
+        with self._lock:
+            self._subs.append(sub)
+        return sub
+
+    def _detach(self, sub: Subscription) -> None:
+        with self._lock:
+            try:
+                self._subs.remove(sub)
+            except ValueError:
+                pass
+
+    def publish(self, topic: Optional[str] = None, *,
+                msg: Optional[BusMessage] = None, **fields: Any) -> BusMessage:
+        """publish("control.cancel", job_id=...) or publish(msg=BusMessage(...))."""
+        if msg is None:
+            if not topic:
+                raise ValueError("publish() needs a topic or a msg")
+            msg = BusMessage(topic=topic, **fields)
+        with self._lock:
+            subs = list(self._subs)
+        for sub in subs:
+            if not sub.closed and sub._matches(msg):
+                sub._offer(msg)
+        return msg
+
+
+bus = Bus()
+
+_WIRED: dict[tuple[int, int], threading.Thread] = {}
+_WIRED_LOCK = threading.Lock()
+
+
+def wire_cancel(the_bus: Optional[Bus] = None, store=None) -> threading.Thread:
+    """F1.3 — control messages act through the same substrate everywhere:
+    a daemon thread subscribes to control.cancel and calls JobStore.cancel,
+    which fires the cancel handle the owning stream attached. Idempotent per
+    (bus, store) pair; call it from every process entrypoint that serves jobs
+    (flask app factory, worker agent main)."""
+    from hugpy_control.jobs import job_store as _default_store
+    the_bus = the_bus or bus
+    store = store if store is not None else _default_store
+    key = (id(the_bus), id(store))
+    with _WIRED_LOCK:
+        th = _WIRED.get(key)
+        if th is not None and th.is_alive():
+            return th
+        sub = the_bus.subscribe(TOPIC_CONTROL_CANCEL)
+
+        def _run() -> None:
+            for m in sub:
+                if m.job_id:
+                    store.cancel(m.job_id,
+                                 reason=str(m.payload.get("reason") or ""))
+
+        th = threading.Thread(target=_run, name="comms-cancel", daemon=True)
+        th.start()
+        _WIRED[key] = th
+        return th
+
+
+def wire_job_events(the_bus: Optional[Bus] = None, store=None,
+                    source: Optional[str] = None) -> None:
+    """Publish job lifecycle transitions onto the bus (job.created /
+    job.status / job.done) via the store's on_change hook — the store never
+    imports the bus, this adapter is the one seam between them."""
+    from hugpy_control.jobs import job_store as _default_store, TERMINAL_STATUSES
+    the_bus = the_bus or bus
+    store = store if store is not None else _default_store
+
+    def _on_change(job, prior: str) -> None:
+        status = job.to_dict()["status"]
+        if not prior:
+            topic = TOPIC_JOB_CREATED
+        elif status in TERMINAL_STATUSES:
+            topic = TOPIC_JOB_DONE
+        else:
+            topic = TOPIC_JOB_STATUS
+        the_bus.publish(topic, source=source, principal=job.principal,
+                        channel=job.channel, target=job.worker,
+                        job_id=job.id,
+                        payload={"status": status, "prior": prior,
+                                 "kind": job.kind, "model_key": job.model_key})
+
+    store.on_change = _on_change

hugpy_control/calllog.py

@@ -0,0 +1,131 @@
+"""CALL-LOG-20260910: append-only call log (one JSON line per event) + tail reader.
+
+Events: phase "start" (job created; carries client/ua/route when a request
+context exists) and phase "end" (job finished; carries status, worker, tokens,
+duration_ms, error). The reader merges the two by job id, newest first.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import socket
+import threading
+import time
+
+log = logging.getLogger(__name__)
+_LOCK = threading.Lock()
+_HOST = socket.gethostname()
+
+
+def path() -> str:
+    env = (os.environ.get("HUGPY_CALL_LOG") or "").strip()
+    if env:
+        return env
+    db = (os.environ.get("HUGPY_COMMS_DB") or "").strip()
+    base = os.path.dirname(db) if db else ""
+    if not base:
+        try:
+            from hugpy_platform.constants import DEFAULT_ROOT
+            base = os.path.join(str(DEFAULT_ROOT), "comms")
+        except Exception:  # noqa: BLE001
+            base = os.path.expanduser("~/.hugpy")
+    return os.path.join(base, "calls.jsonl")
+
+
+def _request_context() -> dict:
+    try:
+        from flask import has_request_context, request
+        if not has_request_context():
+            return {}
+        xff = (request.headers.get("X-Forwarded-For") or "").split(",")[0].strip()
+        return {
+            "client": xff or request.remote_addr,
+            "ua": (request.headers.get("User-Agent") or "")[:160],
+            "route": request.path,
+            "method": request.method,
+            "host": request.host,
+        }
+    except Exception:  # noqa: BLE001
+        return {}
+
+
+def _err_text(err) -> str | None:
+    if err is None:
+        return None
+    for attr in ("message", "msg"):
+        v = getattr(err, attr, None)
+        if v:
+            return str(v)[:400]
+    if isinstance(err, dict):
+        return str(err.get("message") or err)[:400]
+    return str(err)[:400]
+
+
+def record(phase: str, job, **extra) -> None:
+    """Append one event for `job`. Never raises — logging must not break serving."""
+    try:
+        row = {
+            "ts": time.time(), "phase": phase, "node": _HOST,
+            "id": getattr(job, "id", None), "kind": getattr(job, "kind", None),
+            "model_key": getattr(job, "model_key", None), "model": getattr(job, "model_name", None),
+            "principal": getattr(job, "principal", None), "transport": getattr(job, "transport", None),
+            "channel": getattr(job, "channel", None), "worker": getattr(job, "worker", None),
+            "status": getattr(job, "status", None), "tokens": getattr(job, "tokens", 0),
+            "started_ts": getattr(job, "started_ts", None),
+        }
+        if phase == "start":
+            row.update(_request_context())
+        else:
+            st = getattr(job, "started_ts", None)
+            row["duration_ms"] = int((time.time() - st) * 1000) if st else None
+            row["error"] = _err_text(getattr(job, "error", None))
+        row.update(extra)
+        line = json.dumps(row, default=str)
+        p = path()
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        with _LOCK, open(p, "a", encoding="utf-8") as fh:
+            fh.write(line + "\n")
+    except Exception:  # noqa: BLE001
+        log.debug("call log write failed", exc_info=True)
+
+
+def read(limit: int = 300, since: float | None = None, tail_bytes: int = 4 * 1024 * 1024) -> list[dict]:
+    """Newest-first merged calls from the tail of the log."""
+    p = path()
+    try:
+        with open(p, "rb") as fh:
+            fh.seek(0, 2)
+            size = fh.tell()
+            fh.seek(max(0, size - tail_bytes))
+            data = fh.read().decode("utf-8", errors="replace")
+    except OSError:
+        return []
+    lines = data.split("\n")
+    if size > tail_bytes and lines:
+        lines = lines[1:]                       # drop the partial first line
+    calls: dict = {}
+    order: list = []
+    for line in lines:
+        if not line.strip():
+            continue
+        try:
+            ev = json.loads(line)
+        except ValueError:
+            continue
+        jid = ev.get("id") or f"anon-{ev.get('ts')}"
+        if jid not in calls:
+            calls[jid] = {}
+            order.append(jid)
+        cur = calls[jid]
+        if ev.get("phase") == "start":
+            cur.update({k: v for k, v in ev.items() if k != "phase"})
+            cur.setdefault("started_ts", ev.get("ts"))
+        else:
+            cur.update({k: v for k, v in ev.items() if k not in ("phase", "ts") and v is not None})
+            cur["ended_ts"] = ev.get("ts")
+    rows = [calls[j] for j in order]
+    if since:
+        rows = [r for r in rows if (r.get("started_ts") or r.get("ts") or 0) >= since]
+    rows.sort(key=lambda r: r.get("started_ts") or r.get("ts") or 0, reverse=True)
+    return rows[:limit]

hugpy_control/job_schemas.py

@@ -0,0 +1,35 @@
+"""Compat shim for the download-job schema names.
+
+The Job/JobStore that used to live here (download jobs only) is now the shared,
+all-transport store in :mod:`hugpy_control.jobs` (F5). Download callers keep
+importing from here unchanged; the singleton is the same store every other
+transport enqueues into.
+"""
+# The Job/JobStore that used to live here (download jobs only) is now the
+# shared, all-transport store in hugpy_control.jobs (F5): one
+# schema for chat requests AND downloads, canonical lifecycle
+# pending→processing→streaming→(done|cancelled|failed), with the old
+# queued/running/completed names normalized on write. Download callers keep
+# importing from here unchanged; the singleton is the same store every other
+# transport enqueues into.
+from hugpy_control.jobs import (
+    CANONICAL_STATUSES,
+    LEGACY_FOR_CANONICAL,
+    TERMINAL_STATUSES,
+    Job,
+    JobError,
+    JobStore,
+    job_store,
+    normalize_status,
+)
+
+__all__ = [
+    "CANONICAL_STATUSES",
+    "LEGACY_FOR_CANONICAL",
+    "TERMINAL_STATUSES",
+    "Job",
+    "JobError",
+    "JobStore",
+    "job_store",
+    "normalize_status",
+]