tokenhub 0.1.0__py3-none-any.whl

tokenhub/__init__.py

@@ -0,0 +1 @@
+"""TokenHub package."""

tokenhub/__main__.py

@@ -0,0 +1,8 @@
+"""Module entrypoint for the detached child and interactive command."""
+
+from __future__ import annotations
+
+from tokenhub.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tokenhub/analytics/__init__.py

@@ -0,0 +1 @@
+"""Provider-neutral analytics over normalized local events."""

tokenhub/analytics/breakdown.py

@@ -0,0 +1,78 @@
+"""Agent, model, and session views of the canonical observed workload."""
+
+import hashlib
+from collections import defaultdict
+from datetime import UTC
+from typing import Any
+
+from tokenhub.database.models import UsageEventRecord
+
+_COUNTERS = (
+    "input_total_tokens", "output_total_tokens", "cache_read_tokens",
+    "cache_write_tokens", "reasoning_tokens",
+)
+
+
+def _session_key(event: UsageEventRecord) -> str:
+    # Stable across refreshes, with no original session id or filesystem path.
+    identity = f"{event.connector_id}\0{event.session_id or event.source_id}"
+    return hashlib.sha256(identity.encode()).hexdigest()[:24]
+
+
+def _summary(events: list[UsageEventRecord]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for field in _COUNTERS:
+        values = [getattr(event, field) for event in events if getattr(event, field) is not None]
+        result[field] = sum(values) if values else None
+    workloads = [
+        event.input_total_tokens + event.output_total_tokens
+        for event in events
+        if event.input_total_tokens is not None and event.output_total_tokens is not None
+    ]
+    result.update(
+        workload_tokens=sum(workloads) if workloads else None,
+        event_count=len(events),
+        session_count=len({_session_key(event) for event in events}),
+        model_count=len({event.model_name for event in events if event.model_name is not None}),
+        incomplete_event_count=sum(
+            event.input_total_tokens is None or event.output_total_tokens is None for event in events
+        ),
+        first_seen=min((event.timestamp for event in events), default=None),
+        last_seen=max((event.timestamp for event in events), default=None),
+    )
+    for field in ("first_seen", "last_seen"):
+        result[field] = result[field].replace(tzinfo=UTC).isoformat() if result[field] else None
+    return result
+
+
+def usage_breakdown(events: list[UsageEventRecord]) -> dict[str, Any]:
+    providers: dict[str, list[UsageEventRecord]] = defaultdict(list)
+    models: dict[tuple[str, str | None], list[UsageEventRecord]] = defaultdict(list)
+    sessions: dict[tuple[str, str], list[UsageEventRecord]] = defaultdict(list)
+    for event in events:
+        providers[event.provider].append(event)
+        models[event.provider, event.model_name].append(event)
+        sessions[event.provider, _session_key(event)].append(event)
+    model_rows = []
+    for (provider, model), group in models.items():
+        attributions = {event.model_attribution for event in group}
+        model_rows.append(dict(
+            _summary(group), provider=provider, model_name=model,
+            attribution=next(iter(attributions)) if len(attributions) == 1 else "mixed",
+        ))
+    session_rows = []
+    for (provider, key), group in sessions.items():
+        session_models: dict[str | None, list[UsageEventRecord]] = defaultdict(list)
+        for event in group:
+            session_models[event.model_name].append(event)
+        session_rows.append(dict(
+            _summary(group), provider=provider, session_key=key,
+            models=[dict(_summary(rows), model_name=model) for model, rows in session_models.items()],
+        ))
+    sort_key = lambda row: (-(row["workload_tokens"] or 0), row["provider"], row.get("model_name") or "")
+    return {
+        "totals": _summary(events),
+        "providers": sorted([dict(_summary(group), provider=provider) for provider, group in providers.items()], key=sort_key),
+        "models": sorted(model_rows, key=sort_key),
+        "sessions": sorted(session_rows, key=sort_key),
+    }

tokenhub/analytics/service.py

@@ -0,0 +1,21 @@
+"""Dashboard queries over observed, normalized usage only."""
+
+from datetime import datetime
+from typing import Any
+
+from tokenhub.analytics.breakdown import usage_breakdown
+from tokenhub.database.repositories import UsageRepository
+from tokenhub.domain.models import DashboardSummary
+
+
+class AnalyticsService:
+    def __init__(self, usage_repository: UsageRepository) -> None:
+        self.usage_repository = usage_repository
+
+    def dashboard(self) -> DashboardSummary:
+        return self.usage_repository.dashboard_totals()
+
+    def usage_breakdown(
+        self, start: datetime | None = None, end: datetime | None = None
+    ) -> dict[str, Any]:
+        return usage_breakdown(self.usage_repository.observed_events(start, end))

tokenhub/api/__init__.py

@@ -0,0 +1,7 @@
+"""HTTP surface for the local TokenHub server.
+
+* :mod:`tokenhub.api.container` — startup/shutdown and shared services
+* :mod:`tokenhub.api.routes` — the versioned `/api/v1` routes
+
+The application factory lives in :mod:`tokenhub.app`.
+"""

tokenhub/api/container.py

@@ -0,0 +1,146 @@
+"""Composition root for the local TokenHub application.
+
+The container owns startup and shutdown so that constructing the app has **no
+side effects**: no database file, no migration, and no provider discovery until
+the ASGI lifespan starts. It also owns the two pieces of state that must survive
+across HTTP requests:
+
+* the process-local discovery handoff (``SourceRepository`` pending candidates),
+* the injected ``DiscoveryContext``, which tests replace after ``create_app``.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import threading
+from dataclasses import dataclass
+
+from sqlalchemy import Engine
+from sqlalchemy.orm import Session
+
+from tokenhub.analytics.service import AnalyticsService
+from tokenhub.connectors.protocol import DiscoveryContext
+from tokenhub.connectors.registry import ConnectorRegistry
+from tokenhub.database.migrations import run_migrations
+from tokenhub.database.repositories import SourceRepository, UsageRepository
+from tokenhub.database.session import create_engine_for
+from tokenhub.discovery.service import DiscoveryService
+from tokenhub.ingestion.collection import CollectionService
+from tokenhub.ingestion.service import IngestionService
+from tokenhub.settings import TokenHubSettings
+
+
+@dataclass(frozen=True, slots=True)
+class Services:
+    """Long-lived collaborators shared by the HTTP routes."""
+
+    session: Session
+    source_repository: SourceRepository
+    usage_repository: UsageRepository
+    discovery: DiscoveryService
+    ingestion: IngestionService
+    analytics: AnalyticsService
+    collection: CollectionService
+
+
+class Container:
+    """Startup, shutdown, and request serialization for one app instance."""
+
+    def __init__(self, settings: TokenHubSettings) -> None:
+        self.settings = settings
+        # One SQLite connection backs this single-user local server; the lock
+        # keeps sequential routes from interleaving on it.
+        self.lock = threading.RLock()
+        self._engine: Engine | None = None
+        self._services: Services | None = None
+        self._discovery_context: DiscoveryContext | None = None
+        self.collection_thread: threading.Thread | None = None
+        self._collection_stop = threading.Event()
+
+    @property
+    def services(self) -> Services:
+        """Collaborators for the running app; raises before :meth:`start`."""
+        if self._services is None:
+            raise RuntimeError("container has not started")
+        return self._services
+
+    @property
+    def discovery_context(self) -> DiscoveryContext:
+        """Discovery inputs, defaulting to the real home and process environment."""
+        if self._discovery_context is not None:
+            return self._discovery_context
+        return DiscoveryContext(
+            home=self.settings.home_directory,
+            environment=os.environ,
+            which=shutil.which,
+        )
+
+    @discovery_context.setter
+    def discovery_context(self, context: DiscoveryContext) -> None:
+        """Replace discovery inputs, including after startup."""
+        self._discovery_context = context
+        if self._services is not None:
+            self._services.discovery.context = context
+
+    def start(self) -> None:
+        """Migrate the database, then build the services that routes depend on."""
+        run_migrations(self.settings)
+        engine = create_engine_for(self.settings)
+        session = Session(engine)
+        registry = ConnectorRegistry.default()
+        source_repository = SourceRepository(session)
+        source_repository.upgrade_codex_parser_versions()
+        usage_repository = UsageRepository(session)
+        self._engine = engine
+        discovery = DiscoveryService(registry, source_repository, self.discovery_context)
+        ingestion = IngestionService(source_repository, usage_repository, registry)
+        self._services = Services(
+            session=session,
+            source_repository=source_repository,
+            usage_repository=usage_repository,
+            discovery=discovery,
+            ingestion=ingestion,
+            analytics=AnalyticsService(usage_repository),
+            collection=CollectionService(source_repository, usage_repository, discovery, ingestion),
+        )
+        self._collect_safely()
+        self._collection_stop.clear()
+        self.collection_thread = threading.Thread(
+            target=self._collect_periodically, name="tokenhub-collection", daemon=True
+        )
+        self.collection_thread.start()
+
+    def collect(self) -> None:
+        with self.lock:
+            self.services.collection.run_once()
+
+    def _collect_periodically(self) -> None:
+        while not self._collection_stop.wait(self.settings.scan_interval_seconds):
+            self._collect_safely()
+
+    def _collect_safely(self) -> None:
+        try:
+            self.collect()
+        except Exception:  # noqa: BLE001 - startup and later collection must remain available
+            # Do not log exception text, which may contain a provider path.
+            logging.getLogger(__name__).error("Automatic collection failed; retrying on the next scan")
+            with self.lock:
+                self.services.session.rollback()
+                self.services.collection.failed_source_count = max(
+                    1, self.services.collection.failed_source_count
+                )
+
+    def stop(self) -> None:
+        """Release the session and engine; the database file stays behind."""
+        self._collection_stop.set()
+        if self.collection_thread is not None:
+            self.collection_thread.join()
+            self.collection_thread = None
+        if self._services is not None:
+            self._services.session.close()
+            self._services = None
+        if self._engine is not None:
+            self._engine.dispose()
+            self._engine = None

tokenhub/api/routes.py

@@ -0,0 +1,267 @@
+"""Thin, versioned HTTP routes over the TokenHub service layer.
+
+Routes own transport concerns only — status codes, JSON projection, and error
+mapping. All persistence, discovery, parsing, and aggregation stay in the
+services so the local API cannot drift from the tested behavior.
+
+Every projection here is deliberately path-free and record-free: the HTTP
+surface exposes identifiers, states, counts, and timestamps, never a canonical
+path, approved root, provider path, credential, prompt, or raw record.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Annotated
+
+from fastapi import APIRouter, HTTPException, Query, Request
+
+from tokenhub.api.container import Container
+
+# The service layer's own return type for an approval. The HTTP layer only
+# reads it — it never queries the ORM itself — so this is a data dependency on
+# the service contract, not a persistence leak into transport code.
+from tokenhub.database.models import SourceRecord
+from tokenhub.domain.models import (
+    DashboardSummary,
+    ImportOutcome,
+    RebuildOutcome,
+)
+from tokenhub.ingestion.service import (
+    SourceNotApprovedError,
+    SourceNotFoundError,
+    UnsupportedSourceError,
+)
+from tokenhub.security.redaction import redact_sensitive
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1")
+
+_ERROR_MAP: tuple[tuple[type[Exception], int, str], ...] = (
+    (SourceNotFoundError, 404, "Unknown source"),
+    (SourceNotApprovedError, 409, "Source is not approved"),
+    (UnsupportedSourceError, 422, "Source is not supported"),
+    (ValueError, 400, "Invalid source path"),
+    (OSError, 400, "Source is unavailable"),
+)
+
+
+def container_for(request: Request) -> Container:
+    """The app's container, injected by the factory."""
+    return request.app.state.container
+
+
+def _service_failure(error: Exception) -> HTTPException:
+    """Map a service exception to a status code and a path-free message.
+
+    Known service failures get a fixed, path-free detail. Anything else becomes
+    a generic internal error: a route must never hand a provider path, a raw
+    record, or a credential to the client, and the log line goes through the
+    central redactor.
+    """
+    for error_type, status, detail in _ERROR_MAP:
+        if isinstance(error, error_type):
+            return HTTPException(status_code=status, detail=redact_sensitive(detail))
+    logger.error("unhandled TokenHub service failure: %s", redact_sensitive(str(error)))
+    return HTTPException(status_code=500, detail="Internal error")
+
+
+def _freshness_payload(summary: DashboardSummary) -> list[dict[str, object]]:
+    return [
+        {
+            "source_id": item.source_id,
+            "state": item.state.value,
+            "latest_event_at": (
+                item.latest_event_at.isoformat()
+                if item.latest_event_at is not None
+                else None
+            ),
+            "unsupported_records": item.unsupported_records,
+        }
+        for item in summary.source_freshness
+    ]
+
+
+def _quality_payload(summary: DashboardSummary) -> dict[str, int]:
+    return {quality.value: count for quality, count in summary.quality_counts.items()}
+
+
+def _summary_payload(summary: DashboardSummary) -> dict[str, object]:
+    return {
+        "workload_tokens": summary.workload_tokens,
+        "input_total_tokens": summary.input_total_tokens,
+        "output_total_tokens": summary.output_total_tokens,
+        "cache_read_tokens": summary.cache_read_tokens,
+        "cache_write_tokens": summary.cache_write_tokens,
+        "reasoning_tokens": summary.reasoning_tokens,
+        "event_count": summary.event_count,
+        "quality_counts": _quality_payload(summary),
+        "source_freshness": _freshness_payload(summary),
+    }
+
+
+def _import_payload(outcome: ImportOutcome) -> dict[str, object]:
+    return {
+        "inserted_events": outcome.inserted_events,
+        "duplicate_events": outcome.duplicate_events,
+        "partial_final_record": outcome.partial_final_record,
+        "unsupported_records": outcome.unsupported_records,
+    }
+
+
+def _approval_payload(source: SourceRecord) -> dict[str, object]:
+    return {
+        "source_id": source.source_id,
+        "provider": source.provider,
+        "display_name": source.display_name,
+        "state": source.state,
+    }
+
+
+@router.get("/status")
+def status() -> dict[str, str]:
+    """Liveness for the local UI and for startup checks."""
+    return {"status": "ok"}
+
+
+@router.get("/discovery")
+def discovery(request: Request) -> dict[str, object]:
+    """Report detected providers. Presence only — no source file is read."""
+    container = container_for(request)
+    with container.lock:
+        results = container.services.discovery.discover()
+    return {
+        "providers": [
+            {
+                "connector_id": result.connector_id,
+                "display_name": result.display_name,
+                "provider": result.provider.value if result.provider is not None else None,
+                "state": result.state.value,
+                "confidence": result.confidence.value,
+                "evidence_codes": list(result.evidence_codes),
+                "sources": [source.model_dump(mode="json") for source in result.sources],
+            }
+            for result in results
+        ]
+    }
+
+
+@router.get("/collection")
+def collection_status(request: Request) -> dict[str, object]:
+    container = container_for(request)
+    with container.lock:
+        return container.services.collection.status(container.settings.scan_interval_seconds)
+
+
+def _collection_connector(provider: str) -> str:
+    connectors = {
+        "codex": "codex-local",
+        "claude_code": "claude-code-local",
+        "hermes": "hermes-local",
+        "vscode_copilot": "vscode-copilot-local",
+        "antigravity": "antigravity-local",
+    }
+    if provider not in connectors:
+        raise HTTPException(status_code=422, detail="Source is not supported")
+    return connectors[provider]
+
+
+@router.post("/collection/{provider}/enable")
+def enable_provider_collection(provider: str, request: Request) -> dict[str, object]:
+    """Explicitly include existing and future sources under the discovered root."""
+    container = container_for(request)
+    connector_id = _collection_connector(provider)
+    with container.lock:
+        try:
+            container.services.collection.enable(connector_id)
+        except Exception as error:
+            raise _service_failure(error) from error
+        return container.services.collection.status(container.settings.scan_interval_seconds)
+
+
+@router.post("/collection/{provider}/disable")
+def disable_provider_collection(provider: str, request: Request) -> dict[str, object]:
+    """Stop automatically approving new sources; existing approvals remain."""
+    container = container_for(request)
+    connector_id = _collection_connector(provider)
+    with container.lock:
+        container.services.source_repository.disable_auto_import(connector_id)
+        return container.services.collection.status(container.settings.scan_interval_seconds)
+
+
+@router.post("/sources/{source_id}/approve")
+def approve_source(source_id: str, request: Request) -> dict[str, object]:
+    """Approve reading one discovered source; requires a same-origin request."""
+    container = container_for(request)
+    with container.lock:
+        try:
+            source = container.services.ingestion.approve(source_id)
+        except Exception as error:  # mapped to a safe status below
+            raise _service_failure(error) from error
+    return _approval_payload(source)
+
+
+@router.post("/sources/{source_id}/rescan")
+def rescan_source(source_id: str, request: Request) -> dict[str, object]:
+    """Import new records from an approved source."""
+    container = container_for(request)
+    with container.lock:
+        try:
+            outcome = container.services.ingestion.rescan(source_id)
+        except Exception as error:  # mapped to a safe status below
+            raise _service_failure(error) from error
+    return _import_payload(outcome)
+
+
+@router.post("/rebuild")
+def rebuild(request: Request) -> dict[str, object]:
+    """Re-derive normalized usage for every approved source."""
+    container = container_for(request)
+    with container.lock:
+        try:
+            outcome: RebuildOutcome = container.services.ingestion.rebuild()
+        except Exception as error:  # mapped to a safe status below
+            raise _service_failure(error) from error
+    return {
+        "inserted_events": outcome.inserted_events,
+        "failed_source_ids": list(outcome.failed_source_ids),
+    }
+
+
+@router.get("/dashboard")
+def dashboard(request: Request) -> dict[str, object]:
+    """Observed delta totals; unknown metrics stay null rather than becoming zero."""
+    container = container_for(request)
+    with container.lock:
+        summary = container.services.analytics.dashboard()
+    return _summary_payload(summary)
+
+
+@router.get("/data-quality")
+def data_quality(request: Request) -> dict[str, object]:
+    """The same quality and freshness view the dashboard shows."""
+    container = container_for(request)
+    with container.lock:
+        summary = container.services.analytics.dashboard()
+    return {
+        "quality_counts": _quality_payload(summary),
+        "source_freshness": _freshness_payload(summary),
+    }
+
+
+@router.get("/usage")
+def usage(
+    request: Request,
+    start: Annotated[datetime | None, Query(alias="from")] = None,
+    end: Annotated[datetime | None, Query(alias="to")] = None,
+) -> dict[str, object]:
+    """Path-free usage grouped by agent, recorded model, and opaque session."""
+    if any(value is not None and value.utcoffset() is None for value in (start, end)):
+        raise HTTPException(status_code=422, detail="Date bounds require a timezone")
+    if start is not None and end is not None and start >= end:
+        raise HTTPException(status_code=422, detail="Date range must end after it begins")
+    container = container_for(request)
+    with container.lock:
+        return container.services.analytics.usage_breakdown(start, end)

tokenhub/app.py

@@ -0,0 +1,77 @@
+"""FastAPI application factory for the local TokenHub server.
+
+Two guards run before every route, in this order:
+
+1. **Host** — only loopback host forms with a canonical port are accepted, so a
+   DNS-rebinding page cannot reach the API. Applies to the static UI as well.
+2. **Origin** — state-changing routes must carry the request's own loopback
+   origin; a missing, delegated, or duplicated Origin is refused before any
+   service sees the request.
+
+Neither guard adds CORS headers, and forwarded-host/proto headers are never
+consulted: the server is not behind a proxy by design.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, Request, Response
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.middleware.base import RequestResponseEndpoint
+
+from tokenhub.api.container import Container
+from tokenhub.api.routes import router
+from tokenhub.runtime import RuntimeControl, runtime_router
+from tokenhub.security.http import is_loopback_host, is_same_origin
+from tokenhub.settings import TokenHubSettings
+
+#: The built UI is package data so installed wheels and source builds serve it.
+FRONTEND_DIST = Path(__file__).resolve().parent / "web"
+
+_READ_ONLY_METHODS = frozenset({"GET", "HEAD"})
+
+
+def create_app(
+    settings: TokenHubSettings | None = None,
+    runtime_control: RuntimeControl | None = None,
+) -> FastAPI:
+    """Build the app. No database, migration, or discovery work happens here."""
+    container = Container(settings if settings is not None else TokenHubSettings())
+
+    @asynccontextmanager
+    async def lifespan(_: FastAPI) -> AsyncIterator[None]:
+        container.start()
+        try:
+            yield
+        finally:
+            container.stop()
+
+    app = FastAPI(title="TokenHub", lifespan=lifespan)
+    app.state.container = container
+
+    @app.middleware("http")
+    async def local_request_guard(
+        request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        port = container.settings.port
+        hosts = request.headers.getlist("host")
+        if len(hosts) != 1 or not is_loopback_host(hosts[0], port):
+            return JSONResponse(status_code=400, content={"detail": "Invalid Host header"})
+        if request.method not in _READ_ONLY_METHODS:
+            origins = request.headers.getlist("origin")
+            if len(origins) != 1 or not is_same_origin(origins[0], hosts[0], port):
+                return JSONResponse(
+                    status_code=403, content={"detail": "Invalid Origin header"}
+                )
+        return await call_next(request)
+
+    app.include_router(router)
+    if runtime_control is not None:
+        app.include_router(runtime_router(runtime_control))
+    if FRONTEND_DIST.is_dir():
+        app.mount("/", StaticFiles(directory=FRONTEND_DIST, html=True), name="frontend")
+    return app

tokenhub/cli.py

@@ -0,0 +1,65 @@
+"""Commands for the local TokenHub dashboard and its background server."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import webbrowser
+from collections.abc import Sequence
+
+from tokenhub.runtime.manager import RuntimeManager
+from tokenhub.server import run_server
+from tokenhub.settings import TokenHubSettings
+
+
+def loopback_url(settings: TokenHubSettings) -> str:
+    """The only URL this server is reachable at."""
+    host = "[::1]" if settings.host == "::1" else settings.host
+    return f"http://{host}:{settings.port}/"
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Start or manage the local dashboard; start opens it in a browser."""
+    args = list(sys.argv[1:] if argv is None else argv)
+    if args and args[0] == "_serve":
+        if len(args) != 2:
+            raise ValueError("internal server requires one port")
+        token = os.environ.pop("TOKENHUB_INTERNAL_CONTROL_TOKEN")
+        run_server(TokenHubSettings(port=int(args[1])), token)
+        return 0
+
+    parser = argparse.ArgumentParser(prog="tokenhub")
+    parser.add_argument(
+        "command", nargs="?", choices=("start", "status", "open", "stop"), default="start"
+    )
+    parser.add_argument("--port", type=int, help="loopback port for a new server")
+    parser.add_argument("--no-open", action="store_true", help="do not open a browser after start")
+    options = parser.parse_args(args)
+    manager = RuntimeManager(TokenHubSettings().data_directory)
+    try:
+        if options.command == "stop":
+            print("Token Hub stopped" if manager.stop() else "Token Hub is not running")
+            return 0
+        if options.command == "status":
+            url = manager.status()
+            print(url if url else "Token Hub is not running")
+            return 0 if url else 1
+        if options.command == "open":
+            url = manager.status()
+            if url is None:
+                print("Token Hub is not running; run tokenhub start")
+                return 1
+        else:
+            url = manager.start(options.port)
+    except (OSError, RuntimeError, ValueError) as error:
+        print(f"Token Hub: {error}", file=sys.stderr)
+        return 1
+
+    print(url)
+    if options.command == "open" or not options.no_open:
+        try:
+            webbrowser.open(url)
+        except Exception as error:  # noqa: BLE001 - browser implementations vary by platform
+            print(f"Could not open browser: {error}. Open {url}", file=sys.stderr)
+    return 0

tokenhub/connectors/__init__.py

@@ -0,0 +1 @@
+"""Provider connectors for safe local discovery and approved scans."""