lorebase-keeper 0.2.3__tar.gz

lorebase_keeper-0.2.3/.gitignore

@@ -0,0 +1,79 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+.eggs/
+.venv/
+venv/
+.python-eggs/
+pip-wheel-metadata/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.pyre/
+.pytype/
+.hypothesis/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+.tox/
+.nox/
+*.prof
+
+# Node
+node_modules/
+.pnpm-store/
+.npm/
+.yarn/
+!.yarn/releases
+.pnp.*
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+.vite/
+*.tsbuildinfo
+apps/web/dist/
+apps/web/.vitest/
+
+# Environment and secrets
+.env
+.env.*
+!.env.example
+*.pem
+*.key
+secrets/
+
+# Docker
+docker-compose.override.yaml
+docker-compose.override.yml
+.docker/
+
+# Data and local artifacts
+*.sqlite
+*.db
+/tmp/
+/data/
+tests/perf/results/
+
+# Editors and OS
+.idea/
+.vscode/*
+!.vscode/extensions.json
+!.vscode/settings.example.json
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+*.code-workspace
+.history/
+
+# Session worktrees
+.claude/

lorebase_keeper-0.2.3/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.5
+Name: lorebase-keeper
+Version: 0.2.3
+Summary: Lorebase Keeper: REST API service
+Requires-Python: >=3.12
+Requires-Dist: asyncpg>=0.29
+Requires-Dist: fastapi>=0.115
+Requires-Dist: httpx>=0.27
+Requires-Dist: lorebase-platform<0.3,>=0.2
+Requires-Dist: lorebase-store<0.3,>=0.2
+Requires-Dist: opentelemetry-api>=1.25
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyjwt[crypto]>=2.8
+Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: uvicorn>=0.30
+Provides-Extra: mssql
+Requires-Dist: pymssql>=2.3; extra == 'mssql'
+Provides-Extra: mysql
+Requires-Dist: asyncmy>=0.2.10; extra == 'mysql'
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=15; extra == 'parquet'

lorebase_keeper-0.2.3/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "lorebase-keeper"
+version = "0.2.3"
+description = "Lorebase Keeper: REST API service"
+requires-python = ">=3.12"
+dependencies = [
+  "lorebase-store>=0.2,<0.3",
+  "lorebase-platform>=0.2,<0.3",
+  "fastapi>=0.115",
+  "uvicorn>=0.30",
+  "pydantic>=2.7",
+  "opentelemetry-api>=1.25",
+  "asyncpg>=0.29",
+  "httpx>=0.27",
+  "PyJWT[crypto]>=2.8",
+  "python-multipart>=0.0.9",  # form parser of Starlette: file uploads of POST schema:infer
+]
+
+[project.optional-dependencies]
+parquet = ["pyarrow>=15"]  # Parquet uploads of POST schema:infer and schema:preview
+mysql = ["asyncmy>=0.2.10"]  # MySQL and MariaDB sources of POST schema:infer
+mssql = ["pymssql>=2.3"]  # SQL Server sources of POST schema:infer
+
+[project.scripts]
+lorebase-keeper = "lorebase_keeper.__main__:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lorebase_keeper"]

lorebase_keeper-0.2.3/src/lorebase_keeper/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.3"

lorebase_keeper-0.2.3/src/lorebase_keeper/__main__.py

@@ -0,0 +1,456 @@
+"""Run Keeper, or prepare an installation, with the settings Keeper reads.
+
+python -m lorebase_keeper [serve]
+    Serve the API; host and port come from the `keeper` settings.
+python -m lorebase_keeper bootstrap [--principal ID] [--name NAME] [--force]
+    Create the first installation administrator key and print it once (see bootstrap.py).
+python -m lorebase_keeper principal create ID [--name NAME]
+    Add a service principal with a chosen id; nothing happens when it exists.
+python -m lorebase_keeper audit verify (DOMAIN | --installation | --all) [--after-seq N] [--json]
+        [--reads] [--checkpoints PATH ... --checkpoint-key FILE]
+    Recompute the hash chains of the audit log (the read stream with --reads) and report the
+    first broken entry; with --checkpoints, also compare the log with the signed checkpoints
+    Scribe published (files or directories of them) to detect removed or rewritten entries.
+    The exit status is 1 when an entry does not verify, a checkpointed entry is missing or
+    changed, or a checkpoint is invalid.
+python -m lorebase_keeper sources vault [--json] [--check]
+    List the Sources of every Domain's default branch that map vault Attributes (upgrade
+    check for pii.import, design 13.5); with --check the exit status is 1 when any is listed.
+
+The installation commands connect with `database.maintenance_dsn` when it is set, else with
+`database.dsn`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from collections.abc import Awaitable, Callable, Coroutine, Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import uvicorn
+from sqlalchemy.ext.asyncio import async_sessionmaker
+
+from lorebase_keeper.app import create_app
+from lorebase_keeper.bootstrap import (
+    DEFAULT_ADMIN_NAME,
+    BootstrapError,
+    bootstrap,
+    create_service_principal,
+)
+from lorebase_keeper.services.pii_import import installation_vault_sources
+from lorebase_keeper.settings import keeper_settings
+from lorebase_platform.logging import configure_logging
+from lorebase_platform.settings import ConfigError, Settings, load_settings
+from lorebase_platform.telemetry import setup_telemetry
+from lorebase_store.auth import AuditStream, Chains, ChainVerification, SqlAuthStore
+from lorebase_store.auth.checkpoints import (
+    Checkpoint,
+    CheckpointError,
+    CheckpointFinding,
+    compare,
+    load_public_key,
+    parse_document,
+)
+from lorebase_store.db import create_engine
+from lorebase_store.settings import DatabaseSettings, store_settings
+
+SERVICE_NAME = "keeper"
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="python -m lorebase_keeper",
+        description="Run Keeper or prepare an installation with Keeper's settings.",
+    )
+    commands = parser.add_subparsers(dest="command")
+    commands.add_parser("serve", help="serve the API (the default)")
+    boot = commands.add_parser(
+        "bootstrap",
+        help="create the first installation administrator key and print it once",
+        description="Create a key for an installation administrator (keeper.auth."
+        "installation_admins) and print it on standard output, once. Refused when an "
+        "installation administrator already holds a usable key, unless --force is given.",
+    )
+    boot.add_argument(
+        "--principal",
+        help="administrator principal id (default: the first of keeper.auth.installation_admins)",
+    )
+    boot.add_argument(
+        "--name",
+        default=DEFAULT_ADMIN_NAME,
+        help="display name when the principal is created",
+    )
+    boot.add_argument(
+        "--force",
+        action="store_true",
+        help="add a key even when an administrator already holds one (existing keys stay valid)",
+    )
+    principal = commands.add_parser("principal", help="manage service principals")
+    principal_commands = principal.add_subparsers(dest="principal_command", required=True)
+    create = principal_commands.add_parser(
+        "create",
+        help="add a service principal with a chosen id",
+        description="Add a service principal with a chosen id, for example one that a schema "
+        "names in Operation permissions. Nothing happens when it already exists.",
+    )
+    create.add_argument("principal_id", metavar="ID")
+    create.add_argument("--name", help="display name (default: the id)")
+    audit = commands.add_parser("audit", help="check the audit log")
+    audit_commands = audit.add_subparsers(dest="audit_command", required=True)
+    verify = audit_commands.add_parser(
+        "verify",
+        help="recompute the audit hash chains and report the first broken entry",
+        description="Recompute the hash chain of a Domain's audit log, of the installation-wide "
+        "chain or of every chain, reading entries in batches. Exits with status 1 when an entry "
+        "does not verify.",
+    )
+    chain = verify.add_mutually_exclusive_group(required=True)
+    chain.add_argument("domain", nargs="?", metavar="DOMAIN", help="Domain id")
+    chain.add_argument(
+        "--installation", action="store_true", help="the chain of entries without a Domain"
+    )
+    chain.add_argument("--all", action="store_true", help="every chain")
+    verify.add_argument(
+        "--after-seq",
+        type=int,
+        default=0,
+        metavar="N",
+        help="start after entry N, trusting it (the last_seq of an earlier run); default 0",
+    )
+    verify.add_argument("--json", action="store_true", help="print the result as JSON")
+    verify.add_argument(
+        "--reads", action="store_true", help="the read stream (audit_read_log) instead of actions"
+    )
+    verify.add_argument(
+        "--checkpoints",
+        action="append",
+        default=[],
+        type=Path,
+        metavar="PATH",
+        help="checkpoint file, or directory searched for *.json (repeatable)",
+    )
+    verify.add_argument(
+        "--checkpoint-key",
+        type=Path,
+        metavar="FILE",
+        help="PEM Ed25519 public key the checkpoints are signed with",
+    )
+    verify.add_argument(
+        "--max-age",
+        type=float,
+        metavar="SECONDS",
+        help="fail when the newest checkpoint is older (publication stopped)",
+    )
+    sources = commands.add_parser("sources", help="check Sources")
+    sources_commands = sources.add_subparsers(dest="sources_command", required=True)
+    vault = sources_commands.add_parser(
+        "vault",
+        help="list the Sources that map vault Attributes",
+        description="List the Sources of every Domain's default branch that map columns to "
+        "vault Attributes. Defining, changing or triggering such a Source needs pii.import; "
+        "Sources defined before that permission existed still sync on their schedule: confirm "
+        "each, or disable it (remove its schedule or deprecate it) with a schema change.",
+    )
+    vault.add_argument("--json", action="store_true", help="print the Sources as JSON")
+    vault.add_argument(
+        "--check", action="store_true", help="exit with status 1 when any Source is listed"
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    """Dispatch to serving (no arguments) or to an installation command."""
+    args = _parser().parse_args(argv)
+    if args.command == "bootstrap":
+        _run_admin(lambda settings: _bootstrap(settings, args))
+    elif args.command == "principal":
+        _run_admin(lambda settings: _create_principal(settings, args))
+    elif args.command == "audit":
+        _run_admin(lambda settings: _verify_audit(settings, args))
+    elif args.command == "sources":
+        _run_admin(lambda settings: _vault_sources(settings, args))
+    else:
+        serve()
+
+
+def serve() -> None:
+    """Load settings, configure logging and telemetry, and serve until stopped."""
+    try:
+        settings = load_settings()
+        keeper = keeper_settings(settings)
+        configure_logging(settings.logging, service=SERVICE_NAME)
+        telemetry = setup_telemetry(settings.telemetry)
+        app = create_app(settings, telemetry=telemetry)
+    except ConfigError as exc:
+        sys.exit(str(exc))
+    try:
+        uvicorn.run(
+            app,
+            host=keeper.host,
+            port=keeper.port,
+            log_config=None,
+            access_log=False,
+            server_header=False,
+            # The client address comes from `keeper.trusted_proxies` (client_address.py) only.
+            proxy_headers=False,
+        )
+    finally:
+        telemetry.shutdown()
+
+
+def _run_admin(command: Callable[[Settings], Coroutine[Any, Any, None]]) -> None:
+    """Load settings and run an installation command; problems exit with a message."""
+    try:
+        settings = load_settings()
+        configure_logging(settings.logging, service=SERVICE_NAME)
+        asyncio.run(command(settings))
+    except (ConfigError, BootstrapError) as exc:
+        sys.exit(f"lorebase_keeper: {exc}")
+
+
+def _store(settings: Settings) -> tuple[SqlAuthStore, Callable[[], Awaitable[None]]]:
+    """Auth store of the installation commands: the owner role of `database.maintenance_dsn`
+    when it is set (the services may connect as a role subject to row-level security), else
+    `database.dsn`."""
+    database = settings.section("database", DatabaseSettings)
+    owner = database.maintenance_dsn
+    engine = create_engine(
+        database, url=None if owner is None else owner.get_secret_value(), pooled=False
+    )
+    return SqlAuthStore(async_sessionmaker(engine, expire_on_commit=False)), engine.dispose
+
+
+async def _bootstrap(settings: Settings, args: argparse.Namespace) -> None:
+    auth = keeper_settings(settings).auth
+    store, dispose = _store(settings)
+    try:
+        created = await bootstrap(
+            store,
+            auth,
+            now=datetime.now(UTC),
+            principal_id=args.principal,
+            display_name=args.name,
+            force=args.force,
+        )
+    finally:
+        await dispose()
+    expiry = created.expires_at.date().isoformat() if created.expires_at else "never"
+    action = "created" if created.principal_created else "existing"
+    print(
+        f"API key {created.prefix}... for {action} principal {created.principal_id} "
+        f"(expires {expiry}); it is printed once on standard output",
+        file=sys.stderr,
+    )
+    print(created.token)
+
+
+async def _create_principal(settings: Settings, args: argparse.Namespace) -> None:
+    store, dispose = _store(settings)
+    try:
+        added = await create_service_principal(
+            store, args.principal_id, args.name or args.principal_id, now=datetime.now(UTC)
+        )
+    finally:
+        await dispose()
+    state = "created" if added else "already exists"
+    print(f"service principal {args.principal_id} {state}", file=sys.stderr)
+
+
+def _chain_of(args: argparse.Namespace) -> str | Chains | None:
+    if args.all:
+        return Chains.ALL
+    return None if args.installation else str(args.domain)
+
+
+def verification_report(
+    chain: str | Chains | None,
+    result: ChainVerification,
+    stream: AuditStream = AuditStream.ACTIONS,
+    anchored: Anchoring | None = None,
+) -> dict[str, Any]:
+    """JSON form of a verification, as `audit verify --json` prints it."""
+    broken = (
+        None
+        if result.ok
+        else {"seq": result.bad_seq, "domain_id": result.bad_domain_id, "reason": result.reason}
+    )
+    report: dict[str, Any] = {
+        "stream": stream.value,
+        "chain": chain.value if isinstance(chain, Chains) else chain,
+        "intact": result.ok and (anchored is None or anchored.ok),
+        "checked": result.checked,
+        "chains": result.chains,
+        "last_seq": result.last_seq,
+        "first_broken": broken,
+    }
+    if anchored is not None:
+        report["checkpoints"] = {
+            "checked": anchored.checked,
+            "newest_at": anchored.newest.isoformat() if anchored.newest else None,
+            "newest_age_s": anchored.age_s,
+            "problems": anchored.problems,
+            "invalid": anchored.invalid,
+            "findings": [
+                {
+                    "checkpoint_at": f.checkpoint_at.isoformat(),
+                    "domain_id": f.domain_id,
+                    "seq": f.seq,
+                    "problem": f.problem,
+                }
+                for f in anchored.findings
+            ],
+        }
+    return report
+
+
+@dataclass(frozen=True)
+class Anchoring:
+    """Comparison of the log with checkpoints: how many were compared, the files that are
+    not valid checkpoints, and the checkpointed entries the log no longer holds as they were."""
+
+    checked: int
+    invalid: list[str]
+    findings: list[CheckpointFinding]
+    newest: datetime | None = None
+    max_age_s: float | None = None
+    now: datetime = field(default_factory=lambda: datetime.now(UTC))
+
+    @property
+    def age_s(self) -> float | None:
+        """Age of the newest checkpoint compared."""
+        return None if self.newest is None else (self.now - self.newest).total_seconds()
+
+    @property
+    def problems(self) -> list[str]:
+        """Why the comparison does not vouch for the log, besides the findings."""
+        found = [f"invalid checkpoint {problem}" for problem in self.invalid]
+        if self.checked == 0:
+            found.append("no valid checkpoint of this stream was found")
+        age = self.age_s
+        if self.max_age_s is not None and age is not None and age > self.max_age_s:
+            found.append(f"the newest checkpoint is {age:.0f} s old (--max-age {self.max_age_s:g})")
+        return found
+
+    @property
+    def ok(self) -> bool:
+        return not self.problems and not self.findings
+
+
+def _checkpoint_files(paths: Sequence[Path]) -> list[Path]:
+    files: list[Path] = []
+    for path in paths:
+        files.extend(sorted(path.rglob("*.json")) if path.is_dir() else [path])
+    return files
+
+
+def _load_checkpoints(
+    paths: Sequence[Path], key_file: Path, stream: AuditStream
+) -> tuple[list[Checkpoint], list[str]]:
+    """Valid checkpoints of `stream` and the files that are not valid checkpoints."""
+    try:
+        key = load_public_key(key_file.read_text(encoding="utf-8"))
+    except (OSError, CheckpointError) as error:
+        raise BootstrapError(f"--checkpoint-key: {error}") from None
+    found: list[Checkpoint] = []
+    invalid: list[str] = []
+    for file in _checkpoint_files(paths):
+        try:
+            checkpoint = parse_document(file.read_bytes(), key)
+        except (OSError, CheckpointError) as error:
+            invalid.append(f"{file}: {error}")
+            continue
+        if checkpoint.stream is stream:
+            found.append(checkpoint)
+    return found, invalid
+
+
+async def _verify_audit(settings: Settings, args: argparse.Namespace) -> None:
+    if args.after_seq < 0:
+        raise BootstrapError("--after-seq must not be negative")
+    if args.checkpoints and args.checkpoint_key is None:
+        raise BootstrapError("--checkpoints needs --checkpoint-key")
+    chain = _chain_of(args)
+    stream = AuditStream.READS if args.reads else AuditStream.ACTIONS
+    checkpoints, invalid = (
+        _load_checkpoints(args.checkpoints, args.checkpoint_key, stream)
+        if args.checkpoints
+        else ([], [])
+    )
+    anchored: Anchoring | None = None
+    store, dispose = _store(settings)
+    try:
+        async with store.unit() as unit:
+            repository = unit.read_audit if args.reads else unit.audit
+            result = await repository.verify(after_seq=args.after_seq, domain_id=chain)
+            if args.checkpoints:
+                domains = None if isinstance(chain, Chains) else [chain]
+                findings = await compare(repository, checkpoints, domain_ids=domains)
+                newest = max((c.at for c in checkpoints), default=None)
+                anchored = Anchoring(len(checkpoints), invalid, findings, newest, args.max_age)
+    finally:
+        await dispose()
+    if args.json:
+        print(json.dumps(verification_report(chain, result, stream, anchored)))
+    elif result.ok:
+        last = "-" if result.last_seq is None else result.last_seq
+        print(
+            f"audit log intact: {result.checked} entries in {result.chains} chain(s), "
+            f"last seq {last}"
+        )
+    else:
+        where = result.bad_domain_id or "installation"
+        print(
+            f"audit log broken at seq {result.bad_seq} (chain {where}): {result.reason}; "
+            f"{result.checked} entries verified before it"
+        )
+    if anchored is not None and not args.json:
+        age = "-" if anchored.age_s is None else f"{anchored.age_s:.0f} s"
+        print(f"checkpoints compared: {anchored.checked}, newest {age} old")
+        for problem in anchored.problems:
+            print(problem)
+        for finding in anchored.findings:
+            where = finding.domain_id or "installation"
+            print(
+                f"entry seq {finding.seq} (chain {where}) of the checkpoint of "
+                f"{finding.checkpoint_at.isoformat()} is {finding.problem}"
+            )
+    if not result.ok or (anchored is not None and not anchored.ok):
+        sys.exit(1)
+
+
+async def _vault_sources(settings: Settings, args: argparse.Namespace) -> None:
+    database = settings.section("database", DatabaseSettings)
+    owner = database.maintenance_dsn
+    engine = create_engine(
+        database, url=None if owner is None else owner.get_secret_value(), pooled=False
+    )
+    try:
+        found = await installation_vault_sources(
+            async_sessionmaker(engine, expire_on_commit=False),
+            store_settings(settings).writes,
+        )
+    finally:
+        await engine.dispose()
+    if args.json:
+        print(json.dumps([source.document() for source in found]))
+    elif not found:
+        print("no Source maps vault Attributes")
+    else:
+        for source in found:
+            schedule = source.schedule or "manual"
+            print(
+                f"{source.domain} {source.api_name} ({source.source}): {source.kind}."
+                f"{', '.join(source.attributes)}; schedule {schedule}; {source.status}"
+            )
+    if args.check and found:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

lorebase_keeper-0.2.3/src/lorebase_keeper/agents/__init__.py

@@ -0,0 +1 @@
+"""Agent-facing reads: context packs, Record explanations and the change feed (design 11)."""