sunstone-data-platform 1.0.0__py3-none-any.whl

sunstone_data/discovery.py

@@ -0,0 +1,317 @@
+"""Discovery and caching of data-platform identity configuration.
+
+graph-server publishes the identity settings a client needs at
+``/ui/config.json``. That endpoint needs no auth, so a user configures one
+setting — the graph-server base URL — and the rest is discovered. From the
+``issuer``, standard OIDC discovery supplies the remaining OAuth2 endpoints.
+
+Clients fetch this once and cache it locally with a TTL.
+
+Payload (served by ``crates/graph-server/src/ui.rs``)::
+
+    {
+      "issuer": "https://auth.example.com/realms/sunstone",
+      "clientId": "example-cli",
+      "sparqlUrl": "/sparql",
+      "branch": "main",
+      "limits": {...}
+    }
+
+There is no schema version to check. Unknown keys are ignored and missing
+keys become ``None``, so the server can add fields without breaking clients.
+``sparqlUrl`` is served as a relative path and is resolved here against the
+base URL, so callers always get an absolute URL.
+
+This does *not* describe object storage. Warehouse location and S3/GCS
+credentials are configured explicitly (env vars, ``.env``, or the
+``data-platform`` section) — see :mod:`sunstone_data.config`.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from dataclasses import asdict, dataclass, fields
+from pathlib import Path
+from typing import Any
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+logger = logging.getLogger(__name__)
+
+CONFIG_PATH = "/ui/config.json"
+DEFAULT_TTL_SECONDS = 16 * 3600
+FETCH_TIMEOUT_SECONDS = 10.0
+
+#: TOML subtable name owned by the data-platform EnvSectionProvider.
+SECTION_NAME = "data-platform"
+
+
+@dataclass(frozen=True)
+class DiscoveredConfig:
+    """Identity config published by graph-server, with TTL bookkeeping."""
+
+    issuer: str | None = None
+    """OIDC issuer URL. Standard OIDC discovery on it supplies the rest."""
+
+    client_id: str | None = None
+    """OAuth2 client id the CLI authenticates as."""
+
+    sparql_url: str | None = None
+    """Absolute SPARQL endpoint URL, resolved against the graph-server base."""
+
+    fetched_at: float = 0.0
+    expires_at: float = 0.0
+    source_url: str = ""
+
+
+def discovery_url(graph_url: str) -> str:
+    """Build the config URL for a graph-server base URL."""
+    parts = urlsplit(graph_url)
+    if not parts.scheme or not parts.netloc:
+        raise ValueError(f"Invalid graph-server URL: {graph_url!r}")
+    return urlunsplit((parts.scheme, parts.netloc, CONFIG_PATH, "", ""))
+
+
+def _cache_dir() -> Path:
+    base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    return Path(base) / "sunstone" / "data-platform"
+
+
+def cache_path(env_name: str) -> Path:
+    """Per-env cache file path under XDG_CACHE_HOME (or ~/.cache)."""
+    return _cache_dir() / f"{env_name}.json"
+
+
+def _ttl_seconds() -> int:
+    raw = os.environ.get("SUNSTONE_DATA_DISCOVERY_TTL_SECONDS")
+    if raw is None:
+        return DEFAULT_TTL_SECONDS
+    try:
+        return max(0, int(raw))
+    except ValueError:
+        logger.warning("Invalid SUNSTONE_DATA_DISCOVERY_TTL_SECONDS=%r; using default", raw)
+        return DEFAULT_TTL_SECONDS
+
+
+def _parse_payload(raw: bytes, source_url: str) -> dict:
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Discovery response from {source_url} is not valid JSON: {e}") from e
+    if not isinstance(payload, dict):
+        raise ValueError(f"Discovery response from {source_url} is not a JSON object")
+    return payload
+
+
+def fetch_discovery(graph_url: str, *, timeout: float = FETCH_TIMEOUT_SECONDS) -> DiscoveredConfig:
+    """Fetch and parse graph-server's config payload. Raises on any failure."""
+    url = discovery_url(graph_url)
+    request = urllib.request.Request(url, headers={"Accept": "application/json"})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as resp:
+            raw = resp.read()
+    except urllib.error.HTTPError as e:
+        raise RuntimeError(f"Discovery HTTP {e.code} from {url}: {e.reason}") from e
+    except urllib.error.URLError as e:
+        raise RuntimeError(f"Discovery request to {url} failed: {e.reason}") from e
+    except TimeoutError as e:
+        raise RuntimeError(f"Discovery request to {url} timed out after {timeout}s") from e
+
+    payload = _parse_payload(raw, url)
+    sparql = payload.get("sparqlUrl") or None
+    now = time.time()
+    return DiscoveredConfig(
+        issuer=payload.get("issuer") or None,
+        client_id=payload.get("clientId") or None,
+        # Served relative (``/sparql``); urljoin leaves an absolute one alone.
+        sparql_url=urljoin(url, sparql) if sparql else None,
+        fetched_at=now,
+        expires_at=now + _ttl_seconds(),
+        source_url=url,
+    )
+
+
+def _write_cache_atomic(path: Path, config: DiscoveredConfig) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(asdict(config), f, indent=2)
+        os.replace(tmp_name, path)
+    except Exception:
+        Path(tmp_name).unlink(missing_ok=True)
+        raise
+
+
+def write_cache(env_name: str, config: DiscoveredConfig) -> Path:
+    """Persist a discovered config to the per-env cache file."""
+    path = cache_path(env_name)
+    _write_cache_atomic(path, config)
+    return path
+
+
+def read_cache(env_name: str, *, ignore_expiry: bool = False) -> DiscoveredConfig | None:
+    """Load a cached discovered config. Returns None if missing or expired."""
+    path = cache_path(env_name)
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    field_names = {f.name for f in fields(DiscoveredConfig)}
+    kwargs = {k: v for k, v in data.items() if k in field_names}
+    try:
+        config = DiscoveredConfig(**kwargs)
+    except TypeError:
+        return None
+    if not ignore_expiry and config.expires_at and time.time() >= config.expires_at:
+        return None
+    return config
+
+
+def _discovery_disabled() -> bool:
+    return os.environ.get("SUNSTONE_DATA_DISABLE_DISCOVERY", "").strip().lower() in ("1", "true", "yes")
+
+
+def get_discovered(
+    env_name: str,
+    graph_url: str,
+    *,
+    force_refresh: bool = False,
+) -> DiscoveredConfig | None:
+    """Return cached discovered config, fetching if absent, stale, or forced.
+
+    Behavior:
+        - ``SUNSTONE_DATA_DISABLE_DISCOVERY=1``: short-circuits to ``None`` even
+          when forced (use to skip discovery in tests or offline runs).
+        - Cache hit (fresh) and not forced: return cached.
+        - Forced or cache miss/stale: fetch. On success, cache and return.
+        - Forced + fetch fails: raise.
+        - Unforced + fetch fails: log warning, return stale cache if any
+          (with ignore_expiry), else None.
+    """
+    if _discovery_disabled():
+        return None
+
+    if not force_refresh:
+        cached = read_cache(env_name)
+        if cached is not None:
+            return cached
+
+    try:
+        config = fetch_discovery(graph_url)
+    except (RuntimeError, ValueError):
+        if force_refresh:
+            raise
+        stale = read_cache(env_name, ignore_expiry=True)
+        if stale is not None:
+            logger.warning("Discovery fetch failed; using stale cache for env %r", env_name)
+            return stale
+        logger.warning("Discovery fetch failed and no cache available for env %r", env_name)
+        return None
+
+    write_cache(env_name, config)
+    return config
+
+
+def clear_cache(env_name: str | None = None) -> None:
+    """Remove cache files. With no arg, removes all."""
+    if env_name is None:
+        directory = _cache_dir()
+        if directory.is_dir():
+            for p in directory.glob("*.json"):
+                p.unlink(missing_ok=True)
+    else:
+        cache_path(env_name).unlink(missing_ok=True)
+
+
+@dataclass(frozen=True)
+class InitResult:
+    """Outcome of a single env in :func:`init_environments`."""
+
+    name: str
+    graph_url: str
+    refreshed: bool  # True when discovery fetch succeeded
+    refresh_error: str | None  # str on fetch failure, None otherwise
+
+
+def _graph_url(env_def: dict[str, Any]) -> str | None:
+    """Return an env definition's graph_url.
+
+    sunstone-py 1.12+ keeps it in the ``data-platform`` subtable. Top-level
+    keys are still accepted for configs written by older versions or by hand.
+    """
+    section = env_def.get(SECTION_NAME) if isinstance(env_def.get(SECTION_NAME), dict) else {}
+    return section.get("graph_url") or env_def.get("graph_url")
+
+
+def init_environments(
+    *,
+    user_config: Path | None = None,
+    system_config: Path | None = None,
+    project_config: Path | None = None,
+) -> list[InitResult]:
+    """Cache discovery for every configured environment that has a graph_url.
+
+    The package ships no environment URLs. Managed machines get them from the
+    system config (``/etc/sunstone/data_platform.toml``, delivered by Fleet
+    from the fleet-config repo); others declare them in user or project
+    config, or set ``GRAPH_URL``.
+    """
+    from sunstone.env import list_environments
+
+    envs = list_environments(
+        system_config=system_config,
+        user_config=user_config,
+        project_config=project_config,
+    )
+    results: list[InitResult] = []
+    for name, env_def in envs.items():
+        graph_url = _graph_url(env_def)
+        if not graph_url:
+            continue
+        refreshed = False
+        refresh_error: str | None = None
+        try:
+            if get_discovered(name, graph_url, force_refresh=True) is not None:
+                refreshed = True
+        except (RuntimeError, ValueError) as e:
+            refresh_error = str(e)
+        results.append(InitResult(name=name, graph_url=graph_url, refreshed=refreshed, refresh_error=refresh_error))
+    return results
+
+
+def refresh_environment(
+    name: str,
+    *,
+    user_config: Path | None = None,
+    system_config: Path | None = None,
+    project_config: Path | None = None,
+) -> DiscoveredConfig:
+    """Force a discovery fetch and cache write for the named env.
+
+    Raises if the env is not defined, lacks a graph_url, or the fetch fails.
+    """
+    from sunstone.env import list_environments
+
+    envs = list_environments(
+        system_config=system_config,
+        user_config=user_config,
+        project_config=project_config,
+    )
+    if name not in envs:
+        raise ValueError(f"Environment '{name}' is not defined in any config file")
+    graph_url = _graph_url(envs[name])
+    if not graph_url:
+        raise ValueError(f"Environment '{name}' has no graph_url")
+    config = get_discovered(name, graph_url, force_refresh=True)
+    if config is None:
+        raise RuntimeError(f"Discovery for '{name}' returned no config")
+    return config

sunstone_data/env_commands.py

@@ -0,0 +1,97 @@
+"""Register data-platform-specific commands on sunstone-py's ``env`` CLI group.
+
+Importing this module attaches ``init`` and ``refresh`` subcommands to
+``sunstone.cli.env_app``. The base sunstone-py library knows nothing about
+catalogs or discovery; those concerns live here in sunstone-data.
+
+The plugin module (:mod:`sunstone_data.plugin`) imports this at module load
+so registration happens during entry-point discovery, before sunstone-py's
+top-level CLI dispatches.
+"""
+
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+_registered = False
+
+
+def _register() -> None:
+    global _registered
+    if _registered:
+        return
+    # Set the flag before any work so we never re-enter, even if imports raise.
+    _registered = True
+    try:
+        import typer
+        from sunstone.cli import env_app
+    except Exception:
+        # sunstone-py CLI is not available in this process (e.g. library-only
+        # usage). Nothing to register; this is fine.
+        logger.debug("sunstone.cli.env_app not available; skipping env command registration", exc_info=True)
+        return
+
+    from sunstone_data.discovery import init_environments, refresh_environment
+
+    @env_app.command("init")
+    def env_init() -> None:  # noqa: D401 — Typer command docstring becomes --help text
+        """Cache discovery for every configured environment.
+
+        Environments come from config files, not from this package. Managed
+        machines get them from /etc/sunstone/data_platform.toml.
+        """
+        try:
+            results = init_environments()
+        except (OSError, RuntimeError) as e:
+            typer.echo(f"Error: {e}", err=True)
+            raise typer.Exit(1)
+
+        if not results:
+            typer.echo(
+                "No environment with a graph_url is configured. Managed machines get one from "
+                "/etc/sunstone/data_platform.toml; otherwise add one with 'sunstone env add'.",
+                err=True,
+            )
+            raise typer.Exit(1)
+
+        any_refresh_failed = False
+        for r in results:
+            if r.refreshed:
+                typer.echo(f"Discovery cached for '{r.name}' ({r.graph_url})")
+            else:
+                any_refresh_failed = True
+                detail = f": {r.refresh_error}" if r.refresh_error else ""
+                typer.echo(f"Warning: discovery fetch failed for '{r.name}'{detail}", err=True)
+
+        if any_refresh_failed:
+            typer.echo(
+                "Re-run with 'sunstone env refresh <name>' once graph-server is reachable.",
+                err=True,
+            )
+
+    @env_app.command("refresh")
+    def env_refresh(
+        name: str = typer.Argument(..., help="Environment name to refresh"),
+    ) -> None:
+        """Force-fetch graph-server's config payload and update the cache."""
+        try:
+            config = refresh_environment(name)
+        except (OSError, RuntimeError, ValueError) as e:
+            typer.echo(f"Error: {e}", err=True)
+            raise typer.Exit(1)
+        typer.echo(f"Refreshed discovery for '{name}' from {config.source_url}")
+        if config.issuer:
+            typer.echo(f"  issuer:    {config.issuer}")
+        if config.client_id:
+            typer.echo(f"  client id: {config.client_id}")
+        if config.sparql_url:
+            typer.echo(f"  sparql:    {config.sparql_url}")
+
+
+# Register at module import so the env commands attach during entry-point
+# discovery, before the top-level CLI dispatches the user's command. The
+# idempotency guard above makes any later call (e.g. from
+# ``SunstoneDataPlugin.cli_groups``) a safe no-op.
+_register()

sunstone_data/env_section.py

@@ -0,0 +1,86 @@
+"""Typed environment section schema for the data platform.
+
+sunstone-py 1.12+ replaces the typed ``DataEnvironment`` with a generic
+``Environment(name, source, vars, sections)``. Plugins own typed slices of
+the config via the ``EnvSectionProvider`` protocol: each plugin claims a
+TOML subtable name and returns a callable that validates the subtable.
+
+This module defines the ``data-platform`` section. A user config that
+declares::
+
+    [environments.dev."data-platform"]
+    graph_url = "https://graph.example.com"
+    s3_endpoint = "http://localhost:9000"
+    s3_access_key = "rustfsadmin"
+    s3_secret_key = "rustfsadmin12345"
+
+is resolved by sunstone-py into ``env.section("data-platform")``
+returning a :class:`DataPlatformSection`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable
+
+
+@dataclass(frozen=True)
+class DataPlatformSection:
+    """Typed data-platform config slice from the active environment.
+
+    All fields are optional; consumers should treat unset (``None``)
+    fields as "fall back to ``os.environ`` or a built-in default".
+
+    Extra unknown keys in the TOML subtable raise ``TypeError`` at
+    construction time, which sunstone-py wraps into a ``ValueError``
+    indicating which environment / section caused the failure.
+    """
+
+    graph_url: str | None = None
+    """graph-server base URL — the one setting a user has to configure.
+
+    ``GET {graph_url}/ui/config.json`` supplies the identity settings
+    (``issuer``, ``clientId``, ``sparqlUrl``); see
+    :mod:`sunstone_data.discovery`.
+    """
+
+    client_id: str | None = None
+    """OAuth2 client id for the client-credentials (service) auth flow.
+
+    Pair with ``client_secret``. Setting both makes the client authenticate
+    as a service (spec §4 Pattern B) instead of using a cached human login,
+    so CI is never silently authenticated as whoever last logged in.
+    """
+
+    client_secret: str | None = None
+    """OAuth2 client secret. Pair with ``client_id``."""
+
+    s3_endpoint: str | None = None
+    """S3-compatible endpoint URL (e.g. RustFS).
+
+    Object storage is not discoverable — graph-server publishes nothing
+    about it — but PyIceberg and DuckDB read data files straight from the
+    bucket, so these stay user-configurable.
+    """
+
+    s3_access_key: str | None = None
+    """S3 access key id. Pair with ``s3_secret_key``."""
+
+    s3_secret_key: str | None = None
+    """S3 secret access key. Pair with ``s3_access_key``."""
+
+
+class DataPlatformSectionProvider:
+    """``EnvSectionProvider`` implementation for the data-platform plugin.
+
+    Registered via the ``sunstone.plugins`` entry point as part of
+    :class:`sunstone_data.plugin.SunstoneDataPlugin`.
+    """
+
+    SECTION_NAME = "data-platform"
+
+    def env_section_name(self) -> str:
+        return self.SECTION_NAME
+
+    def env_section_model(self) -> Callable[..., DataPlatformSection]:
+        return DataPlatformSection

sunstone_data/graph/__init__.py

@@ -0,0 +1,36 @@
+"""RDF graph vocabulary and canonical serialization.
+
+`terms` holds the namespace IRIs for every RDF vocabulary this project uses.
+`dataset` is the only place that turns a set of triples into bytes: sorted
+N-Triples, with deterministic fragment IRIs standing in for blank nodes.
+"""
+
+from sunstone_data.graph.dataset import (
+    IRI,
+    DatasetDescription,
+    DatasetVersion,
+    Graph,
+    Literal,
+    Triple,
+    build_graph,
+    description_to_graph,
+    fragment_iri,
+    graph_to_description,
+    parse_ntriples,
+    serialize,
+)
+
+__all__ = [
+    "IRI",
+    "DatasetDescription",
+    "DatasetVersion",
+    "Graph",
+    "Literal",
+    "Triple",
+    "build_graph",
+    "description_to_graph",
+    "fragment_iri",
+    "graph_to_description",
+    "parse_ntriples",
+    "serialize",
+]