sunstone-data-platform 1.0.0__py3-none-any.whl

sunstone_data/cli/commands/snapshots.py

@@ -0,0 +1,77 @@
+"""``sunstone data history`` — the Iceberg snapshot log (spec 180 §8).
+
+This reads Iceberg table metadata through the catalog, not the graph server.
+"""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+from sunstone_data.cli._errors import cli_errors
+
+
+@data_app.command("history")
+def history_cmd(
+    table: str = typer.Argument(..., help="Table reference (namespace.table[@branch])"),
+    branch: str = typer.Option("main", "--branch", "-b", help="Catalog branch to read"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Show a table's Iceberg snapshot log, oldest snapshot first.
+
+    Examples:
+        sunstone data history un.health.cases
+        sunstone data history un.health.cases@experiment --json
+    """
+    with cli_errors(json_output):
+        from datetime import datetime, timezone
+
+        from sunstone_data.catalog import get_catalog
+        from sunstone_data.query import TableRef
+
+        ref = TableRef.parse(table)
+        iceberg_table = get_catalog(ref.version_ref or branch).load_table(f"{ref.namespace}.{ref.table}")
+
+        rows = []
+        for entry in iceberg_table.history():
+            snapshot = iceberg_table.snapshot_by_id(entry.snapshot_id)
+            summary = getattr(snapshot, "summary", None)
+            operation = getattr(summary, "operation", None)
+            parent = getattr(snapshot, "parent_snapshot_id", None)
+            rows.append(
+                {
+                    "snapshot_id": str(entry.snapshot_id),
+                    "parent_snapshot_id": str(parent) if parent is not None else None,
+                    "timestamp": datetime.fromtimestamp(entry.timestamp_ms / 1000, tz=timezone.utc).isoformat(),
+                    "operation": getattr(operation, "value", operation),
+                    "summary": dict(getattr(summary, "additional_properties", {}) or {}),
+                }
+            )
+
+        if json_output:
+            print(json.dumps(rows, indent=2))
+            return
+
+        if not rows:
+            console.print("[yellow]No snapshots found[/yellow]")
+            return
+
+        from rich.table import Table
+
+        out = Table(box=None, pad_edge=False)
+        out.add_column("Timestamp", style="cyan")
+        out.add_column("Snapshot")
+        out.add_column("Parent")
+        out.add_column("Operation", style="magenta")
+        out.add_column("Records added", justify="right")
+        for row in rows:
+            out.add_row(
+                row["timestamp"],
+                row["snapshot_id"],
+                row["parent_snapshot_id"] or "",
+                row["operation"] or "",
+                row["summary"].get("added-records", ""),
+            )
+        console.print(out)

sunstone_data/cli/commands/sparql.py

@@ -0,0 +1,68 @@
+"""``sunstone data sparql`` — run a query through the graph-server proxy (spec 180 §8).
+
+`execute_query` is also what ``sunstone data lineage`` runs its PROV query
+with, so both commands render SPARQL results the same way.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+from sunstone_data.cli._errors import cli_errors
+
+
+def execute_query(query: str, json_output: bool, empty_message: str = "No results") -> None:
+    """Run `query` through the proxy and write the answer to stdout.
+
+    With `json_output` the response body is passed through untouched. A
+    SPARQL results document is rendered as a table, an ASK answer as
+    ``true``/``false``, and anything else (CONSTRUCT, DESCRIBE, an error
+    page) as its raw text.
+    """
+    from sunstone_data.graph.client import get_graph_client
+
+    response = get_graph_client().sparql(query)
+
+    if json_output:
+        print(response.text)
+        return
+
+    try:
+        payload = response.json()
+    except ValueError:
+        print(response.text)
+        return
+
+    if "boolean" in payload:
+        console.print(str(payload["boolean"]).lower())
+        return
+
+    bindings = payload.get("results", {}).get("bindings", [])
+    if not bindings:
+        console.print(f"[yellow]{empty_message}[/yellow]")
+        return
+
+    from rich.table import Table
+
+    variables = payload.get("head", {}).get("vars") or sorted({k for b in bindings for k in b})
+    table = Table(box=None, pad_edge=False)
+    for variable in variables:
+        table.add_column(variable, style="cyan", overflow="fold")
+    for binding in bindings:
+        table.add_row(*(binding.get(v, {}).get("value", "") for v in variables))
+    console.print(table)
+
+
+@data_app.command("sparql")
+def sparql_cmd(
+    query: str = typer.Argument(..., help="SPARQL query text"),
+    json_output: bool = typer.Option(False, "--json", help="Print the raw response body"),
+) -> None:
+    """Run a SPARQL query through the graph-server proxy.
+
+    Examples:
+        sunstone data sparql 'SELECT * WHERE { GRAPH ?g { ?s ?p ?o } } LIMIT 10'
+    """
+    with cli_errors(json_output):
+        execute_query(query, json_output)

sunstone_data/cli/main.py

@@ -0,0 +1,30 @@
+"""Entry point for the ``sunstone`` CLI.
+
+Kept intentionally thin so that ``sunstone`` / ``sunstone --help`` and the
+sunstone-py-owned subcommands (``sunstone env`` etc.) don't pay the cost
+of pulling pyiceberg, pandas, s3fs, and the rest of the data-platform
+stack just to print a usage line.
+
+Layout:
+
+* :mod:`sunstone_data.cli._app` — defines ``app``, ``data_app``, ``console``
+  (Typer + Rich only, ~1ms to import).
+* :mod:`sunstone_data.cli.commands` — one module per ``sunstone data ...``
+  subcommand. Each module registers its command at import time via
+  ``@data_app.command(...)`` and lazy-imports its heavy dependencies
+  inside the function body.
+
+This module re-exports ``app`` (the entry point) and ``data_app`` (used by
+:mod:`sunstone_data.plugin`).
+"""
+
+from __future__ import annotations
+
+from sunstone_data.cli import commands  # noqa: F401  (import side-effect: registers commands)
+from sunstone_data.cli._app import app, console, data_app
+
+__all__ = ["app", "console", "data_app"]
+
+
+if __name__ == "__main__":
+    app()

sunstone_data/config.py

@@ -0,0 +1,250 @@
+"""Configuration loading for Sunstone Data Platform."""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Platform configuration settings."""
+
+    # Iceberg REST catalog
+    catalog_uri: str = "http://localhost:8181"
+    warehouse: str = "s3://iceberg/"
+
+    # graph-server base URL. Identity settings are discovered from it at
+    # ``/ui/config.json``; see sunstone_data.discovery.
+    graph_url: str = ""
+
+    # OAuth2 client-credentials (service) identity. Setting both selects the
+    # client-credentials flow over any cached human login; see
+    # sunstone_data.auth.client_creds.
+    client_id: str = ""
+    client_secret: str = ""
+
+    # S3-compatible storage (RustFS locally, Hetzner/R2 remote)
+    s3_endpoint: str = "http://localhost:9000"
+    s3_access_key: str = ""
+    s3_secret_key: str = ""
+    s3_region: str = "us-east-1"  # "auto" for Cloudflare R2
+
+    # GCS storage
+    gcs_project_id: str = ""
+    gcs_endpoint: str = ""  # empty = real GCS; set for fake-gcs-server
+
+    # Data-owner organization modeling (import + org tree)
+    org_roots: tuple[str, ...] = ()
+    owner_rules_path: str = ""
+    org_base_uri: str = "https://sunstone.institute/org"
+
+    # Base IRI the Iceberg REST catalog mints dataset IRIs under. Must match
+    # the catalog's own DATA_BASE_IRI (CatalogConfig::from_env in
+    # crates/iceberg-catalog/src/app.rs, same default), or named-version
+    # lookups address a graph the catalog never wrote.
+    data_base_iri: str = "https://data.sunstoneinstitute.ai"
+
+    @property
+    def storage_backend(self) -> str:
+        """Return 'gcs' for gs:// warehouses, 's3' otherwise."""
+        return "gcs" if self.warehouse.startswith("gs://") else "s3"
+
+
+def _parse_org_roots(raw: str) -> tuple[str, ...]:
+    """Split a comma-separated ``SUNSTONE_ORG_ROOTS`` value into a tuple.
+
+    Empty / unset yields an empty tuple, which disables root-allowlist
+    enforcement (any owner root is accepted).
+    """
+    return tuple(part.strip() for part in raw.split(",") if part.strip())
+
+
+def _r2_defaults() -> dict[str, str]:
+    """Derive S3 client settings from ``R2_*`` env vars (docker ``r2`` profile).
+
+    The ``r2`` docker-compose profile is configured entirely from four
+    ``R2_*`` variables (see ``.env.r2.example``). A PyIceberg/DuckDB client
+    reads Iceberg data files directly from object storage, so it needs the
+    same bucket, endpoint, and credentials Nessie was given. Rather than make
+    R2 users duplicate those four values into ``S3_*``/``ICEBERG_WAREHOUSE``,
+    derive them here. Explicit ``S3_*``/``ICEBERG_WAREHOUSE``/``CATALOG_URI``
+    env vars still take precedence over these derived defaults.
+
+    Returns an empty dict when ``R2_ACCOUNT_ID``/``R2_BUCKET`` are not set, so
+    non-R2 setups are unaffected.
+    """
+    account_id = os.getenv("R2_ACCOUNT_ID", "")
+    bucket = os.getenv("R2_BUCKET", "")
+    if not (account_id and bucket):
+        return {}
+    return {
+        "catalog_uri": "http://localhost:19122",
+        "warehouse": f"s3://{bucket}/",
+        "s3_endpoint": f"https://{account_id}.r2.cloudflarestorage.com",
+        "s3_access_key": os.getenv("R2_ACCESS_KEY_ID", ""),
+        "s3_secret_key": os.getenv("R2_SECRET_ACCESS_KEY", ""),
+        "s3_region": "auto",  # R2 ignores region but the SDK requires a value
+    }
+
+
+def _try_resolve_environment() -> Any | None:
+    """Resolve the active environment via sunstone-py.
+
+    Returns an ``Environment`` (sunstone-py 1.12+) if available, else None.
+    Never raises — missing sunstone-py or resolution errors are logged at
+    debug level and return None so the caller falls back to ``.env`` files.
+    """
+    try:
+        from sunstone.env import resolve_environment
+
+        return resolve_environment()
+    except ImportError:
+        logger.debug("sunstone-py not installed, skipping environment resolution")
+        return None
+    except Exception:
+        logger.debug("Failed to resolve environment", exc_info=True)
+        return None
+
+
+def current_environment_name() -> str:
+    """Name of the active sunstone-py environment, or ``"default"`` if none is active.
+
+    Per-environment caches (discovery, login sessions) are keyed by this name.
+    """
+    env = _try_resolve_environment()
+    return getattr(env, "name", None) or "default"
+
+
+def _section_field(env: Any, attr: str) -> str | None:
+    """Read ``attr`` from the data-platform section if present.
+
+    Returns ``None`` when either the section is missing (legacy config that
+    only declared top-level keys) or the field on the section is unset.
+    Callers should fall back to ``os.environ`` in that case.
+    """
+    try:
+        section = env.section("data-platform")
+    except KeyError:
+        return None
+    value = getattr(section, attr, None)
+    return value or None
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    """Load settings, preferring sunstone-py's active environment over ``.env`` vars.
+
+    Resolution order (highest precedence wins):
+        1. Real ``os.environ`` entries set by the user / CI.
+        2. The active sunstone-py ``Environment``'s ``data-platform`` section.
+        3. ``.env`` file in the project root.
+        4. Built-in defaults.
+
+    Storage settings are not discovered. graph-server's ``/ui/config.json``
+    publishes identity settings only, so ``warehouse`` and the S3/GCS fields
+    resolve from the section, env vars, or built-in defaults.
+
+    Legacy configs that declared ``graph_url`` etc. at the *top level* of an
+    ``[environments.<name>]`` block (rather than under
+    ``[environments.<name>."data-platform"]``) are handled transparently:
+    sunstone-py's ``Environment.activate()`` flattens top-level scalars into
+    ``os.environ`` (e.g. ``CATALOG_URL``, ``S3_ENDPOINT``) so the ``os.environ``
+    fallback covers them. Top-level CLI callbacks call ``activate_environment``
+    automatically; library callers that want the same behaviour should call
+    ``sunstone.env.activate_environment()`` once before ``get_settings()``.
+
+    Settings are cached after first load. Call ``get_settings.cache_clear()``
+    to force a reload (e.g. in tests).
+    """
+    env = _try_resolve_environment()
+    if env is not None:
+        # The data-platform section is the new home for typed fields. Each
+        # lookup falls back to the corresponding flattened env var so that
+        # both new section-style and legacy top-level configs work.
+        graph_url = _section_field(env, "graph_url") or os.environ.get("GRAPH_URL") or ""
+        client_id = _section_field(env, "client_id") or os.environ.get("SUNSTONE_CLIENT_ID") or ""
+        client_secret = _section_field(env, "client_secret") or os.environ.get("SUNSTONE_CLIENT_SECRET") or ""
+        section_s3_endpoint = _section_field(env, "s3_endpoint")
+        section_s3_access_key = _section_field(env, "s3_access_key")
+        section_s3_secret_key = _section_field(env, "s3_secret_key")
+
+        # The Iceberg REST catalog is a different service from graph_url, so
+        # it keeps its own env var rather than being derived from the graph
+        # URL. ``CATALOG_URL`` is a legacy flattened spelling, still read.
+        catalog_uri = os.environ.get("CATALOG_URI") or os.environ.get("CATALOG_URL") or "http://localhost:8181"
+        warehouse = os.getenv("ICEBERG_WAREHOUSE") or "s3://iceberg/"
+        gcs_project_id = os.getenv("GCS_PROJECT_ID") or ""
+        gcs_endpoint = os.getenv("GCS_ENDPOINT") or ""
+        s3_endpoint = section_s3_endpoint or os.environ.get("S3_ENDPOINT") or ""
+        s3_access_key = section_s3_access_key or os.environ.get("S3_ACCESS_KEY") or ""
+        s3_secret_key = section_s3_secret_key or os.environ.get("S3_SECRET_KEY") or ""
+        s3_region = os.environ.get("S3_REGION") or "us-east-1"
+
+        return Settings(
+            catalog_uri=catalog_uri,
+            warehouse=warehouse,
+            graph_url=graph_url,
+            client_id=client_id,
+            client_secret=client_secret,
+            s3_endpoint=s3_endpoint,
+            s3_access_key=s3_access_key,
+            s3_secret_key=s3_secret_key,
+            s3_region=s3_region,
+            gcs_project_id=gcs_project_id,
+            gcs_endpoint=gcs_endpoint,
+            org_roots=_parse_org_roots(os.getenv("SUNSTONE_ORG_ROOTS", "")),
+            owner_rules_path=os.getenv("SUNSTONE_OWNER_RULES_PATH", ""),
+            org_base_uri=os.getenv("SUNSTONE_ORG_BASE_URI", "https://sunstone.institute/org"),
+            data_base_iri=os.getenv("DATA_BASE_IRI", "https://data.sunstoneinstitute.ai"),
+        )
+
+    # Fallback: load from .env file
+    project_root = Path(__file__).parent.parent.parent
+    root_env = project_root / ".env"
+
+    if root_env.exists():
+        load_dotenv(root_env)
+
+    # R2 profile: a present R2_* set seeds the warehouse/endpoint/creds the
+    # client needs. Explicit S3_*/ICEBERG_WAREHOUSE/CATALOG_URI vars override.
+    r2 = _r2_defaults()
+
+    catalog_uri = os.getenv("CATALOG_URI") or r2.get("catalog_uri") or "http://localhost:8181"
+    return Settings(
+        catalog_uri=catalog_uri,
+        warehouse=os.getenv("ICEBERG_WAREHOUSE") or r2.get("warehouse") or "s3://iceberg/",
+        graph_url=os.getenv("GRAPH_URL", ""),
+        client_id=os.getenv("SUNSTONE_CLIENT_ID", ""),
+        client_secret=os.getenv("SUNSTONE_CLIENT_SECRET", ""),
+        s3_endpoint=os.getenv("S3_ENDPOINT") or r2.get("s3_endpoint") or "http://localhost:9000",
+        s3_access_key=(
+            os.getenv("S3_ACCESS_KEY")
+            or r2.get("s3_access_key")
+            or os.getenv("OBJECT_STORE_ACCESS_KEY")
+            or os.getenv("AWS_ACCESS_KEY_ID")
+            or ""
+        ),
+        s3_secret_key=(
+            os.getenv("S3_SECRET_KEY")
+            or r2.get("s3_secret_key")
+            or os.getenv("OBJECT_STORE_SECRET_KEY")
+            or os.getenv("AWS_SECRET_ACCESS_KEY")
+            or ""
+        ),
+        s3_region=os.getenv("S3_REGION") or r2.get("s3_region") or "us-east-1",
+        gcs_project_id=os.getenv("GCS_PROJECT_ID", ""),
+        gcs_endpoint=os.getenv("GCS_ENDPOINT", ""),
+        org_roots=_parse_org_roots(os.getenv("SUNSTONE_ORG_ROOTS", "")),
+        owner_rules_path=os.getenv("SUNSTONE_OWNER_RULES_PATH", ""),
+        org_base_uri=os.getenv("SUNSTONE_ORG_BASE_URI", "https://sunstone.institute/org"),
+        data_base_iri=os.getenv("DATA_BASE_IRI", "https://data.sunstoneinstitute.ai"),
+    )