sunstone-data-platform 1.0.0__py3-none-any.whl

sunstone_data/auth/store.py

@@ -0,0 +1,97 @@
+"""Per-environment cache for a human login session.
+
+Holds the OAuth2 refresh token from ``sunstone data login`` so a user logs in
+once per environment. Access tokens are never stored here — they are minted on
+demand and kept in memory only.
+
+The file lives next to the discovery cache but one level down, so the two
+never collide::
+
+    $XDG_CACHE_HOME/sunstone/data-platform/auth/{env_name}.json
+
+It is written atomically and chmod'ed ``0600``: a refresh token is a
+long-lived credential, so no other user on the machine may read it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import time
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+
+#: Mode a session file must end up with — owner read/write only.
+SESSION_FILE_MODE = 0o600
+
+
+@dataclass(frozen=True)
+class Session:
+    """A cached human login session for one environment."""
+
+    refresh_token: str
+    """OAuth2 refresh token, exchanged for access tokens on demand."""
+
+    issuer: str
+    """OIDC issuer the refresh token was minted by."""
+
+    client_id: str
+    """OAuth2 client id the refresh token belongs to."""
+
+    cached_at: float = field(default_factory=time.time)
+    """Unix time the session was written."""
+
+
+def _auth_dir() -> Path:
+    base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    return Path(base) / "sunstone" / "data-platform" / "auth"
+
+
+def session_path(env_name: str) -> Path:
+    """Per-env session file path under XDG_CACHE_HOME (or ~/.cache)."""
+    return _auth_dir() / f"{env_name}.json"
+
+
+def write_session(env_name: str, session: Session) -> Path:
+    """Persist a login session, atomically and mode 0600.
+
+    Returns the path written.
+    """
+    path = session_path(env_name)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    try:
+        os.chmod(tmp_name, SESSION_FILE_MODE)
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(asdict(session), f, indent=2)
+        os.replace(tmp_name, path)
+    except Exception:
+        Path(tmp_name).unlink(missing_ok=True)
+        raise
+    # mkstemp is already 0600, but the mode is a security property of this
+    # file — assert it on the final path rather than trusting the tempfile.
+    os.chmod(path, SESSION_FILE_MODE)
+    return path
+
+
+def read_session(env_name: str) -> Session | None:
+    """Load a cached login session. Returns None if missing or unreadable."""
+    try:
+        with open(session_path(env_name), encoding="utf-8") as f:
+            data = json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    field_names = {f.name for f in fields(Session)}
+    kwargs = {k: v for k, v in data.items() if k in field_names}
+    try:
+        return Session(**kwargs)
+    except TypeError:
+        return None
+
+
+def clear_session(env_name: str) -> None:
+    """Remove the cached session for an environment, if any."""
+    session_path(env_name).unlink(missing_ok=True)

sunstone_data/catalog.py

@@ -0,0 +1,175 @@
+"""PyIceberg catalog connection to the Iceberg REST catalog."""
+
+import httpx
+from pyiceberg.catalog import load_catalog
+from pyiceberg.catalog.rest import RestCatalog
+from pyiceberg.exceptions import NamespaceAlreadyExistsError
+
+from sunstone_data.config import Settings, current_environment_name, get_settings
+
+
+def _resolve_token(settings: Settings) -> str | None:
+    """Return a bearer token for iceberg-catalog, or None if none is configured.
+
+    iceberg-catalog shares graph-server's auth middleware, which always fails
+    closed on authentication: a missing or unverifiable token is a 401 even
+    with ``AUTH_ENFORCE=false`` (that only stages the role decision). So every
+    request PyIceberg makes needs a real token, resolved the same way
+    :func:`sunstone_data.graph.client.get_graph_client` resolves one.
+
+    Unlike ``GraphClient``, PyIceberg's ``RestCatalog`` has no per-request
+    refresh hook — it reads the static ``token`` property once at construction
+    — so this resolves once, synchronously, before the catalog is built.
+    """
+    from sunstone_data.auth.client_creds import resolve_access_token
+
+    with httpx.Client() as client:
+        return resolve_access_token(settings, current_environment_name(), client)
+
+
+def _build_catalog_properties(settings: Settings, branch: str, token: str | None = None) -> dict:
+    """Build catalog properties dict for the given settings and branch.
+
+    Args:
+        settings: Platform configuration settings.
+        branch: Branch to connect to.
+        token: Bearer token PyIceberg sends as ``Authorization: Bearer ...``.
+            Omitted from the properties when None.
+
+    Returns:
+        Dict of catalog properties suitable for passing to load_catalog.
+    """
+    props: dict = {
+        "type": "rest",
+        "uri": f"{settings.catalog_uri}/iceberg/{branch}/",
+        "warehouse": settings.warehouse,
+    }
+
+    if token is not None:
+        props["token"] = token
+
+    if settings.storage_backend == "gcs":
+        if settings.gcs_project_id:
+            props["gcs.project-id"] = settings.gcs_project_id
+        if settings.gcs_endpoint:
+            props["gcs.service.host"] = settings.gcs_endpoint
+    else:
+        props["s3.endpoint"] = settings.s3_endpoint
+        props["s3.access-key-id"] = settings.s3_access_key
+        props["s3.secret-access-key"] = settings.s3_secret_key
+        props["s3.region"] = settings.s3_region
+        props["s3.path-style-access"] = "true"
+
+    return props
+
+
+def get_catalog(branch: str = "main") -> RestCatalog:
+    """Create PyIceberg catalog connection to the Iceberg REST catalog.
+
+    Args:
+        branch: Branch to connect to. Defaults to "main".
+
+    Returns:
+        RestCatalog: PyIceberg REST catalog connected to the Iceberg REST catalog.
+    """
+    settings = get_settings()
+    return load_catalog("iceberg", **_build_catalog_properties(settings, branch, _resolve_token(settings)))
+
+
+def create_namespace_if_not_exists(catalog: RestCatalog, namespace: str) -> tuple[str, ...]:
+    """Create ``namespace`` and any missing parent levels.
+
+    A dotted namespace like ``un.health`` is a *nested* Iceberg namespace whose
+    parent ``un`` must exist first — the catalog rejects creating a child under a
+    missing parent. We create each level of the hierarchy in order, ignoring
+    levels that already exist. (Owner root-prefixing routinely produces nested
+    namespaces, so single-level creation is not enough.)
+
+    Args:
+        catalog: PyIceberg REST catalog instance.
+        namespace: Dot-separated namespace name (e.g. ``un.health``).
+
+    Returns:
+        tuple: The namespace identifier as a tuple of its levels.
+    """
+    parts = tuple(namespace.split("."))
+    for depth in range(1, len(parts) + 1):
+        level = parts[:depth]
+        try:
+            catalog.create_namespace(level)
+        except NamespaceAlreadyExistsError:
+            pass
+    return parts
+
+
+def get_table_metadata(table_identifier: str) -> dict[str, str]:
+    """Get metadata properties from Iceberg table.
+
+    Args:
+        table_identifier: Full table identifier (namespace.table).
+
+    Returns:
+        Dict of metadata properties (sunstone.import.* and sunstone.prov.*).
+
+    Raises:
+        ValueError: If table not found.
+    """
+    catalog = get_catalog()
+
+    # Parse identifier
+    parts = table_identifier.split(".")
+    if len(parts) != 2:
+        raise ValueError(f"Invalid table identifier: {table_identifier}. Expected namespace.table")
+
+    try:
+        table = catalog.load_table(table_identifier)
+    except Exception as e:
+        raise ValueError(f"Table not found: {table_identifier}") from e
+
+    # Extract sunstone.* properties
+    props = {}
+    for key, value in table.metadata.properties.items():
+        if key.startswith("sunstone."):
+            props[key] = value
+
+    return props
+
+
+def list_tables_with_metadata(namespace: str | None = None) -> list[dict]:
+    """List tables with basic metadata.
+
+    Args:
+        namespace: Optional namespace filter. Lists all if None.
+
+    Returns:
+        List of dicts with table_identifier and basic metadata.
+    """
+    catalog = get_catalog()
+
+    results = []
+    if namespace:
+        namespaces = [namespace]
+    else:
+        namespaces = [ns[0] for ns in catalog.list_namespaces()]
+
+    for ns in namespaces:
+        for table_id in catalog.list_tables(ns):
+            # table_id is a tuple like ('namespace', 'table_name')
+            identifier = f"{table_id[0]}.{table_id[1]}"
+            try:
+                table = catalog.load_table(identifier)
+                props = table.metadata.properties
+                results.append(
+                    {
+                        "identifier": identifier,
+                        "source": props.get("sunstone.import.source", "unknown"),
+                        "timestamp": props.get("sunstone.import.timestamp", "unknown"),
+                        "format": props.get("sunstone.import.format", "unknown"),
+                        "row_count": props.get("sunstone.import.row_count", "?"),
+                    }
+                )
+            except Exception:
+                # Skip tables that can't be loaded
+                pass
+
+    return results

sunstone_data/cli/__init__.py

@@ -0,0 +1,5 @@
+"""CLI module for Sunstone Data Platform."""
+
+from sunstone_data.cli.main import app
+
+__all__ = ["app"]

sunstone_data/cli/_app.py

@@ -0,0 +1,26 @@
+"""Shared Typer app, sub-app, and Rich console.
+
+Kept deliberately light: only ``typer`` and ``rich.console`` are imported
+here so that loading the entry point (``sunstone_data.cli.main``) doesn't
+drag in pyiceberg, pandas, s3fs, or any other data-platform internals.
+Heavy imports live inside the individual command bodies under
+:mod:`sunstone_data.cli.commands`.
+"""
+
+from __future__ import annotations
+
+import typer
+from rich.console import Console
+
+app = typer.Typer(
+    name="sunstone",
+    help="Sunstone Data Platform CLI",
+    no_args_is_help=True,
+)
+
+data_app = typer.Typer(help="Data operations")
+app.add_typer(data_app, name="data")
+
+console = Console()
+
+__all__ = ["app", "data_app", "console"]

sunstone_data/cli/_errors.py

@@ -0,0 +1,31 @@
+"""One error-reporting shape for every ``sunstone data`` command.
+
+Without ``--json`` a failure is a red one-liner; with ``--json`` it is an
+error object on stdout, so a script never has to parse Rich output. Either
+way the process exits 1.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+import typer
+
+from sunstone_data.cli._app import console
+
+
+@contextmanager
+def cli_errors(json_output: bool) -> Iterator[None]:
+    """Report anything raised inside the block and exit 1."""
+    try:
+        yield
+    except typer.Exit:
+        raise  # a command asking for a specific exit code, not a failure
+    except Exception as e:
+        if json_output:
+            print(json.dumps({"error": str(e), "type": type(e).__name__}))
+        else:
+            console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)

sunstone_data/cli/commands/__init__.py

@@ -0,0 +1,22 @@
+"""CLI sub-command modules for ``sunstone data ...``.
+
+Importing this package runs each command module, which decorates its
+function with ``@data_app.command(...)``. Decorators only register a
+function reference; nothing inside the function body executes until the
+command is actually invoked. The heavy imports (pyiceberg, s3fs, the
+versioning stack, ...) live inside each function body, so the entry
+point can register the whole CLI surface in well under 100ms.
+"""
+
+from sunstone_data.cli.commands import (  # noqa: F401  (import side-effect: registers commands)
+    data_auth,
+    data_import,
+    data_info,
+    data_publish,
+    ls,
+    prov,
+    refs,
+    show,
+    snapshots,
+    sparql,
+)

sunstone_data/cli/commands/data_auth.py

@@ -0,0 +1,120 @@
+"""``sunstone data login`` / ``logout`` / ``whoami`` — human sign-in sessions.
+
+``login`` runs the PKCE browser flow (:mod:`sunstone_data.auth.pkce`) and
+caches only the refresh token, per environment, mode 0600. Access tokens are
+minted on demand and never touch disk, so ``whoami`` reports what was cached,
+not live token claims.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+
+_ENV_OPTION_HELP = "Environment to act on (defaults to the active sunstone environment)."
+
+
+def _resolve_env(env: str | None) -> str:
+    from sunstone_data.config import current_environment_name
+
+    return env or current_environment_name()
+
+
+@data_app.command("login")
+def login_cmd(
+    env: str | None = typer.Option(None, "--env", help=_ENV_OPTION_HELP),
+) -> None:
+    """Sign in to graph-server in a browser and cache the session."""
+    import httpx
+
+    from sunstone_data.auth import pkce, store
+    from sunstone_data.auth.client_creds import discover_authorization_endpoint, discover_token_endpoint
+    from sunstone_data.config import get_settings
+    from sunstone_data.discovery import get_discovered
+
+    env_name = _resolve_env(env)
+    try:
+        settings = get_settings()
+        # graph-server publishes the issuer and the client id its own login
+        # button uses (``/ui/config.json``) — "configuration is discovered,
+        # not typed" (spec 180 §4).
+        discovered = get_discovered(env_name, settings.graph_url)
+        if discovered is None or not discovered.issuer or not discovered.client_id:
+            raise RuntimeError(
+                f"No OIDC issuer and client id discovered for environment {env_name!r}; "
+                f"cannot authenticate (graph_url={settings.graph_url!r})"
+            )
+        issuer = discovered.issuer
+        client_id = discovered.client_id
+        with httpx.Client() as client:
+            authorization_endpoint = discover_authorization_endpoint(issuer, client)
+            token_endpoint = discover_token_endpoint(issuer, client)
+
+            def open_browser(url: str) -> None:
+                import webbrowser
+
+                console.print(f"Opening your browser to sign in.\nIf it does not open, visit:\n  {url}")
+                webbrowser.open(url)
+
+            tokens = pkce.login(
+                authorization_endpoint=authorization_endpoint,
+                token_endpoint=token_endpoint,
+                client_id=client_id,
+                client=client,
+                open_browser=open_browser,
+            )
+        store.write_session(
+            env_name,
+            store.Session(
+                refresh_token=tokens.refresh_token,
+                issuer=issuer,
+                client_id=client_id,
+            ),
+        )
+    except Exception as e:
+        console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+    console.print(f"[green]Logged in to '{env_name}'.[/green]")
+
+
+@data_app.command("logout")
+def logout_cmd(
+    env: str | None = typer.Option(None, "--env", help=_ENV_OPTION_HELP),
+) -> None:
+    """Forget the cached session for an environment."""
+    from sunstone_data.auth import store
+
+    env_name = _resolve_env(env)
+    was_logged_in = store.read_session(env_name) is not None
+    store.clear_session(env_name)
+    if was_logged_in:
+        console.print(f"Logged out of '{env_name}'.")
+    else:
+        console.print(f"Not logged in to '{env_name}'; nothing to do.")
+
+
+@data_app.command("whoami")
+def whoami_cmd(
+    env: str | None = typer.Option(None, "--env", help=_ENV_OPTION_HELP),
+) -> None:
+    """Show the cached session for an environment.
+
+    Reports what ``login`` stored. The refresh token itself is never printed.
+    """
+    import time
+
+    from sunstone_data.auth import store
+
+    env_name = _resolve_env(env)
+    session = store.read_session(env_name)
+    if session is None:
+        console.print(f"Not logged in to '{env_name}'. Run `sunstone data login`.")
+        return
+
+    age_hours = max(0.0, time.time() - session.cached_at) / 3600
+    age = f"{age_hours * 60:.0f}m ago" if age_hours < 1 else f"{age_hours:.1f}h ago"
+    console.print(f"Environment: {env_name}")
+    console.print(f"Issuer:      {session.issuer}")
+    console.print(f"Client:      {session.client_id}")
+    console.print(f"Logged in:   {age}")

sunstone_data/cli/commands/data_import.py

@@ -0,0 +1,157 @@
+"""``sunstone data import`` — bring CSV/Excel files into the platform."""
+
+from __future__ import annotations
+
+import json
+import sys
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+
+
+def _format_result(result, as_json: bool = False, branch: str = "main") -> None:
+    """Pretty-print (or JSON-print) a single ImportResult."""
+    from sunstone_data.metadata.provenance_rdf import activity_iri
+    from sunstone_data.query import TableRef
+    from sunstone_data.resolve import dataset_iri
+
+    iri = dataset_iri(TableRef(namespace=result.namespace, table=result.table_name, branch=branch))
+    activity = activity_iri(iri, result.metadata.activity_id)
+
+    if as_json:
+        print(
+            json.dumps(
+                {
+                    "table": f"{result.namespace}.{result.table_name}",
+                    "rows": result.row_count,
+                    "columns": result.column_count,
+                    "checksum": result.checksum,
+                    "original": result.original_path,
+                    "already_existed": result.already_existed,
+                    "branch": branch,
+                    "dataset_iri": iri,
+                    "snapshot_id": result.snapshot_id,
+                    "activity_iri": activity,
+                }
+            )
+        )
+        return
+
+    if result.already_existed:
+        console.print(f"[yellow]Already imported:[/yellow] {result.namespace}.{result.table_name}")
+    else:
+        console.print(f"[green]Imported:[/green] {result.namespace}.{result.table_name}")
+
+    console.print(f"  Rows: {result.row_count:,}, Columns: {result.column_count}")
+    console.print(f"  Schema: {result.schema_summary}")
+    console.print(f"  Checksum: {result.checksum[:16]}...")
+    console.print(f"  Snapshot: {result.snapshot_id}")
+    console.print(f"  Dataset: [cyan]{iri}[/cyan]")
+    console.print(f"  Activity: [cyan]{activity}[/cyan]")
+    if branch != "main":
+        console.print(f"  Branch: [cyan]{branch}[/cyan]")
+
+
+@data_app.command("import")
+def import_cmd(
+    source: str = typer.Argument(..., help="Local file path or URL to import"),
+    name: str | None = typer.Option(None, "--name", "-n", help="Override table name"),
+    namespace: str | None = typer.Option(None, "--namespace", help="Target namespace"),
+    owner: str | None = typer.Option(
+        None,
+        "--owner",
+        help="Owning organization path, e.g. un/who/emissions-div. Overrides the source-URL rules file.",
+    ),
+    tag: list[str] | None = typer.Option(None, "--tag", "-t", help="Add tags (key:value)"),
+    branch: str = typer.Option("main", "--branch", "-b", help="Branch to import to"),
+    force_schema: bool = typer.Option(False, "--force-schema", help="Allow unsafe schema changes"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Import CSV or Excel file into the data platform."""
+    from rich.progress import (
+        BarColumn,
+        DownloadColumn,
+        Progress,
+        SpinnerColumn,
+        TextColumn,
+        TimeRemainingColumn,
+        TransferSpeedColumn,
+    )
+
+    from sunstone_data.import_ import run_import
+
+    is_url = source.startswith(("http://", "https://"))
+
+    try:
+        if sys.stdout.isatty() and not json_output:
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                DownloadColumn(),
+                TransferSpeedColumn(),
+                TimeRemainingColumn(),
+                console=console,
+                transient=True,
+            ) as progress:
+                if is_url:
+                    task = progress.add_task("Downloading...", total=None)
+                else:
+                    task = progress.add_task("Reading...", total=None)
+
+                def update_progress(downloaded: int, total: int | None) -> None:
+                    if total:
+                        progress.update(task, completed=downloaded, total=total)
+                    else:
+                        progress.update(task, advance=0)
+
+                result = run_import(
+                    source=source,
+                    name_override=name,
+                    namespace_override=namespace,
+                    owner=owner,
+                    tags=tag,
+                    branch=branch,
+                    force_schema=force_schema,
+                    progress_callback=update_progress,
+                )
+        else:
+            result = run_import(
+                source=source,
+                name_override=name,
+                namespace_override=namespace,
+                owner=owner,
+                tags=tag,
+                branch=branch,
+                force_schema=force_schema,
+            )
+
+        if isinstance(result, list):
+            for r in result:
+                _format_result(r, json_output, branch)
+        else:
+            _format_result(result, json_output, branch)
+
+    except FileNotFoundError as e:
+        if json_output:
+            print(json.dumps({"error": str(e)}))
+        else:
+            console.print(f"[red]File not found:[/red] {e}")
+        raise typer.Exit(1)
+
+    except ValueError as e:
+        if json_output:
+            print(json.dumps({"error": str(e)}))
+        else:
+            console.print(f"[red]Invalid input:[/red] {e}")
+        raise typer.Exit(1)
+
+    except Exception as e:
+        if json_output:
+            print(json.dumps({"error": str(e), "type": type(e).__name__}))
+        else:
+            console.print(f"[red]Error:[/red] {e}")
+            if console.is_terminal:
+                console.print_exception()
+        raise typer.Exit(1)