sunstone-data-platform 1.0.0__py3-none-any.whl

sunstone_data/cli/commands/data_info.py

@@ -0,0 +1,166 @@
+"""``sunstone data info`` — view metadata for imported datasets."""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+
+
+@data_app.command("info")
+def info_cmd(
+    table: str = typer.Argument(None, help="Table identifier (namespace.table) or omit to list all"),
+    minimal: bool = typer.Option(False, "--minimal", "-m", help="Minimal output (source, date, rows, checksum)"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Show all metadata including PROV URIs"),
+    namespace: str | None = typer.Option(None, "--namespace", "-n", help="Filter by namespace when listing"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """View metadata for imported datasets.
+
+    Without TABLE argument, lists all datasets in catalog.
+    With TABLE argument, shows detailed metadata for that dataset.
+    """
+    try:
+        if table:
+            _show_table_info(table, minimal, verbose, json_output)
+        else:
+            _show_table_list(namespace, json_output)
+    except ValueError as e:
+        if json_output:
+            print(json.dumps({"error": str(e)}))
+        else:
+            console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)
+    except Exception as e:
+        if json_output:
+            print(json.dumps({"error": str(e), "type": type(e).__name__}))
+        else:
+            console.print(f"[red]Error:[/red] {e}")
+            if console.is_terminal:
+                console.print_exception()
+        raise typer.Exit(1)
+
+
+def _show_table_info(table: str, minimal: bool, verbose: bool, json_output: bool) -> None:
+    """Show metadata for a single table."""
+    from rich.panel import Panel
+    from rich.table import Table
+
+    from sunstone_data.catalog import get_table_metadata
+
+    props = get_table_metadata(table)
+
+    if json_output:
+        print(json.dumps(props, indent=2))
+        return
+
+    if minimal:
+        print(f"Source: {props.get('sunstone.import.source', 'unknown')}")
+        print(f"Imported: {props.get('sunstone.import.timestamp', 'unknown')}")
+        row_count = props.get("sunstone.import.row_count", "?")
+        col_count = props.get("sunstone.import.column_count", "?")
+        print(f"Rows: {row_count}, Columns: {col_count}")
+        print(f"Checksum: {props.get('sunstone.import.checksum', 'unknown')}")
+        return
+
+    info_table = Table(show_header=False, box=None, padding=(0, 1))
+    info_table.add_column("Key", style="cyan")
+    info_table.add_column("Value")
+
+    info_table.add_row("Source", props.get("sunstone.import.source", "unknown"))
+    info_table.add_row(
+        "Imported by",
+        f"{props.get('sunstone.import.user', 'unknown')} at {props.get('sunstone.import.timestamp', 'unknown')}",
+    )
+    info_table.add_row(
+        "Format",
+        f"{props.get('sunstone.import.format', 'unknown')}, Encoding: {props.get('sunstone.import.encoding', 'unknown')}",
+    )
+    info_table.add_row(
+        "Size",
+        f"{props.get('sunstone.import.file_size', '?')} bytes, "
+        f"Rows: {props.get('sunstone.import.row_count', '?')}, "
+        f"Columns: {props.get('sunstone.import.column_count', '?')}",
+    )
+
+    schema_str = props.get("sunstone.import.schema_original", "")
+    if schema_str:
+        columns = schema_str.split(", ")
+        if len(columns) > 5 and not verbose:
+            schema_preview = ", ".join(columns[:5]) + f", ... ({len(columns)} total)"
+        else:
+            schema_preview = schema_str
+        info_table.add_row("Schema", schema_preview)
+
+    checksum = props.get("sunstone.import.checksum", "unknown")
+    if not verbose and len(checksum) > 16:
+        checksum = checksum[:16] + "..."
+    info_table.add_row("Checksum", checksum)
+
+    if verbose:
+        info_table.add_row("", "")
+        info_table.add_row("[bold]Extended Info[/bold]", "")
+
+        if "sunstone.import.source_normalized" in props:
+            info_table.add_row("Normalized source", props["sunstone.import.source_normalized"])
+
+        if "sunstone.import.activity_id" in props:
+            info_table.add_row("Activity ID", props["sunstone.import.activity_id"])
+
+        if "sunstone.import.system_version" in props:
+            info_table.add_row("System version", props["sunstone.import.system_version"])
+
+        if "sunstone.import.parameters" in props:
+            info_table.add_row("Parameters", props["sunstone.import.parameters"])
+
+        info_table.add_row("", "")
+        info_table.add_row("[bold]PROV URIs[/bold]", "")
+
+        prov_keys = sorted([k for k in props if k.startswith("sunstone.prov.")])
+        for key in prov_keys:
+            short_key = key.replace("sunstone.prov.", "")
+            info_table.add_row(f"  {short_key}", props[key])
+
+    panel = Panel(info_table, title=f"[bold]{table}[/bold]", border_style="blue")
+    console.print(panel)
+
+
+def _show_table_list(namespace: str | None, json_output: bool) -> None:
+    """Show list of tables with metadata."""
+    from rich.tree import Tree
+
+    from sunstone_data.catalog import list_tables_with_metadata
+
+    tables = list_tables_with_metadata(namespace)
+
+    if not tables:
+        if json_output:
+            print(json.dumps([]))
+        else:
+            console.print("[yellow]No datasets found[/yellow]")
+        return
+
+    if json_output:
+        print(json.dumps(tables, indent=2))
+        return
+
+    by_namespace: dict[str, list[dict]] = {}
+    for t in tables:
+        ns, _ = t["identifier"].split(".", 1)
+        if ns not in by_namespace:
+            by_namespace[ns] = []
+        by_namespace[ns].append(t)
+
+    tree = Tree("[bold]Datasets[/bold]")
+    for ns in sorted(by_namespace.keys()):
+        ns_branch = tree.add(f"[cyan]{ns}[/cyan]")
+        for t in by_namespace[ns]:
+            _, table_name = t["identifier"].split(".", 1)
+            ts = t["timestamp"]
+            if "T" in ts:
+                ts = ts.split("T")[0]
+            ns_branch.add(f"{table_name} ({ts}, {t['row_count']} rows)")
+
+    console.print(tree)

sunstone_data/cli/commands/data_publish.py

@@ -0,0 +1,69 @@
+"""``sunstone data publish`` — publish a dataset's metadata to the graph."""
+
+from __future__ import annotations
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+
+
+@data_app.command("publish")
+def publish_cmd(
+    table: str = typer.Argument(..., help="Table identifier (namespace.table)"),
+    branch: str = typer.Option("main", "--branch", "-b", help="Branch to publish on"),
+    version: str | None = typer.Option(
+        None, "--version", help="Publish the table's current snapshot under this version label, e.g. v1.0"
+    ),
+    replace: bool = typer.Option(
+        False, "--replace", help="Retract previously published triples instead of merging into what is there"
+    ),
+) -> None:
+    """Publish metadata for a dataset into its named graph.
+
+    The default merges, so nothing already in the graph is lost. `--replace`
+    retracts what this command published before — and only that.
+    """
+    from datetime import datetime, timezone
+
+    from sunstone.lineage import Metadata
+
+    from sunstone_data.catalog import get_catalog
+    from sunstone_data.graph.dataset import DatasetDescription, DatasetVersion
+    from sunstone_data.publish import publish
+    from sunstone_data.query import TableRef
+    from sunstone_data.resolve import dataset_iri
+
+    try:
+        ref = TableRef.parse(table)
+        identifier = f"{ref.namespace}.{ref.table}"
+        versions: tuple[DatasetVersion, ...] = ()
+
+        if version is not None:
+            iceberg = get_catalog(branch).load_table(identifier)
+            snapshot = iceberg.current_snapshot()
+            if snapshot is None:
+                raise ValueError(f"{identifier} has no snapshot to publish as {version}")
+            versions = (
+                DatasetVersion(
+                    version=version,
+                    snapshot_id=str(snapshot.snapshot_id),
+                    metadata_location=iceberg.metadata_location,
+                    import_activity_id=iceberg.properties.get("sunstone.import.activity_id"),
+                    generated_at=datetime.fromtimestamp(snapshot.timestamp_ms / 1000, tz=timezone.utc).isoformat(),
+                ),
+            )
+
+        etag = publish(
+            DatasetDescription(
+                dataset_iri=dataset_iri(ref),
+                metadata=Metadata(slug=identifier),
+                versions=versions,
+                current_version=version,
+            ),
+            branch=branch,
+            replace=replace,
+        )
+        console.print(f"Published [cyan]{dataset_iri(ref)}[/cyan] on {branch} (ETag {etag})")
+    except Exception as e:
+        console.print(f"[red]Error:[/red] {e}")
+        raise typer.Exit(1)

sunstone_data/cli/commands/ls.py

@@ -0,0 +1,49 @@
+"""``sunstone data ls`` — list namespaces and tables (spec 180 §8)."""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+from sunstone_data.cli._errors import cli_errors
+
+
+@data_app.command("ls")
+def ls_cmd(
+    namespace: str = typer.Argument(None, help="Only list tables in this namespace"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """List namespaces and the tables in them.
+
+    Examples:
+        sunstone data ls
+        sunstone data ls un.health
+    """
+    with cli_errors(json_output):
+        from sunstone_data.catalog import list_tables_with_metadata
+
+        tables = list_tables_with_metadata(namespace)
+
+        if json_output:
+            print(json.dumps(tables, indent=2))
+            return
+
+        if not tables:
+            console.print("[yellow]No datasets found[/yellow]")
+            return
+
+        from rich.tree import Tree
+
+        by_namespace: dict[str, list[tuple[str, dict]]] = {}
+        for entry in tables:
+            ns, name = entry["identifier"].rsplit(".", 1)
+            by_namespace.setdefault(ns, []).append((name, entry))
+
+        tree = Tree("[bold]Datasets[/bold]")
+        for ns in sorted(by_namespace):
+            node = tree.add(f"[cyan]{ns}[/cyan]")
+            for name, entry in sorted(by_namespace[ns]):
+                node.add(f"{name} ({entry.get('row_count', '?')} rows)")
+        console.print(tree)

sunstone_data/cli/commands/prov.py

@@ -0,0 +1,58 @@
+"""``sunstone data lineage`` — a SPARQL query over PROV (spec 180 §8).
+
+The facts this asks for are the ones `graph.dataset.description_to_graph`
+writes when a version is published: each version pins a snapshot, the
+snapshot was generated by an import activity, and the qualified generation
+carries the instant. Until import provenance is asserted end to end (a
+separate task), a dataset published without generation facts answers with
+the version rows and empty activity columns.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from sunstone_data.cli._app import data_app
+from sunstone_data.cli._errors import cli_errors
+
+_QUERY = """PREFIX dcat: <{dcat}>
+PREFIX prov: <{prov}>
+PREFIX si: <{si}>
+
+SELECT ?version ?snapshot ?activity ?generatedAt WHERE {{
+  GRAPH <{iri}> {{
+    <{iri}> dcat:hasVersion ?node .
+    ?node dcat:version ?version .
+    OPTIONAL {{
+      ?node si:hasSnapshot ?snapshot .
+      OPTIONAL {{ ?snapshot prov:wasGeneratedBy ?activity }}
+      OPTIONAL {{ ?snapshot prov:qualifiedGeneration/prov:atTime ?generatedAt }}
+    }}
+  }}
+}}
+ORDER BY ?version
+"""
+
+
+@data_app.command("lineage")
+def lineage_cmd(
+    table: str = typer.Argument(..., help="Table reference (namespace.table)"),
+    json_output: bool = typer.Option(False, "--json", help="Print the raw SPARQL response body"),
+) -> None:
+    """Show where a dataset's versions came from.
+
+    Examples:
+        sunstone data lineage un.health.cases
+        sunstone data lineage un.health.cases --json
+    """
+    with cli_errors(json_output):
+        from sunstone_data.cli.commands.sparql import execute_query
+        from sunstone_data.graph import terms
+        from sunstone_data.query import TableRef
+        from sunstone_data.resolve import dataset_iri
+
+        # The dataset IRI is minted from the ref's namespace and table name,
+        # both already validated by TableRef.parse, so it is safe to inline.
+        iri = dataset_iri(TableRef.parse(table))
+        query = _QUERY.format(dcat=terms.DCAT, prov=terms.PROV, si=terms.DCAT_SI, iri=iri)
+        execute_query(query, json_output, empty_message="No lineage recorded for this dataset")

sunstone_data/cli/commands/refs.py

@@ -0,0 +1,142 @@
+"""graph-server refs: ``branches``, ``branch create|delete``, ``tags``,
+``tag create|delete`` (spec 180 §8).
+
+Branch and tag names go into a URL path, so they are validated here before
+the request is built.
+"""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+from sunstone_data.cli._errors import cli_errors
+
+branch_app = typer.Typer(help="Manage graph-server branches", no_args_is_help=True)
+tag_app = typer.Typer(help="Manage graph-server tags", no_args_is_help=True)
+data_app.add_typer(branch_app, name="branch")
+data_app.add_typer(tag_app, name="tag")
+
+
+def _client():
+    from sunstone_data.graph.client import get_graph_client
+
+    return get_graph_client()
+
+
+def _print_rows(rows: list[dict], columns: list[str], empty: str, json_output: bool) -> None:
+    if json_output:
+        print(json.dumps(rows, indent=2))
+        return
+    if not rows:
+        console.print(f"[yellow]{empty}[/yellow]")
+        return
+
+    from rich.table import Table
+
+    out = Table(box=None, pad_edge=False)
+    for column in columns:
+        out.add_column(column.replace("_", " ").capitalize(), style="cyan" if column == "name" else None)
+    for row in rows:
+        out.add_row(*("" if row.get(c) is None else str(row.get(c)) for c in columns))
+    console.print(out)
+
+
+@data_app.command("branches")
+def branches_cmd(
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """List graph-server branches."""
+    with cli_errors(json_output):
+        rows = _client().list_branches()
+        _print_rows(rows, ["name", "parent", "head_txn", "created_by"], "No branches found", json_output)
+
+
+@branch_app.command("create")
+def branch_create_cmd(
+    name: str = typer.Argument(..., help="Name of the branch to create"),
+    parent: str = typer.Option(None, "--parent", "-p", help="Branch to fork from (server default: main)"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Create a branch."""
+    with cli_errors(json_output):
+        from sunstone_data.identifiers import validate_ref_name
+
+        validate_ref_name(name, "branch")
+        if parent is not None:
+            validate_ref_name(parent, "branch")
+        _client().create_branch(name, parent)
+
+        if json_output:
+            print(json.dumps({"name": name, "parent": parent}))
+        else:
+            console.print(f"[green]Created branch[/green] {name}")
+
+
+@branch_app.command("delete")
+def branch_delete_cmd(
+    name: str = typer.Argument(..., help="Name of the branch to delete"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Delete a branch."""
+    with cli_errors(json_output):
+        from sunstone_data.identifiers import validate_ref_name
+
+        validate_ref_name(name, "branch")
+        _client().delete_branch(name)
+
+        if json_output:
+            print(json.dumps({"name": name, "deleted": True}))
+        else:
+            console.print(f"[green]Deleted branch[/green] {name}")
+
+
+@data_app.command("tags")
+def tags_cmd(
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """List graph-server tags."""
+    with cli_errors(json_output):
+        rows = _client().list_tags()
+        _print_rows(rows, ["name", "branch", "at_txn"], "No tags found", json_output)
+
+
+@tag_app.command("create")
+def tag_create_cmd(
+    name: str = typer.Argument(..., help="Name of the tag to create"),
+    branch: str = typer.Option("main", "--branch", "-b", help="Branch to tag"),
+    at_txn: int = typer.Option(None, "--at-txn", help="Txn to pin (default: the branch head)"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Tag a branch at a txn."""
+    with cli_errors(json_output):
+        from sunstone_data.identifiers import validate_ref_name
+
+        validate_ref_name(name, "tag")
+        validate_ref_name(branch, "branch")
+        created = _client().create_tag(name, branch, at_txn)
+
+        if json_output:
+            print(json.dumps(created))
+        else:
+            console.print(f"[green]Created tag[/green] {name} on {branch} at txn {created.get('at_txn')}")
+
+
+@tag_app.command("delete")
+def tag_delete_cmd(
+    name: str = typer.Argument(..., help="Name of the tag to delete"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+) -> None:
+    """Delete a tag."""
+    with cli_errors(json_output):
+        from sunstone_data.identifiers import validate_ref_name
+
+        validate_ref_name(name, "tag")
+        _client().delete_tag(name)
+
+        if json_output:
+            print(json.dumps({"name": name, "deleted": True}))
+        else:
+            console.print(f"[green]Deleted tag[/green] {name}")

sunstone_data/cli/commands/show.py

@@ -0,0 +1,139 @@
+"""``sunstone data show`` — dataset metadata as a table, Turtle or JSON-LD (spec 180 §8).
+
+Turtle comes straight off graph-server, which serializes it. The table and
+JSON-LD renderings are built here from the N-Triples form of the same graph.
+"""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from sunstone_data.cli._app import console, data_app
+from sunstone_data.cli._errors import cli_errors
+
+FORMATS = ("table", "turtle", "jsonld")
+
+
+@data_app.command("show")
+def show_cmd(
+    table: str = typer.Argument(..., help="Table reference (namespace.table[@branch])"),
+    output_format: str = typer.Option("table", "--format", "-f", help=f"One of: {', '.join(FORMATS)}"),
+    branch: str = typer.Option("main", "--branch", "-b", help="Branch to read the graph from"),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON-LD (same as --format jsonld)"),
+) -> None:
+    """Show a dataset's metadata graph.
+
+    Examples:
+        sunstone data show un.health.cases
+        sunstone data show un.health.cases --format turtle
+        sunstone data show un.health.cases@experiment --json
+    """
+    with cli_errors(json_output):
+        chosen = "jsonld" if json_output else output_format
+        if chosen not in FORMATS:
+            raise ValueError(f"Unknown format {output_format!r}. Use one of: {', '.join(FORMATS)}")
+
+        from sunstone_data.graph.client import N_TRIPLES, get_graph_client
+        from sunstone_data.query import TableRef
+        from sunstone_data.resolve import dataset_iri
+
+        ref = TableRef.parse(table)
+        iri = dataset_iri(ref)
+        client = get_graph_client()
+        # A ref after `@` names a graph-server branch here: the graph is
+        # per-branch, and versions/snapshots are inside it, not selectors on it.
+        source_branch = ref.version_ref or branch
+
+        if chosen == "turtle":
+            print(client.get_graph(source_branch, iri))
+            return
+
+        from sunstone_data.graph.dataset import parse_ntriples
+
+        graph = parse_ntriples(client.get_graph(source_branch, iri, accept=N_TRIPLES))
+
+        if chosen == "jsonld":
+            print(json.dumps(to_jsonld(graph), indent=2))
+            return
+
+        _print_table(graph, iri)
+
+
+def to_jsonld(graph) -> dict:
+    """Render a graph as a small JSON-LD document.
+
+    One node object per subject, keyed by ``@id``, with predicates as CURIEs
+    where the namespace is known. Deliberately not a JSON-LD processor: no
+    framing, no nesting, no blank nodes (this project's serializer emits
+    none). Values stay expanded — ``{"@id": ...}`` for IRIs, ``{"@value":
+    ...}`` with ``@type``/``@language`` for literals — so nothing is lost.
+    """
+    from sunstone_data.graph.dataset import IRI, RDF_TYPE
+    from sunstone_data.graph.terms import PREFIXES
+
+    nodes: dict[str, dict] = {}
+    for triple in sorted(graph, key=lambda t: (t.subject.value, t.predicate.value, t.obj.value)):
+        node = nodes.setdefault(triple.subject.value, {"@id": triple.subject.value})
+        if triple.predicate == RDF_TYPE:
+            node.setdefault("@type", []).append(triple.obj.value)
+            continue
+        value = {"@id": triple.obj.value} if isinstance(triple.obj, IRI) else _literal(triple.obj)
+        node.setdefault(_curie(triple.predicate.value), []).append(value)
+
+    return {"@context": dict(PREFIXES), "@graph": [nodes[k] for k in sorted(nodes)]}
+
+
+def _literal(term) -> dict:
+    value = {"@value": term.value}
+    if term.language is not None:
+        value["@language"] = term.language
+    elif term.datatype is not None:
+        value["@type"] = term.datatype
+    return value
+
+
+def _curie(iri: str) -> str:
+    """Shorten `iri` to ``prefix:local`` for a known namespace, else leave it."""
+    from sunstone_data.graph.terms import PREFIXES
+
+    for prefix, namespace in sorted(PREFIXES.items(), key=lambda kv: -len(kv[1])):
+        if iri.startswith(namespace):
+            return f"{prefix}:{iri[len(namespace) :]}"
+    return iri
+
+
+def _relative(iri: str, base: str) -> str:
+    """Show an IRI inside the dataset's graph relative to the dataset itself."""
+    if iri == base:
+        return "<>"
+    if iri.startswith(base):
+        return iri[len(base) :]
+    return _curie(iri)
+
+
+def _print_table(graph, iri: str) -> None:
+    from rich.table import Table
+
+    from sunstone_data.graph.dataset import IRI
+
+    if not graph:
+        console.print("[yellow]No metadata found[/yellow]")
+        return
+
+    out = Table(box=None, pad_edge=False)
+    out.add_column("Subject", style="cyan", overflow="fold")
+    out.add_column("Predicate", style="magenta", overflow="fold")
+    out.add_column("Object", overflow="fold")
+    for triple in sorted(graph, key=lambda t: (t.subject.value, t.predicate.value, t.obj.value)):
+        if isinstance(triple.obj, IRI):
+            obj = _relative(triple.obj.value, iri)
+        else:
+            obj = triple.obj.value
+            if triple.obj.language:
+                obj += f"@{triple.obj.language}"
+            elif triple.obj.datatype:
+                obj += f" ({_curie(triple.obj.datatype)})"
+        out.add_row(_relative(triple.subject.value, iri), _curie(triple.predicate.value), obj)
+    console.print(out)