sunstone-data-platform 1.0.0__py3-none-any.whl

sunstone_pipeline/resources/__init__.py

@@ -0,0 +1,6 @@
+"""Dagster resources wrapping sunstone_data services."""
+
+from sunstone_pipeline.resources.iceberg import IcebergResource
+from sunstone_pipeline.resources.refs import RefsResource
+
+__all__ = ["IcebergResource", "RefsResource"]

sunstone_pipeline/resources/iceberg.py

@@ -0,0 +1,29 @@
+"""Dagster resource wrapping sunstone_data Iceberg catalog access."""
+
+import dagster as dg
+
+from sunstone_data.catalog import get_catalog
+
+
+class IcebergResource(dg.ConfigurableResource):
+    """Provides access to the Iceberg catalog via sunstone_data.
+
+    Wraps get_catalog() so assets receive catalog access as an
+    injectable resource. All data state lives in Iceberg — this
+    resource is a thin pass-through.
+    """
+
+    branch: str = "main"
+
+    def get_table(self, table_identifier: str):
+        """Load an Iceberg table by identifier (e.g., 'oslo.population')."""
+        catalog = get_catalog(self.branch)
+        return catalog.load_table(table_identifier)
+
+    def get_table_property(self, table_identifier: str, key: str) -> str | None:
+        """Read a single property from an Iceberg table's metadata.
+
+        Returns None if the property doesn't exist.
+        """
+        table = self.get_table(table_identifier)
+        return table.metadata.properties.get(key)

sunstone_pipeline/resources/refs.py

@@ -0,0 +1,88 @@
+"""Dagster resource for per-run branch isolation on graph-server refs.
+
+Branches are graph-server refs (``graph.branch``). The Iceberg REST catalog
+serves each branch at ``{catalog_uri}/iceberg/{branch}/``, so a branch created
+here is both a graph ref and an isolated Iceberg catalog view. Branch CRUD goes
+through :class:`sunstone_data.graph.client.GraphClient`; catalog access on the
+branch goes through :func:`sunstone_data.catalog.get_catalog`.
+"""
+
+import dagster as dg
+from pyiceberg.catalog.rest import RestCatalog
+
+from sunstone_data.catalog import get_catalog
+from sunstone_data.graph.client import get_graph_client
+
+
+class RefsResource(dg.ConfigurableResource):
+    """Manages graph-server branches for pipeline run isolation.
+
+    A pipeline run creates an isolated branch, runs its transforms against
+    the Iceberg catalog on that branch, and abandons the branch on failure.
+    """
+
+    parent: str = "main"
+
+    @staticmethod
+    def branch_name(pipeline_name: str, run_id: str) -> str:
+        """Return the branch name for a run.
+
+        graph-server and iceberg-catalog both take the branch as a single URL
+        path segment, so the name uses ``-`` as separator and rejects ``/``.
+
+        Raises:
+            ValueError: If ``pipeline_name`` or ``run_id`` contains ``/``.
+        """
+        if "/" in pipeline_name or "/" in run_id:
+            raise ValueError(f"pipeline name and run id must not contain '/': {pipeline_name!r}, {run_id!r}")
+        return f"pipeline-{pipeline_name}-{run_id}"
+
+    def create_pipeline_branch(self, pipeline_name: str, run_id: str) -> str:
+        """Create an isolated branch for a pipeline run, forked from ``parent``.
+
+        Args:
+            pipeline_name: Name of the pipeline (e.g. ``trusthere``).
+            run_id: Unique run identifier.
+
+        Returns:
+            Branch name (e.g. ``pipeline-trusthere-run-1``).
+
+        Raises:
+            RuntimeError: If graph-server rejects the request.
+        """
+        name = self.branch_name(pipeline_name, run_id)
+        try:
+            get_graph_client().create_branch(name, self.parent)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create branch '{name}': {e}") from e
+        return name
+
+    def catalog(self, branch_name: str) -> RestCatalog:
+        """Return a PyIceberg catalog scoped to ``branch_name``."""
+        return get_catalog(branch_name)
+
+    def merge_to_main(self, branch_name: str) -> None:
+        """Merge a pipeline branch into ``parent``.
+
+        graph-server has no branch-merge endpoint yet (its ref API is create,
+        list and delete), so this always fails. Pipelines that need their
+        output on ``main`` must write there directly until the server grows a
+        merge.
+
+        Raises:
+            NotImplementedError: Always.
+        """
+        raise NotImplementedError(
+            f"graph-server has no branch merge; cannot merge '{branch_name}' into '{self.parent}'"
+        )
+
+    def abandon_branch(self, branch_name: str) -> None:
+        """Delete a pipeline branch without merging (run failed).
+
+        Raises:
+            RuntimeError: If graph-server rejects the request.
+        """
+        try:
+            get_graph_client().delete_branch(branch_name)
+        except Exception as e:
+            raise RuntimeError(f"Failed to delete branch '{branch_name}': {e}") from e