isa-data 1.0.0__py3-none-any.whl

isa_data/services/data_fabric_service/federation/iceberg/hms_catalog.py

@@ -0,0 +1,404 @@
+"""
+HMS / Iceberg catalog client — facade over PyIceberg's ``HiveCatalog``.
+
+When the isA_Data lakehouse is backed by Hive Metastore + Iceberg, snapshot
+and manifest provenance may previously have been reachable only through an
+external governance platform. This client gives isA_Data a direct read path:
+list/describe Iceberg tables AND walk the snapshot history straight from the
+metastore.
+
+Why PyIceberg's ``HiveCatalog`` (and not ``pyhive``/``hmsclient``)
+-----------------------------------------------------------------
+A raw Thrift HMS client (``pyhive`` / ``hmsclient``) only returns the *Hive*
+view of a table — storage descriptor, columns, the ``metadata_location``
+table property. It would still leave us to fetch and parse the Iceberg
+metadata JSON, manifest lists and manifest files by hand. PyIceberg's
+``HiveCatalog`` resolves the metastore entry AND loads the Iceberg table
+object, exposing ``.snapshots()``, ``.history()`` and per-snapshot manifest
+metadata in one client. One dependency, both acceptance criteria covered.
+
+Configuration (env vars, resolved by :class:`HmsCatalogConfig.from_env`)
+------------------------------------------------------------------------
+    HMS_THRIFT_URI       — Hive Metastore Thrift URI, e.g. ``thrift://hms:9083``
+    HMS_CATALOG_NAME     — logical catalog name (default: ``hms``)
+    HMS_WAREHOUSE        — optional warehouse path (e.g. ``s3a://lakehouse/wh``)
+    HMS_S3_ENDPOINT      — optional S3 endpoint for reading manifest files
+    HMS_S3_ACCESS_KEY    — optional S3 access key
+    HMS_S3_SECRET_KEY    — optional S3 secret key
+
+No hosts or credentials are hardcoded — an unconfigured environment yields a
+config whose :attr:`HmsCatalogConfig.configured` is ``False``, and the client
+degrades gracefully instead of crashing.
+
+Graceful degradation
+--------------------
+``pyiceberg`` is an optional dependency. It is imported lazily inside
+:meth:`HmsCatalogClient.connect` — importing this module never fails, so the
+FastAPI app boots even where ``pyiceberg[hive]`` is not installed. Operations
+that genuinely need a live catalog raise a typed
+:class:`IcebergCatalogError`; nothing crashes the process.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from isa_data.core.logging import get_logger
+
+from .errors import (
+    IcebergConnectionError,
+    IcebergDependencyMissingError,
+    IcebergTableNotFoundError,
+)
+
+logger = get_logger(__name__)
+
+DEFAULT_CATALOG_NAME = "hms"
+
+
+@dataclass
+class HmsCatalogConfig:
+    """Connection config for a Hive Metastore-backed Iceberg catalog.
+
+    Resolved from the environment — never carries hardcoded hosts/secrets.
+    """
+
+    thrift_uri: str = ""
+    catalog_name: str = DEFAULT_CATALOG_NAME
+    warehouse: str = ""
+    s3_endpoint: str = ""
+    s3_access_key: str = ""
+    s3_secret_key: str = ""
+
+    @property
+    def configured(self) -> bool:
+        """True only when a Thrift URI is present — the minimum to connect."""
+        return bool(self.thrift_uri)
+
+    @classmethod
+    def from_env(cls) -> "HmsCatalogConfig":
+        """Load configuration from environment variables."""
+        return cls(
+            thrift_uri=os.getenv("HMS_THRIFT_URI", ""),
+            catalog_name=os.getenv("HMS_CATALOG_NAME", DEFAULT_CATALOG_NAME),
+            warehouse=os.getenv("HMS_WAREHOUSE", ""),
+            s3_endpoint=os.getenv("HMS_S3_ENDPOINT", ""),
+            s3_access_key=os.getenv("HMS_S3_ACCESS_KEY", ""),
+            s3_secret_key=os.getenv("HMS_S3_SECRET_KEY", ""),
+        )
+
+    def to_pyiceberg_properties(self) -> Dict[str, str]:
+        """Render PyIceberg ``load_catalog`` properties from this config."""
+        props: Dict[str, str] = {
+            "type": "hive",
+            "uri": self.thrift_uri,
+        }
+        if self.warehouse:
+            props["warehouse"] = self.warehouse
+        if self.s3_endpoint:
+            props["s3.endpoint"] = self.s3_endpoint
+        if self.s3_access_key:
+            props["s3.access-key-id"] = self.s3_access_key
+        if self.s3_secret_key:
+            props["s3.secret-access-key"] = self.s3_secret_key
+        return props
+
+
+@dataclass
+class IcebergSnapshotInfo:
+    """A single Iceberg snapshot plus its manifest provenance.
+
+    Captures everything needed to express snapshot/manifest provenance as
+    lineage: the snapshot id, its parent, the operation that produced it,
+    and the manifest list / manifest files it references.
+    """
+
+    snapshot_id: int
+    parent_snapshot_id: Optional[int]
+    timestamp_ms: int
+    operation: str = ""
+    manifest_list: str = ""
+    summary: Dict[str, Any] = field(default_factory=dict)
+    manifests: List[str] = field(default_factory=list)
+    schema_id: Optional[int] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "snapshot_id": self.snapshot_id,
+            "parent_snapshot_id": self.parent_snapshot_id,
+            "timestamp_ms": self.timestamp_ms,
+            "operation": self.operation,
+            "manifest_list": self.manifest_list,
+            "summary": self.summary,
+            "manifests": self.manifests,
+            "schema_id": self.schema_id,
+        }
+
+
+@dataclass
+class IcebergTableMetadata:
+    """Describe-table result for an Iceberg table read via HMS."""
+
+    namespace: str
+    table_name: str
+    location: str = ""
+    metadata_location: str = ""
+    current_snapshot_id: Optional[int] = None
+    columns: List[Dict[str, Any]] = field(default_factory=list)
+    partition_spec: List[str] = field(default_factory=list)
+    properties: Dict[str, Any] = field(default_factory=dict)
+    snapshots: List[IcebergSnapshotInfo] = field(default_factory=list)
+
+    @property
+    def identifier(self) -> str:
+        """Fully-qualified ``namespace.table`` identifier."""
+        return f"{self.namespace}.{self.table_name}"
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "namespace": self.namespace,
+            "table_name": self.table_name,
+            "identifier": self.identifier,
+            "location": self.location,
+            "metadata_location": self.metadata_location,
+            "current_snapshot_id": self.current_snapshot_id,
+            "columns": self.columns,
+            "partition_spec": self.partition_spec,
+            "properties": self.properties,
+            "snapshots": [s.to_dict() for s in self.snapshots],
+        }
+
+
+class HmsCatalogClient:
+    """Read-only client over a Hive Metastore-backed Iceberg catalog.
+
+    Lifecycle::
+
+        client = HmsCatalogClient.from_env()
+        client.connect()                      # lazy-imports pyiceberg
+        namespaces = client.list_namespaces()
+        tables = client.list_tables("analytics")
+        meta = client.describe_table("analytics", "orders")
+
+    Every catalog operation raises a typed :class:`IcebergCatalogError`
+    subclass on failure; the constructor and module import never raise.
+    """
+
+    def __init__(self, config: Optional[HmsCatalogConfig] = None) -> None:
+        self._config = config or HmsCatalogConfig()
+        self._catalog: Any = None
+
+    @classmethod
+    def from_env(cls) -> "HmsCatalogClient":
+        """Create a client from environment variables."""
+        return cls(HmsCatalogConfig.from_env())
+
+    @property
+    def config(self) -> HmsCatalogConfig:
+        return self._config
+
+    @property
+    def connected(self) -> bool:
+        return self._catalog is not None
+
+    # ------------------------------------------------------------------
+    # Connection
+    # ------------------------------------------------------------------
+
+    def connect(self) -> None:
+        """Load the PyIceberg ``HiveCatalog``.
+
+        Raises:
+            IcebergDependencyMissingError: ``pyiceberg`` is not installed.
+            IcebergConnectionError: no Thrift URI configured, or the
+                catalog could not be loaded.
+        """
+        if self._catalog is not None:
+            return
+
+        if not self._config.configured:
+            raise IcebergConnectionError(
+                "HMS not configured — set HMS_THRIFT_URI "
+                "(e.g. thrift://metastore-host:9083)"
+            )
+
+        # Lazy import — keeps `pyiceberg` optional. Importing this module
+        # must never fail just because the extra is not installed.
+        try:
+            from pyiceberg.catalog import load_catalog
+        except ImportError as exc:
+            raise IcebergDependencyMissingError(
+                "pyiceberg is not installed — install the HMS extra with "
+                "`pip install 'pyiceberg[hive]'` to read Iceberg metadata "
+                "via Hive Metastore"
+            ) from exc
+
+        try:
+            self._catalog = load_catalog(
+                self._config.catalog_name,
+                **self._config.to_pyiceberg_properties(),
+            )
+        except Exception as exc:  # pragma: no cover - network-specific
+            raise IcebergConnectionError(
+                f"Failed to load Hive Metastore catalog "
+                f"'{self._config.catalog_name}' at "
+                f"'{self._config.thrift_uri}': {exc}"
+            ) from exc
+
+        logger.info(
+            "hms_catalog_connected",
+            catalog=self._config.catalog_name,
+            thrift_uri=self._config.thrift_uri,
+        )
+
+    def _require_catalog(self) -> Any:
+        """Return the loaded catalog, connecting on first use."""
+        if self._catalog is None:
+            self.connect()
+        return self._catalog
+
+    # ------------------------------------------------------------------
+    # Table listing / describe
+    # ------------------------------------------------------------------
+
+    def list_namespaces(self) -> List[str]:
+        """List all namespaces (Hive databases) in the catalog."""
+        catalog = self._require_catalog()
+        try:
+            raw = catalog.list_namespaces()
+        except Exception as exc:  # pragma: no cover - network-specific
+            raise IcebergConnectionError(f"Failed to list namespaces: {exc}") from exc
+        return [".".join(ns) for ns in raw]
+
+    def list_tables(self, namespace: str) -> List[str]:
+        """List fully-qualified Iceberg table identifiers in a namespace."""
+        catalog = self._require_catalog()
+        try:
+            raw = catalog.list_tables(namespace)
+        except Exception as exc:  # pragma: no cover - network-specific
+            raise IcebergConnectionError(
+                f"Failed to list tables in namespace '{namespace}': {exc}"
+            ) from exc
+        return [".".join(ident) for ident in raw]
+
+    def describe_table(self, namespace: str, table_name: str) -> IcebergTableMetadata:
+        """Load full metadata for one Iceberg table, including snapshots.
+
+        Raises:
+            IcebergTableNotFoundError: the table does not exist.
+            IcebergConnectionError: the metastore could not be reached.
+        """
+        catalog = self._require_catalog()
+        identifier = f"{namespace}.{table_name}"
+        try:
+            table = catalog.load_table(identifier)
+        except Exception as exc:
+            # PyIceberg raises NoSuchTableError; match by name to avoid a
+            # hard dependency on the exception class across versions.
+            if "NoSuchTable" in type(exc).__name__ or "not exist" in str(exc):
+                raise IcebergTableNotFoundError(
+                    f"Iceberg table '{identifier}' not found in HMS"
+                ) from exc
+            raise IcebergConnectionError(
+                f"Failed to load table '{identifier}': {exc}"
+            ) from exc
+
+        return self._table_to_metadata(namespace, table_name, table)
+
+    # ------------------------------------------------------------------
+    # Snapshot / manifest provenance
+    # ------------------------------------------------------------------
+
+    def get_snapshots(
+        self, namespace: str, table_name: str
+    ) -> List[IcebergSnapshotInfo]:
+        """Return the full snapshot history for a table.
+
+        Each snapshot carries its parent id and manifest provenance — the
+        raw material for snapshot lineage edges.
+        """
+        return self.describe_table(namespace, table_name).snapshots
+
+    # ------------------------------------------------------------------
+    # PyIceberg object -> dataclass mapping
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _table_to_metadata(
+        namespace: str, table_name: str, table: Any
+    ) -> IcebergTableMetadata:
+        """Map a PyIceberg ``Table`` to :class:`IcebergTableMetadata`."""
+        meta = table.metadata
+
+        columns: List[Dict[str, Any]] = []
+        try:
+            for fld in table.schema().fields:
+                columns.append(
+                    {
+                        "name": fld.name,
+                        "type": str(fld.field_type),
+                        "required": bool(fld.required),
+                        "field_id": fld.field_id,
+                    }
+                )
+        except Exception:  # pragma: no cover - schema-shape defensive
+            columns = []
+
+        partition_spec: List[str] = []
+        try:
+            for pf in table.spec().fields:
+                partition_spec.append(pf.name)
+        except Exception:  # pragma: no cover - spec-shape defensive
+            partition_spec = []
+
+        current_snapshot_id = getattr(meta, "current_snapshot_id", None)
+
+        snapshots = [
+            HmsCatalogClient._snapshot_to_info(snap)
+            for snap in getattr(meta, "snapshots", []) or []
+        ]
+
+        return IcebergTableMetadata(
+            namespace=namespace,
+            table_name=table_name,
+            location=getattr(meta, "location", "") or "",
+            metadata_location=getattr(table, "metadata_location", "") or "",
+            current_snapshot_id=current_snapshot_id,
+            columns=columns,
+            partition_spec=partition_spec,
+            properties=dict(getattr(meta, "properties", {}) or {}),
+            snapshots=snapshots,
+        )
+
+    @staticmethod
+    def _snapshot_to_info(snap: Any) -> IcebergSnapshotInfo:
+        """Map a PyIceberg ``Snapshot`` to :class:`IcebergSnapshotInfo`."""
+        summary_obj = getattr(snap, "summary", None)
+        operation = ""
+        summary: Dict[str, Any] = {}
+        if summary_obj is not None:
+            # PyIceberg Summary exposes `.operation` and behaves like a dict.
+            operation = str(getattr(summary_obj, "operation", "") or "")
+            try:
+                summary = dict(summary_obj)
+            except (TypeError, ValueError):
+                summary = {}
+
+        manifests: List[str] = []
+        # `manifests(io)` needs a FileIO to read the manifest list. When an
+        # IO is wired we could enumerate manifest file paths; we keep the
+        # manifest-list pointer unconditionally so provenance is never lost.
+        manifest_list = getattr(snap, "manifest_list", "") or ""
+
+        return IcebergSnapshotInfo(
+            snapshot_id=getattr(snap, "snapshot_id", 0),
+            parent_snapshot_id=getattr(snap, "parent_snapshot_id", None),
+            timestamp_ms=getattr(snap, "timestamp_ms", 0) or 0,
+            operation=operation,
+            manifest_list=manifest_list,
+            summary=summary,
+            manifests=manifests,
+            schema_id=getattr(snap, "schema_id", None),
+        )