phlo-openmetadata 0.1.0__tar.gz

phlo_openmetadata-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: phlo-openmetadata
+Version: 0.1.0
+Summary: OpenMetadata integration for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: phlo-dbt>=0.1.0
+Requires-Dist: phlo-nessie>=0.1.0
+Requires-Dist: phlo-postgres>=0.1.0
+Requires-Dist: phlo-trino>=0.1.0
+Requires-Dist: pyyaml>=6.0.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+OpenMetadata integration for Phlo.

phlo_openmetadata-0.1.0/README.md

@@ -0,0 +1,70 @@
+# phlo-openmetadata
+
+OpenMetadata integration for Phlo.
+
+## Description
+
+Syncs table metadata, lineage, and quality check results to OpenMetadata for data governance and discovery.
+
+## Installation
+
+```bash
+pip install phlo-openmetadata
+# or
+phlo plugin install openmetadata
+```
+
+## Configuration
+
+| Variable                             | Default               | Description                |
+| ------------------------------------ | --------------------- | -------------------------- |
+| `OPENMETADATA_HOST`                  | `openmetadata-server` | OpenMetadata server host   |
+| `OPENMETADATA_PORT`                  | `8585`                | OpenMetadata API port      |
+| `OPENMETADATA_USERNAME`              | `admin`               | Admin username             |
+| `OPENMETADATA_PASSWORD`              | `admin`               | Admin password             |
+| `OPENMETADATA_VERIFY_SSL`            | `false`               | Verify SSL certificates    |
+| `OPENMETADATA_SYNC_ENABLED`          | `true`                | Enable automatic sync      |
+| `OPENMETADATA_SYNC_INTERVAL_SECONDS` | `300`                 | Min interval between syncs |
+
+## Auto-Configuration
+
+This package is **fully auto-configured**:
+
+| Feature               | How It Works                                                     |
+| --------------------- | ---------------------------------------------------------------- |
+| **Hook Registration** | Receives `lineage.edges`, `quality.result`, `publish.end` events |
+| **Lineage Sync**      | Automatically syncs lineage edges to OpenMetadata                |
+| **Quality Results**   | Syncs quality check results as test cases                        |
+| **Table Metadata**    | Syncs published tables with documentation                        |
+
+### Event Flow
+
+```
+Pipeline Events → HookBus → OpenMetadataHookPlugin → OpenMetadata API
+```
+
+## Usage
+
+### CLI Commands
+
+```bash
+# Manually sync all tables
+phlo openmetadata sync
+
+# Sync specific table
+phlo openmetadata sync --table bronze.users
+```
+
+### Programmatic
+
+```python
+from phlo_openmetadata.client import OpenMetadataClient
+
+client = OpenMetadataClient()
+client.sync_table_metadata("bronze.users")
+```
+
+## Entry Points
+
+- `phlo.plugins.cli` - Provides `openmetadata` CLI commands
+- `phlo.plugins.hooks` - Provides `OpenMetadataHookPlugin` for sync

phlo_openmetadata-0.1.0/pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phlo-openmetadata"
+version = "0.1.0"
+description = "OpenMetadata integration for Phlo"
+readme = {text = "OpenMetadata integration for Phlo.", content-type = "text/plain"}
+requires-python = ">=3.11"
+authors = [
+    {name = "Phlo Team", email = "team@phlo.dev"},
+]
+license = {text = "MIT"}
+dependencies = [
+    "phlo>=0.1.0",
+    "phlo-dbt>=0.1.0",
+    "phlo-nessie>=0.1.0",
+    "phlo-postgres>=0.1.0",
+    "phlo-trino>=0.1.0",
+    "pyyaml>=6.0.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.entry-points."phlo.plugins.cli"]
+openmetadata = "phlo_openmetadata.cli_plugin:OpenMetadataCliPlugin"
+
+[project.entry-points."phlo.plugins.hooks"]
+openmetadata = "phlo_openmetadata.hooks_plugin:OpenMetadataHookPlugin"
+
+[project.entry-points."phlo.plugins.services"]
+openmetadata = "phlo_openmetadata.plugin:OpenMetadataServicePlugin"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+phlo_openmetadata = [
+  "service.yaml",
+  "openmetadata-setup.yaml",
+  "openmetadata-mysql-setup.yaml",
+  "openmetadata-elasticsearch-setup.yaml",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

phlo_openmetadata-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_openmetadata-0.1.0/src/phlo_openmetadata/__init__.py

@@ -0,0 +1,18 @@
+from phlo_openmetadata.dbt_sync import DbtManifestParser
+from phlo_openmetadata.openmetadata import (
+    OpenMetadataClient,
+    OpenMetadataColumn,
+    OpenMetadataLineageEdge,
+    OpenMetadataTable,
+)
+from phlo_openmetadata.settings import OpenMetadataSettings, get_settings
+
+__all__ = [
+    "DbtManifestParser",
+    "OpenMetadataSettings",
+    "OpenMetadataClient",
+    "OpenMetadataColumn",
+    "OpenMetadataLineageEdge",
+    "OpenMetadataTable",
+    "get_settings",
+]

phlo_openmetadata-0.1.0/src/phlo_openmetadata/cli_openmetadata.py

@@ -0,0 +1,105 @@
+"""OpenMetadata CLI commands."""
+
+from __future__ import annotations
+
+import json
+import sys
+
+import click
+from rich.console import Console
+
+from phlo_openmetadata.dbt_sync import DbtManifestParser
+from phlo_openmetadata.nessie_sync import sync_nessie_tables_to_openmetadata
+from phlo_openmetadata.openmetadata import OpenMetadataClient
+from phlo_openmetadata.settings import get_settings
+
+console = Console()
+
+
+@click.group()
+def openmetadata():
+    """Manage OpenMetadata integration (optional)."""
+
+
+@openmetadata.command()
+def health() -> None:
+    """Check OpenMetadata connectivity using configured credentials."""
+    cfg = get_settings()
+    client = OpenMetadataClient(
+        base_url=cfg.openmetadata_uri(),
+        username=cfg.openmetadata_username,
+        password=cfg.openmetadata_password,
+        verify_ssl=cfg.openmetadata_verify_ssl,
+        timeout=10,
+        service_name=cfg.openmetadata_service_name,
+        service_type=cfg.openmetadata_service_type,
+        database_name=cfg.openmetadata_database(),
+    )
+    ok = client.health_check()
+    if ok:
+        console.print("[green]OpenMetadata is reachable[/green]")
+        return
+    console.print("[red]OpenMetadata is not reachable[/red]")
+    sys.exit(1)
+
+
+@openmetadata.command()
+@click.option("--include-namespace", multiple=True, help="Only sync these namespaces (repeatable)")
+@click.option("--exclude-namespace", multiple=True, help="Skip these namespaces (repeatable)")
+@click.option("--dbt/--no-dbt", default=True, help="Also sync dbt models (if manifest exists)")
+@click.option("--dbt-schema", default=None, help="Limit dbt sync to a schema (e.g., bronze)")
+def sync(
+    include_namespace: tuple[str, ...],
+    exclude_namespace: tuple[str, ...],
+    dbt: bool,
+    dbt_schema: str | None,
+) -> None:
+    """Sync Nessie catalog (and optionally dbt docs) into OpenMetadata."""
+    cfg = get_settings()
+    client = OpenMetadataClient(
+        base_url=cfg.openmetadata_uri(),
+        username=cfg.openmetadata_username,
+        password=cfg.openmetadata_password,
+        verify_ssl=cfg.openmetadata_verify_ssl,
+        timeout=30,
+        service_name=cfg.openmetadata_service_name,
+        service_type=cfg.openmetadata_service_type,
+        database_name=cfg.openmetadata_database(),
+    )
+
+    if not client.health_check():
+        console.print("[red]OpenMetadata is not reachable[/red]")
+        sys.exit(1)
+
+    from phlo_nessie.catalog_scanner import NessieTableScanner
+
+    scanner = NessieTableScanner.from_config()
+    nessie_stats = sync_nessie_tables_to_openmetadata(
+        scanner,
+        client,
+        include_namespaces=list(include_namespace) or None,
+        exclude_namespaces=list(exclude_namespace) or None,
+    )
+    console.print(f"[green]Nessie sync[/green]: {nessie_stats}")
+
+    if dbt:
+        try:
+            parser = DbtManifestParser.from_config()
+            if dbt_schema:
+                dbt_stats = parser.sync_to_openmetadata(client, schema_name=dbt_schema)
+            else:
+                # Default to syncing schemas present in manifest by iterating models.
+                manifest = parser.load_manifest()
+                schemas = {m.get("schema") for m in parser.get_models(manifest).values()}
+                schemas = {s for s in schemas if isinstance(s, str)}
+                dbt_stats = {"created": 0, "failed": 0}
+                for schema_name in sorted(schemas):
+                    partial = parser.sync_to_openmetadata(client, schema_name=schema_name)
+                    dbt_stats["created"] += partial.get("created", 0)
+                    dbt_stats["failed"] += partial.get("failed", 0)
+            console.print(f"[green]dbt sync[/green]: {dbt_stats}")
+        except FileNotFoundError:
+            console.print("[yellow]dbt manifest not found; skipping dbt sync[/yellow]")
+        except json.JSONDecodeError as e:
+            console.print(f"[red]Failed to parse dbt manifest: {e}[/red]")
+            sys.exit(1)

phlo_openmetadata-0.1.0/src/phlo_openmetadata/cli_plugin.py

@@ -0,0 +1,21 @@
+"""CLI plugin for OpenMetadata integration."""
+
+from __future__ import annotations
+
+import click
+from phlo.plugins.base import CliCommandPlugin, PluginMetadata
+
+from phlo_openmetadata.cli_openmetadata import openmetadata
+
+
+class OpenMetadataCliPlugin(CliCommandPlugin):
+    @property
+    def metadata(self) -> PluginMetadata:
+        return PluginMetadata(
+            name="openmetadata",
+            version="0.1.0",
+            description="CLI commands for OpenMetadata synchronization",
+        )
+
+    def get_cli_commands(self) -> list[click.Command]:
+        return [openmetadata]

phlo_openmetadata-0.1.0/src/phlo_openmetadata/dbt_sync.py

@@ -0,0 +1,301 @@
+"""
+dbt manifest parser and synchronizer.
+
+Parses dbt manifest.json and catalog.json to extract model documentation,
+column descriptions, and tests for syncing to OpenMetadata.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Optional
+
+from phlo.logging import get_logger
+
+from phlo_openmetadata.openmetadata import OpenMetadataColumn, OpenMetadataTable
+from phlo_dbt.settings import get_settings
+
+logger = get_logger(__name__)
+
+
+class DbtManifestParser:
+    """
+    Parses dbt manifest.json for metadata extraction.
+
+    Extracts model descriptions, column-level documentation, tests,
+    and freshness policies for syncing to OpenMetadata.
+    """
+
+    def __init__(self, manifest_path: str, catalog_path: Optional[str] = None):
+        """
+        Initialize dbt manifest parser.
+
+        Args:
+            manifest_path: Path to dbt manifest.json
+            catalog_path: Path to dbt catalog.json (optional, for column docs)
+        """
+        self.manifest_path = Path(manifest_path)
+        self.catalog_path = Path(catalog_path) if catalog_path else None
+        self.manifest = None
+        self.catalog = None
+
+    @classmethod
+    def from_config(cls) -> DbtManifestParser:
+        """Create parser from application config."""
+        config = get_settings()
+        return cls(
+            manifest_path=config.dbt_manifest_path,
+            catalog_path=config.dbt_catalog_path,
+        )
+
+    def load_manifest(self) -> dict[str, Any]:
+        """
+        Load and parse dbt manifest.json.
+
+        Returns:
+            Parsed manifest dictionary
+
+        Raises:
+            FileNotFoundError: If manifest file not found
+            json.JSONDecodeError: If manifest is invalid JSON
+        """
+        if not self.manifest_path.exists():
+            raise FileNotFoundError(f"dbt manifest not found: {self.manifest_path}")
+
+        try:
+            with open(self.manifest_path) as f:
+                self.manifest = json.load(f)
+            logger.info(f"Loaded dbt manifest from {self.manifest_path}")
+            return self.manifest
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON in manifest: {e}")
+            raise
+
+    def load_catalog(self) -> dict[str, Any]:
+        """
+        Load and parse dbt catalog.json for column documentation.
+
+        Returns:
+            Parsed catalog dictionary, or empty dict if not found
+
+        Raises:
+            json.JSONDecodeError: If catalog is invalid JSON
+        """
+        if not self.catalog_path or not self.catalog_path.exists():
+            logger.warning(
+                f"dbt catalog not found at {self.catalog_path}, "
+                "column-level docs will not be available"
+            )
+            return {}
+
+        try:
+            with open(self.catalog_path) as f:
+                self.catalog = json.load(f)
+            logger.info(f"Loaded dbt catalog from {self.catalog_path}")
+            return self.catalog
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON in catalog: {e}")
+            raise
+
+    def get_models(self, manifest: Optional[dict[str, Any]] = None) -> dict[str, dict[str, Any]]:
+        """
+        Extract all models from manifest.
+
+        Args:
+            manifest: Parsed manifest dict (uses loaded manifest if not provided)
+
+        Returns:
+            Dictionary mapping model unique_id to model metadata
+        """
+        if manifest is None:
+            manifest = self.manifest or self.load_manifest()
+
+        models = {}
+        for unique_id, model in manifest.get("nodes", {}).items():
+            if unique_id.startswith("model."):
+                models[unique_id] = model
+                logger.debug(f"Found model: {model.get('name')}")
+
+        return models
+
+    def get_model_columns(
+        self,
+        model_name: str,
+        schema_name: str,
+        catalog: Optional[dict[str, Any]] = None,
+    ) -> dict[str, dict[str, Any]]:
+        """
+        Get column information for a model from catalog.json.
+
+        Args:
+            model_name: Model name
+            schema_name: Schema name
+            catalog: Parsed catalog dict (uses loaded catalog if not provided)
+
+        Returns:
+            Dictionary mapping column name to column metadata
+        """
+        if catalog is None:
+            catalog = self.catalog or self.load_catalog()
+
+        if not catalog:
+            return {}
+
+        def normalize_columns(columns: Any) -> dict[str, dict[str, Any]]:
+            if isinstance(columns, dict):
+                return columns
+            if isinstance(columns, list):
+                normalized: dict[str, dict[str, Any]] = {}
+                for entry in columns:
+                    if not isinstance(entry, dict):
+                        continue
+                    name = entry.get("name")
+                    if isinstance(name, str) and name:
+                        normalized[name] = entry
+                return normalized
+            return {}
+
+        if isinstance(catalog.get("nodes"), dict):
+            nodes: dict[str, Any] = catalog.get("nodes", {})
+            for node in nodes.values():
+                if not isinstance(node, dict):
+                    continue
+                metadata = node.get("metadata") or {}
+                if not isinstance(metadata, dict):
+                    continue
+                if metadata.get("name") != model_name or metadata.get("schema") != schema_name:
+                    continue
+                return normalize_columns(node.get("columns"))
+
+            return {}
+
+        key = f"{schema_name}.{model_name}"
+        model_entry = catalog.get(key, {})
+        return (
+            normalize_columns(model_entry.get("columns")) if isinstance(model_entry, dict) else {}
+        )
+
+    def get_model_tests(
+        self,
+        model_unique_id: str,
+        manifest: Optional[dict[str, Any]] = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Extract tests associated with a model.
+
+        Args:
+            model_unique_id: Model unique_id (e.g., model.project.table)
+            manifest: Parsed manifest dict
+
+        Returns:
+            List of test metadata dicts
+        """
+        if manifest is None:
+            manifest = self.manifest or self.load_manifest()
+
+        tests = []
+        for unique_id, node in manifest.get("nodes", {}).items():
+            if unique_id.startswith("test.") and "test_metadata" in node:
+                depends = node.get("depends_on", {}).get("nodes", [])
+                if model_unique_id in depends:
+                    tests.append(node)
+        return tests
+
+    def extract_openmetadata_table(
+        self,
+        model: dict[str, Any],
+        schema_name: str,
+        columns_info: Optional[dict[str, Any]] = None,
+    ) -> OpenMetadataTable:
+        """
+        Convert dbt model metadata to OpenMetadataTable format.
+
+        Args:
+            model: dbt model metadata
+            schema_name: Schema name
+            columns_info: Optional column info from catalog.json
+
+        Returns:
+            OpenMetadataTable object
+        """
+        name = model.get("name", "unknown")
+        description = model.get("description")
+
+        columns = []
+        model_columns = model.get("columns", {}) or {}
+
+        for idx, (col_name, col_meta) in enumerate(model_columns.items()):
+            col_desc = col_meta.get("description")
+            data_type = "UNKNOWN"
+
+            if columns_info and col_name in columns_info:
+                data_type = columns_info[col_name].get("type", "UNKNOWN")
+
+            columns.append(
+                OpenMetadataColumn(
+                    name=col_name,
+                    description=col_desc,
+                    dataType=data_type,
+                    ordinalPosition=idx,
+                )
+            )
+
+        tags = []
+        for tag in model.get("tags", []) or []:
+            tags.append({"name": tag})
+
+        freshness = model.get("freshness")
+        if freshness and isinstance(freshness, dict):
+            warn_after = freshness.get("warn_after", {})
+            if isinstance(warn_after, dict):
+                count = warn_after.get("count")
+                period = warn_after.get("period")
+                if count and period:
+                    tags.append({"name": f"freshness_warn_after_{count}_{period}"})
+
+        return OpenMetadataTable(
+            name=name,
+            description=description,
+            columns=columns if columns else None,
+            tags=tags if tags else None,
+        )
+
+    def sync_to_openmetadata(
+        self,
+        om_client: Any,  # OpenMetadataClient
+        schema_name: str,
+        model_filter: Optional[list[str]] = None,
+    ) -> dict[str, int]:
+        """
+        Sync dbt models to OpenMetadata.
+
+        Args:
+            om_client: OpenMetadataClient instance
+            schema_name: Target OpenMetadata schema name
+            model_filter: Optional list of dbt model names to sync
+
+        Returns:
+            Stats dict with created/failed counts
+        """
+        stats = {"created": 0, "failed": 0}
+
+        manifest = self.load_manifest()
+        catalog = self.load_catalog()
+
+        models = self.get_models(manifest)
+        for unique_id, model in models.items():
+            model_name = model.get("name")
+            if model_filter and model_name not in model_filter:
+                continue
+
+            try:
+                columns_info = self.get_model_columns(model_name, schema_name, catalog)
+                om_table = self.extract_openmetadata_table(model, schema_name, columns_info)
+                om_client.create_or_update_table(schema_name, om_table)
+                stats["created"] += 1
+            except Exception as e:
+                logger.error(f"Failed to sync model {model_name}: {e}")
+                stats["failed"] += 1
+
+        return stats