perspective-cli 0.1.0__py3-none-any.whl

perspective/__init__.py

@@ -0,0 +1 @@
+"""Perspective CLI."""

perspective/config.py

@@ -0,0 +1,240 @@
+"""Manage Perspective instance configuration."""
+
+from contextlib import suppress
+from pathlib import Path
+from urllib.parse import urljoin
+
+from requests.models import Response
+from rich.panel import Panel
+import typer
+import yaml
+
+from perspective.models.configs import Config
+from perspective.utils import console, send_request
+from perspective.utils.options import ConfigDir, DryRun, Force, PerspectiveURL
+
+
+app = typer.Typer(
+    name="config", no_args_is_help=True, pretty_exceptions_show_locals=False
+)
+
+CONFIG_YAML_EXAMPLE = """# Example:
+#
+# groups:
+#   - meta_key: "domain"
+#     slug: "domains"
+#     label_plural: "Domains"
+#     label_singular: "Domain"
+#     icon: "Cube"
+#     in_sidebar: true
+#     visible: true
+#   - meta_key: "true_source"
+#     slug: "sources"
+#     label_plural: "Sources"
+#     label_singular: "Source"
+#     icon: "Cloud"
+#     in_sidebar: true
+"""
+
+OWNERS_YAML_EXAMPLE = """# Example:
+#
+# owners:
+#   - email: "some@one.com"
+#     first_name: "Dave"
+#     last_name: "Smith"
+#     title: "Director"
+#   - email: "other@person.com"
+#     first_name: "Michelle"
+#     last_name: "Dunne"
+#     title: "CTO"
+#   - email: "someone@else.com"
+#     first_name: "Dana"
+#     last_name: "Pawlak"
+#     title: "HR Manager"
+"""
+
+
+def init_config(config_dir: Path | str = "./.perspective", force: bool = False) -> None:
+    """Initialize configuration files in the specified directory.
+
+    Args:
+        config_dir (Path | str, optional): The directory where configuration files
+            will be created. Defaults to "./.perspective".
+        force (bool, optional): If True, existing configuration files will be
+            overwritten. Defaults to False.
+
+    Raises:
+        FileExistsError: If configuration files already exist and `force` is not set to
+            True.
+    """
+    config_dir = Path(config_dir)
+
+    config_path = config_dir / "config.yaml"
+    owners_path = config_dir / "owners.yaml"
+
+    if force:
+        config_path.unlink(missing_ok=True)
+        owners_path.unlink(missing_ok=True)
+        with suppress(FileNotFoundError):
+            config_dir.rmdir()
+
+    if not config_path.exists() and not owners_path.exists():
+        config_dir.mkdir(exist_ok=True)
+        config_path.touch(exist_ok=False)
+        owners_path.touch(exist_ok=False)
+    else:
+        raise FileExistsError
+
+    config_path.write_text(CONFIG_YAML_EXAMPLE)
+    owners_path.write_text(OWNERS_YAML_EXAMPLE)
+
+
+def get_config(config_dir: Path | str = "./.perspective") -> Config | None:
+    """Retrieve configuration data from YAML files in the specified directory.
+
+    Args:
+        config_dir (Path | str, optional): The directory containing the
+            configuration files. Defaults to "./.perspective".
+
+    Returns:
+        optional[Config]: The configuration object if the configuration is successfully
+            loaded, otherwise None.
+
+    Raises:
+        FileNotFoundError: If the configuration files are missing.
+        typer.Abort: If there is an error parsing the YAML files.
+    """
+    config_dir = Path(config_dir)
+
+    config_path = config_dir / "config.yaml"
+    owners_path = config_dir / "owners.yaml"
+
+    config_missing = True
+    owners_missing = True
+
+    config_dict = {}
+    config_data = {}
+    owners_data = {}
+
+    if config_path.exists():
+        config_missing = False
+        with config_path.open("r") as f:
+            try:
+                config_data: dict | None = yaml.safe_load(f)
+            except yaml.YAMLError as e:
+                console.print(f"Error parsing YAML file: {e}")
+                raise typer.Abort() from e
+
+    if owners_path.exists():
+        owners_missing = False
+        with owners_path.open("r") as f:
+            try:
+                owners_data: dict | None = yaml.safe_load(f)
+
+            except yaml.YAMLError as e:
+                console.print(f"Error parsing YAML file: {e}")
+                raise typer.Abort() from e
+
+    if config_missing and owners_missing:
+        raise FileNotFoundError
+
+    if config_data is not None:
+        config_dict.update(config_data)
+
+    if owners_data is not None:
+        config_dict.update(owners_data)
+
+    return Config(**config_dict)
+
+
+def send_config(config: Config, perspective_url: str, verify: bool = True) -> Response:
+    """Send configuration data to a specified URL.
+
+    Args:
+        config (Config): The configuration data to be sent.
+        perspective_url (str): The URL where the configuration data will be sent.
+        verify (bool): Whether to verify the server's TLS certificate. Defaults to True.
+
+    Returns:
+        Response: The response from the server after sending the configuration data.
+
+    Raises:
+        typer.Exit: If there is an error in sending the configuration data.
+    """
+    console.print(Panel("[yellow]Sending config info to Perspective...[/yellow]"))
+
+    return send_request(
+        url=urljoin(perspective_url, "config/"),
+        payload=config.model_dump(by_alias=True),
+        verify=verify,
+    )
+
+
+@app.command(help="Initialize the configuration.")
+def init(config_dir: Path = ConfigDir, force: bool = Force) -> None:
+    """Initialize the configuration.
+
+    Args:
+        config_dir (Path): The directory to write the configuration files.
+        force (bool): If True, overwrite the configuration if it already exists.
+
+    Raises FileExistsError if the configuration already exists and 'force' is not True.
+    """
+    try:
+        init_config(config_dir=config_dir, force=force)
+        console.print(f"[green]Config initialized at[/green] {config_dir}")
+    except FileExistsError as e:
+        console.print(
+            f"[red]Error![/red] [red]Config files already exist at[/red] {config_dir}\n"
+            f"[yellow]If you want to override run with flag [/yellow][red]--force/-f[/red]"
+        )
+        raise typer.Exit(1) from e
+
+
+@app.command(help="Display the current configuration information.")
+def show(config_dir: Path = ConfigDir) -> None:
+    """Display current configuration from the specified directory."""
+    try:
+        config = get_config(config_dir=config_dir)
+        console.print(config)
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error![/red] [red]Config files not found at[/red] {config_dir}\n"
+            "[yellow]To generate config files use [/yellow][white]'luma config init'[/white]"
+        )
+        raise typer.Exit(1) from e
+
+
+@app.command(help="Send the current configuration information to luma")
+def send(
+    config_dir: Path = ConfigDir,
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+) -> None:
+    """Send configuration to the specified Perspective URL.
+
+    In dry run mode, the configuration is printed but not sent.
+    """
+    try:
+        config = get_config(config_dir=config_dir)
+
+        if dry_run:
+            console.print(config.model_dump(by_alias=True))
+            return
+
+        if config:
+            response = send_config(config=config, perspective_url=perspective_url)
+            if not response.ok:
+                raise typer.Exit(1)
+        else:
+            console.print(
+                f"[red]No Config detected under {config_dir}[/red]\n"
+                f"[yellow]To generate config files use [/yellow][white]'perspective config init'[/white]"
+            )
+
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error![/red] [red]Config files not found at[/red] {config_dir}\n"
+            f"[yellow]To generate config files use [/yellow][white]'perspective config init'[/white]"
+        )
+        raise typer.Exit(1) from e

perspective/exceptions.py

@@ -0,0 +1,15 @@
+"""Exceptions for CLI commands."""
+
+
+class DbtArtifactValidationError(Exception):
+    """Exception raised for errors in the dbt artifact validation process."""
+
+    def __init__(self, version: str | None = None):  # noqa: D107
+        message = f"Unsupported dbt version: {version}."
+        super().__init__(message)
+
+
+class ExtractionError(Exception):
+    """Raised when an error occurs during data extraction."""
+
+    pass

perspective/ingest/dbt.py

@@ -0,0 +1,150 @@
+"""Ingest dbt metadata into Luma."""
+
+from collections.abc import Callable
+import json
+from pathlib import Path
+import time
+from typing import Any
+from urllib.parse import urljoin
+
+from dbt_artifacts_parser.parser import (
+    parse_catalog,
+    parse_manifest,
+    parse_run_results,
+)
+from pydantic import ValidationError
+from rich.panel import Panel
+from typer import Context, Exit, Typer
+
+from perspective.exceptions import DbtArtifactValidationError
+from perspective.utils import (
+    IngestionStatus,
+    check_ingestion_results,
+    check_ingestion_status,
+    console,
+    send_request,
+)
+from perspective.utils.options import (
+    DryRun,
+    Follow,
+    IsDbtTestResultsIngestion,
+    MetadataDir,
+    PerspectiveURL,
+)
+
+
+app = Typer(no_args_is_help=True, pretty_exceptions_show_locals=False)
+
+
+def _validate_artifacts(artifacts: dict[str, Callable]) -> dict[str, Any]:
+    """Validate and parse dbt artifacts."""
+    file_names = [Path(path).name for path in artifacts]
+    console.print(
+        Panel(
+            f"📦 Validating dbt artifacts: [cyan]{file_names}[/cyan] for ingestion..."
+        )
+    )
+    parsed_artifacts = {}
+    for artifact_path, parsing_func in artifacts.items():
+        try:
+            with Path(artifact_path).open("r", encoding="utf-8") as f:
+                artifact = json.load(f)
+        except FileNotFoundError as e:
+            msg = f"Artifact file not found: {artifact_path}"
+            console.print("[red]✘ " + msg + "[/red]")
+            raise DbtArtifactValidationError(msg) from e
+        try:
+            parsed_artifact = parsing_func(artifact)
+        except ValidationError as e:
+            msg = f"Artifact validation failed for {artifact_path}: {e!r}"
+            console.print("[red]✘ " + msg + "[/red]")
+            raise DbtArtifactValidationError(msg) from e
+        parsed_artifacts[artifact_path] = parsed_artifact.model_dump(
+            by_alias=True, mode="json"
+        )
+        console.print(f"[green]✔ {artifact_path} validated and loaded.[/green]")
+
+    return parsed_artifacts
+
+
+@app.callback(invoke_without_command=True)
+def ingest(
+    ctx: Context,  # noqa: ARG001
+    metadata_dir: Path = MetadataDir,
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+    follow: bool = Follow,
+    is_test_results_ingestion: bool = IsDbtTestResultsIngestion,
+) -> None:
+    """Validate and ingest dbt artifacts from the specified directory.
+
+    By default, ingests `manifest.json` and `catalog.json`. If `--test-results` flag is
+    set, only ingests `run_results.json` instead.
+    """
+    # Validate and load artifacts
+    if is_test_results_ingestion:
+        artifacts = {str(metadata_dir / "run_results.json"): parse_run_results}
+        parsed_artifacts = _validate_artifacts(artifacts)
+        payload = parsed_artifacts[str(metadata_dir / "run_results.json")]
+        url = urljoin(perspective_url, "catalog/ingest/dbt/run_results/")
+    else:
+        artifacts = {
+            str(metadata_dir / "manifest.json"): parse_manifest,
+            str(metadata_dir / "catalog.json"): parse_catalog,
+        }
+        parsed_artifacts = _validate_artifacts(artifacts)
+        payload = {
+            "manifest_json": parsed_artifacts[str(metadata_dir / "manifest.json")],
+            "catalog_json": parsed_artifacts[str(metadata_dir / "catalog.json")],
+        }
+        url = urljoin(perspective_url, "catalog/ingest/dbt/")
+
+    # If in dry run mode, print the bundle and exit.
+    if dry_run:
+        console.print("Dry run mode: Payload with the following keys would be sent:")
+        console.print(list(payload.keys()))
+        raise Exit(0)
+
+    # Send ingestion request.
+    response, ingestion_uuid = send_request(
+        url=url,
+        method="POST",
+        payload=payload,
+        verify=False,
+        return_response_key="ingestion_uuid",
+    )
+
+    # Wait until ingestion is complete.
+    if follow and ingestion_uuid:
+        ingestion_status = None
+
+        with console.status("Waiting...", spinner="dots"):
+            for _ in range(30):
+                ingestion_status = check_ingestion_status(
+                    perspective_url, ingestion_uuid, verify=False
+                )
+                if ingestion_status == IngestionStatus.successful.value:
+                    response = check_ingestion_results(perspective_url, ingestion_uuid)
+                    console.print()
+                    console.print(f"Ingestion results for ID {ingestion_uuid}:")
+                    console.print()
+                    console.print(response)
+                    return
+
+                if ingestion_status == IngestionStatus.failed.value:
+                    console.print()
+                    console.print(f"Ingestion failed for ID {ingestion_uuid}")
+                    return
+
+                if ingestion_status == IngestionStatus.pending.value:
+                    time.sleep(1)
+
+        if ingestion_status != IngestionStatus.successful.value:
+            console.print(
+                f"Ingestion did not complete successfully within the wait period. Status: {ingestion_status}"
+            )
+
+
+# Run the application.
+if __name__ == "__main__":
+    app()

perspective/ingest/ingest.py

@@ -0,0 +1,164 @@
+"""Ingest model metadata from various data sources."""
+
+from importlib import import_module
+import json
+from time import sleep
+from urllib.parse import urljoin
+
+from loguru import logger
+from typer import Exit, Typer
+
+from perspective.ingest import dbt, postgres
+from perspective.utils import (
+    IngestionStatus,
+    check_ingestion_results,
+    check_ingestion_status,
+    console,
+    send_request,
+)
+from perspective.utils.options import (
+    DryRun,
+    Follow,
+    FollowTimeout,
+    IngestionId,
+    PerspectiveURL,
+)
+
+
+app = Typer(name="ingest", no_args_is_help=True, pretty_exceptions_show_locals=False)
+
+
+@app.command()
+def status(
+    ingestion_id: IngestionId, perspective_url: str = PerspectiveURL
+) -> str | dict:
+    """Retrieve the status of an ingestion."""
+    return check_ingestion_status(perspective_url, ingestion_id)
+
+
+# Add sub-apps for custom source implementations
+app.add_typer(dbt.app, name="dbt", help="Ingest metadata from dbt.")
+app.add_typer(postgres.app, name="postgres", help="Ingest metadata from Postgres.")
+
+
+@app.command()
+def powerbi(
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+    follow: bool = Follow,
+    follow_timeout: int = FollowTimeout,
+) -> None:
+    """Ingest metadata from Power BI."""
+    return _ingest_generic_source(
+        "powerbi", perspective_url, dry_run, follow, follow_timeout
+    )
+
+
+@app.command()
+def qlik_sense(
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+    follow: bool = Follow,
+    follow_timeout: int = FollowTimeout,
+) -> None:
+    """Ingest metadata from Qlik Sense."""
+    return _ingest_generic_source(
+        "qlik_sense", perspective_url, dry_run, follow, follow_timeout
+    )
+
+
+@app.command()
+def sap(
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+    follow: bool = Follow,
+    follow_timeout: int = FollowTimeout,
+) -> None:
+    """Ingest metadata from SAP."""
+    return _ingest_generic_source(
+        "sap", perspective_url, dry_run, follow, follow_timeout
+    )
+
+
+def _ingest_generic_source(
+    source_name: str,
+    perspective_url: str,
+    dry_run: bool,
+    follow: bool,
+    follow_timeout: int,
+) -> tuple[list, list]:
+    """Generic ingestion logic for standard sources."""
+    try:
+        module = import_module(
+            f"perspective.ingest.sources.database.{source_name}.pipeline"
+        )
+        source_type = "database"
+    except ModuleNotFoundError:
+        try:
+            module = import_module(
+                f"perspective.ingest.sources.bi.{source_name}.pipeline"
+            )
+            source_type = "dashboard"
+        except ModuleNotFoundError:
+            console.print(f"Unsupported metadata source: {source_name}")
+            raise
+
+    endpoint = urljoin(perspective_url, f"/ingest/{source_type}/")
+    manifests = module.pipeline()
+    responses = []
+    ingestion_ids = []
+    for manifest in manifests:
+        payload = json.loads(manifest.json(by_alias=True))
+        n_items = len(payload["payload"])
+
+        logger.debug(
+            f"Sending {n_items} items from {source_name} metadata source to Perspective Catalog at {perspective_url}..."
+        )
+
+        # If in dry run mode, print the bundle and exit.
+        if dry_run:
+            console.print(payload)
+            raise Exit(0)
+
+        # Send ingestion request.
+        response, ingestion_id = send_request(
+            url=endpoint,
+            payload=payload,
+            return_response_key="ingestion_id",
+            verify=False,
+        )
+
+        if follow and ingestion_id:
+            ingestion_status = None
+
+            with console.status("Waiting...", spinner="dots"):
+                for _ in range(follow_timeout):
+                    ingestion_status = check_ingestion_status(
+                        perspective_url, ingestion_id
+                    )
+                    if ingestion_status == IngestionStatus.successful.value:
+                        response = check_ingestion_results(
+                            perspective_url, ingestion_id
+                        )
+                        console.print()
+                        console.print(f"Ingestion results for ID {ingestion_id}:")
+                        console.print()
+                        console.print(response)
+
+                    if ingestion_status == IngestionStatus.failed.value:
+                        console.print()
+                        console.print(f"Ingestion failed for ID {ingestion_id}")
+
+                    if ingestion_status == IngestionStatus.pending.value:
+                        sleep(1)
+
+            if ingestion_status != IngestionStatus.successful.value:
+                status_human_readable = IngestionStatus(ingestion_status).name
+                console.print(
+                    f"Ingestion did not complete successfully within {follow_timeout} seconds. Status: {status_human_readable}."
+                )
+
+        responses.append(response)
+        ingestion_ids.append(ingestion_id)
+
+    return responses, ingestion_ids