perspective-cli 0.1.0__tar.gz

perspective_cli-0.1.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: perspective-cli
+Version: 0.1.0
+Summary: A CLI tool for managing the Perspective AI platform.
+Author-email: Michal Zawadzki <mzawadzki@dyvenia.com>
+Keywords: cli,dbt,perspective,data,catalog,ai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: typer[all]<1.0,>=0.9
+Requires-Dist: psycopg[binary]>=3.2.3
+Requires-Dist: requests<3.0,>=2.20
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: rich<13.8,>=13.7
+Requires-Dist: loguru>=0.7.2
+Requires-Dist: websocket-client>=1.8.0
+Requires-Dist: requests-ntlm>=1.3.0
+Requires-Dist: websockets>=10.4
+Requires-Dist: pydantic[email]==2.11.7
+Requires-Dist: azure-identity>=1.17.1
+Requires-Dist: dlt[duckdb]>=1.11.0
+Requires-Dist: duckdb>1.1.3
+Requires-Dist: dbt-artifacts-parser<1.0,>=0.12.0
+Provides-Extra: sap
+Requires-Dist: pyrfc==2.5.0; extra == "sap"
+
+# perspective-cli
+
+## Installation
+
+### `pip`
+
+```bash
+pip install perspective-cli
+```
+
+### `uv`
+
+```bash
+uv add perspective-cli
+```
+
+## Next Steps
+
+Proceed to the [official documentation](dev.meetperspective.com/docs/catalog/) for next steps and detailed guides on how to utilize `perspective-cli` effectively.

perspective_cli-0.1.0/README.md

@@ -0,0 +1,19 @@
+# perspective-cli
+
+## Installation
+
+### `pip`
+
+```bash
+pip install perspective-cli
+```
+
+### `uv`
+
+```bash
+uv add perspective-cli
+```
+
+## Next Steps
+
+Proceed to the [official documentation](dev.meetperspective.com/docs/catalog/) for next steps and detailed guides on how to utilize `perspective-cli` effectively.

perspective_cli-0.1.0/pyproject.toml

@@ -0,0 +1,167 @@
+[project]
+name = "perspective-cli"
+description = "A CLI tool for managing the Perspective AI platform."
+version = "0.1.0"
+authors = [{ name = "Michal Zawadzki", email = "mzawadzki@dyvenia.com" }]
+requires-python = ">=3.10"
+readme = "README.md"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = ["cli", "dbt", "perspective", "data", "catalog", "ai"]
+dependencies = [
+    "typer[all] >=0.9,<1.0",
+    "psycopg[binary]>=3.2.3",
+    "requests >=2.20,<3.0",
+    "pyyaml >=6.0.1",
+    "rich >=13.7, <13.8",
+    "loguru>=0.7.2",
+    "websocket-client>=1.8.0",
+    "requests-ntlm>=1.3.0",
+    "websockets>=10.4",
+    "pydantic[email]==2.11.7",
+    "azure-identity>=1.17.1",
+    "dlt[duckdb]>=1.11.0",
+    "duckdb>1.1.3",
+    "dbt-artifacts-parser>=0.12.0,<1.0",
+]
+
+[project.scripts]
+perspective = "perspective.main:app"
+
+[project.optional-dependencies]
+# aka extras; eg. pip install perspective-cli[sap].
+sap = ["pyrfc==2.5.0"]
+
+[dependency-groups]
+# Development groups; eg. uv sync --group dev.
+dev = [
+    "pytest",
+    "pytest-cov",
+    "coverage",
+    "multiprocess >= 0.70.16,<0.71",
+    "uvicorn >=0.27.0.post1,<0.28",
+    "fastapi >=0.109.0, <0.110",
+    "pytest-postgresql>=6.1.1",
+    "ruff>=0.9.5",
+    "dbt-osmosis>=1.1.17",
+    "psycopg2>=2.9.10",
+    "responses>=0.25.8",
+]
+
+
+[tool.uv]
+package = true
+no-build-isolation-package = ["pyrfc"]
+
+[tool.ruff]
+preview = true
+
+[tool.ruff.lint]
+pylint.max-args = 9
+
+# Last rule review: ruff 0.1.5
+extend-select = [
+    "I",    # isort
+    "D",    # pydocstyle
+    "W",    # pycodestyle (warnings)
+    "B",    # flake8-bugbear
+    "S",    # flake8-bandit
+    "ANN",  # flake8-annotations
+    "A",    # flake8-builtins
+    "C4",   # flake8-comprehensions
+    "EM",   # flake8-errmsg
+    "T20",  # flake8-print
+    "PT",   # flake8-pytest-style
+    "RET",  # flake8-return
+    "SIM",  # flake8-simplify
+    "ARG",  # flake8-unused-arguments
+    "PTH",  # flake8-use-pathlib
+    "N",    # pep8-naming
+    "UP",   # pyupgrade
+    "C901", # mccabe
+    "FURB", # refurb
+    "TRY",  # tryceratops
+    # "PD", # https://docs.astral.sh/ruff/rules/#pandas-vet-pd
+    "PL",  # pylint
+    "RUF", # Ruff-specific rules
+]
+
+ignore = [
+    "W191",
+    "D206",
+    "D300",
+    "ANN101", # Type annotation for `self`.
+    "ANN102", # Type annotation for `cls`.
+    "ANN204", # Return type annotation for `__init__`.
+    "D101",   # Missing docstring in public class (we use __init__()).
+]
+
+[tool.ruff.lint.extend-per-file-ignores]
+"tests/**" = [
+    "S101",
+    "ANN201",
+    "ANN202",
+    "ANN001",
+    "D103",
+    "D100",
+    "N802",
+    "N803",
+    "B905",
+    "D102",
+    "PLR2004",
+    "PT004",
+    "ARG001",
+    "PLC2701", # Private function imports.
+]
+
+[tool.ruff.lint.mccabe]
+# Ignore rules known to be conflicting between the ruff linter and formatter.
+# See https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
+max-complexity = 10
+
+[tool.ruff.lint.isort]
+force-sort-within-sections = true
+lines-after-imports = 2
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.pycodestyle]
+max-doc-length = 88
+
+[tool.coverage.paths]
+source = ["src", "*/site-packages"]
+tests = ["tests", "*/tests"]
+
+[tool.coverage.run]
+branch = true
+source = ["perspective", "tests"]
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 100
+
+[tool.mypy]
+strict = true
+warn_unreachable = true
+pretty = true
+show_column_numbers = true
+show_error_context = true
+
+# For checking whether the docstrings match function signature.
+# https://peps.python.org/pep-0727/ should basically solve this in Python 3.13.
+[tool.pydoclint]
+style = "google"
+arg-type-hints-in-docstring = false
+check-return-types = false
+check-yield-types = false
+allow-init-docstring = true
+
+# [build-system]
+# requires = ["hatchling"]
+# build-backend = "hatchling.build"

perspective_cli-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

perspective_cli-0.1.0/src/perspective/__init__.py

@@ -0,0 +1 @@
+"""Perspective CLI."""

perspective_cli-0.1.0/src/perspective/config.py

@@ -0,0 +1,240 @@
+"""Manage Perspective instance configuration."""
+
+from contextlib import suppress
+from pathlib import Path
+from urllib.parse import urljoin
+
+from requests.models import Response
+from rich.panel import Panel
+import typer
+import yaml
+
+from perspective.models.configs import Config
+from perspective.utils import console, send_request
+from perspective.utils.options import ConfigDir, DryRun, Force, PerspectiveURL
+
+
+app = typer.Typer(
+    name="config", no_args_is_help=True, pretty_exceptions_show_locals=False
+)
+
+CONFIG_YAML_EXAMPLE = """# Example:
+#
+# groups:
+#   - meta_key: "domain"
+#     slug: "domains"
+#     label_plural: "Domains"
+#     label_singular: "Domain"
+#     icon: "Cube"
+#     in_sidebar: true
+#     visible: true
+#   - meta_key: "true_source"
+#     slug: "sources"
+#     label_plural: "Sources"
+#     label_singular: "Source"
+#     icon: "Cloud"
+#     in_sidebar: true
+"""
+
+OWNERS_YAML_EXAMPLE = """# Example:
+#
+# owners:
+#   - email: "some@one.com"
+#     first_name: "Dave"
+#     last_name: "Smith"
+#     title: "Director"
+#   - email: "other@person.com"
+#     first_name: "Michelle"
+#     last_name: "Dunne"
+#     title: "CTO"
+#   - email: "someone@else.com"
+#     first_name: "Dana"
+#     last_name: "Pawlak"
+#     title: "HR Manager"
+"""
+
+
+def init_config(config_dir: Path | str = "./.perspective", force: bool = False) -> None:
+    """Initialize configuration files in the specified directory.
+
+    Args:
+        config_dir (Path | str, optional): The directory where configuration files
+            will be created. Defaults to "./.perspective".
+        force (bool, optional): If True, existing configuration files will be
+            overwritten. Defaults to False.
+
+    Raises:
+        FileExistsError: If configuration files already exist and `force` is not set to
+            True.
+    """
+    config_dir = Path(config_dir)
+
+    config_path = config_dir / "config.yaml"
+    owners_path = config_dir / "owners.yaml"
+
+    if force:
+        config_path.unlink(missing_ok=True)
+        owners_path.unlink(missing_ok=True)
+        with suppress(FileNotFoundError):
+            config_dir.rmdir()
+
+    if not config_path.exists() and not owners_path.exists():
+        config_dir.mkdir(exist_ok=True)
+        config_path.touch(exist_ok=False)
+        owners_path.touch(exist_ok=False)
+    else:
+        raise FileExistsError
+
+    config_path.write_text(CONFIG_YAML_EXAMPLE)
+    owners_path.write_text(OWNERS_YAML_EXAMPLE)
+
+
+def get_config(config_dir: Path | str = "./.perspective") -> Config | None:
+    """Retrieve configuration data from YAML files in the specified directory.
+
+    Args:
+        config_dir (Path | str, optional): The directory containing the
+            configuration files. Defaults to "./.perspective".
+
+    Returns:
+        optional[Config]: The configuration object if the configuration is successfully
+            loaded, otherwise None.
+
+    Raises:
+        FileNotFoundError: If the configuration files are missing.
+        typer.Abort: If there is an error parsing the YAML files.
+    """
+    config_dir = Path(config_dir)
+
+    config_path = config_dir / "config.yaml"
+    owners_path = config_dir / "owners.yaml"
+
+    config_missing = True
+    owners_missing = True
+
+    config_dict = {}
+    config_data = {}
+    owners_data = {}
+
+    if config_path.exists():
+        config_missing = False
+        with config_path.open("r") as f:
+            try:
+                config_data: dict | None = yaml.safe_load(f)
+            except yaml.YAMLError as e:
+                console.print(f"Error parsing YAML file: {e}")
+                raise typer.Abort() from e
+
+    if owners_path.exists():
+        owners_missing = False
+        with owners_path.open("r") as f:
+            try:
+                owners_data: dict | None = yaml.safe_load(f)
+
+            except yaml.YAMLError as e:
+                console.print(f"Error parsing YAML file: {e}")
+                raise typer.Abort() from e
+
+    if config_missing and owners_missing:
+        raise FileNotFoundError
+
+    if config_data is not None:
+        config_dict.update(config_data)
+
+    if owners_data is not None:
+        config_dict.update(owners_data)
+
+    return Config(**config_dict)
+
+
+def send_config(config: Config, perspective_url: str, verify: bool = True) -> Response:
+    """Send configuration data to a specified URL.
+
+    Args:
+        config (Config): The configuration data to be sent.
+        perspective_url (str): The URL where the configuration data will be sent.
+        verify (bool): Whether to verify the server's TLS certificate. Defaults to True.
+
+    Returns:
+        Response: The response from the server after sending the configuration data.
+
+    Raises:
+        typer.Exit: If there is an error in sending the configuration data.
+    """
+    console.print(Panel("[yellow]Sending config info to Perspective...[/yellow]"))
+
+    return send_request(
+        url=urljoin(perspective_url, "config/"),
+        payload=config.model_dump(by_alias=True),
+        verify=verify,
+    )
+
+
+@app.command(help="Initialize the configuration.")
+def init(config_dir: Path = ConfigDir, force: bool = Force) -> None:
+    """Initialize the configuration.
+
+    Args:
+        config_dir (Path): The directory to write the configuration files.
+        force (bool): If True, overwrite the configuration if it already exists.
+
+    Raises FileExistsError if the configuration already exists and 'force' is not True.
+    """
+    try:
+        init_config(config_dir=config_dir, force=force)
+        console.print(f"[green]Config initialized at[/green] {config_dir}")
+    except FileExistsError as e:
+        console.print(
+            f"[red]Error![/red] [red]Config files already exist at[/red] {config_dir}\n"
+            f"[yellow]If you want to override run with flag [/yellow][red]--force/-f[/red]"
+        )
+        raise typer.Exit(1) from e
+
+
+@app.command(help="Display the current configuration information.")
+def show(config_dir: Path = ConfigDir) -> None:
+    """Display current configuration from the specified directory."""
+    try:
+        config = get_config(config_dir=config_dir)
+        console.print(config)
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error![/red] [red]Config files not found at[/red] {config_dir}\n"
+            "[yellow]To generate config files use [/yellow][white]'luma config init'[/white]"
+        )
+        raise typer.Exit(1) from e
+
+
+@app.command(help="Send the current configuration information to luma")
+def send(
+    config_dir: Path = ConfigDir,
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+) -> None:
+    """Send configuration to the specified Perspective URL.
+
+    In dry run mode, the configuration is printed but not sent.
+    """
+    try:
+        config = get_config(config_dir=config_dir)
+
+        if dry_run:
+            console.print(config.model_dump(by_alias=True))
+            return
+
+        if config:
+            response = send_config(config=config, perspective_url=perspective_url)
+            if not response.ok:
+                raise typer.Exit(1)
+        else:
+            console.print(
+                f"[red]No Config detected under {config_dir}[/red]\n"
+                f"[yellow]To generate config files use [/yellow][white]'perspective config init'[/white]"
+            )
+
+    except FileNotFoundError as e:
+        console.print(
+            f"[red]Error![/red] [red]Config files not found at[/red] {config_dir}\n"
+            f"[yellow]To generate config files use [/yellow][white]'perspective config init'[/white]"
+        )
+        raise typer.Exit(1) from e

perspective_cli-0.1.0/src/perspective/exceptions.py

@@ -0,0 +1,15 @@
+"""Exceptions for CLI commands."""
+
+
+class DbtArtifactValidationError(Exception):
+    """Exception raised for errors in the dbt artifact validation process."""
+
+    def __init__(self, version: str | None = None):  # noqa: D107
+        message = f"Unsupported dbt version: {version}."
+        super().__init__(message)
+
+
+class ExtractionError(Exception):
+    """Raised when an error occurs during data extraction."""
+
+    pass

perspective_cli-0.1.0/src/perspective/ingest/dbt.py

@@ -0,0 +1,150 @@
+"""Ingest dbt metadata into Luma."""
+
+from collections.abc import Callable
+import json
+from pathlib import Path
+import time
+from typing import Any
+from urllib.parse import urljoin
+
+from dbt_artifacts_parser.parser import (
+    parse_catalog,
+    parse_manifest,
+    parse_run_results,
+)
+from pydantic import ValidationError
+from rich.panel import Panel
+from typer import Context, Exit, Typer
+
+from perspective.exceptions import DbtArtifactValidationError
+from perspective.utils import (
+    IngestionStatus,
+    check_ingestion_results,
+    check_ingestion_status,
+    console,
+    send_request,
+)
+from perspective.utils.options import (
+    DryRun,
+    Follow,
+    IsDbtTestResultsIngestion,
+    MetadataDir,
+    PerspectiveURL,
+)
+
+
+app = Typer(no_args_is_help=True, pretty_exceptions_show_locals=False)
+
+
+def _validate_artifacts(artifacts: dict[str, Callable]) -> dict[str, Any]:
+    """Validate and parse dbt artifacts."""
+    file_names = [Path(path).name for path in artifacts]
+    console.print(
+        Panel(
+            f"📦 Validating dbt artifacts: [cyan]{file_names}[/cyan] for ingestion..."
+        )
+    )
+    parsed_artifacts = {}
+    for artifact_path, parsing_func in artifacts.items():
+        try:
+            with Path(artifact_path).open("r", encoding="utf-8") as f:
+                artifact = json.load(f)
+        except FileNotFoundError as e:
+            msg = f"Artifact file not found: {artifact_path}"
+            console.print("[red]✘ " + msg + "[/red]")
+            raise DbtArtifactValidationError(msg) from e
+        try:
+            parsed_artifact = parsing_func(artifact)
+        except ValidationError as e:
+            msg = f"Artifact validation failed for {artifact_path}: {e!r}"
+            console.print("[red]✘ " + msg + "[/red]")
+            raise DbtArtifactValidationError(msg) from e
+        parsed_artifacts[artifact_path] = parsed_artifact.model_dump(
+            by_alias=True, mode="json"
+        )
+        console.print(f"[green]✔ {artifact_path} validated and loaded.[/green]")
+
+    return parsed_artifacts
+
+
+@app.callback(invoke_without_command=True)
+def ingest(
+    ctx: Context,  # noqa: ARG001
+    metadata_dir: Path = MetadataDir,
+    perspective_url: str = PerspectiveURL,
+    dry_run: bool = DryRun,
+    follow: bool = Follow,
+    is_test_results_ingestion: bool = IsDbtTestResultsIngestion,
+) -> None:
+    """Validate and ingest dbt artifacts from the specified directory.
+
+    By default, ingests `manifest.json` and `catalog.json`. If `--test-results` flag is
+    set, only ingests `run_results.json` instead.
+    """
+    # Validate and load artifacts
+    if is_test_results_ingestion:
+        artifacts = {str(metadata_dir / "run_results.json"): parse_run_results}
+        parsed_artifacts = _validate_artifacts(artifacts)
+        payload = parsed_artifacts[str(metadata_dir / "run_results.json")]
+        url = urljoin(perspective_url, "catalog/ingest/dbt/run_results/")
+    else:
+        artifacts = {
+            str(metadata_dir / "manifest.json"): parse_manifest,
+            str(metadata_dir / "catalog.json"): parse_catalog,
+        }
+        parsed_artifacts = _validate_artifacts(artifacts)
+        payload = {
+            "manifest_json": parsed_artifacts[str(metadata_dir / "manifest.json")],
+            "catalog_json": parsed_artifacts[str(metadata_dir / "catalog.json")],
+        }
+        url = urljoin(perspective_url, "catalog/ingest/dbt/")
+
+    # If in dry run mode, print the bundle and exit.
+    if dry_run:
+        console.print("Dry run mode: Payload with the following keys would be sent:")
+        console.print(list(payload.keys()))
+        raise Exit(0)
+
+    # Send ingestion request.
+    response, ingestion_uuid = send_request(
+        url=url,
+        method="POST",
+        payload=payload,
+        verify=False,
+        return_response_key="ingestion_uuid",
+    )
+
+    # Wait until ingestion is complete.
+    if follow and ingestion_uuid:
+        ingestion_status = None
+
+        with console.status("Waiting...", spinner="dots"):
+            for _ in range(30):
+                ingestion_status = check_ingestion_status(
+                    perspective_url, ingestion_uuid, verify=False
+                )
+                if ingestion_status == IngestionStatus.successful.value:
+                    response = check_ingestion_results(perspective_url, ingestion_uuid)
+                    console.print()
+                    console.print(f"Ingestion results for ID {ingestion_uuid}:")
+                    console.print()
+                    console.print(response)
+                    return
+
+                if ingestion_status == IngestionStatus.failed.value:
+                    console.print()
+                    console.print(f"Ingestion failed for ID {ingestion_uuid}")
+                    return
+
+                if ingestion_status == IngestionStatus.pending.value:
+                    time.sleep(1)
+
+        if ingestion_status != IngestionStatus.successful.value:
+            console.print(
+                f"Ingestion did not complete successfully within the wait period. Status: {ingestion_status}"
+            )
+
+
+# Run the application.
+if __name__ == "__main__":
+    app()