perspective-cli 0.1.0__py3-none-any.whl

perspective/ingest/sources/database/sap/transform.py

@@ -0,0 +1,85 @@
+"""Extract a database table manifest from SAP metadata.
+
+We use the local DuckDB instance to join the two source tables in order to be able
+to produce models expected by Luma.
+
+Manifests are saved in batches (50,000 tables per batch) in order to limit payload size.
+"""
+
+from itertools import batched
+
+import dlt
+from loguru import logger
+
+from lumaCLI.metadata.models.database import (
+    DatabaseTable,
+    DatabaseTableColumn,
+    DatabaseTableManifest,
+    DatabaseTableSchemaMetadata,
+)
+
+
+pipeline = dlt.pipeline(
+    pipeline_name="sap", destination="duckdb", dataset_name="bronze"
+)
+
+custom_tables_query = """
+SELECT tabname, fieldname, datatype, leng, ddtext
+FROM custom_tables_details
+LEFT JOIN column_details ON custom_tables_details.fieldname = column_details.rollname
+ORDER BY tabname
+"""
+
+standard_tables_query = """
+SELECT tabname, fieldname, datatype, leng, ddtext
+FROM standard_tables_details
+LEFT JOIN column_details ON standard_tables_details.fieldname = column_details.rollname
+ORDER BY tabname
+"""
+
+
+def transform():
+    with (
+        pipeline.sql_client() as client,
+        client.execute_query(standard_tables_query) as cursor,
+    ):
+        table_to_columns = {}
+        # Create a table -> columns mapping.
+        for i, df in enumerate(cursor.iter_df(chunk_size=50000)):
+            logger.info(
+                f"Extracting table->column mappings from chunk {i} ({len(df)} rows)..."
+            )
+            for _, row in df.iterrows():
+                table_name = row["tabname"]
+                column = DatabaseTableColumn(
+                    name=row["fieldname"],
+                    type=row["datatype"],
+                    length=str(row["leng"]),
+                    description=row["ddtext"],
+                )
+                if table_name not in table_to_columns:
+                    table_to_columns[table_name] = {"columns": []}
+                table_to_columns[table_name]["columns"].append(column)
+
+    # Create table manifests from the mapping.
+    tables = (
+        DatabaseTable(
+            name=table_name, columns=table_to_columns[table_name]["columns"], type="sap"
+        )
+        for table_name in table_to_columns
+    )
+    batch_size = 50000
+    for i, table_batch in enumerate(batched(tables, batch_size)):
+        logger.info(f"Generating manifest for table batch {i}...")
+
+        manifest = DatabaseTableManifest(
+            metadata=DatabaseTableSchemaMetadata(schema="database_table", version=1),
+            payload=list(table_batch),
+        )
+
+        yield manifest
+
+        # logger.info(f"Writing {len(manifest.payload)} tables to batch {i} manifest...")
+
+        # with open(f"database_table_manifest__batch_{i}.json", "w") as f:
+        #     f.write(manifest.json())

perspective/main.py

@@ -0,0 +1,74 @@
+"""A command-line interface (CLI) for the Perspective application.
+
+Provides commands for database operations, configuration management, and more.
+"""
+
+import importlib.metadata
+import sys
+
+from loguru import logger
+import typer
+import urllib3
+
+from perspective import config
+from perspective.ingest import ingest
+
+
+# Set the logging level to INFO by default.
+logger.remove()
+logger.add(sys.stdout, level="INFO")
+
+
+__version__ = importlib.metadata.version("perspective-cli")
+
+# Disable warnings related to insecure requests for specific cases
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+# Create a Typer application with configured properties
+app = typer.Typer(
+    name="perspective",
+    no_args_is_help=True,
+    pretty_exceptions_enable=True,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_short=True,
+)
+
+
+def version_callback(show_version: bool) -> str | None:
+    """Print the version of the application.
+
+    Args:
+        show_version (bool): If True, shows the version.
+    """
+    if show_version:
+        typer.echo(__version__)
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        None,
+        "--version",
+        "-v",
+        callback=version_callback,
+        is_eager=True,
+        help="Show the version and exit.",
+    ),
+) -> None:
+    """Main function for the Typer application.
+
+    Args:
+        version (bool): Flag to show the version and exit.
+        config_dir (str): The directory containing the Perspective configuration file.
+    """
+    pass
+
+
+app.add_typer(
+    config.app, name="config", help="Manage Perspective instance configuration."
+)
+app.add_typer(
+    ingest.app, name="ingest", help="Ingest model metadata from various data sources."
+)

perspective/models/configs.py

@@ -0,0 +1,422 @@
+"""Pydantic models for Perspective instance configuration."""
+
+from typing import Any
+
+from pydantic import BaseModel, EmailStr, field_validator, model_validator
+
+
+allowed_icons = (
+    "AcademicCap",
+    "AdjustmentsHorizontal",
+    "AdjustmentsVertical",
+    "ArchiveBox",
+    "ArchiveBoxArrowDown",
+    "ArchiveBoxXMark",
+    "ArrowDown",
+    "ArrowDownCircle",
+    "ArrowDownLeft",
+    "ArrowDownOnSquare",
+    "ArrowDownOnSquareStack",
+    "ArrowDownRight",
+    "ArrowDownTray",
+    "ArrowLeft",
+    "ArrowLeftCircle",
+    "ArrowLeftOnRectangle",
+    "ArrowLongDown",
+    "ArrowLongLeft",
+    "ArrowLongRight",
+    "ArrowLongUp",
+    "ArrowPath",
+    "ArrowPathRoundedSquare",
+    "ArrowRight",
+    "ArrowRightCircle",
+    "ArrowRightOnRectangle",
+    "ArrowSmallDown",
+    "ArrowSmallLeft",
+    "ArrowSmallRight",
+    "ArrowSmallUp",
+    "ArrowTopRightOnSquare",
+    "ArrowTrendingDown",
+    "ArrowTrendingUp",
+    "ArrowUp",
+    "ArrowUpCircle",
+    "ArrowUpLeft",
+    "ArrowUpOnSquare",
+    "ArrowUpOnSquareStack",
+    "ArrowUpRight",
+    "ArrowUpTray",
+    "ArrowUturnDown",
+    "ArrowUturnLeft",
+    "ArrowUturnRight",
+    "ArrowUturnUp",
+    "ArrowsPointingIn",
+    "ArrowsPointingOut",
+    "ArrowsRightLeft",
+    "ArrowsUpDown",
+    "AtSymbol",
+    "Backspace",
+    "Backward",
+    "Banknotes",
+    "Bars2",
+    "Bars3",
+    "Bars3BottomLeft",
+    "Bars3BottomRight",
+    "Bars3CenterLeft",
+    "Bars4",
+    "BarsArrowDown",
+    "BarsArrowUp",
+    "Battery0",
+    "Battery100",
+    "Battery50",
+    "Beaker",
+    "Bell",
+    "BellAlert",
+    "BellSlash",
+    "BellSnooze",
+    "Bolt",
+    "BoltSlash",
+    "BookOpen",
+    "Bookmark",
+    "BookmarkSlash",
+    "BookmarkSquare",
+    "Briefcase",
+    "BugAnt",
+    "BuildingLibrary",
+    "BuildingOffice",
+    "BuildingOffice2",
+    "BuildingStorefront",
+    "Cake",
+    "Calculator",
+    "Calendar",
+    "CalendarDays",
+    "Camera",
+    "ChartBar",
+    "ChartBarSquare",
+    "ChartPie",
+    "ChatBubbleBottomCenter",
+    "ChatBubbleBottomCenterText",
+    "ChatBubbleLeft",
+    "ChatBubbleLeftEllipsis",
+    "ChatBubbleLeftRight",
+    "ChatBubbleOvalLeft",
+    "ChatBubbleOvalLeftEllipsis",
+    "Check",
+    "CheckBadge",
+    "CheckCircle",
+    "ChevronDoubleDown",
+    "ChevronDoubleLeft",
+    "ChevronDoubleRight",
+    "ChevronDoubleUp",
+    "ChevronDown",
+    "ChevronLeft",
+    "ChevronRight",
+    "ChevronUp",
+    "ChevronUpDown",
+    "CircleStack",
+    "Clipboard",
+    "ClipboardDocument",
+    "ClipboardDocumentCheck",
+    "ClipboardDocumentList",
+    "Clock",
+    "Cloud",
+    "CloudArrowDown",
+    "CloudArrowUp",
+    "CodeBracket",
+    "CodeBracketSquare",
+    "Cog",
+    "Cog6Tooth",
+    "Cog8Tooth",
+    "CommandLine",
+    "ComputerDesktop",
+    "CpuChip",
+    "CreditCard",
+    "Cube",
+    "CubeTransparent",
+    "CurrencyBangladeshi",
+    "CurrencyDollar",
+    "CurrencyEuro",
+    "CurrencyPound",
+    "CurrencyRupee",
+    "CurrencyYen",
+    "CursorArrowRays",
+    "CursorArrowRipple",
+    "DevicePhoneMobile",
+    "DeviceTablet",
+    "Document",
+    "DocumentArrowDown",
+    "DocumentArrowUp",
+    "DocumentChartBar",
+    "DocumentCheck",
+    "DocumentDuplicate",
+    "DocumentMagnifyingGlass",
+    "DocumentMinus",
+    "DocumentPlus",
+    "DocumentText",
+    "EllipsisHorizontal",
+    "EllipsisHorizontalCircle",
+    "EllipsisVertical",
+    "Envelope",
+    "EnvelopeOpen",
+    "ExclamationCircle",
+    "ExclamationTriangle",
+    "Eye",
+    "EyeDropper",
+    "EyeSlash",
+    "FaceFrown",
+    "FaceSmile",
+    "Film",
+    "FingerPrint",
+    "Fire",
+    "Flag",
+    "Folder",
+    "FolderArrowDown",
+    "FolderMinus",
+    "FolderOpen",
+    "FolderPlus",
+    "Forward",
+    "Funnel",
+    "Gif",
+    "Gift",
+    "GiftTop",
+    "GlobeAlt",
+    "GlobeAmericas",
+    "GlobeAsiaAustralia",
+    "GlobeEuropeAfrica",
+    "HandRaised",
+    "HandThumbDown",
+    "HandThumbUp",
+    "Hashtag",
+    "Heart",
+    "Home",
+    "HomeModern",
+    "Identification",
+    "Inbox",
+    "InboxArrowDown",
+    "InboxStack",
+    "InformationCircle",
+    "Key",
+    "Language",
+    "Lifebuoy",
+    "LightBulb",
+    "Link",
+    "ListBullet",
+    "LockClosed",
+    "LockOpen",
+    "MagnifyingGlass",
+    "MagnifyingGlassCircle",
+    "MagnifyingGlassMinus",
+    "MagnifyingGlassPlus",
+    "Map",
+    "MapPin",
+    "Megaphone",
+    "Microphone",
+    "Minus",
+    "MinusCircle",
+    "MinusSmall",
+    "Moon",
+    "MusicalNote",
+    "Newspaper",
+    "NoSymbol",
+    "PaintBrush",
+    "PaperAirplane",
+    "PaperClip",
+    "Pause",
+    "PauseCircle",
+    "Pencil",
+    "PencilSquare",
+    "Phone",
+    "PhoneArrowDownLeft",
+    "PhoneArrowUpRight",
+    "PhoneXMark",
+    "Photo",
+    "Play",
+    "PlayCircle",
+    "PlayPause",
+    "Plus",
+    "PlusCircle",
+    "PlusSmall",
+    "Power",
+    "PresentationChartBar",
+    "PresentationChartLine",
+    "Printer",
+    "PuzzlePiece",
+    "QrCode",
+    "QuestionMarkCircle",
+    "QueueList",
+    "Radio",
+    "ReceiptPercent",
+    "ReceiptRefund",
+    "RectangleGroup",
+    "RectangleStack",
+    "RocketLaunch",
+    "Rss",
+    "Scale",
+    "Scissors",
+    "Server",
+    "ServerStack",
+    "Share",
+    "ShieldCheck",
+    "ShieldExclamation",
+    "ShoppingBag",
+    "ShoppingCart",
+    "Signal",
+    "SignalSlash",
+    "Sparkles",
+    "SpeakerWave",
+    "SpeakerXMark",
+    "Square2Stack",
+    "Square3Stack3d",
+    "Squares2x2",
+    "SquaresPlus",
+    "Star",
+    "Stop",
+    "StopCircle",
+    "Sun",
+    "Swatch",
+    "TableCells",
+    "Tag",
+    "Ticket",
+    "Trash",
+    "Trophy",
+    "Truck",
+    "Tv",
+    "User",
+    "UserCircle",
+    "UserGroup",
+    "UserMinus",
+    "UserPlus",
+    "Users",
+    "Variable",
+    "VideoCamera",
+    "VideoCameraSlash",
+    "ViewColumns",
+    "ViewfinderCircle",
+    "Wallet",
+    "Wifi",
+    "Window",
+    "Wrench",
+    "WrenchScrewdriver",
+    "XCircle",
+    "XMark",
+)
+
+
+class Group(BaseModel):
+    """Represents a group with metadata, slug, labels, and an optional icon.
+
+    Attributes:
+        meta_key (str): A key for metadata associated with the group.
+        slug (str): A slug for the group, used in URLs or as an identifier.
+        label_plural (str): The plural label for the group.
+        label_singular (str): The singular label for the group.
+        icon (str, optional): An optional icon for the group. Must be one of the allowed
+            icons.
+        in_sidebar (bool): Whether the group should be displayed in the sidebar.
+        visible (bool): Whether the group should be displayed in Perspective UI.
+
+    Methods:
+        icon_validator: Validates that the provided icon (if any) is in the allowed set.
+    """
+
+    meta_key: str
+    slug: str
+    label_plural: str
+    label_singular: str
+    icon: str | None = None
+    in_sidebar: bool = True
+    visible: bool = True
+
+    @field_validator("icon")
+    def icon_validator(cls, v) -> str | None:  # noqa: N805, ANN001
+        """Validates the icon attribute.
+
+        Ensures that if an icon is provided, it is one of the pre-approved icons.
+
+        Args:
+            v (str): The icon to validate.
+
+        Returns:
+            str: The validated icon.
+
+        Raises:
+            ValueError: If the icon is not in the allowed set.
+        """
+        if v is not None and v not in allowed_icons:
+            msg = "Icon must be one of the allowed icons."
+            raise ValueError(msg)
+        return v
+
+
+class Owner(BaseModel):
+    """Represents an owner with email, name, and title.
+
+    Attributes:
+        email (EmailStr): The email address of the owner.
+        first_name (str): The first name of the owner.
+        last_name (str): The last name of the owner.
+        title (str): The title of the owner.
+    """
+
+    email: EmailStr
+    first_name: str
+    last_name: str
+    title: str
+
+
+class MetadataField(BaseModel):
+    """Represents metadata field configuration.
+
+    Attributes:
+        name (str): The name of the field.
+        default (str): The default value to use.
+    """
+
+    name: str
+    default: Any
+
+
+class Config(BaseModel):
+    """Configuration model holding information about groups and owners.
+
+    Attributes:
+        groups (list[Group] | None): A list of Group objects. Ensures uniqueness of
+            meta_keys and slugs among groups.
+        owners (list[Owner] | None): A list of Owner objects.
+
+    Methods:
+        validate_unique: Validates the uniqueness of meta_keys and slugs within the
+            groups.
+    """
+
+    groups: list[Group] | None
+    owners: list[Owner] | None
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_unique(cls, values: dict) -> dict:
+        """Validates the uniqueness of 'meta_key' and 'slug' for each group.
+
+        Args:
+            values (dict): The values to validate.
+
+        Returns:
+            dict: The validated values.
+
+        Raises:
+            ValueError: If 'meta_key' or 'slug' is not unique across all groups.
+        """
+        groups = values.get("groups")
+        if groups:
+            # Check for unique meta_key
+
+            meta_keys = {group["meta_key"] for group in groups}
+            if len(meta_keys) != len(groups):
+                msg = "meta_key must be unique for each group."
+                raise ValueError(msg)
+
+            # Check for unique slug
+            slugs = {group["slug"] for group in groups}
+            if len(slugs) != len(groups):
+                msg = "slug must be unique for each group."
+                raise ValueError(msg)
+        return values

perspective/models/dashboards.py

@@ -0,0 +1,44 @@
+"""Pydantic models for dashboard metadata."""
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class DataModel(BaseModel):
+    name: str
+    schema_field: str = Field(..., alias="schema")
+    database: str
+    columns: list[dict[str, str]]
+    tags: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class DashboardOwner(BaseModel):
+    user_id: str  #  Typically an email or a uuid.
+    username: (
+        str | None
+    )  # Typically an email address. Empty in PBI if a group or app is the owner.
+    name: str  # Typically the human name.
+
+
+class Dashboard(BaseModel):
+    external_id: str
+    url: str
+    type: Literal["powerbi", "qliksense"]
+    name: str
+    workspace: str
+    created_at: datetime | None
+    modified_at: datetime
+    owners: list[DashboardOwner]
+    parent_models: list[DataModel]
+
+
+class DashboardSchemaMetadata(BaseModel):
+    schema_field: str = Field(..., alias="schema")
+    version: int
+
+
+class DashboardManifest(BaseModel):
+    metadata: DashboardSchemaMetadata
+    payload: list[Dashboard]

perspective/models/databases.py

@@ -0,0 +1,26 @@
+"""Pydantic models for database table metadata."""
+
+from pydantic import BaseModel, Field
+
+
+class DatabaseTableColumn(BaseModel):
+    name: str
+    type: str
+    length: str
+    description: str | None
+
+
+class DatabaseTable(BaseModel):
+    type: str
+    name: str
+    columns: list[DatabaseTableColumn]
+
+
+class DatabaseTableSchemaMetadata(BaseModel):
+    schema_field: str = Field(..., alias="schema")
+    version: int
+
+
+class DatabaseTableManifest(BaseModel):
+    metadata: DatabaseTableSchemaMetadata
+    payload: list[DatabaseTable]

perspective/utils/__init__.py

@@ -0,0 +1,3 @@
+"""Utility functions for the Perspective CLI."""
+
+from .utils import *  # noqa: F403