acryl-datahub 1.2.0.2rc1__py3-none-any.whl → 1.2.0.2rc3__py3-none-any.whl

datahub/ingestion/source/grafana/grafana_api.py

@@ -0,0 +1,142 @@
+"""API client for Grafana metadata extraction"""
+
+import logging
+from typing import Dict, List, Optional, Union
+
+import requests
+import urllib3.exceptions
+from pydantic import SecretStr
+
+from datahub.ingestion.source.grafana.models import Dashboard, Folder
+from datahub.ingestion.source.grafana.report import GrafanaSourceReport
+
+logger = logging.getLogger(__name__)
+
+
+class GrafanaAPIClient:
+    """Client for making requests to Grafana API"""
+
+    def __init__(
+        self,
+        base_url: str,
+        token: SecretStr,
+        verify_ssl: bool,
+        page_size: int,
+        report: GrafanaSourceReport,
+    ) -> None:
+        self.base_url = base_url
+        self.verify_ssl = verify_ssl
+        self.page_size = page_size
+        self.report = report
+        self.session = self._create_session(token)
+
+    def _create_session(self, token: SecretStr) -> requests.Session:
+        session = requests.Session()
+        session.headers.update(
+            {
+                "Authorization": f"Bearer {token.get_secret_value()}",
+                "Accept": "application/json",
+                "Content-Type": "application/json",
+            }
+        )
+        session.verify = self.verify_ssl
+
+        # If SSL verification is disabled, suppress the warnings
+        if not self.verify_ssl:
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+            self.report.warning(
+                title="SSL Configuration Warning",
+                message="SSL Verification is recommended.",
+            )
+
+        return session
+
+    def get_folders(self) -> List[Folder]:
+        """Fetch all folders from Grafana with pagination."""
+        folders: List[Folder] = []
+        page = 1
+        per_page = self.page_size
+
+        while True:
+            try:
+                response = self.session.get(
+                    f"{self.base_url}/api/folders",
+                    params={"page": page, "limit": per_page},
+                )
+                response.raise_for_status()
+
+                batch = response.json()
+                if not batch:
+                    break
+
+                folders.extend(Folder.parse_obj(folder) for folder in batch)
+                page += 1
+            except requests.exceptions.RequestException as e:
+                self.report.report_failure(
+                    title="Folder Fetch Error",
+                    message="Failed to fetch folders on page",
+                    context=str(page),
+                    exc=e,
+                )
+                self.report.report_permission_warning()  # Likely a permission issue
+                break
+
+        return folders
+
+    def get_dashboard(self, uid: str) -> Optional[Dashboard]:
+        """Fetch a specific dashboard by UID"""
+        try:
+            response = self.session.get(f"{self.base_url}/api/dashboards/uid/{uid}")
+            response.raise_for_status()
+            return Dashboard.parse_obj(response.json())
+        except requests.exceptions.RequestException as e:
+            self.report.warning(
+                title="Dashboard Fetch Error",
+                message="Failed to fetch dashboard",
+                context=uid,
+                exc=e,
+            )
+            if e.response and e.response.status_code in (401, 403):
+                self.report.report_permission_warning()
+            return None
+
+    def get_dashboards(self) -> List[Dashboard]:
+        """Fetch all dashboards from search endpoint with pagination."""
+        dashboards: List[Dashboard] = []
+        page = 1
+        per_page = self.page_size
+
+        while True:
+            try:
+                params: Dict[str, Union[str, int]] = {
+                    "type": "dash-db",
+                    "page": page,
+                    "limit": per_page,
+                }
+                response = self.session.get(
+                    f"{self.base_url}/api/search",
+                    params=params,
+                )
+                response.raise_for_status()
+
+                batch = response.json()
+                if not batch:
+                    break
+
+                for result in batch:
+                    dashboard = self.get_dashboard(result["uid"])
+                    if dashboard:
+                        dashboards.append(dashboard)
+                page += 1
+            except requests.exceptions.RequestException as e:
+                self.report.report_failure(
+                    title="Dashboard Search Error",
+                    message="Failed to fetch dashboards on page",
+                    context=str(page),
+                    exc=e,
+                )
+                if e.response and e.response.status_code in (401, 403):
+                    self.report.report_permission_warning()
+                break
+
+        return dashboards

datahub/ingestion/source/grafana/grafana_config.py

@@ -0,0 +1,104 @@
+from typing import Dict, Optional
+
+from pydantic import Field, SecretStr, validator
+
+from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.source_common import (
+    DatasetLineageProviderConfigBase,
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionConfigBase,
+)
+from datahub.utilities import config_clean
+
+
+class PlatformConnectionConfig(
+    EnvConfigMixin,
+    PlatformInstanceConfigMixin,
+):
+    """Platform connection configuration for mapping Grafana datasources to their actual platforms."""
+
+    platform: str = Field(
+        description="The platform name (e.g., 'postgres', 'mysql', 'snowflake')"
+    )
+    database: Optional[str] = Field(default=None, description="Default database name")
+    database_schema: Optional[str] = Field(
+        default=None, description="Default schema name"
+    )
+
+
+class GrafanaSourceConfig(
+    DatasetLineageProviderConfigBase,
+    StatefulIngestionConfigBase,
+    PlatformInstanceConfigMixin,
+    EnvConfigMixin,
+):
+    """Configuration for Grafana source"""
+
+    platform: str = Field(default="grafana", hidden_from_docs=True)
+    url: str = Field(
+        description="Grafana URL in the format http://your-grafana-instance with no trailing slash"
+    )
+    service_account_token: SecretStr = Field(
+        description="Service account token for Grafana"
+    )
+    verify_ssl: bool = Field(
+        default=True,
+        description="Whether to verify SSL certificates when connecting to Grafana",
+    )
+
+    # API pagination configuration
+    page_size: int = Field(
+        default=100,
+        description="Number of items to fetch per API call when paginating through folders and dashboards",
+    )
+
+    # Extraction mode configuration
+    basic_mode: bool = Field(
+        default=False,
+        description="Enable basic extraction mode for users with limited permissions. "
+        "In basic mode, only dashboard metadata is extracted without detailed panel information, "
+        "lineage, or folder hierarchy. This requires only basic dashboard read permissions.",
+    )
+
+    # Content filtering
+    dashboard_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex pattern to filter dashboards for ingestion",
+    )
+    folder_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Regex pattern to filter folders for ingestion",
+    )
+
+    # Feature toggles
+    ingest_tags: bool = Field(
+        default=True, description="Whether to ingest dashboard and chart tags"
+    )
+    ingest_owners: bool = Field(
+        default=True, description="Whether to ingest dashboard ownership information"
+    )
+
+    include_lineage: bool = Field(
+        default=True,
+        description="Whether to extract lineage between charts and data sources. "
+        "When enabled, the source will parse SQL queries and datasource configurations "
+        "to build lineage relationships.",
+    )
+    include_column_lineage: bool = Field(
+        default=True,
+        description="Whether to extract column-level lineage from SQL queries. "
+        "Only applicable when include_lineage is enabled.",
+    )
+
+    # Platform connection mappings
+    connection_to_platform_map: Dict[str, PlatformConnectionConfig] = Field(
+        default_factory=dict,
+        description="Map of Grafana datasource types/UIDs to platform connection configs for lineage extraction",
+    )
+
+    @validator("url", allow_reuse=True)
+    def remove_trailing_slash(cls, v):
+        return config_clean.remove_trailing_slashes(v)