openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.9.2.0__py3-none-any.whl

metadata/ingestion/source/api/rest/connection.py

@@ -28,6 +28,11 @@ from metadata.generated.schema.entity.services.connections.testConnectionResult
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.api.rest.parser import (
+    OpenAPIParseError,
+    parse_openapi_schema,
+    validate_openapi_schema,
+)
 from metadata.utils.constants import THREE_MIN
 
 
@@ -75,16 +80,17 @@ def test_connection(
 
     def custom_schema_exec():
         try:
-            if client.json() is not None and client.json().get("openapi") is not None:
+            schema = parse_openapi_schema(client)
+            if validate_openapi_schema(schema):
                 return []
 
             raise InvalidOpenAPISchemaError(
-                "Provided schema is not valid OpenAPI JSON schema"
-            )
-        except:
-            raise InvalidOpenAPISchemaError(
-                "Provided schema is not valid OpenAPI JSON schema"
+                "Provided schema is not valid OpenAPI specification"
             )
+        except OpenAPIParseError as e:
+            raise InvalidOpenAPISchemaError(f"Failed to parse OpenAPI schema: {e}")
+        except Exception as e:
+            raise InvalidOpenAPISchemaError(f"Error validating OpenAPI schema: {e}")
 
     test_fn = {"CheckURL": custom_url_exec, "CheckSchema": custom_schema_exec}
 

metadata/ingestion/source/api/rest/metadata.py

@@ -40,6 +40,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.api.api_service import ApiServiceSource
 from metadata.ingestion.source.api.rest.models import RESTCollection, RESTEndpoint
+from metadata.ingestion.source.api.rest.parser import parse_openapi_schema
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_collection
 from metadata.utils.helpers import clean_uri
@@ -77,7 +78,7 @@ class RestSource(ApiServiceSource):
         Here is where filtering happens
         """
         try:
-            self.json_response = self.connection.json()
+            self.json_response = parse_openapi_schema(self.connection)
             collections_list = []
             tags_collection_set = set()
             if self.json_response.get("tags", []):

metadata/ingestion/source/api/rest/parser.py

@@ -0,0 +1,103 @@
+#  Copyright 2024 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+OpenAPI schema parser for both JSON and YAML formats
+"""
+import json
+from typing import Any, Dict
+
+import yaml
+from requests.models import Response
+
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class OpenAPIParseError(Exception):
+    """
+    Exception raised when OpenAPI schema cannot be parsed as either JSON or YAML
+    """
+
+
+def parse_openapi_schema(response: Response) -> Dict[str, Any]:
+    """
+    Parse OpenAPI schema from HTTP response.
+    Supports both JSON and YAML formats.
+
+    Args:
+        response: HTTP response containing OpenAPI schema
+
+    Returns:
+        Parsed OpenAPI schema as dictionary
+
+    Raises:
+        OpenAPIParseError: If content cannot be parsed as either JSON or YAML
+    """
+    content = response.text
+    content_type = response.headers.get("content-type", "").lower()
+
+    # Try to determine format from content-type header
+    if "json" in content_type:
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError as e:
+            logger.warning(f"Failed to parse as JSON despite content-type: {e}")
+    elif "yaml" in content_type or "yml" in content_type:
+        try:
+            return yaml.safe_load(content)
+        except yaml.YAMLError as e:
+            logger.warning(f"Failed to parse as YAML despite content-type: {e}")
+
+    # If content-type is not definitive or parsing failed, try both formats
+
+    # First try JSON (backward compatibility)
+    try:
+        parsed = json.loads(content)
+        logger.debug("Successfully parsed OpenAPI schema as JSON")
+        return parsed
+    except json.JSONDecodeError:
+        logger.debug("Content is not valid JSON, trying YAML")
+
+    # Then try YAML
+    try:
+        parsed = yaml.safe_load(content)
+        if parsed is None:
+            raise OpenAPIParseError("YAML parsing returned None")
+        logger.debug("Successfully parsed OpenAPI schema as YAML")
+        return parsed
+    except yaml.YAMLError as e:
+        logger.error(f"Failed to parse as YAML: {e}")
+
+    # If both formats fail, raise an error
+    raise OpenAPIParseError(
+        "Failed to parse OpenAPI schema as either JSON or YAML. "
+        "Please ensure the schema is valid OpenAPI specification."
+    )
+
+
+def validate_openapi_schema(schema: Dict[str, Any]) -> bool:
+    """
+    Validate that the parsed schema is a valid OpenAPI specification.
+
+    Args:
+        schema: Parsed schema dictionary
+
+    Returns:
+        True if schema appears to be valid OpenAPI, False otherwise
+    """
+    # Check for required OpenAPI fields
+    if not isinstance(schema, dict):
+        return False
+
+    # OpenAPI 3.x uses "openapi" field, OpenAPI 2.x uses "swagger" field
+    return schema.get("openapi") is not None or schema.get("swagger") is not None

metadata/ingestion/source/dashboard/grafana/__init__.py

@@ -0,0 +1,10 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

metadata/ingestion/source/dashboard/grafana/client.py

@@ -0,0 +1,223 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Grafana API client
+"""
+from typing import List, Optional, Union
+
+import requests
+from requests import Session
+
+from metadata.ingestion.source.dashboard.grafana.models import (
+    GrafanaDashboardResponse,
+    GrafanaDatasource,
+    GrafanaFolder,
+    GrafanaSearchResult,
+)
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+API_TIMEOUT = 30
+
+
+class GrafanaApiClient:
+    """
+    Grafana API client for fetching dashboards, folders, and datasources
+    """
+
+    def __init__(
+        self,
+        host_port: str,
+        api_key: str,
+        verify_ssl: bool = True,
+        page_size: int = 100,
+    ):
+        self.host_port = str(host_port).rstrip("/")
+        self.api_key = api_key
+        self.verify_ssl = verify_ssl
+        self.page_size = page_size
+        self._session: Optional[Session] = None
+
+        # Log a warning if not using Service Account Token format
+        if not api_key.startswith("glsa_"):
+            logger.warning(
+                "Token does not appear to be a Service Account Token (should start with 'glsa_'). "
+                "Legacy API keys are no longer supported by Grafana as of January 2025. "
+                "Please create a Service Account Token in Grafana."
+            )
+
+    @property
+    def session(self) -> Session:
+        """Get or create HTTP session with authentication"""
+        if not self._session:
+            self._session = Session()
+            self._session.headers.update(
+                {
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Accept": "application/json",
+                    "Content-Type": "application/json",
+                }
+            )
+            self._session.verify = self.verify_ssl
+        return self._session
+
+    def _make_request(
+        self, method: str, endpoint: str, **kwargs
+    ) -> Optional[requests.Response]:
+        """Make HTTP request with error handling"""
+        url = f"{self.host_port}/api{endpoint}"
+
+        try:
+            response = self.session.request(
+                method=method, url=url, timeout=API_TIMEOUT, **kwargs
+            )
+            response.raise_for_status()
+            return response
+        except requests.exceptions.HTTPError as err:
+            if err.response.status_code in (401, 403):
+                logger.warning(
+                    f"Permission denied for {endpoint}. "
+                    f"Status: {err.response.status_code}"
+                )
+            else:
+                logger.error(f"HTTP error for {endpoint}: {err}")
+            return None
+        except Exception as err:
+            logger.error(f"Error making request to {endpoint}: {err}")
+            return None
+
+    def get_folders(self) -> List[GrafanaFolder]:
+        """Get all folders with pagination"""
+
+        try:
+            folders = []
+            page = 1
+            while True:
+                response = self._make_request(
+                    "GET", "/folders", params={"page": page, "limit": self.page_size}
+                )
+
+                if not response:
+                    break
+
+                page_data = response.json()
+                if not page_data:
+                    break
+
+                folders.extend([GrafanaFolder(**folder) for folder in page_data])
+
+                # Check if we need to fetch more pages
+                if len(page_data) < self.page_size:
+                    break
+
+                page += 1
+            return folders
+
+        except Exception as err:
+            logger.error(f"Error fetching folders from Grafana: {err}")
+
+    def search_dashboards(
+        self, folder_id: Optional[int] = None
+    ) -> List[GrafanaSearchResult]:
+        """Search for dashboards with optional folder filter"""
+        try:
+            dashboards = []
+            page = 1
+
+            while True:
+                params = {
+                    "type": "dash-db",
+                    "page": page,
+                    "limit": self.page_size,
+                }
+
+                if folder_id is not None:
+                    params["folderIds"] = folder_id
+
+                response = self._make_request("GET", "/search", params=params)
+
+                if not response:
+                    break
+
+                page_data = response.json()
+                if not page_data:
+                    break
+
+                dashboards.extend([GrafanaSearchResult(**dash) for dash in page_data])
+
+                # Check if we need to fetch more pages
+                if len(page_data) < self.page_size:
+                    break
+
+                page += 1
+
+            return dashboards
+        except Exception as err:
+            logger.error(f"Error fetching dashboards from Grafana: {err}")
+
+    def get_dashboard(self, uid: str) -> Optional[GrafanaDashboardResponse]:
+        """Get detailed dashboard information by UID"""
+        try:
+            response = self._make_request("GET", f"/dashboards/uid/{uid}")
+            if response:
+                return GrafanaDashboardResponse(**response.json())
+            return None
+        except Exception as err:
+            logger.error(f"Error fetching dashboard details from Grafana: {err}")
+            return None
+
+    def get_datasources(self) -> List[GrafanaDatasource]:
+        """Get all datasources"""
+        try:
+            response = self._make_request("GET", "/datasources")
+            if response:
+                return [GrafanaDatasource(**ds) for ds in response.json()]
+            return []
+        except Exception as err:
+            logger.error(f"Error fetching datasources from Grafana: {err}")
+            return []
+
+    def get_datasource(
+        self, datasource_id: Union[int, str]
+    ) -> Optional[GrafanaDatasource]:
+        """Get datasource by ID or UID"""
+        try:
+            # Try by ID first if it's numeric
+            if isinstance(datasource_id, int) or datasource_id.isdigit():
+                response = self._make_request("GET", f"/datasources/{datasource_id}")
+            else:
+                # Try by UID
+                response = self._make_request(
+                    "GET", f"/datasources/uid/{datasource_id}"
+                )
+
+            if response:
+                return GrafanaDatasource(**response.json())
+
+        except Exception as err:
+            logger.error(f"Error fetching datasource from Grafana: {err}")
+            return None
+
+    def test_connection(self) -> bool:
+        """Test connection to Grafana API"""
+        try:
+            response = self._make_request("GET", "/org")
+            return response is not None
+        except Exception as err:
+            logger.error(f"Failed to test Grafana connection: {err}")
+            return False
+
+    def close(self):
+        """Close HTTP session"""
+        if self._session:
+            self._session.close()
+            self._session = None

metadata/ingestion/source/dashboard/grafana/connection.py

@@ -0,0 +1,66 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Source connection handler for Grafana
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.dashboard.grafanaConnection import (
+    GrafanaConnection,
+)
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
+from metadata.ingestion.connections.test_connections import test_connection_steps
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.dashboard.grafana.client import GrafanaApiClient
+from metadata.utils.constants import THREE_MIN
+
+
+def get_connection(connection: GrafanaConnection) -> GrafanaApiClient:
+    """
+    Create connection to Grafana
+    """
+    return GrafanaApiClient(
+        host_port=connection.hostPort,
+        api_key=connection.apiKey.get_secret_value(),
+        verify_ssl=connection.verifySSL or True,
+        page_size=connection.pageSize or 100,
+    )
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    client: GrafanaApiClient,
+    service_connection: GrafanaConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
+    """
+    Test connection to Grafana instance
+    """
+
+    def custom_executor():
+        if not client.test_connection():
+            raise Exception("Failed to connect to Grafana")
+
+    test_fn = {"GetDashboards": custom_executor}
+
+    return test_connection_steps(
+        metadata=metadata,
+        test_fn=test_fn,
+        service_type=service_connection.type.value,
+        automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
+    )