openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/ometa/ometa_api.py

@@ -33,6 +33,7 @@ from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
 from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
+from metadata.ingestion.ometa.mixins.csv_mixin import CSVMixin
 from metadata.ingestion.ometa.mixins.custom_property_mixin import (
     OMetaCustomPropertyMixin,
 )
@@ -44,9 +45,11 @@ from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
 from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
     OMetaIngestionPipelineMixin,
 )
+from metadata.ingestion.ometa.mixins.logs_mixin import OMetaLogsMixin
 from metadata.ingestion.ometa.mixins.mlmodel_mixin import OMetaMlModelMixin
 from metadata.ingestion.ometa.mixins.patch_mixin import OMetaPatchMixin
 from metadata.ingestion.ometa.mixins.pipeline_mixin import OMetaPipelineMixin
+from metadata.ingestion.ometa.mixins.profile_mixin import OMetaProfileMixin
 from metadata.ingestion.ometa.mixins.query_mixin import OMetaQueryMixin
 from metadata.ingestion.ometa.mixins.role_policy_mixin import OMetaRolePolicyMixin
 from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin
@@ -54,6 +57,7 @@ from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
 from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin
 from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin
 from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin
+from metadata.ingestion.ometa.mixins.tag_glossary_mixin import OMetaTagGlossaryMixin
 from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin
 from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin
 from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
@@ -108,6 +112,7 @@ class OpenMetadataSettings(BaseSettings):
 
 
 class OpenMetadata(
+    CSVMixin,
     OMetaPipelineMixin,
     OMetaMlModelMixin,
     OMetaTableMixin,
@@ -122,6 +127,7 @@ class OpenMetadata(
     OMetaTestsMixin,
     DataInsightMixin,
     OMetaIngestionPipelineMixin,
+    OMetaLogsMixin,
     OMetaUserMixin,
     OMetaQueryMixin,
     OMetaRolePolicyMixin,
@@ -129,6 +135,8 @@ class OpenMetadata(
     OMetaCustomPropertyMixin,
     OMetaSuggestionsMixin,
     OMetaDomainMixin,
+    OMetaProfileMixin,
+    OMetaTagGlossaryMixin,
     Generic[T, C],
 ):
     """
@@ -337,7 +345,22 @@ class OpenMetadata(
         return entity_class(**resp)
 
     def create_or_update(self, data: C) -> T:
-        """Run a PUT requesting via create request C"""
+        """
+        Run a PUT request via create request C.
+
+        Note: This method uses PUT operations with server-side business rules that may prevent
+        certain field overwrites across various entity types for data integrity reasons. If you
+        need to override existing metadata fields, consider using patch methods instead:
+
+        - For descriptions: Use patch_description(force=True)
+        - For general metadata: Use patch(override_metadata=True)
+
+        Args:
+            data: Create request object
+
+        Returns:
+            Updated or created entity
+        """
         return self._create(data=data, method="put")
 
     def create(self, data: C) -> T:

metadata/ingestion/ometa/routes.py

@@ -193,6 +193,7 @@ from metadata.generated.schema.tests.basic import TestCaseResult
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.generated.schema.tests.testDefinition import TestDefinition
 from metadata.generated.schema.tests.testSuite import TestSuite
+from metadata.generated.schema.type.entityProfile import EntityProfile
 
 ROUTES = {
     MlModel.__name__: "/mlmodels",
@@ -327,4 +328,8 @@ ROUTES = {
     # Data Contracts
     DataContract.__name__: "/dataContracts",
     CreateDataContractRequest.__name__: "/dataContracts",
+    # Drives
+    DriveService.__name__: "/services/driveServices",
+    CreateDriveServiceRequest.__name__: "/services/driveServices",
+    EntityProfile.__name__: "/entity/profiles",
 }

metadata/ingestion/source/api/rest/connection.py

@@ -28,6 +28,11 @@ from metadata.generated.schema.entity.services.connections.testConnectionResult
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.api.rest.parser import (
+    OpenAPIParseError,
+    parse_openapi_schema,
+    validate_openapi_schema,
+)
 from metadata.utils.constants import THREE_MIN
 
 
@@ -75,16 +80,17 @@ def test_connection(
 
     def custom_schema_exec():
         try:
-            if client.json() is not None and client.json().get("openapi") is not None:
+            schema = parse_openapi_schema(client)
+            if validate_openapi_schema(schema):
                 return []
 
             raise InvalidOpenAPISchemaError(
-                "Provided schema is not valid OpenAPI JSON schema"
-            )
-        except:
-            raise InvalidOpenAPISchemaError(
-                "Provided schema is not valid OpenAPI JSON schema"
+                "Provided schema is not valid OpenAPI specification"
             )
+        except OpenAPIParseError as e:
+            raise InvalidOpenAPISchemaError(f"Failed to parse OpenAPI schema: {e}")
+        except Exception as e:
+            raise InvalidOpenAPISchemaError(f"Error validating OpenAPI schema: {e}")
 
     test_fn = {"CheckURL": custom_url_exec, "CheckSchema": custom_schema_exec}
 

metadata/ingestion/source/api/rest/metadata.py

@@ -40,6 +40,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.api.api_service import ApiServiceSource
 from metadata.ingestion.source.api.rest.models import RESTCollection, RESTEndpoint
+from metadata.ingestion.source.api.rest.parser import parse_openapi_schema
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_collection
 from metadata.utils.helpers import clean_uri
@@ -77,7 +78,7 @@ class RestSource(ApiServiceSource):
         Here is where filtering happens
         """
         try:
-            self.json_response = self.connection.json()
+            self.json_response = parse_openapi_schema(self.connection)
             collections_list = []
             tags_collection_set = set()
             if self.json_response.get("tags", []):
@@ -252,8 +253,9 @@ class RestSource(ApiServiceSource):
         return None
 
     def _get_request_schema(self, info: dict) -> Optional[APISchema]:
-        """fetch request schema"""
+        """fetch request schema - supports both OpenAPI 3.0 and Swagger 2.0"""
         try:
+            # Try OpenAPI 3.0 format first (requestBody)
             schema_ref = (
                 info.get("requestBody", {})
                 .get("content", {})
@@ -261,41 +263,215 @@ class RestSource(ApiServiceSource):
                 .get("schema", {})
                 .get("$ref")
             )
-            if not schema_ref:
-                logger.debug("No request schema found for the endpoint")
-                return None
-            return APISchema(schemaFields=self.process_schema_fields(schema_ref))
+
+            if schema_ref:
+                return APISchema(schemaFields=self.process_schema_fields(schema_ref))
+
+            # Try Swagger 2.0 format (parameters with "in": "body")
+            parameters = info.get("parameters", [])
+            for param in parameters:
+                if param.get("in") == "body" and "schema" in param:
+                    schema_ref = param["schema"].get("$ref")
+                    if schema_ref:
+                        return APISchema(
+                            schemaFields=self.process_schema_fields(schema_ref)
+                        )
+
+            # Try to get query/path parameters for GET/DELETE requests
+            # This handles Swagger 2.0 and OpenAPI 3.0 query parameters
+            param_fields = []
+            for param in parameters:
+                # Resolve parameter $ref if present
+                if "$ref" in param:
+                    param = self._resolve_parameter_ref(param.get("$ref"))
+                    if not param:
+                        continue
+
+                if param.get("in") in ["query", "path"]:
+                    field = self._convert_parameter_to_field(param)
+                    if field:
+                        param_fields.append(field)
+
+            if param_fields:
+                return APISchema(schemaFields=param_fields)
+
+            logger.debug("No request schema found for the endpoint")
+            return None
         except Exception as err:
             logger.warning(f"Error while parsing request schema: {err}")
         return None
 
-    def _get_response_schema(self, info: dict) -> Optional[APISchema]:
-        """fetch response schema"""
+    def _resolve_parameter_ref(self, param_ref: str) -> Optional[dict]:
+        """Resolve parameter $ref to actual parameter definition"""
         try:
-            schema_ref = (
-                info.get("responses", {})
-                .get("200", {})
-                .get("content", {})
-                .get("application/json", {})
-                .get("schema", {})
-                .get("$ref", {})
+            # Parameter refs look like: "#/parameters/ParameterName"
+            if not param_ref or not param_ref.startswith("#/parameters/"):
+                return None
+
+            param_name = param_ref.split("/")[-1]
+
+            # Swagger 2.0: parameters at root level
+            if self.json_response.get("parameters"):
+                return self.json_response.get("parameters", {}).get(param_name)
+
+            # OpenAPI 3.0: components.parameters
+            if self.json_response.get("components"):
+                return (
+                    self.json_response.get("components", {})
+                    .get("parameters", {})
+                    .get(param_name)
+                )
+
+            logger.debug(f"Parameter reference '{param_name}' not found")
+            return None
+        except Exception as err:
+            logger.warning(f"Error resolving parameter reference: {err}")
+            return None
+
+    def _parse_openapi_type(self, openapi_type: Optional[str]) -> DataTypeTopic:
+        """
+        Parse OpenAPI type string to DataTypeTopic enum.
+        Shared type conversion logic used across the codebase.
+        """
+        if not openapi_type:
+            return DataTypeTopic.UNKNOWN
+
+        # Handle INTEGER -> INT conversion
+        normalized_type = (
+            "INT" if openapi_type.upper() == "INTEGER" else openapi_type.upper()
+        )
+
+        # Check if type exists in DataTypeTopic enum
+        if normalized_type in DataTypeTopic.__members__:
+            return DataTypeTopic[normalized_type]
+
+        return DataTypeTopic.UNKNOWN
+
+    def _convert_parameter_to_field(self, param: dict) -> Optional[FieldModel]:
+        """Convert OpenAPI/Swagger parameter to FieldModel for query/path parameters"""
+        try:
+            param_name = param.get("name")
+            if not param_name:
+                return None
+
+            # Get type from parameter (Swagger 2.0 format)
+            param_type = param.get("type")
+
+            # Get type from schema (OpenAPI 3.0 format)
+            if not param_type and "schema" in param:
+                param_type = param["schema"].get("type")
+
+            data_type = self._parse_openapi_type(param_type)
+
+            # Handle array items
+            children = None
+            if data_type == DataTypeTopic.ARRAY:
+                items = param.get("items") or param.get("schema", {}).get("items")
+                if items:
+                    item_type = items.get("type")
+                    child_data_type = self._parse_openapi_type(item_type)
+                    children = [FieldModel(name="item", dataType=child_data_type)]
+
+            return FieldModel(
+                name=param_name,
+                dataType=data_type,
+                children=children,
+                description=param.get("description"),
             )
-            if not schema_ref:
-                logger.debug("Trying to parse response schema from schema property")
-                schema_ref = (
-                    info.get("responses", {})
-                    .get("200", {})
-                    .get("content", {})
-                    .get("application/json", {})
-                    .get("schema", {})
-                    .get("properties", {})
-                    .get("data", {})
-                    .get("$ref")
+        except Exception as err:
+            logger.warning(f"Error converting parameter to field: {err}")
+            return None
+
+    def _process_inline_schema(self, properties: dict) -> Optional[APISchema]:
+        """Process inline schema properties (schemas without $ref)"""
+        try:
+            fields = []
+            for prop_name, prop_def in properties.items():
+                prop_type = prop_def.get("type")
+
+                data_type = self._parse_openapi_type(prop_type)
+
+                # Handle array items
+                children = None
+                if data_type == DataTypeTopic.ARRAY:
+                    items = prop_def.get("items", {})
+                    if items:
+                        item_type = items.get("type")
+                        child_data_type = self._parse_openapi_type(item_type)
+                        children = [FieldModel(name="item", dataType=child_data_type)]
+
+                fields.append(
+                    FieldModel(
+                        name=prop_name,
+                        dataType=data_type,
+                        children=children,
+                        description=prop_def.get("description"),
+                    )
                 )
-            if not schema_ref:
+
+            return APISchema(schemaFields=fields) if fields else None
+        except Exception as err:
+            logger.warning(f"Error processing inline schema: {err}")
+            return None
+
+    def _extract_schema_from_response(self, response: dict) -> dict:
+        """Extract schema from a response object (supports both OpenAPI 3.0 and Swagger 2.0)"""
+        # OpenAPI 3.0: response.content.application/json.schema
+        schema = (
+            response.get("content", {}).get("application/json", {}).get("schema", {})
+        )
+        # Swagger 2.0: response.schema
+        if not schema:
+            schema = response.get("schema", {})
+        return schema
+
+    def _get_response_schema(self, info: dict) -> Optional[APISchema]:
+        """fetch response schema - supports OpenAPI 3.0, Swagger 2.0, arrays, and inline schemas"""
+        try:
+            # Try response code 200 first
+            response_200 = info.get("responses", {}).get("200", {})
+            schema = self._extract_schema_from_response(response_200)
+
+            # Fallback: Try other success response codes if 200 not found
+            if not schema:
+                responses = info.get("responses", {})
+                for code in ["201", "202", "203", "204"]:
+                    if code in responses:
+                        logger.debug(f"Using response code {code} as 200 not found")
+                        schema = self._extract_schema_from_response(responses[code])
+                        if schema:
+                            break
+
+            if not schema:
                 logger.debug("No response schema found for the endpoint")
                 return None
-            return APISchema(schemaFields=self.process_schema_fields(schema_ref))
+
+            # Case 1: Direct $ref (object response)
+            schema_ref = schema.get("$ref")
+            if schema_ref:
+                return APISchema(schemaFields=self.process_schema_fields(schema_ref))
+
+            # Case 2: Array response with $ref in items
+            if schema.get("type") == "array":
+                items_ref = schema.get("items", {}).get("$ref")
+                if items_ref:
+                    logger.debug(f"Processing array response schema: {items_ref}")
+                    return APISchema(schemaFields=self.process_schema_fields(items_ref))
+
+            # Case 3: Nested $ref in schema.properties.data
+            schema_ref = schema.get("properties", {}).get("data", {}).get("$ref")
+            if schema_ref:
+                logger.debug("Found response schema in schema.properties.data")
+                return APISchema(schemaFields=self.process_schema_fields(schema_ref))
+
+            # Case 4: Inline schema with properties (no $ref)
+            properties = schema.get("properties", {})
+            if properties:
+                logger.debug("Processing inline response schema with properties")
+                return self._process_inline_schema(properties)
+
+            logger.debug("No processable response schema found for the endpoint")
+            return None
         except Exception as err:
             logger.warning(f"Error while parsing response schema: {err}")
         return None
@@ -307,20 +483,34 @@ class RestSource(ApiServiceSource):
             if parent_refs is None:
                 parent_refs = []
             schema_name = schema_ref.split("/")[-1]
-            schema_fields = (
-                self.json_response.get("components").get("schemas").get(schema_name)
-            )
+
+            # Support both OpenAPI 3.0 (components.schemas) and Swagger 2.0 (definitions)
+            schema_fields = None
+            if self.json_response.get("components"):
+                # OpenAPI 3.0: components.schemas.{SchemaName}
+                schema_fields = (
+                    self.json_response.get("components", {})
+                    .get("schemas", {})
+                    .get(schema_name)
+                )
+            elif self.json_response.get("definitions"):
+                # Swagger 2.0: definitions.{SchemaName}
+                schema_fields = self.json_response.get("definitions", {}).get(
+                    schema_name
+                )
+
+            if not schema_fields:
+                logger.warning(
+                    f"Schema '{schema_name}' not found in components.schemas or definitions"
+                )
+                return None
+
             parent_refs.append(schema_ref)
             fetched_fields = []
             for key, val in schema_fields.get("properties", {}).items():
                 dtype = val.get("type")
                 if dtype:
-                    dtype = "INT" if dtype.upper() == "INTEGER" else dtype
-                    parsed_dtype = (
-                        DataTypeTopic[dtype.upper()]
-                        if dtype.upper() in DataTypeTopic.__members__
-                        else DataTypeTopic.UNKNOWN
-                    )
+                    parsed_dtype = self._parse_openapi_type(dtype)
                     children = None
                     if parsed_dtype.value == DataTypeTopic.ARRAY.value:
                         # If field of array type then parse children

metadata/ingestion/source/api/rest/parser.py

@@ -0,0 +1,103 @@
+#  Copyright 2024 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+OpenAPI schema parser for both JSON and YAML formats
+"""
+import json
+from typing import Any, Dict
+
+import yaml
+from requests.models import Response
+
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+class OpenAPIParseError(Exception):
+    """
+    Exception raised when OpenAPI schema cannot be parsed as either JSON or YAML
+    """
+
+
+def parse_openapi_schema(response: Response) -> Dict[str, Any]:
+    """
+    Parse OpenAPI schema from HTTP response.
+    Supports both JSON and YAML formats.
+
+    Args:
+        response: HTTP response containing OpenAPI schema
+
+    Returns:
+        Parsed OpenAPI schema as dictionary
+
+    Raises:
+        OpenAPIParseError: If content cannot be parsed as either JSON or YAML
+    """
+    content = response.text
+    content_type = response.headers.get("content-type", "").lower()
+
+    # Try to determine format from content-type header
+    if "json" in content_type:
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError as e:
+            logger.warning(f"Failed to parse as JSON despite content-type: {e}")
+    elif "yaml" in content_type or "yml" in content_type:
+        try:
+            return yaml.safe_load(content)
+        except yaml.YAMLError as e:
+            logger.warning(f"Failed to parse as YAML despite content-type: {e}")
+
+    # If content-type is not definitive or parsing failed, try both formats
+
+    # First try JSON (backward compatibility)
+    try:
+        parsed = json.loads(content)
+        logger.debug("Successfully parsed OpenAPI schema as JSON")
+        return parsed
+    except json.JSONDecodeError:
+        logger.debug("Content is not valid JSON, trying YAML")
+
+    # Then try YAML
+    try:
+        parsed = yaml.safe_load(content)
+        if parsed is None:
+            raise OpenAPIParseError("YAML parsing returned None")
+        logger.debug("Successfully parsed OpenAPI schema as YAML")
+        return parsed
+    except yaml.YAMLError as e:
+        logger.error(f"Failed to parse as YAML: {e}")
+
+    # If both formats fail, raise an error
+    raise OpenAPIParseError(
+        "Failed to parse OpenAPI schema as either JSON or YAML. "
+        "Please ensure the schema is valid OpenAPI specification."
+    )
+
+
+def validate_openapi_schema(schema: Dict[str, Any]) -> bool:
+    """
+    Validate that the parsed schema is a valid OpenAPI specification.
+
+    Args:
+        schema: Parsed schema dictionary
+
+    Returns:
+        True if schema appears to be valid OpenAPI, False otherwise
+    """
+    # Check for required OpenAPI fields
+    if not isinstance(schema, dict):
+        return False
+
+    # OpenAPI 3.x uses "openapi" field, OpenAPI 2.x uses "swagger" field
+    return schema.get("openapi") is not None or schema.get("swagger") is not None

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -79,6 +79,8 @@ class DomodashboardSource(DashboardServiceSource):
         return cls(config, metadata)
 
     def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]:
+        if not self.source_config.includeOwners:
+            logger.debug("Skipping owner information as includeOwners is False")
         dashboards = self.client.domo.page_list()
         dashboard_list = []
         for dashboard in dashboards:
@@ -104,15 +106,23 @@ class DomodashboardSource(DashboardServiceSource):
     def get_owner_ref(
         self, dashboard_details: DomoDashboardDetails
     ) -> Optional[EntityReferenceList]:
-        for owner in dashboard_details.owners or []:
-            try:
-                owner_details = self.client.domo.users_get(owner.id)
-                if owner_details.get("email"):
-                    return self.metadata.get_reference_by_email(owner_details["email"])
-            except Exception as exc:
-                logger.warning(
-                    f"Error while getting details of user {owner.displayName} - {exc}"
-                )
+        try:
+            if not self.source_config.includeOwners:
+                return None
+            for owner in dashboard_details.owners or []:
+                try:
+                    owner_details = self.client.domo.users_get(owner.id)
+                    if owner_details.get("email"):
+                        return self.metadata.get_reference_by_email(
+                            owner_details["email"]
+                        )
+                except Exception as exc:
+                    logger.warning(
+                        f"Error while getting details of user {owner.displayName} - {exc}"
+                    )
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not fetch owner data due to {err}")
         return None
 
     def yield_dashboard(

metadata/ingestion/source/dashboard/grafana/__init__.py

@@ -0,0 +1,10 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.