openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/ingestion/models/custom_basemodel_validation.py

@@ -13,19 +13,103 @@ Validation logic for Custom Pydantic BaseModel
 """
 
 import logging
+from enum import Enum
+from typing import Any, Callable, Dict, Optional
 
 logger = logging.getLogger("metadata")
 
-
 RESTRICTED_KEYWORDS = ["::", ">"]
 RESERVED_COLON_KEYWORD = "__reserved__colon__"
 RESERVED_ARROW_KEYWORD = "__reserved__arrow__"
 RESERVED_QUOTE_KEYWORD = "__reserved__quote__"
 
-CREATE_ADJACENT_MODELS = {"ProfilerResponse", "SampleData"}
-NAME_FIELDS = {"EntityName", "str", "ColumnName", "TableData"}
-FETCH_MODELS = {"Table", "CustomColumnName", "DashboardDataModel"}
-FIELD_NAMES = {"name", "columns", "root"}
+
+class TransformDirection(Enum):
+    """Direction of name transformation"""
+
+    ENCODE = "encode"  # For storage (Create operations) - replace separators
+    DECODE = "decode"  # For display (Fetch operations) - revert separators
+
+
+def is_service_level_create_model(model_name: str) -> bool:
+    """
+    Check if a model is a Service-level Create model that should NOT be transformed.
+    Service-level models follow the pattern: Create*ServiceRequest where * is the service name
+    This is scalable and requires no maintenance for new services.
+    """
+    if not model_name.startswith("Create") or not model_name.endswith("ServiceRequest"):
+        return False
+
+    # Extract the middle part (service name) - must not be empty
+    # "CreateServiceRequest" -> middle = "" (invalid)
+    # "CreateDatabaseServiceRequest" -> middle = "Database" (valid)
+    middle = model_name[
+        6:-14
+    ]  # Remove "Create" (6 chars) and "ServiceRequest" (14 chars)
+    return len(middle) > 0
+
+
+# Explicit configuration for entity name transformations
+# This dictionary will be populated lazily to avoid circular imports
+TRANSFORMABLE_ENTITIES: Dict[Any, Dict[str, Any]] = {}
+
+
+def _initialize_transformable_entities():
+    """Initialize the transformable entities dictionary lazily to avoid circular imports"""
+    # Import all model classes here to avoid circular dependency at module load time
+    from metadata.generated.schema.api.data.createDashboardDataModel import (
+        CreateDashboardDataModelRequest,
+    )
+    from metadata.generated.schema.api.data.createTable import CreateTableRequest
+    from metadata.generated.schema.entity.data.dashboardDataModel import (
+        DashboardDataModel,
+    )
+    from metadata.generated.schema.entity.data.table import (
+        ColumnName,
+        ColumnProfile,
+        Table,
+        TableData,
+    )
+    from metadata.profiler.api.models import ProfilerResponse
+    from metadata.utils.entity_link import CustomColumnName
+
+    # Now populate the dictionary with the imported classes
+    TRANSFORMABLE_ENTITIES.update(
+        {
+            # Fetch models - decode reserved keywords back to original characters
+            Table: {
+                "fields": {"name", "columns", "children", "tableConstraints"},
+                "direction": TransformDirection.DECODE,
+            },
+            DashboardDataModel: {
+                "fields": {"name", "columns", "children"},
+                "direction": TransformDirection.DECODE,
+            },
+            CustomColumnName: {
+                "fields": {"root"},
+                "direction": TransformDirection.DECODE,
+            },
+            # Create/Store models - encode special characters to reserved keywords
+            ProfilerResponse: {
+                "fields": {"name", "profile"},
+                "direction": TransformDirection.ENCODE,
+            },
+            TableData: {"fields": {"columns"}, "direction": TransformDirection.ENCODE},
+            ColumnName: {"fields": {"root"}, "direction": TransformDirection.ENCODE},
+            CreateTableRequest: {
+                "fields": {"name", "columns", "children", "tableConstraints"},
+                "direction": TransformDirection.ENCODE,
+            },
+            CreateDashboardDataModelRequest: {
+                "fields": {"name", "columns", "children"},
+                "direction": TransformDirection.ENCODE,
+            },
+            ColumnProfile: {
+                "fields": {"name"},
+                "direction": TransformDirection.ENCODE,
+            },
+        }
+    )
 
 
 def revert_separators(value):
@@ -44,42 +128,84 @@ def replace_separators(value):
     )
 
 
-def validate_name_and_transform(values, modification_method, field_name: str = None):
-    """
-    Validate the name and transform it if needed.
-    """
-    if isinstance(values, str) and field_name in FIELD_NAMES:
-        values = modification_method(values)
-    elif (
-        hasattr(values, "root")
-        and isinstance(values.root, str)
-        and field_name in FIELD_NAMES
+def get_entity_config(model: Optional[Any]) -> Optional[Dict[str, Any]]:
+    """Get transformation configuration for entity"""
+    _initialize_transformable_entities()  # Ensure entities are loaded
+    return TRANSFORMABLE_ENTITIES.get(model)
+
+
+def get_transformer(model: Optional[Any]) -> Optional[Callable]:
+    """Get the appropriate transformer function for model"""
+    config = get_entity_config(model)
+    if not config:
+        return None
+
+    direction = config.get("direction")
+    if direction == TransformDirection.ENCODE:
+        return replace_separators
+    elif direction == TransformDirection.DECODE:
+        return revert_separators
+    return None
+
+
+def transform_all_names(obj, transformer):
+    """Transform all name fields recursively"""
+    if not obj:
+        return
+
+    # Transform name field if it exists (supports both obj.name.root and obj.root)
+    name = getattr(obj, "name", None)
+    if name and hasattr(name, "root") and name.root is not None:
+        name.root = transformer(name.root)
+    elif hasattr(obj, "root") and obj.root is not None:
+        obj.root = transformer(obj.root)
+
+    # Transform nested collections in a single loop each
+    for attr_name in ["columns", "children"]:
+        if hasattr(obj, attr_name):
+            attr_value = getattr(obj, attr_name)
+            if attr_value is not None:
+                for item in attr_value:
+                    transform_all_names(item, transformer)
+
+    # Transform table constraints
+    if hasattr(obj, "tableConstraints"):
+        table_constraints = getattr(obj, "tableConstraints")
+        if table_constraints is not None:
+            for constraint in table_constraints:
+                if hasattr(constraint, "columns"):
+                    constraint.columns = [
+                        transformer(col) for col in constraint.columns
+                    ]
+
+    if transformer == replace_separators and type(name) == str:
+        obj.name = transformer(name)
+
+
+def transform_entity_names(entity: Any, model: Optional[Any]) -> Any:
+    """Transform entity names"""
+    model_name = model.__name__
+    if not entity or (
+        model_name.startswith("Create") and is_service_level_create_model(model_name)
     ):
-        values.root = modification_method(values.root)
-    elif hasattr(type(values), "model_fields"):
-        for key in type(values).model_fields.keys():
-            if getattr(values, key):
-                if getattr(values, key).__class__.__name__ in NAME_FIELDS:
-                    setattr(
-                        values,
-                        key,
-                        validate_name_and_transform(
-                            getattr(values, key),
-                            modification_method=modification_method,
-                            field_name=key,
-                        ),
-                    )
-                elif isinstance(getattr(values, key), list):
-                    setattr(
-                        values,
-                        key,
-                        [
-                            validate_name_and_transform(
-                                item,
-                                modification_method=modification_method,
-                                field_name=key,
-                            )
-                            for item in getattr(values, key)
-                        ],
-                    )
-    return values
+        return entity
+
+    # Root attribute handling
+    if hasattr(entity, "root") and entity.root is not None:
+        entity.root = (
+            replace_separators(entity.root)
+            if model_name.startswith("Create")
+            else revert_separators(entity.root)
+        )
+        return entity
+
+    # Get model-specific transformer
+    transformer = get_transformer(model)
+    if not transformer:
+        # Fallback to original logic for backward compatibility
+        transformer = (
+            replace_separators if model_name.startswith("Create") else revert_separators
+        )
+
+    transform_all_names(entity, transformer)
+    return entity

metadata/ingestion/models/custom_pydantic.py

@@ -26,13 +26,7 @@ from pydantic.types import SecretStr
 from pydantic_core.core_schema import SerializationInfo
 from typing_extensions import Annotated
 
-from metadata.ingestion.models.custom_basemodel_validation import (
-    CREATE_ADJACENT_MODELS,
-    FETCH_MODELS,
-    replace_separators,
-    revert_separators,
-    validate_name_and_transform,
-)
+from metadata.ingestion.models.custom_basemodel_validation import transform_entity_names
 
 logger = logging.getLogger("metadata")
 
@@ -75,25 +69,18 @@ class BaseModel(PydanticBaseModel):
     @classmethod
     def parse_name(cls, values):  # pylint: disable=inconsistent-return-statements
         """
-        Primary entry point to process values based on their class.
+        Transform entity names using hybrid configuration system.
         """
 
         if not values:
-            return
+            return values
 
         try:
-
-            if cls.__name__ in CREATE_ADJACENT_MODELS or cls.__name__.startswith(
-                "Create"
-            ):
-                values = validate_name_and_transform(values, replace_separators)
-            elif cls.__name__ in FETCH_MODELS:
-                values = validate_name_and_transform(values, revert_separators)
-
+            # Try new hybrid system first
+            return transform_entity_names(entity=values, model=cls)
         except Exception as exc:
             logger.warning("Exception while parsing Basemodel: %s", exc)
-            raise exc
-        return values
+            return values
 
     def model_dump_json(  # pylint: disable=too-many-arguments
         self,

metadata/ingestion/models/patch_request.py

@@ -88,6 +88,8 @@ ALLOWED_COMMON_PATCH_FIELDS = {
     "owners": True,
     "tags": True,
     "sourceHash": True,
+    "entityStatus": True,
+    "reviewers": True,
     # Table Entity Fields
     "tableType": True,
     "columns": {"__all__": ALLOWED_COLUMN_FIELDS},

metadata/ingestion/ometa/client.py

@@ -335,7 +335,7 @@ class REST:
         return self._request("POST", path, data, json)
 
     @calculate_execution_time(context="PUT")
-    def put(self, path, data=None):
+    def put(self, path, data=None, json=None, headers=None):
         """
         PUT method
 
@@ -346,7 +346,7 @@ class REST:
         Returns:
             Response
         """
-        return self._request("PUT", path, data)
+        return self._request("PUT", path, data, json=json, headers=headers)
 
     @calculate_execution_time(context="PATCH")
     def patch(self, path, data=None):

metadata/ingestion/ometa/mixins/csv_mixin.py

@@ -0,0 +1,180 @@
+"""
+CSV import/export mixin for OpenMetadata client.
+"""
+import logging
+from typing import Dict, Type, TypeVar
+
+from metadata.generated.schema.entity.data.database import Database
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.glossary import Glossary
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.teams.team import Team
+from metadata.generated.schema.entity.teams.user import User
+from metadata.ingestion.ometa.client import APIError
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class CSVMixin:
+    """
+    OpenMetadata API methods for CSV import and export operations.
+
+    Implements export_csv and import_csv following the same pattern as Java SDK.
+    """
+
+    def export_csv(self, entity: Type[T], name: str) -> str:
+        """
+        Export entity data in CSV format.
+
+        Args:
+            entity: Entity type (e.g., Glossary, Team, User)
+            name: Name of the entity to export
+
+        Returns:
+            CSV data as string
+        """
+        try:
+            # Determine the API path based on entity type
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/export"
+
+            response = self.client.get(path)
+            if isinstance(response, str):
+                return response
+            if isinstance(response, dict):
+                return response.get("data", "")
+            return getattr(response, "text", "")
+        except APIError as err:
+            logger.error(f"Failed to export CSV for {entity.__name__} '{name}': {err}")
+            raise
+
+    def export_csv_async(self, entity: Type[T], name: str) -> str:
+        """
+        Export entity data in CSV format asynchronously.
+
+        Args:
+            entity: Entity type
+            name: Name of the entity to export
+
+        Returns:
+            Job ID for the async export
+        """
+        try:
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/exportAsync"
+
+            response = self.client.get(path)
+            # The async endpoint returns a job ID
+            if isinstance(response, dict):
+                return response.get("jobId", "")
+            return getattr(response, "text", str(response))
+        except APIError as err:
+            logger.error(
+                f"Failed to start async CSV export for {entity.__name__} '{name}': {err}"
+            )
+            raise
+
+    def import_csv(
+        self, entity: Type[T], name: str, csv_data: str, dry_run: bool = False
+    ) -> Dict:
+        """
+        Import entity data from CSV format.
+
+        Args:
+            entity: Entity type
+            name: Name of the entity to import into
+            csv_data: CSV data as string
+            dry_run: If True, validate without actually importing
+
+        Returns:
+            Import result with statistics
+        """
+        try:
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/import"
+            if dry_run:
+                path = f"{path}?dryRun=true"
+
+            response = self.client.put(
+                path,
+                csv_data,
+                headers={"Content-Type": "text/plain"},
+            )
+
+            if isinstance(response, dict):
+                return response
+            if isinstance(response, str):
+                return {"message": response}
+            return {"message": getattr(response, "text", "")}
+        except APIError as err:
+            logger.error(f"Failed to import CSV for {entity.__name__} '{name}': {err}")
+            raise
+
+    def import_csv_async(
+        self, entity: Type[T], name: str, csv_data: str, dry_run: bool = False
+    ) -> str:
+        """
+        Import entity data from CSV format asynchronously.
+
+        Args:
+            entity: Entity type
+            name: Name of the entity to import into
+            csv_data: CSV data as string
+            dry_run: If True, validate without actually importing
+
+        Returns:
+            Job ID for the async import
+        """
+        try:
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/importAsync"
+            if dry_run:
+                path = f"{path}?dryRun=true"
+
+            response = self.client.put(
+                path,
+                csv_data,
+                headers={"Content-Type": "text/plain"},
+            )
+
+            if isinstance(response, dict):
+                return response.get("jobId", "")
+            return getattr(response, "text", str(response))
+        except APIError as err:
+            logger.error(
+                f"Failed to start async CSV import for {entity.__name__} '{name}': {err}"
+            )
+            raise
+
+    def _get_csv_endpoint(self, entity: Type[T]) -> str:
+        """
+        Get the API endpoint for CSV operations based on entity type.
+
+        Args:
+            entity: Entity type
+
+        Returns:
+            API endpoint path
+        """
+        # Map entity types to their API endpoints
+        entity_endpoints = {
+            Glossary: "glossaries",
+            Team: "teams",
+            User: "users",
+            Table: "tables",
+            Database: "databases",
+            DatabaseSchema: "databaseSchemas",
+            DatabaseService: "databaseServices",
+            # Add more entity types as needed
+        }
+
+        endpoint = entity_endpoints.get(entity)
+        if not endpoint:
+            raise ValueError(
+                f"CSV operations not supported for entity type {entity.__name__}"
+            )
+
+        return endpoint

metadata/ingestion/ometa/mixins/domain_mixin.py

@@ -9,11 +9,15 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Domain and Data Product specific operations"""
+import traceback
 from typing import Dict, List
 
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.client import REST
+from metadata.utils.logger import ometa_logger
+
+logger = ometa_logger()
 
 
 class AssetsRequest(BaseModel):
@@ -57,6 +61,50 @@ class OMetaDomainMixin:
         """
         return self._handle_data_product_assets(name, assets, "remove")
 
+    def get_data_product_assets(
+        self, name: str, limit: int = 10, offset: int = 0
+    ) -> Dict:
+        """
+        Get paginated list of assets for a data product
+
+        Args:
+            name: Name of the data product
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/dataProducts/name/{name}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get data product assets due to {exc}")
+            return {}
+
+    def get_domain_assets(self, name: str, limit: int = 10, offset: int = 0) -> Dict:
+        """
+        Get paginated list of assets for a domain
+
+        Args:
+            name: Name of the domain
+            limit: Maximum number of assets to return (default 10, max 1000)
+            offset: Offset from which to start returning results (default 0)
+
+        Returns:
+            API response as a dictionary containing paginated assets
+        """
+        try:
+            path = f"/domains/name/{name}/assets"
+            params = {"limit": limit, "offset": offset}
+            return self.client.get(path, params)
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Could not get domain assets due to {exc}")
+            return {}
+
     def _handle_data_product_assets(
         self,
         name: str,

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -39,7 +39,9 @@ from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.client import REST, APIError
 from metadata.ingestion.ometa.utils import quote
 from metadata.ingestion.source.models import TableView
+from metadata.utils import fqn
 from metadata.utils.elasticsearch import ES_INDEX_MAP, get_entity_from_es_result
+from metadata.utils.execution_time_tracker import calculate_execution_time_generator
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -404,6 +406,7 @@ class ESMixin(Generic[T]):
                     f"Error while getting {hit.source['fullyQualifiedName']} - {exc}"
                 )
 
+    @calculate_execution_time_generator(context="ES.FetchViewDefinition")
     def yield_es_view_def(
         self,
         service_name: str,
@@ -519,10 +522,13 @@ class ESMixin(Generic[T]):
         fetch table from es when with/without `db_service_name`
         """
         try:
+            prepended_fqn = fqn.prefix_entity_for_wildcard_search(
+                entity_type=entity_type, fqn=fqn_search_string
+            )
             entity_result = get_entity_from_es_result(
                 entity_list=self.es_search_from_fqn(
                     entity_type=entity_type,
-                    fqn_search_string=fqn_search_string,
+                    fqn_search_string=prepended_fqn,
                 ),
                 fetch_multiple_entities=fetch_multiple_entities,
             )

metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py

@@ -133,3 +133,29 @@ class OMetaIngestionPipelineMixin:
             return parse_ingestion_pipeline_config_gracefully(resp)
 
         return None
+
+    def extract_pipeline_id_from_fqn(
+        self, ingestion_pipeline_fqn: str
+    ) -> Optional[str]:
+        """
+        Extract pipeline ID from FQN by fetching the pipeline entity
+
+        Args:
+            ingestion_pipeline_fqn (str): Fully qualified name of the ingestion pipeline
+
+        Returns:
+            Optional[str]: Pipeline ID if found, None otherwise
+        """
+        try:
+            pipeline = self.get_by_name(
+                entity=IngestionPipeline, fqn=ingestion_pipeline_fqn
+            )
+            if pipeline and hasattr(pipeline, "id"):
+                return str(
+                    pipeline.id.root if hasattr(pipeline.id, "root") else pipeline.id
+                )
+        except Exception as e:
+            logger.error(
+                f"Failed to extract pipeline ID from FQN {ingestion_pipeline_fqn}: {e}"
+            )
+        return None

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -383,7 +383,7 @@ class OMetaLineageMixin(Generic[T]):
             connection_type = database_service.serviceType.value
             add_lineage_request = get_lineage_by_query(
                 metadata=self,
-                service_name=database_service.name.root,
+                service_names=database_service.name.root,
                 dialect=ConnectionTypeDialectMapper.dialect_of(connection_type),
                 query=sql,
                 database_name=database_name,