openmetadata-ingestion 1.9.13.0__py3-none-any.whl → 1.10.1.0__py3-none-any.whl

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/status.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  type/status.json
+#   timestamp: 2025-10-09T20:05:36+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class EntityStatus(Enum):
+    Draft = 'Draft'
+    In_Review = 'In Review'
+    Approved = 'Approved'
+    Deprecated = 'Deprecated'
+    Rejected = 'Rejected'
+    Unprocessed = 'Unprocessed'

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 
@@ -69,3 +69,10 @@ class TagLabel(BaseModel):
     href: Annotated[
         Optional[basic.Href], Field(None, description='Link to the tag resource.')
     ]
+    reason: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='An explanation of why this tag was proposed, specially for autoclassification tags',
+        ),
+    ]

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-10-08T20:25:03+00:00
+#   timestamp: 2025-10-09T20:05:36+00:00
 
 from __future__ import annotations
 

metadata/ingestion/models/patch_request.py

@@ -88,6 +88,8 @@ ALLOWED_COMMON_PATCH_FIELDS = {
     "owners": True,
     "tags": True,
     "sourceHash": True,
+    "entityStatus": True,
+    "reviewers": True,
     # Table Entity Fields
     "tableType": True,
     "columns": {"__all__": ALLOWED_COLUMN_FIELDS},

metadata/ingestion/ometa/client.py

@@ -335,7 +335,7 @@ class REST:
         return self._request("POST", path, data, json)
 
     @calculate_execution_time(context="PUT")
-    def put(self, path, data=None):
+    def put(self, path, data=None, json=None, headers=None):
         """
         PUT method
 
@@ -346,7 +346,7 @@ class REST:
         Returns:
             Response
         """
-        return self._request("PUT", path, data)
+        return self._request("PUT", path, data, json=json, headers=headers)
 
     @calculate_execution_time(context="PATCH")
     def patch(self, path, data=None):

metadata/ingestion/ometa/mixins/csv_mixin.py

@@ -0,0 +1,180 @@
+"""
+CSV import/export mixin for OpenMetadata client.
+"""
+import logging
+from typing import Dict, Type, TypeVar
+
+from metadata.generated.schema.entity.data.database import Database
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.glossary import Glossary
+from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.teams.team import Team
+from metadata.generated.schema.entity.teams.user import User
+from metadata.ingestion.ometa.client import APIError
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class CSVMixin:
+    """
+    OpenMetadata API methods for CSV import and export operations.
+
+    Implements export_csv and import_csv following the same pattern as Java SDK.
+    """
+
+    def export_csv(self, entity: Type[T], name: str) -> str:
+        """
+        Export entity data in CSV format.
+
+        Args:
+            entity: Entity type (e.g., Glossary, Team, User)
+            name: Name of the entity to export
+
+        Returns:
+            CSV data as string
+        """
+        try:
+            # Determine the API path based on entity type
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/export"
+
+            response = self.client.get(path)
+            if isinstance(response, str):
+                return response
+            if isinstance(response, dict):
+                return response.get("data", "")
+            return getattr(response, "text", "")
+        except APIError as err:
+            logger.error(f"Failed to export CSV for {entity.__name__} '{name}': {err}")
+            raise
+
+    def export_csv_async(self, entity: Type[T], name: str) -> str:
+        """
+        Export entity data in CSV format asynchronously.
+
+        Args:
+            entity: Entity type
+            name: Name of the entity to export
+
+        Returns:
+            Job ID for the async export
+        """
+        try:
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/exportAsync"
+
+            response = self.client.get(path)
+            # The async endpoint returns a job ID
+            if isinstance(response, dict):
+                return response.get("jobId", "")
+            return getattr(response, "text", str(response))
+        except APIError as err:
+            logger.error(
+                f"Failed to start async CSV export for {entity.__name__} '{name}': {err}"
+            )
+            raise
+
+    def import_csv(
+        self, entity: Type[T], name: str, csv_data: str, dry_run: bool = False
+    ) -> Dict:
+        """
+        Import entity data from CSV format.
+
+        Args:
+            entity: Entity type
+            name: Name of the entity to import into
+            csv_data: CSV data as string
+            dry_run: If True, validate without actually importing
+
+        Returns:
+            Import result with statistics
+        """
+        try:
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/import"
+            if dry_run:
+                path = f"{path}?dryRun=true"
+
+            response = self.client.put(
+                path,
+                csv_data,
+                headers={"Content-Type": "text/plain"},
+            )
+
+            if isinstance(response, dict):
+                return response
+            if isinstance(response, str):
+                return {"message": response}
+            return {"message": getattr(response, "text", "")}
+        except APIError as err:
+            logger.error(f"Failed to import CSV for {entity.__name__} '{name}': {err}")
+            raise
+
+    def import_csv_async(
+        self, entity: Type[T], name: str, csv_data: str, dry_run: bool = False
+    ) -> str:
+        """
+        Import entity data from CSV format asynchronously.
+
+        Args:
+            entity: Entity type
+            name: Name of the entity to import into
+            csv_data: CSV data as string
+            dry_run: If True, validate without actually importing
+
+        Returns:
+            Job ID for the async import
+        """
+        try:
+            endpoint = self._get_csv_endpoint(entity)
+            path = f"/{endpoint}/name/{name}/importAsync"
+            if dry_run:
+                path = f"{path}?dryRun=true"
+
+            response = self.client.put(
+                path,
+                csv_data,
+                headers={"Content-Type": "text/plain"},
+            )
+
+            if isinstance(response, dict):
+                return response.get("jobId", "")
+            return getattr(response, "text", str(response))
+        except APIError as err:
+            logger.error(
+                f"Failed to start async CSV import for {entity.__name__} '{name}': {err}"
+            )
+            raise
+
+    def _get_csv_endpoint(self, entity: Type[T]) -> str:
+        """
+        Get the API endpoint for CSV operations based on entity type.
+
+        Args:
+            entity: Entity type
+
+        Returns:
+            API endpoint path
+        """
+        # Map entity types to their API endpoints
+        entity_endpoints = {
+            Glossary: "glossaries",
+            Team: "teams",
+            User: "users",
+            Table: "tables",
+            Database: "databases",
+            DatabaseSchema: "databaseSchemas",
+            DatabaseService: "databaseServices",
+            # Add more entity types as needed
+        }
+
+        endpoint = entity_endpoints.get(entity)
+        if not endpoint:
+            raise ValueError(
+                f"CSV operations not supported for entity type {entity.__name__}"
+            )
+
+        return endpoint

metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py

@@ -133,3 +133,29 @@ class OMetaIngestionPipelineMixin:
             return parse_ingestion_pipeline_config_gracefully(resp)
 
         return None
+
+    def extract_pipeline_id_from_fqn(
+        self, ingestion_pipeline_fqn: str
+    ) -> Optional[str]:
+        """
+        Extract pipeline ID from FQN by fetching the pipeline entity
+
+        Args:
+            ingestion_pipeline_fqn (str): Fully qualified name of the ingestion pipeline
+
+        Returns:
+            Optional[str]: Pipeline ID if found, None otherwise
+        """
+        try:
+            pipeline = self.get_by_name(
+                entity=IngestionPipeline, fqn=ingestion_pipeline_fqn
+            )
+            if pipeline and hasattr(pipeline, "id"):
+                return str(
+                    pipeline.id.root if hasattr(pipeline.id, "root") else pipeline.id
+                )
+        except Exception as e:
+            logger.error(
+                f"Failed to extract pipeline ID from FQN {ingestion_pipeline_fqn}: {e}"
+            )
+        return None

metadata/ingestion/ometa/mixins/logs_mixin.py

@@ -0,0 +1,336 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Mixin class for centralized log streaming to S3 storage
+
+This mixin provides methods to send ingestion logs to OpenMetadata server's
+S3 storage backend. It's designed to be used by OpenMetadata class and
+integrates with the existing logging infrastructure.
+"""
+
+import base64
+import gzip
+import json
+import os
+import socket
+import time
+from typing import Optional
+from uuid import UUID
+
+from metadata.ingestion.ometa.client import REST
+from metadata.ingestion.ometa.utils import model_str
+from metadata.utils.constants import UTF_8
+from metadata.utils.logger import ometa_logger
+
+logger = ometa_logger()
+
+
+class OMetaLogsMixin:
+    """
+    OpenMetadata API methods for log streaming to S3.
+
+    This mixin provides centralized methods for shipping logs to the
+    OpenMetadata server's S3 storage backend. It reuses the existing
+    OpenMetadata client instance to avoid creating new sessions.
+    """
+
+    client: REST
+
+    def send_logs_to_s3(
+        self,
+        pipeline_fqn: str,
+        run_id: UUID,
+        log_content: str,
+        compress: bool = False,
+    ) -> bool:
+        """
+        Send logs to S3 storage via OpenMetadata server endpoint.
+
+        Args:
+            pipeline_fqn: Fully qualified name of the ingestion pipeline
+            run_id: Unique identifier for the pipeline run
+            log_content: The log content to send
+            compress: Whether to compress logs before sending
+
+        Returns:
+            bool: True if logs were sent successfully, False otherwise
+        """
+        try:
+            # Extract the UUID string value from the object if it has a .root attribute
+            # Build the API endpoint
+            url = (
+                f"/services/ingestionPipelines/logs/{pipeline_fqn}/{model_str(run_id)}"
+            )
+
+            # Prepare log batch data matching Java LogBatch structure
+            log_batch = {
+                "logs": log_content,
+                "timestamp": int(time.time() * 1000),  # milliseconds since epoch
+                "connectorId": f"{socket.gethostname()}-{os.getpid()}",
+                "compressed": False,
+                "lineCount": log_content.count("\n") + 1,
+            }
+
+            # Apply compression if requested and content is large enough
+            if compress and len(log_content) > 10240:  # Compress if > 10KB
+                compressed_data = gzip.compress(log_content.encode(UTF_8))
+                log_batch["logs"] = base64.b64encode(compressed_data).decode(UTF_8)
+                log_batch["compressed"] = True
+
+            # Send logs using the existing client
+            response = self.client.post(
+                url,
+                data=json.dumps(log_batch),
+            )
+
+            # The REST client returns None for successful requests with empty response body (HTTP 200 with no content)
+            # It also returns None on errors, but those are caught by the exception handler
+            # Since we're not in the exception handler, None here means successful upload with empty response
+            if response is None or response:
+                logger.debug(
+                    f"Successfully sent {log_batch['lineCount']} log lines for pipeline {pipeline_fqn}"
+                )
+                return True
+
+            logger.warning(f"Unexpected response from log upload: {response}")
+            return False
+
+        except Exception as e:
+            logger.error(f"Failed to send logs to S3 for pipeline {pipeline_fqn}: {e}")
+            return False
+
+    def send_logs_batch(
+        self,
+        pipeline_fqn: str,
+        run_id: UUID,
+        log_content: str,
+        enable_compression: bool = False,
+    ) -> dict:
+        """
+        Send logs batch to S3 storage, handling both new and legacy approaches.
+
+        This method consolidates all log sending logic, including fallback
+        for backward compatibility.
+
+        Args:
+            pipeline_fqn: Fully qualified name of the ingestion pipeline
+            run_id: Unique identifier for the pipeline run
+            log_content: The log content to send
+            enable_compression: Whether to compress logs before sending
+
+        Returns:
+            dict: Metrics including lines sent and bytes sent
+
+        Raises:
+            Exception: If logs cannot be sent via any method
+        """
+        metrics = {"logs_sent": 0, "bytes_sent": 0}
+
+        try:
+            success = self.send_logs_to_s3(
+                pipeline_fqn=pipeline_fqn,
+                run_id=run_id,
+                log_content=log_content,
+                compress=enable_compression and len(log_content) > 10240,
+            )
+
+            if success:
+                # Update metrics
+                line_count = log_content.count("\n") + 1
+                metrics["logs_sent"] = line_count
+                metrics["bytes_sent"] = len(log_content)
+
+                logger.debug(f"Successfully shipped {line_count} log lines to server")
+                return metrics
+
+            # If send_logs_to_s3 fails, try fallback method
+            logger.warning("Primary log shipping failed, trying fallback method")
+
+            # Build payload
+            log_batch = {
+                "logs": log_content,
+                "timestamp": int(time.time() * 1000),
+                "connectorId": f"{socket.gethostname()}-{os.getpid()}",
+                "compressed": False,
+                "lineCount": log_content.count("\n") + 1,
+            }
+
+            if enable_compression and len(log_content) > 10240:
+                compressed_data = gzip.compress(log_content.encode(UTF_8))
+                log_batch["logs"] = base64.b64encode(compressed_data).decode(UTF_8)
+                log_batch["compressed"] = True
+
+            # Use the metadata client's REST interface directly
+            self.client.post(
+                f"/services/ingestionPipelines/logs/{pipeline_fqn}/{model_str(run_id)}",
+                data=json.dumps(log_batch),
+            )
+
+            # Update metrics
+            metrics["logs_sent"] = log_batch["lineCount"]
+            metrics["bytes_sent"] = len(json.dumps(log_batch))
+
+            logger.debug(
+                f"Successfully shipped {log_batch['lineCount']} log lines to server"
+            )
+            return metrics
+
+        except Exception as e:
+            logger.error(f"Failed to send logs batch for pipeline {pipeline_fqn}: {e}")
+        return metrics
+
+    def create_log_stream(
+        self,
+        pipeline_fqn: str,
+        run_id: UUID,
+    ) -> Optional[str]:
+        """
+        Initialize a log stream for a pipeline run.
+
+        This method can be used to set up a log stream session with the server,
+        potentially getting a session ID or token for subsequent log shipments.
+
+        Args:
+            pipeline_fqn: Fully qualified name of the ingestion pipeline
+            run_id: Unique identifier for the pipeline run
+
+        Returns:
+            Optional[str]: Stream session ID if applicable, None otherwise
+        """
+        try:
+
+            # Initialize log stream with the server
+            url = (
+                f"/services/ingestionPipelines/logs/{pipeline_fqn}/{model_str(run_id)}"
+            )
+
+            init_data = {
+                "connectorId": f"{socket.gethostname()}-{os.getpid()}",
+                "timestamp": int(time.time() * 1000),
+            }
+
+            response = self.client.post(
+                url,
+                data=json.dumps(init_data),
+            )
+
+            # Return session ID if provided by the server
+            if response and isinstance(response, dict):
+                return response.get("sessionId")
+
+            return None
+
+        except Exception as e:
+            logger.warning(
+                f"Failed to initialize log stream for pipeline {pipeline_fqn}: {e}"
+            )
+            return None
+
+    def close_log_stream(
+        self,
+        pipeline_fqn: str,
+        run_id: UUID,
+        session_id: Optional[str] = None,
+    ) -> bool:
+        """
+        Close a log stream for a pipeline run.
+
+        This method signals the server that log streaming has completed
+        for a particular pipeline run.
+
+        Args:
+            pipeline_fqn: Fully qualified name of the ingestion pipeline
+            run_id: Unique identifier for the pipeline run
+            session_id: Optional session ID from create_log_stream
+
+        Returns:
+            bool: True if stream was closed successfully, False otherwise
+        """
+        try:
+
+            url = f"/services/ingestionPipelines/logs/{pipeline_fqn}/{model_str(run_id)}/close"
+
+            close_data = {
+                "connectorId": f"{socket.gethostname()}-{os.getpid()}",
+                "timestamp": int(time.time() * 1000),
+            }
+
+            if session_id:
+                close_data["sessionId"] = session_id
+
+            self.client.post(
+                url,
+                data=json.dumps(close_data),
+            )
+
+            logger.debug(f"Successfully closed log stream for pipeline {pipeline_fqn}")
+
+            return True
+
+        except Exception as e:
+            logger.warning(
+                f"Failed to close log stream for pipeline {pipeline_fqn}: {e}"
+            )
+            return False
+
+    def get_logs_from_s3(
+        self,
+        pipeline_fqn: str,
+        run_id: UUID,
+        offset: int = 0,
+        limit: int = 1000,
+    ) -> Optional[str]:
+        """
+        Retrieve logs from S3 storage for a pipeline run.
+
+        Args:
+            pipeline_fqn: Fully qualified name of the ingestion pipeline
+            run_id: Unique identifier for the pipeline run
+            offset: Starting offset for log retrieval
+            limit: Maximum number of log lines to retrieve
+
+        Returns:
+            Optional[str]: Log content if available, None otherwise
+        """
+        try:
+
+            url = (
+                f"/services/ingestionPipelines/logs/{pipeline_fqn}/{model_str(run_id)}"
+            )
+
+            params = {
+                "offset": offset,
+                "limit": limit,
+            }
+
+            response = self.client.get(url, data=params)
+
+            if response and isinstance(response, dict):
+                log_data = response.get("logs", "")
+
+                # Decompress if necessary
+                if response.get("compressed"):
+                    try:
+                        decoded = base64.b64decode(log_data)
+                        log_data = gzip.decompress(decoded).decode(UTF_8)
+                    except Exception as e:
+                        logger.error(f"Failed to decompress logs: {e}")
+                        return None
+
+                return log_data
+
+            return None
+
+        except Exception as e:
+            logger.error(
+                f"Failed to retrieve logs from S3 for pipeline {pipeline_fqn}: {e}"
+            )
+            return None