openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 
@@ -34,6 +34,7 @@ class Source(Enum):
     PipelineLineage = 'PipelineLineage'
     DashboardLineage = 'DashboardLineage'
     DbtLineage = 'DbtLineage'
+    SparkLineage = 'SparkLineage'
 
 
 class LineageDetails(BaseModel):

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 
@@ -36,4 +36,8 @@ class EntityReference(BaseModel):
     deleted: Optional[bool] = Field(
         None, description='If true the entity referred to has been soft-deleted.'
     )
+    inherited: Optional[bool] = Field(
+        None,
+        description='If true the relationship indicated by this entity reference is inherited from the parent entity.',
+    )
     href: Optional[basic.Href] = Field(None, description='Link to the entity resource.')

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,11 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import List, Optional
+from typing import Any, List, Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -15,15 +15,14 @@ class ParameterType(Enum):
     AllIndexElasticSearch = 'AllIndexElasticSearch'
     SpecificIndexElasticSearch = 'SpecificIndexElasticSearch'
     ReadFromParamContext = 'ReadFromParamContext'
+    ReadFromParamContextPerEntity = 'ReadFromParamContextPerEntity'
 
 
 class ParamAdditionalContext(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    data: Optional[List[str]] = Field(
-        None, description='List of Entities', unique_items=True
-    )
+    data: Optional[Any] = Field(None, description='List of Entities')
 
 
 class Function(BaseModel):
@@ -34,7 +33,9 @@ class Function(BaseModel):
     input: Optional[str] = Field(
         None, description='Description of input taken by the function.'
     )
-    description: Optional[str] = Field(None, description='Description fo the function.')
+    description: Optional[str] = Field(
+        None, description='Description for the function.'
+    )
     examples: Optional[List] = Field(
         None, description='Examples of the function to help users author conditions.'
     )

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 
@@ -35,7 +35,7 @@ class ParsedData(BaseModel):
         None, description='Database schema of the associated with query'
     )
     duration: Optional[float] = Field(
-        None, description='How long did the query took to run in seconds.'
+        None, description='How long did the query took to run in milliseconds.'
     )
 
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 
@@ -43,7 +43,7 @@ class TableQuery(BaseModel):
         None, description='Database schema of the associated with query'
     )
     duration: Optional[float] = Field(
-        None, description='How long did the query took to run in seconds.'
+        None, description='How long did the query took to run in milliseconds.'
     )
 
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-01-30T14:23:01+00:00
+#   timestamp: 2024-02-06T05:39:24+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -420,8 +420,9 @@ class OpenMetadataValidationAction(ValidationAction):
             test_case = self.ometa_conn.get_or_create_test_case(
                 test_case_fqn,
                 entity_link=get_entity_link(
-                    table_entity.fullyQualifiedName.__root__,
-                    fqn.split_test_case_fqn(test_case_fqn).column,
+                    Table,
+                    fqn=table_entity.fullyQualifiedName.__root__,
+                    column_name=fqn.split_test_case_fqn(test_case_fqn).column,
                 ),
                 test_suite_fqn=test_suite.fullyQualifiedName.__root__,
                 test_definition_fqn=test_definition.fullyQualifiedName.__root__,

metadata/ingestion/api/delete.py

@@ -14,7 +14,10 @@ Delete methods
 import traceback
 from typing import Dict, Iterable, Optional, Type
 
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.ometa.ometa_api import OpenMetadata, T
 from metadata.utils.logger import utils_logger
@@ -52,6 +55,6 @@ def delete_entity_from_source(
             left=StackTraceError(
                 name="Delete Entity",
                 error=f"Error deleting {entity_type.__class__}: {exc}",
-                stack_trace=traceback.format_exc(),
+                stackTrace=traceback.format_exc(),
             )
         )

metadata/ingestion/api/models.py

@@ -15,21 +15,15 @@ from typing import Generic, Optional, TypeVar
 
 from pydantic import BaseModel
 
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+
 # Entities are instances of BaseModel
 Entity = BaseModel
 T = TypeVar("T")
 
 
-class StackTraceError(BaseModel):
-    """
-    Class that represents a failure status
-    """
-
-    name: str
-    error: str
-    stack_trace: Optional[str]
-
-
 class Either(BaseModel, Generic[T]):
     """
     Any execution should return us Either an Entity of an error for us to handle

metadata/ingestion/api/parser.py

@@ -93,7 +93,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
     WorkflowConfig,
 )
-from metadata.ingestion.ometa.provider_registry import PROVIDER_CLASS_MAP
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -196,7 +195,7 @@ def get_connection_class(
         Type[PipelineConnection],
         Type[MlModelConnection],
     ],
-) -> T:
+) -> Type[T]:
     """
     Build the connection class path, import and return it
     :param source_type: e.g., Glue
@@ -251,7 +250,7 @@ def _parse_validation_err(validation_error: ValidationError) -> str:
     return "\t - " + "\n\t - ".join(missing_fields + extra_fields + invalid_fields)
 
 
-def _unsafe_parse_config(config: dict, cls: T, message: str) -> None:
+def _unsafe_parse_config(config: dict, cls: Type[T], message: str) -> None:
     """
     Given a config dictionary and the class it should match,
     try to parse it or log the given message
@@ -354,39 +353,6 @@ def parse_workflow_source(config_dict: dict) -> None:
     parse_source_config(config_dict)
 
 
-def parse_server_config(config_dict: dict) -> None:
-    """
-    Validate the parsing of openMetadataServerConfig.
-    This is valuable to make sure there are no issues
-    when setting up auth providers.
-
-    :param config_dict: JSON configuration
-    """
-    # Unsafe access to the keys. Allow a KeyError if the config is not well formatted
-    auth_provider = config_dict["workflowConfig"]["openMetadataServerConfig"][
-        "authProvider"
-    ]
-    logger.debug(
-        f"Error parsing the Workflow Server Configuration with {auth_provider} auth provider"
-    )
-
-    # If the error comes from the security config:
-    auth_class = PROVIDER_CLASS_MAP.get(auth_provider)
-    # throw an error if the keys are not present
-    security_config = config_dict["workflowConfig"]["openMetadataServerConfig"][
-        "securityConfig"
-    ]
-
-    _unsafe_parse_config(
-        config=security_config,
-        cls=auth_class,
-        message="Error parsing the workflow security config",
-    )
-
-    # If the security config is properly configured, let's raise the ValidationError of the whole WorkflowConfig
-    WorkflowConfig.parse_obj(config_dict["workflowConfig"])
-
-
 def parse_workflow_config_gracefully(
     config_dict: dict,
 ) -> OpenMetadataWorkflowConfig:
@@ -413,7 +379,7 @@ def parse_workflow_config_gracefully(
     except ValidationError as original_error:
         try:
             parse_workflow_source(config_dict)
-            parse_server_config(config_dict)
+            WorkflowConfig.parse_obj(config_dict["workflowConfig"])
         except (ValidationError, InvalidWorkflowException) as scoped_error:
             if isinstance(scoped_error, ValidationError):
                 # Let's catch validations of internal Workflow models, not the Workflow itself

metadata/ingestion/api/status.py

@@ -17,7 +17,10 @@ from typing import Any, Dict, List
 
 from pydantic import BaseModel, Field
 
-from metadata.ingestion.api.models import StackTraceError
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.ingestion.models.patch_request import PatchedEntity, PatchRequest
 from metadata.utils.logger import get_log_name, ingestion_logger
 
 logger = ingestion_logger()
@@ -31,6 +34,7 @@ class Status(BaseModel):
     source_start_time: Any
 
     records: List[Any] = Field(default_factory=list)
+    updated_records: List[Any] = Field(default_factory=list)
     warnings: List[Any] = Field(default_factory=list)
     filtered: List[Dict[str, str]] = Field(default_factory=list)
     failures: List[StackTraceError] = Field(default_factory=list)
@@ -47,7 +51,14 @@ class Status(BaseModel):
         are not worth keeping record of.
         """
         if log_name := get_log_name(record):
-            self.records.append(log_name)
+            if isinstance(record, (PatchRequest, PatchedEntity)):
+                self.updated_records.append(log_name)
+            else:
+                self.records.append(log_name)
+
+    def updated(self, record: Any) -> None:
+        if log_name := get_log_name(record):
+            self.updated_records.append(log_name)
 
     def warning(self, key: str, reason: str) -> None:
         self.warnings.append({key: reason})
@@ -63,7 +74,7 @@ class Status(BaseModel):
         Add a failure to the list of failures
         """
         logger.warning(error.error)
-        logger.debug(error.stack_trace)
+        logger.debug(error.stackTrace)
         self.failures.append(error)
 
     def fail_all(self, failures: List[StackTraceError]) -> None:
@@ -76,7 +87,7 @@ class Status(BaseModel):
 
     def calculate_success(self) -> float:
         source_success = max(
-            len(self.records), 1
+            len(self.records) + len(self.updated_records), 1
         )  # To avoid ZeroDivisionError using minimum value as 1
         source_failed = len(self.failures)
         return round(source_success * 100 / (source_success + source_failed), 2)

metadata/ingestion/api/step.py

@@ -16,6 +16,9 @@ import traceback
 from abc import ABC, abstractmethod
 from typing import Iterable, Optional
 
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StepSummary,
+)
 from metadata.ingestion.api.closeable import Closeable
 from metadata.ingestion.api.models import Either, Entity, StackTraceError
 from metadata.ingestion.api.status import Status
@@ -49,11 +52,41 @@ class Step(ABC, Closeable):
     def get_status(self) -> Status:
         return self.status
 
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
     @abstractmethod
     def close(self) -> None:
         pass
 
 
+class Summary(StepSummary):
+    """
+    Auxiliary class to calculate the summary of all statuses
+    """
+
+    @classmethod
+    def from_step(cls, step: Step) -> "Summary":
+        """Compute summary from Step"""
+        return Summary(
+            name=step.name,
+            records=len(step.status.records),
+            updated_records=len(step.status.updated_records),
+            warnings=len(step.status.warnings),
+            errors=len(step.status.failures),
+            filtered=len(step.status.filtered),
+            failures=step.status.failures[0:10] if step.status.failures else None,
+        )
+
+    def __str__(self):
+        return (
+            f"{self.name} Summary: [{self.records} Records, [{self.updated_records} Updated Records,"
+            f" {self.warnings} Warnings, {self.errors} Errors, {self.filtered} Filtered]"
+        )
+
+
 class ReturnStep(Step, ABC):
     """Steps that run by returning a single unit"""
 
@@ -90,7 +123,7 @@ class ReturnStep(Step, ABC):
                 StackTraceError(
                     name="Not an Either",
                     error=error,
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as exc:
@@ -98,7 +131,7 @@ class ReturnStep(Step, ABC):
             logger.warning(error)
             self.status.failed(
                 StackTraceError(
-                    name="Unhandled", error=error, stack_trace=traceback.format_exc()
+                    name="Unhandled", error=error, stackTrace=traceback.format_exc()
                 )
             )
 
@@ -145,7 +178,7 @@ class StageStep(Step, ABC):
                 StackTraceError(
                     name="Not an Either",
                     error=error,
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as exc:
@@ -153,7 +186,7 @@ class StageStep(Step, ABC):
             logger.warning(error)
             self.status.failed(
                 StackTraceError(
-                    name="Unhandled", error=error, stack_trace=traceback.format_exc()
+                    name="Unhandled", error=error, stackTrace=traceback.format_exc()
                 )
             )
 
@@ -194,7 +227,7 @@ class IterStep(Step, ABC):
                 StackTraceError(
                     name="Not an Either",
                     error=error,
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )
         except Exception as exc:
@@ -202,7 +235,7 @@ class IterStep(Step, ABC):
             logger.warning(error)
             self.status.failed(
                 StackTraceError(
-                    name="Unhandled", error=error, stack_trace=traceback.format_exc()
+                    name="Unhandled", error=error, stackTrace=traceback.format_exc()
                 )
             )
 

metadata/ingestion/api/steps.py

@@ -46,18 +46,38 @@ class Source(IterStep, ABC):
     def test_connection(self) -> None:
         pass
 
+    @property
+    def name(self) -> str:
+        return "Source"
+
 
 class Sink(ReturnStep, ABC):
     """All Sinks must inherit this base class."""
 
+    @property
+    def name(self) -> str:
+        return "Sink"
+
 
 class Processor(ReturnStep, ABC):
     """All Processor must inherit this base class"""
 
+    @property
+    def name(self) -> str:
+        return "Processor"
+
 
 class Stage(StageStep, ABC):
     """All Stages must inherit this base class."""
 
+    @property
+    def name(self) -> str:
+        return "Stage"
+
 
 class BulkSink(BulkStep, ABC):
     """All Stages must inherit this base class."""
+
+    @property
+    def name(self) -> str:
+        return "BulkSink"