openmetadata-ingestion 1.3.0.1__py3-none-any.whl → 1.3.1.0__py3-none-any.whl

metadata/generated/schema/tests/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/basic.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/customMetric.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/customMetric.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/resolved.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/resolved.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testCase.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testCase.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testCaseResolutionStatus.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testCaseResolutionStatus.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/tests/testDefinition.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testDefinition.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 
@@ -85,6 +85,7 @@ class TestDefinition(BaseModel):
     owner: Optional[entityReference.EntityReference] = Field(
         None, description='Owner of this TestCase definition.'
     )
+    provider: Optional[basic.ProviderType] = basic.ProviderType.user
     version: Optional[entityHistory.EntityVersion] = Field(
         None, description='Metadata version of the entity.'
     )

metadata/generated/schema/tests/testSuite.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  tests/testSuite.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00

metadata/generated/schema/type/auditLog.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/auditLog.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/basic.py

@@ -1,12 +1,12 @@
 # generated by datamodel-codegen:
 #   filename:  type/basic.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 
 from datetime import date, datetime, time
 from enum import Enum
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 from uuid import UUID
 
 from pydantic import AnyUrl, BaseModel, EmailStr, Extra, Field, constr
@@ -86,6 +86,10 @@ class Time(BaseModel):
     )
 
 
+class EnumModel(BaseModel):
+    __root__: List[str] = Field(..., description='List of values in Enum.')
+
+
 class Timezone(BaseModel):
     __root__: str = Field(
         ...,

metadata/generated/schema/type/bulkOperationResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/bulkOperationResult.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEvent.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/changeEventType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEventType.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/collectionDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/collectionDescriptor.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvDocumentation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvDocumentation.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvErrorType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvErrorType.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvFile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvFile.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvImportResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvImportResult.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2024-03-01T08:53:21+00:00

metadata/generated/schema/type/customProperties/enumConfig.py

@@ -0,0 +1,17 @@
+# generated by datamodel-codegen:
+#   filename:  type/customProperties/enumConfig.json
+#   timestamp: 2024-03-01T08:53:21+00:00
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Extra
+
+
+class EnumConfig(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    multiSelect: Optional[bool] = False
+    values: Optional[List[str]] = None

metadata/generated/schema/type/customProperty.py

@@ -0,0 +1,52 @@
+# generated by datamodel-codegen:
+#   filename:  type/customProperty.json
+#   timestamp: 2024-03-01T08:53:21+00:00
+
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from pydantic import BaseModel, Extra, Field
+
+from . import basic, entityReference
+from .customProperties import enumConfig
+
+
+class Format(BaseModel):
+    __root__: str = Field(
+        ..., description='Applies to date interval, date, time format.'
+    )
+
+
+class EntityTypes(BaseModel):
+    __root__: str = Field(
+        ...,
+        description='Applies to Entity References. Entity Types can be used to restrict what type of entities can be configured for a entity reference.',
+    )
+
+
+class CustomPropertyConfig(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    config: Optional[Union[enumConfig.EnumConfig, Format, EntityTypes]] = None
+
+
+class PropertyType(BaseModel):
+    __root__: entityReference.EntityReference = Field(
+        ...,
+        description='Reference to a property type. Only property types are allowed and entity types are not allowed as custom properties to extend an existing entity',
+    )
+
+
+class CustomProperty(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: basic.EntityName = Field(
+        ...,
+        description='Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.',
+    )
+    description: basic.Markdown
+    propertyType: PropertyType
+    customPropertyConfig: Optional[CustomPropertyConfig] = None

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-02-13T09:42:10+00:00
+#   timestamp: 2024-03-01T08:53:21+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -97,10 +97,12 @@ class OpenMetadataValidationAction(ValidationAction):
         database_service_name: Optional[str] = None,
         schema_name: Optional[str] = "default",
         database_name: Optional[str] = None,
+        table_name: Optional[str] = None,
     ):
         super().__init__(data_context)
         self.database_service_name = database_service_name
         self.database_name = database_name
+        self.table_name = table_name
         self.schema_name = schema_name  # for database without schema concept
         self.config_file_path = config_file_path
         self.ometa_conn = self._create_ometa_connection()
@@ -128,6 +130,7 @@ class OpenMetadataValidationAction(ValidationAction):
         """
 
         check_point_spec = self._get_checkpoint_batch_spec(data_asset)
+        table_entity = None
         if isinstance(check_point_spec, SqlAlchemyDatasourceBatchSpec):
             execution_engine_url = self._get_execution_engine_url(data_asset)
             table_entity = self._get_table_entity(
@@ -139,13 +142,10 @@ class OpenMetadataValidationAction(ValidationAction):
             )
 
         elif isinstance(check_point_spec, RuntimeDataBatchSpec):
-            table_name = self._get_metadata_from_validation_suite(
-                validation_result_suite
-            )
             table_entity = self._get_table_entity(
                 self.database_name,
                 self.schema_name,
-                table_name,
+                self.table_name,
             )
 
         if table_entity:
@@ -153,17 +153,6 @@ class OpenMetadataValidationAction(ValidationAction):
             for result in validation_result_suite.results:
                 self._handle_test_case(result, table_entity, test_suite)
 
-    def _get_metadata_from_validation_suite(self, validation_result_suite: dict) -> str:
-        # table_name_1, split on last "_" in the case there are multiple suites for one schema
-        try:
-            name = validation_result_suite["meta"]["expectation_suite_name"]
-            splitted_name = name.rpartition("_")
-            table_name = splitted_name[0]
-            return table_name
-
-        except KeyError:
-            raise KeyError("No suite name present in validation_result_suite")
-
     @staticmethod
     def _get_checkpoint_batch_spec(
         data_asset: Union[Validator, DataAsset, Batch]

metadata/ingestion/api/steps.py

@@ -12,10 +12,15 @@
 Abstract definition of each step
 """
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Iterable, Optional
 
+from metadata.ingestion.api.models import Entity
 from metadata.ingestion.api.step import BulkStep, IterStep, ReturnStep, StageStep
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.execution_time_tracker import (
+    calculate_execution_time,
+    calculate_execution_time_generator,
+)
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -50,6 +55,10 @@ class Source(IterStep, ABC):
     def name(self) -> str:
         return "Source"
 
+    @calculate_execution_time_generator(context="Source")
+    def run(self) -> Iterable[Optional[Entity]]:
+        yield from super().run()
+
 
 class Sink(ReturnStep, ABC):
     """All Sinks must inherit this base class."""
@@ -58,6 +67,10 @@ class Sink(ReturnStep, ABC):
     def name(self) -> str:
         return "Sink"
 
+    @calculate_execution_time(context="Sink")
+    def run(self, record: Entity) -> Optional[Entity]:
+        return super().run(record)
+
 
 class Processor(ReturnStep, ABC):
     """All Processor must inherit this base class"""

metadata/ingestion/api/topology_runner.py

@@ -255,12 +255,16 @@ class TopologyRunnerMixin(Generic[C]):
 
         # If we don't want to write data in OM, we'll return what we fetch from the API.
         # This will be applicable for service entities since we do not want to overwrite the data
+        same_fingerprint = False
         if not stage.overwrite and not self._is_force_overwrite_enabled():
             entity = self.metadata.get_by_name(
                 entity=stage.type_,
                 fqn=entity_fqn,
                 fields=["*"],
             )
+            if entity:
+                same_fingerprint = True
+
         create_entity_request_hash = generate_source_hash(
             create_request=entity_request.right,
         )
@@ -268,7 +272,6 @@ class TopologyRunnerMixin(Generic[C]):
         if hasattr(entity_request.right, "sourceHash"):
             entity_request.right.sourceHash = create_entity_request_hash
 
-        same_fingerprint = False
         if entity is None and stage.use_cache:
             # check if we find the entity in the entities list
             entity_source_hash = self.cache[stage.type_].get(entity_fqn)

metadata/ingestion/models/custom_properties.py

@@ -41,7 +41,6 @@ class CustomPropertyDataTypes(Enum):
 
 class OMetaCustomProperties(BaseModel):
     entity_type: Type[T]
-    custom_property_type: Optional[CustomPropertyDataTypes]
     createCustomPropertyRequest: CreateCustomPropertyRequest