openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/generated/schema/type/changeEvent.py

@@ -1,92 +1,22 @@
 # generated by datamodel-codegen:
 #   filename:  type/changeEvent.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
-from enum import Enum
-from typing import Any, List, Optional
+from typing import Any, Optional
 
 from pydantic import BaseModel, Extra, Field
 
-from . import basic, entityHistory
-
-
-class EventType(Enum):
-    entityCreated = 'entityCreated'
-    entityUpdated = 'entityUpdated'
-    entityNoChange = 'entityNoChange'
-    entitySoftDeleted = 'entitySoftDeleted'
-    entityDeleted = 'entityDeleted'
-    entityRestored = 'entityRestored'
-
-
-class EntityTypes(Enum):
-    All = 'All'
-    Chart = 'Chart'
-    Dashboard = 'Dashboard'
-    Database = 'Database'
-    Database_Schema = 'Database Schema'
-    Glossary = 'Glossary'
-    Glossary_Term = 'Glossary Term'
-    Location = 'Location'
-    Metrics = 'Metrics'
-    Ml_Model = 'Ml Model'
-    Pipeline = 'Pipeline'
-    Report = 'Report'
-    Table = 'Table'
-    Topic = 'Topic'
-    Test_Case = 'Test Case'
-
-
-class FieldTypes(Enum):
-    All = 'All'
-    Display_Name = 'Display Name'
-    Description = 'Description'
-    Owner = 'Owner'
-    Location = 'Location'
-    Tags = 'Tags'
-    Usage_Summary = 'Usage Summary'
-    Followers = 'Followers'
-    Sample_Data = 'Sample Data'
-    Synonyms = 'Synonyms'
-    Glossary = 'Glossary'
-    Test_Case_Result_Success = 'Test Case Result Success'
-    Test_Case_Result_Failed = 'Test Case Result Failed'
-    Test_Case_Result_Aborted = 'Test Case Result Aborted'
-
-
-class Filters(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    eventType: EventType = Field(..., description='Event type that is being requested.')
-    include: Optional[List[str]] = Field(
-        ['all'], description='Field which are allowed to pass', unique_items=True
-    )
-    exclude: Optional[List[str]] = Field(
-        [], description='Field which are not allowed to pass', unique_items=True
-    )
-
-
-class EventFilter(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    entityType: str = Field(
-        ...,
-        description='Entity type for filter , example : topic , table, dashboard, mlmodel , etc',
-    )
-    filters: Optional[List[Filters]] = Field(
-        None, description='List of operations supported by the resource.'
-    )
+from . import basic, changeEventType, entityHistory
 
 
 class ChangeEvent(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    eventType: EventType
+    id: basic.Uuid = Field(..., description='Unique identifier for the event.')
+    eventType: changeEventType.EventType
     entityType: str = Field(
         ...,
         description='Entity type that changed. Use the schema of this entity to process the entity attribute.',

metadata/generated/schema/type/changeEventType.py

@@ -0,0 +1,29 @@
+# generated by datamodel-codegen:
+#   filename:  type/changeEventType.json
+#   timestamp: 2024-03-25T11:44:26+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class EventType(Enum):
+    entityCreated = 'entityCreated'
+    entityUpdated = 'entityUpdated'
+    entityFieldsChanged = 'entityFieldsChanged'
+    entityNoChange = 'entityNoChange'
+    entitySoftDeleted = 'entitySoftDeleted'
+    entityDeleted = 'entityDeleted'
+    entityRestored = 'entityRestored'
+    threadCreated = 'threadCreated'
+    threadUpdated = 'threadUpdated'
+    postCreated = 'postCreated'
+    postUpdated = 'postUpdated'
+    taskResolved = 'taskResolved'
+    taskClosed = 'taskClosed'
+    logicalTestCaseAdded = 'logicalTestCaseAdded'
+    suggestionCreated = 'suggestionCreated'
+    suggestionUpdated = 'suggestionUpdated'
+    suggestionAccepted = 'suggestionAccepted'
+    suggestionRejected = 'suggestionRejected'
+    suggestionDeleted = 'suggestionDeleted'

metadata/generated/schema/type/collectionDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/collectionDescriptor.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvDocumentation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvDocumentation.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvErrorType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvErrorType.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvFile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvFile.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/csvImportResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/csvImportResult.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/customProperties/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2024-03-25T11:44:26+00:00

metadata/generated/schema/type/customProperties/enumConfig.py

@@ -0,0 +1,17 @@
+# generated by datamodel-codegen:
+#   filename:  type/customProperties/enumConfig.json
+#   timestamp: 2024-03-25T11:44:26+00:00
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Extra
+
+
+class EnumConfig(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    multiSelect: Optional[bool] = False
+    values: Optional[List[str]] = None

metadata/generated/schema/type/customProperty.py

@@ -0,0 +1,52 @@
+# generated by datamodel-codegen:
+#   filename:  type/customProperty.json
+#   timestamp: 2024-03-25T11:44:26+00:00
+
+from __future__ import annotations
+
+from typing import Optional, Union
+
+from pydantic import BaseModel, Extra, Field
+
+from . import basic, entityReference
+from .customProperties import enumConfig
+
+
+class Format(BaseModel):
+    __root__: str = Field(
+        ..., description='Applies to date interval, date, time format.'
+    )
+
+
+class EntityTypes(BaseModel):
+    __root__: str = Field(
+        ...,
+        description='Applies to Entity References. Entity Types can be used to restrict what type of entities can be configured for a entity reference.',
+    )
+
+
+class CustomPropertyConfig(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    config: Optional[Union[enumConfig.EnumConfig, Format, EntityTypes]] = None
+
+
+class PropertyType(BaseModel):
+    __root__: entityReference.EntityReference = Field(
+        ...,
+        description='Reference to a property type. Only property types are allowed and entity types are not allowed as custom properties to extend an existing entity',
+    )
+
+
+class CustomProperty(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: basic.EntityName = Field(
+        ...,
+        description='Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.',
+    )
+    description: basic.Markdown
+    propertyType: PropertyType
+    customPropertyConfig: Optional[CustomPropertyConfig] = None

metadata/generated/schema/type/dailyCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/dailyCount.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/databaseConnectionConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/databaseConnectionConfig.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityHistory.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityHistory.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityLineage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityLineage.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -34,6 +34,7 @@ class Source(Enum):
     PipelineLineage = 'PipelineLineage'
     DashboardLineage = 'DashboardLineage'
     DbtLineage = 'DbtLineage'
+    SparkLineage = 'SparkLineage'
 
 
 class LineageDetails(BaseModel):

metadata/generated/schema/type/entityReference.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReference.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -36,4 +36,8 @@ class EntityReference(BaseModel):
     deleted: Optional[bool] = Field(
         None, description='If true the entity referred to has been soft-deleted.'
     )
+    inherited: Optional[bool] = Field(
+        None,
+        description='If true the relationship indicated by this entity reference is inherited from the parent entity.',
+    )
     href: Optional[basic.Href] = Field(None, description='Link to the entity resource.')

metadata/generated/schema/type/entityReferenceList.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityReferenceList.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityRelationship.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityRelationship.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,11 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import List, Optional
+from typing import Any, List, Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -15,15 +15,14 @@ class ParameterType(Enum):
     AllIndexElasticSearch = 'AllIndexElasticSearch'
     SpecificIndexElasticSearch = 'SpecificIndexElasticSearch'
     ReadFromParamContext = 'ReadFromParamContext'
+    ReadFromParamContextPerEntity = 'ReadFromParamContextPerEntity'
 
 
 class ParamAdditionalContext(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    data: Optional[List[str]] = Field(
-        None, description='List of Entities', unique_items=True
-    )
+    data: Optional[Any] = Field(None, description='List of Entities')
 
 
 class Function(BaseModel):
@@ -34,7 +33,9 @@ class Function(BaseModel):
     input: Optional[str] = Field(
         None, description='Description of input taken by the function.'
     )
-    description: Optional[str] = Field(None, description='Description fo the function.')
+    description: Optional[str] = Field(
+        None, description='Description for the function.'
+    )
     examples: Optional[List] = Field(
         None, description='Examples of the function to help users author conditions.'
     )

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -35,7 +35,7 @@ class ParsedData(BaseModel):
         None, description='Database schema of the associated with query'
     )
     duration: Optional[float] = Field(
-        None, description='How long did the query took to run in seconds.'
+        None, description='How long did the query took to run in milliseconds.'
     )
 
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -43,7 +43,7 @@ class TableQuery(BaseModel):
         None, description='Database schema of the associated with query'
     )
     duration: Optional[float] = Field(
-        None, description='How long did the query took to run in seconds.'
+        None, description='How long did the query took to run in milliseconds.'
     )
 
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action.py

@@ -15,6 +15,7 @@ Open Metadata table quality.
 This subpackage needs to be used in Great Expectations
 checkpoints actions.
 """
+import logging
 import traceback
 from datetime import datetime, timezone
 from typing import Dict, List, Optional, Union, cast
@@ -72,9 +73,10 @@ from metadata.great_expectations.utils.ometa_config_handler import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 from metadata.utils.entity_link import get_entity_link
-from metadata.utils.logger import great_expectations_logger
 
-logger = great_expectations_logger()
+logger = logging.getLogger(
+    "great_expectations.validation_operators.validation_operators.openmetadata"
+)
 
 
 class OpenMetadataValidationAction(ValidationAction):
@@ -97,10 +99,12 @@ class OpenMetadataValidationAction(ValidationAction):
         database_service_name: Optional[str] = None,
         schema_name: Optional[str] = "default",
         database_name: Optional[str] = None,
+        table_name: Optional[str] = None,
     ):
         super().__init__(data_context)
         self.database_service_name = database_service_name
         self.database_name = database_name
+        self.table_name = table_name
         self.schema_name = schema_name  # for database without schema concept
         self.config_file_path = config_file_path
         self.ometa_conn = self._create_ometa_connection()
@@ -128,6 +132,7 @@ class OpenMetadataValidationAction(ValidationAction):
         """
 
         check_point_spec = self._get_checkpoint_batch_spec(data_asset)
+        table_entity = None
         if isinstance(check_point_spec, SqlAlchemyDatasourceBatchSpec):
             execution_engine_url = self._get_execution_engine_url(data_asset)
             table_entity = self._get_table_entity(
@@ -139,13 +144,10 @@ class OpenMetadataValidationAction(ValidationAction):
             )
 
         elif isinstance(check_point_spec, RuntimeDataBatchSpec):
-            table_name = self._get_metadata_from_validation_suite(
-                validation_result_suite
-            )
             table_entity = self._get_table_entity(
                 self.database_name,
                 self.schema_name,
-                table_name,
+                self.table_name,
             )
 
         if table_entity:
@@ -153,17 +155,6 @@ class OpenMetadataValidationAction(ValidationAction):
             for result in validation_result_suite.results:
                 self._handle_test_case(result, table_entity, test_suite)
 
-    def _get_metadata_from_validation_suite(self, validation_result_suite: dict) -> str:
-        # table_name_1, split on last "_" in the case there are multiple suites for one schema
-        try:
-            name = validation_result_suite["meta"]["expectation_suite_name"]
-            splitted_name = name.rpartition("_")
-            table_name = splitted_name[0]
-            return table_name
-
-        except KeyError:
-            raise KeyError("No suite name present in validation_result_suite")
-
     @staticmethod
     def _get_checkpoint_batch_spec(
         data_asset: Union[Validator, DataAsset, Batch]
@@ -420,8 +411,9 @@ class OpenMetadataValidationAction(ValidationAction):
             test_case = self.ometa_conn.get_or_create_test_case(
                 test_case_fqn,
                 entity_link=get_entity_link(
-                    table_entity.fullyQualifiedName.__root__,
-                    fqn.split_test_case_fqn(test_case_fqn).column,
+                    Table,
+                    fqn=table_entity.fullyQualifiedName.__root__,
+                    column_name=fqn.split_test_case_fqn(test_case_fqn).column,
                 ),
                 test_suite_fqn=test_suite.fullyQualifiedName.__root__,
                 test_definition_fqn=test_definition.fullyQualifiedName.__root__,
@@ -439,7 +431,7 @@ class OpenMetadataValidationAction(ValidationAction):
                 test_case_fqn=test_case.fullyQualifiedName.__root__,
             )
 
-            logger.info(
+            logger.debug(
                 f"Test case result for {test_case.fullyQualifiedName.__root__} successfully ingested"
             )
 

metadata/ingestion/api/delete.py

@@ -14,7 +14,10 @@ Delete methods
 import traceback
 from typing import Dict, Iterable, Optional, Type
 
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.ometa.ometa_api import OpenMetadata, T
 from metadata.utils.logger import utils_logger
@@ -52,6 +55,6 @@ def delete_entity_from_source(
             left=StackTraceError(
                 name="Delete Entity",
                 error=f"Error deleting {entity_type.__class__}: {exc}",
-                stack_trace=traceback.format_exc(),
+                stackTrace=traceback.format_exc(),
             )
         )