openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtS3Config.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from ...security.credentials import awsCredentials
 
@@ -17,31 +20,40 @@ class DbtConfigType(Enum):
 
 
 class DbtPrefixConfig(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    dbtBucketName: Optional[str] = Field(
-        None,
-        description='Name of the bucket where the dbt files are stored',
-        title='DBT Bucket Name',
-    )
-    dbtObjectPrefix: Optional[str] = Field(
-        None,
-        description='Path of the folder where the dbt files are stored',
-        title='DBT Object Prefix',
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    dbtBucketName: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Name of the bucket where the dbt files are stored',
+            title='DBT Bucket Name',
+        ),
+    ]
+    dbtObjectPrefix: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Path of the folder where the dbt files are stored',
+            title='DBT Object Prefix',
+        ),
+    ]
 
 
 class DbtS3Config(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    dbtConfigType: DbtConfigType = Field(..., description='dbt Configuration type')
-    dbtSecurityConfig: awsCredentials.AWSCredentials = Field(
-        ..., title='DBT S3 Security Config'
-    )
-    dbtPrefixConfig: Optional[DbtPrefixConfig] = Field(
-        None,
-        description='Details of the bucket where the dbt files are stored',
-        title='DBT Prefix Config',
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    dbtConfigType: Annotated[DbtConfigType, Field(description='dbt Configuration type')]
+    dbtSecurityConfig: Annotated[
+        awsCredentials.AWSCredentials, Field(title='DBT S3 Security Config')
+    ]
+    dbtPrefixConfig: Annotated[
+        Optional[DbtPrefixConfig],
+        Field(
+            None,
+            description='Details of the bucket where the dbt files are stored',
+            title='DBT Prefix Config',
+        ),
+    ]

metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/messagingServiceMetadataPipeline.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from ..type import filterPattern
 
@@ -17,24 +20,44 @@ class MessagingMetadataConfigType(Enum):
 
 
 class MessagingServiceMetadataPipeline(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[MessagingMetadataConfigType] = Field(
-        MessagingMetadataConfigType.MessagingMetadata, description='Pipeline type'
-    )
-    topicFilterPattern: Optional[filterPattern.FilterPattern] = Field(
-        None,
-        description='Regex to only fetch topics that matches the pattern.',
-        title='Topic Filter Pattern',
-    )
-    generateSampleData: Optional[bool] = Field(
-        False,
-        description='Option to turn on/off generating sample data during metadata extraction.',
-        title='Generate Sample Data',
-    )
-    markDeletedTopics: Optional[bool] = Field(
-        True,
-        description='Optional configuration to soft delete topics in OpenMetadata if the source topics are deleted. Also, if the topic is deleted, all the associated entities like sample data, lineage, etc., with that topic will be deleted',
-        title='Mark Deleted Topics',
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    type: Annotated[
+        Optional[MessagingMetadataConfigType],
+        Field(
+            MessagingMetadataConfigType.MessagingMetadata, description='Pipeline type'
+        ),
+    ]
+    topicFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch topics that matches the pattern.',
+            title='Topic Filter Pattern',
+        ),
+    ]
+    generateSampleData: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Option to turn on/off generating sample data during metadata extraction.',
+            title='Generate Sample Data',
+        ),
+    ]
+    markDeletedTopics: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to soft delete topics in OpenMetadata if the source topics are deleted. Also, if the topic is deleted, all the associated entities like sample data, lineage, etc., with that topic will be deleted',
+            title='Mark Deleted Topics',
+        ),
+    ]
+    overrideMetadata: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
+            title='Override Metadata',
+        ),
+    ]

metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/metadataToElasticSearchPipeline.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from ..configuration import elasticSearchConfiguration
 
@@ -17,45 +20,64 @@ class MetadataToESConfigType(Enum):
 
 
 class MetadataToElasticSearchPipeline(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: MetadataToESConfigType = Field(..., description='Pipeline type')
-    regionName: Optional[str] = Field(
-        None,
-        description='Region name. Required when using AWS Credentials.',
-        title='AWS Region Name',
-    )
-    caCerts: Optional[str] = Field(
-        None,
-        description='Certificate path to be added in configuration. The path should be local in the Ingestion Container.',
-        title='CA Certificates',
-    )
-    timeout: Optional[int] = Field(30, description='Connection Timeout')
-    useSSL: Optional[bool] = Field(
-        False,
-        description='Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore SSL settings.',
-        title='Use SSL',
-    )
-    verifyCerts: Optional[bool] = Field(
-        False,
-        description='Indicates whether to verify certificates when using SSL connection to ElasticSearch. Ignored by default. Is set to true, make sure to send the certificates in the property `CA Certificates`.',
-        title='Validate Certificates',
-    )
-    useAwsCredentials: Optional[bool] = Field(
-        False,
-        description='Indicates whether to use aws credentials when connecting to OpenSearch in AWS.',
-        title='Use AWS Credentials',
-    )
-    searchIndexMappingLanguage: Optional[
-        elasticSearchConfiguration.SearchIndexMappingLanguage
-    ] = Field(
-        elasticSearchConfiguration.SearchIndexMappingLanguage.EN,
-        description='Recreate Indexes with updated Language',
-    )
-    batchSize: Optional[int] = Field(
-        1000,
-        description='Maximum number of events entities in a batch (Default 1000).',
-        title='Batch Size',
+    model_config = ConfigDict(
+        extra='forbid',
     )
-    recreateIndex: Optional[bool] = Field(True, title='Recreate Indexes')
+    type: Annotated[MetadataToESConfigType, Field(description='Pipeline type')]
+    regionName: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Region name. Required when using AWS Credentials.',
+            title='AWS Region Name',
+        ),
+    ]
+    caCerts: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Certificate path to be added in configuration. The path should be local in the Ingestion Container.',
+            title='CA Certificates',
+        ),
+    ]
+    timeout: Annotated[Optional[int], Field(30, description='Connection Timeout')]
+    useSSL: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore SSL settings.',
+            title='Use SSL',
+        ),
+    ]
+    verifyCerts: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Indicates whether to verify certificates when using SSL connection to ElasticSearch. Ignored by default. Is set to true, make sure to send the certificates in the property `CA Certificates`.',
+            title='Validate Certificates',
+        ),
+    ]
+    useAwsCredentials: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Indicates whether to use aws credentials when connecting to OpenSearch in AWS.',
+            title='Use AWS Credentials',
+        ),
+    ]
+    searchIndexMappingLanguage: Annotated[
+        Optional[elasticSearchConfiguration.SearchIndexMappingLanguage],
+        Field(
+            elasticSearchConfiguration.SearchIndexMappingLanguage.EN,
+            description='Recreate Indexes with updated Language',
+        ),
+    ]
+    batchSize: Annotated[
+        Optional[int],
+        Field(
+            1000,
+            description='Maximum number of events entities in a batch (Default 1000).',
+            title='Batch Size',
+        ),
+    ]
+    recreateIndex: Annotated[Optional[bool], Field(True, title='Recreate Indexes')]

metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/mlmodelServiceMetadataPipeline.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from ..type import filterPattern
 
@@ -17,19 +20,34 @@ class MlModelMetadataConfigType(Enum):
 
 
 class MlModelServiceMetadataPipeline(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[MlModelMetadataConfigType] = Field(
-        MlModelMetadataConfigType.MlModelMetadata, description='Pipeline type'
-    )
-    mlModelFilterPattern: Optional[filterPattern.FilterPattern] = Field(
-        None,
-        description='Regex to only fetch MlModels with names matching the pattern.',
-        title='ML Model Filter Pattern',
-    )
-    markDeletedMlModels: Optional[bool] = Field(
-        True,
-        description='Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., with that MlModels will be deleted',
-        title='Mark Deleted ML Models',
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    type: Annotated[
+        Optional[MlModelMetadataConfigType],
+        Field(MlModelMetadataConfigType.MlModelMetadata, description='Pipeline type'),
+    ]
+    mlModelFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch MlModels with names matching the pattern.',
+            title='ML Model Filter Pattern',
+        ),
+    ]
+    markDeletedMlModels: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., with that MlModels will be deleted',
+            title='Mark Deleted ML Models',
+        ),
+    ]
+    overrideMetadata: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
+            title='Override Metadata',
+        ),
+    ]

metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py

@@ -1,28 +1,37 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/pipelineServiceMetadataPipeline.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import List, Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from ..type import filterPattern
 
 
 class LineageInformation(BaseModel):
-    dbServiceNames: Optional[List[str]] = Field(
-        None,
-        description='List of Database Service Names for creation of lineage',
-        title='Database Service Names List',
-    )
-    storageServiceNames: Optional[List[str]] = Field(
-        None,
-        description='List of Storage Service Names for creation of lineage',
-        title='Storage Service Names List',
-    )
+    dbServiceNames: Annotated[
+        Optional[List[str]],
+        Field(
+            None,
+            description='List of Database Service Names for creation of lineage',
+            title='Database Service Names List',
+        ),
+    ]
+    storageServiceNames: Annotated[
+        Optional[List[str]],
+        Field(
+            None,
+            description='List of Storage Service Names for creation of lineage',
+            title='Storage Service Names List',
+        ),
+    ]
 
 
 class PipelineMetadataConfigType(Enum):
@@ -30,42 +39,82 @@ class PipelineMetadataConfigType(Enum):
 
 
 class PipelineServiceMetadataPipeline(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[PipelineMetadataConfigType] = Field(
-        PipelineMetadataConfigType.PipelineMetadata, description='Pipeline type'
-    )
-    includeLineage: Optional[bool] = Field(
-        True,
-        description='Optional configuration to turn off fetching lineage from pipelines.',
-        title='Include Lineage',
-    )
-    includeOwners: Optional[bool] = Field(
-        True,
-        description="Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.",
-        title='Include Owners',
-    )
-    pipelineFilterPattern: Optional[filterPattern.FilterPattern] = Field(
-        None, description='Regex exclude pipelines.', title='Pipeline Filter Pattern'
-    )
-    lineageInformation: Optional[LineageInformation] = Field(
-        None,
-        description='Details required to generate Lineage',
-        title='Lineage Information',
-    )
-    markDeletedPipelines: Optional[bool] = Field(
-        True,
-        description='Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, etc., with that Pipeline will be deleted',
-        title='Mark Deleted Pipeline',
-    )
-    includeTags: Optional[bool] = Field(
-        True,
-        description='Optional configuration to toggle the tags ingestion.',
-        title='Include Tags',
-    )
-    includeUnDeployedPipelines: Optional[bool] = Field(
-        True,
-        description='Optional configuration to toggle whether the un-deployed pipelines should be ingested or not. If set to false, only deployed pipelines will be ingested.',
-        title='Include UnDeployed Pipelines',
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    type: Annotated[
+        Optional[PipelineMetadataConfigType],
+        Field(PipelineMetadataConfigType.PipelineMetadata, description='Pipeline type'),
+    ]
+    includeLineage: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to turn off fetching lineage from pipelines.',
+            title='Include Lineage',
+        ),
+    ]
+    includeOwners: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description="Set the 'Include Owners' toggle to control whether to include owners to the ingested entity if the owner email matches with a user stored in the OM server as part of metadata ingestion. If the ingested entity already exists and has an owner, the owner will not be overwritten.",
+            title='Include Owners',
+        ),
+    ]
+    pipelineFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex exclude pipelines.',
+            title='Pipeline Filter Pattern',
+        ),
+    ]
+    lineageInformation: Annotated[
+        Optional[LineageInformation],
+        Field(
+            None,
+            description='Details required to generate Lineage',
+            title='Lineage Information',
+        ),
+    ]
+    overrideLineage: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Set the 'Override Lineage' toggle to control whether to override the existing lineage.",
+            title='Override Lineage',
+        ),
+    ]
+    markDeletedPipelines: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, etc., with that Pipeline will be deleted',
+            title='Mark Deleted Pipeline',
+        ),
+    ]
+    includeTags: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to toggle the tags ingestion.',
+            title='Include Tags',
+        ),
+    ]
+    includeUnDeployedPipelines: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to toggle whether the un-deployed pipelines should be ingested or not. If set to false, only deployed pipelines will be ingested.',
+            title='Include UnDeployed Pipelines',
+        ),
+    ]
+    overrideMetadata: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
+            title='Override Metadata',
+        ),
+    ]

metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/searchServiceMetadataPipeline.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from ..type import filterPattern
 
@@ -17,29 +20,50 @@ class SearchMetadataConfigType(Enum):
 
 
 class SearchServiceMetadataPipeline(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[SearchMetadataConfigType] = Field(
-        SearchMetadataConfigType.SearchMetadata, description='Pipeline type'
-    )
-    searchIndexFilterPattern: Optional[filterPattern.FilterPattern] = Field(
-        None,
-        description='Regex to only fetch search indexes that matches the pattern.',
-        title='Search Index Filter Pattern',
-    )
-    markDeletedSearchIndexes: Optional[bool] = Field(
-        True,
-        description='Optional configuration to soft delete search indexes in OpenMetadata if the source search indexes are deleted. Also, if the search index is deleted, all the associated entities like lineage, etc., with that search index will be deleted',
-        title='Mark Deleted Search Index',
-    )
-    includeSampleData: Optional[bool] = Field(
-        True,
-        description='Optional configuration to turn off fetching sample data for search index.',
-        title='Include Sample Data',
-    )
-    sampleSize: Optional[int] = Field(
-        10,
-        description='No. of records of sample data we want to ingest.',
-        title='Sample Size',
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    type: Annotated[
+        Optional[SearchMetadataConfigType],
+        Field(SearchMetadataConfigType.SearchMetadata, description='Pipeline type'),
+    ]
+    searchIndexFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch search indexes that matches the pattern.',
+            title='Search Index Filter Pattern',
+        ),
+    ]
+    markDeletedSearchIndexes: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to soft delete search indexes in OpenMetadata if the source search indexes are deleted. Also, if the search index is deleted, all the associated entities like lineage, etc., with that search index will be deleted',
+            title='Mark Deleted Search Index',
+        ),
+    ]
+    includeSampleData: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to turn off fetching sample data for search index.',
+            title='Include Sample Data',
+        ),
+    ]
+    sampleSize: Annotated[
+        Optional[int],
+        Field(
+            10,
+            description='No. of records of sample data we want to ingest.',
+            title='Sample Size',
+        ),
+    ]
+    overrideMetadata: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
+            title='Override Metadata',
+        ),
+    ]

metadata/generated/schema/metadataIngestion/storage/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00