openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py

@@ -1,15 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/unityCatalogConnection.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
 
-from metadata.ingestion.models.custom_pydantic import CustomSecretStr
+from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .. import connectionBasicType
 
@@ -23,49 +24,72 @@ class DatabricksScheme(Enum):
 
 
 class UnityCatalogConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[DatabricksType] = Field(
-        DatabricksType.UnityCatalog, description='Service Type', title='Service Type'
-    )
-    scheme: Optional[DatabricksScheme] = Field(
-        DatabricksScheme.databricks_connector,
-        description='SQLAlchemy driver scheme options.',
-        title='Connection Scheme',
-    )
-    hostPort: str = Field(
-        ...,
-        description='Host and port of the Databricks service.',
-        title='Host and Port',
-    )
-    token: CustomSecretStr = Field(
-        ..., description='Generated Token to connect to Databricks.', title='Token'
-    )
-    httpPath: Optional[str] = Field(
-        None, description='Databricks compute resources URL.', title='Http Path'
-    )
-    catalog: Optional[str] = Field(
-        None,
-        description='Catalog of the data source(Example: hive_metastore). This is optional parameter, if you would like to restrict the metadata reading to a single catalog. When left blank, OpenMetadata Ingestion attempts to scan all the catalog.',
-        title='Catalog',
-    )
-    databaseSchema: Optional[str] = Field(
-        None,
-        description='Database Schema of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion attempts to scan all the schemas.',
-        title='Database Schema',
-    )
-    connectionTimeout: Optional[int] = Field(
-        120,
-        description='The maximum amount of time (in seconds) to wait for a successful connection to the data source. If the connection attempt takes longer than this timeout period, an error will be returned.',
-        title='Connection Timeout',
-    )
-    connectionOptions: Optional[connectionBasicType.ConnectionOptions] = Field(
-        None, title='Connection Options'
-    )
-    connectionArguments: Optional[connectionBasicType.ConnectionArguments] = Field(
-        None, title='Connection Arguments'
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    type: Annotated[
+        Optional[DatabricksType],
+        Field(
+            DatabricksType.UnityCatalog,
+            description='Service Type',
+            title='Service Type',
+        ),
+    ]
+    scheme: Annotated[
+        Optional[DatabricksScheme],
+        Field(
+            DatabricksScheme.databricks_connector,
+            description='SQLAlchemy driver scheme options.',
+            title='Connection Scheme',
+        ),
+    ]
+    hostPort: Annotated[
+        str,
+        Field(
+            description='Host and port of the Databricks service.',
+            title='Host and Port',
+        ),
+    ]
+    token: Annotated[
+        CustomSecretStr,
+        Field(description='Generated Token to connect to Databricks.', title='Token'),
+    ]
+    httpPath: Annotated[
+        Optional[str],
+        Field(None, description='Databricks compute resources URL.', title='Http Path'),
+    ]
+    catalog: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Catalog of the data source(Example: hive_metastore). This is optional parameter, if you would like to restrict the metadata reading to a single catalog. When left blank, OpenMetadata Ingestion attempts to scan all the catalog.',
+            title='Catalog',
+        ),
+    ]
+    databaseSchema: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Database Schema of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion attempts to scan all the schemas.',
+            title='Database Schema',
+        ),
+    ]
+    connectionTimeout: Annotated[
+        Optional[int],
+        Field(
+            120,
+            description='The maximum amount of time (in seconds) to wait for a successful connection to the data source. If the connection attempt takes longer than this timeout period, an error will be returned.',
+            title='Connection Timeout',
+        ),
+    ]
+    connectionOptions: Annotated[
+        Optional[connectionBasicType.ConnectionOptions],
+        Field(None, title='Connection Options'),
+    ]
+    connectionArguments: Annotated[
+        Optional[connectionBasicType.ConnectionArguments],
+        Field(None, title='Connection Arguments'),
+    ]
     supportsUsageExtraction: Optional[
         connectionBasicType.SupportsUsageExtraction
     ] = None
@@ -73,18 +97,23 @@ class UnityCatalogConnection(BaseModel):
         connectionBasicType.SupportsLineageExtraction
     ] = None
     supportsDBTExtraction: Optional[connectionBasicType.SupportsDBTExtraction] = None
-    supportsMetadataExtraction: Optional[
-        connectionBasicType.SupportsMetadataExtraction
-    ] = Field(None, title='Supports Metadata Extraction')
-    supportsProfiler: Optional[connectionBasicType.SupportsProfiler] = Field(
-        None, title='Supports Profiler'
-    )
-    supportsDatabase: Optional[connectionBasicType.SupportsDatabase] = Field(
-        None, title='Supports Database'
-    )
-    supportsQueryComment: Optional[connectionBasicType.SupportsQueryComment] = Field(
-        None, title='Supports Query Comment'
-    )
-    sampleDataStorageConfig: Optional[
-        connectionBasicType.SampleDataStorageConfig
-    ] = Field(None, title='Storage Config for Sample Data')
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]
+    supportsProfiler: Annotated[
+        Optional[connectionBasicType.SupportsProfiler],
+        Field(None, title='Supports Profiler'),
+    ]
+    supportsDatabase: Annotated[
+        Optional[connectionBasicType.SupportsDatabase],
+        Field(None, title='Supports Database'),
+    ]
+    supportsQueryComment: Annotated[
+        Optional[connectionBasicType.SupportsQueryComment],
+        Field(None, title='Supports Query Comment'),
+    ]
+    sampleDataStorageConfig: Annotated[
+        Optional[connectionBasicType.SampleDataStorageConfig],
+        Field(None, title='Storage Config for Sample Data'),
+    ]

metadata/generated/schema/entity/services/connections/database/verticaConnection.py

@@ -1,15 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/verticaConnection.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
 
-from metadata.ingestion.models.custom_pydantic import CustomSecretStr
+from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .. import connectionBasicType
 
@@ -23,42 +24,58 @@ class VerticaScheme(Enum):
 
 
 class VerticaConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[VerticaType] = Field(
-        VerticaType.Vertica, description='Service Type', title='Service Type'
-    )
-    scheme: Optional[VerticaScheme] = Field(
-        VerticaScheme.vertica_vertica_python,
-        description='SQLAlchemy driver scheme options.',
-        title='Connection Scheme',
-    )
-    username: str = Field(
-        ...,
-        description='Username to connect to Vertica. This user should have privileges to read all the metadata in Vertica.',
-        title='Username',
-    )
-    password: Optional[CustomSecretStr] = Field(
-        None, description='Password to connect to Vertica.', title='Password'
-    )
-    hostPort: str = Field(
-        ..., description='Host and port of the Vertica service.', title='Host and Port'
-    )
-    database: Optional[str] = Field(
-        None,
-        description='Database of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single database. When left blank, OpenMetadata Ingestion attempts to scan all the databases.',
-        title='Database',
+    model_config = ConfigDict(
+        extra='forbid',
     )
-    connectionOptions: Optional[connectionBasicType.ConnectionOptions] = Field(
-        None, title='Connection Options'
-    )
-    connectionArguments: Optional[connectionBasicType.ConnectionArguments] = Field(
-        None, title='Connection Arguments'
-    )
-    supportsMetadataExtraction: Optional[
-        connectionBasicType.SupportsMetadataExtraction
-    ] = Field(None, title='Supports Metadata Extraction')
+    type: Annotated[
+        Optional[VerticaType],
+        Field(VerticaType.Vertica, description='Service Type', title='Service Type'),
+    ]
+    scheme: Annotated[
+        Optional[VerticaScheme],
+        Field(
+            VerticaScheme.vertica_vertica_python,
+            description='SQLAlchemy driver scheme options.',
+            title='Connection Scheme',
+        ),
+    ]
+    username: Annotated[
+        str,
+        Field(
+            description='Username to connect to Vertica. This user should have privileges to read all the metadata in Vertica.',
+            title='Username',
+        ),
+    ]
+    password: Annotated[
+        Optional[CustomSecretStr],
+        Field(None, description='Password to connect to Vertica.', title='Password'),
+    ]
+    hostPort: Annotated[
+        str,
+        Field(
+            description='Host and port of the Vertica service.', title='Host and Port'
+        ),
+    ]
+    database: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Database of the data source. This is optional parameter, if you would like to restrict the metadata reading to a single database. When left blank, OpenMetadata Ingestion attempts to scan all the databases.',
+            title='Database',
+        ),
+    ]
+    connectionOptions: Annotated[
+        Optional[connectionBasicType.ConnectionOptions],
+        Field(None, title='Connection Options'),
+    ]
+    connectionArguments: Annotated[
+        Optional[connectionBasicType.ConnectionArguments],
+        Field(None, title='Connection Arguments'),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]
     supportsUsageExtraction: Optional[
         connectionBasicType.SupportsUsageExtraction
     ] = None
@@ -66,15 +83,19 @@ class VerticaConnection(BaseModel):
         connectionBasicType.SupportsLineageExtraction
     ] = None
     supportsDBTExtraction: Optional[connectionBasicType.SupportsDBTExtraction] = None
-    supportsProfiler: Optional[connectionBasicType.SupportsProfiler] = Field(
-        None, title='Supports Profiler'
-    )
-    supportsDatabase: Optional[connectionBasicType.SupportsDatabase] = Field(
-        None, title='Supports Database'
-    )
-    supportsQueryComment: Optional[connectionBasicType.SupportsQueryComment] = Field(
-        None, title='Supports Query Comment'
-    )
-    sampleDataStorageConfig: Optional[
-        connectionBasicType.SampleDataStorageConfig
-    ] = Field(None, title='Storage Config for Sample Data')
+    supportsProfiler: Annotated[
+        Optional[connectionBasicType.SupportsProfiler],
+        Field(None, title='Supports Profiler'),
+    ]
+    supportsDatabase: Annotated[
+        Optional[connectionBasicType.SupportsDatabase],
+        Field(None, title='Supports Database'),
+    ]
+    supportsQueryComment: Annotated[
+        Optional[connectionBasicType.SupportsQueryComment],
+        Field(None, title='Supports Query Comment'),
+    ]
+    sampleDataStorageConfig: Annotated[
+        Optional[connectionBasicType.SampleDataStorageConfig],
+        Field(None, title='Storage Config for Sample Data'),
+    ]

metadata/generated/schema/entity/services/connections/messaging/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00

metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/customMessagingConnection.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .. import connectionBasicType
 
@@ -17,17 +20,22 @@ class CustomMessagingType(Enum):
 
 
 class CustomMessagingConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: CustomMessagingType = Field(
-        ..., description='Custom messaging service type', title='Service Type'
-    )
-    sourcePythonClass: Optional[str] = Field(
-        None,
-        description='Source Python Class Name to instantiated by the ingestion workflow',
-        title='Source Python Class Name',
-    )
-    connectionOptions: Optional[connectionBasicType.ConnectionOptions] = Field(
-        None, title='Connection Options'
+    model_config = ConfigDict(
+        extra='forbid',
     )
+    type: Annotated[
+        CustomMessagingType,
+        Field(description='Custom messaging service type', title='Service Type'),
+    ]
+    sourcePythonClass: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Source Python Class Name to instantiated by the ingestion workflow',
+            title='Source Python Class Name',
+        ),
+    ]
+    connectionOptions: Annotated[
+        Optional[connectionBasicType.ConnectionOptions],
+        Field(None, title='Connection Options'),
+    ]

metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py

@@ -1,15 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/kafkaConnection.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Any, Dict, Optional
 
-from pydantic import AnyUrl, BaseModel, Extra, Field
+from pydantic import AnyUrl, ConfigDict, Field
+from typing_extensions import Annotated
 
-from metadata.ingestion.models.custom_pydantic import CustomSecretStr
+from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .....security.ssl import verifySSLConfig
 from .. import connectionBasicType
@@ -28,62 +29,101 @@ class KafkaType(Enum):
 
 
 class KafkaConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[KafkaType] = Field(
-        KafkaType.Kafka, description='Service Type', title='Service Type'
-    )
-    bootstrapServers: str = Field(
-        ...,
-        description='Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092',
-        title='Bootstrap Servers',
-    )
-    schemaRegistryURL: Optional[AnyUrl] = Field(
-        None,
-        description='Confluent Kafka Schema Registry URL.',
-        title='Schema Registry URL',
-    )
-    saslUsername: Optional[str] = Field(
-        None,
-        description='sasl.username consumer config property',
-        title='SASL Username',
-    )
-    saslPassword: Optional[CustomSecretStr] = Field(
-        None,
-        description='sasl.password consumer config property',
-        title='SASL Password',
-    )
-    securityProtocol: Optional[SecurityProtocol] = Field(
-        SecurityProtocol.PLAINTEXT,
-        description='security.protocol consumer config property',
-        title='Security Protocol',
-    )
-    saslMechanism: Optional[saslMechanismType.SaslMechanismType] = Field(
-        saslMechanismType.SaslMechanismType.PLAIN,
-        description='sasl.mechanism Consumer Config property',
-        title='SASL Mechanism',
-    )
-    basicAuthUserInfo: Optional[CustomSecretStr] = Field(
-        None,
-        description='basic.auth.user.info schema registry config property, Client HTTP credentials in the form of username:password.',
-        title='Schema Registry Basic Auth User Info',
-    )
-    consumerConfig: Optional[Dict[str, Any]] = Field(
-        {},
-        description='Confluent Kafka Consumer Config. From https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md',
-        title='Consumer Config',
-    )
-    schemaRegistryConfig: Optional[Dict[str, Any]] = Field(
-        {},
-        description='Confluent Kafka Schema Registry Config. From https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient',
-        title='Schema Registry Config',
-    )
-    schemaRegistrySSL: Optional[verifySSLConfig.SslConfig] = Field(
-        None,
-        description='Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry connection.',
-        title='Schema Registry SSL',
+    model_config = ConfigDict(
+        extra='forbid',
     )
-    supportsMetadataExtraction: Optional[
-        connectionBasicType.SupportsMetadataExtraction
-    ] = Field(None, title='Supports Metadata Extraction')
+    type: Annotated[
+        Optional[KafkaType],
+        Field(KafkaType.Kafka, description='Service Type', title='Service Type'),
+    ]
+    bootstrapServers: Annotated[
+        str,
+        Field(
+            description='Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092',
+            title='Bootstrap Servers',
+        ),
+    ]
+    schemaRegistryURL: Annotated[
+        Optional[AnyUrl],
+        Field(
+            None,
+            description='Confluent Kafka Schema Registry URL.',
+            title='Schema Registry URL',
+        ),
+    ]
+    saslUsername: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='sasl.username consumer config property',
+            title='SASL Username',
+        ),
+    ]
+    saslPassword: Annotated[
+        Optional[CustomSecretStr],
+        Field(
+            None,
+            description='sasl.password consumer config property',
+            title='SASL Password',
+        ),
+    ]
+    securityProtocol: Annotated[
+        Optional[SecurityProtocol],
+        Field(
+            SecurityProtocol.PLAINTEXT,
+            description='security.protocol consumer config property',
+            title='Security Protocol',
+        ),
+    ]
+    saslMechanism: Annotated[
+        Optional[saslMechanismType.SaslMechanismType],
+        Field(
+            saslMechanismType.SaslMechanismType.PLAIN,
+            description='sasl.mechanism Consumer Config property',
+            title='SASL Mechanism',
+        ),
+    ]
+    basicAuthUserInfo: Annotated[
+        Optional[CustomSecretStr],
+        Field(
+            None,
+            description='basic.auth.user.info schema registry config property, Client HTTP credentials in the form of username:password.',
+            title='Schema Registry Basic Auth User Info',
+        ),
+    ]
+    consumerConfig: Annotated[
+        Optional[Dict[str, Any]],
+        Field(
+            {},
+            description='Confluent Kafka Consumer Config. From https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md',
+            title='Consumer Config',
+        ),
+    ]
+    schemaRegistryConfig: Annotated[
+        Optional[Dict[str, Any]],
+        Field(
+            {},
+            description='Confluent Kafka Schema Registry Config. From https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient',
+            title='Schema Registry Config',
+        ),
+    ]
+    schemaRegistryTopicSuffixName: Annotated[
+        Optional[str],
+        Field(
+            '-value',
+            description='Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get topic schema from registry.',
+            title='Schema Registry Topic Suffix Name',
+        ),
+    ]
+    schemaRegistrySSL: Annotated[
+        Optional[verifySSLConfig.SslConfig],
+        Field(
+            None,
+            description='Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry connection.',
+            title='Schema Registry SSL',
+        ),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/kinesisConnection.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.credentials import awsCredentials
 from .. import connectionBasicType
@@ -18,15 +21,17 @@ class KinesisType(Enum):
 
 
 class KinesisConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[KinesisType] = Field(
-        KinesisType.Kinesis, description='Service Type', title='Service Type'
-    )
-    awsConfig: awsCredentials.AWSCredentials = Field(
-        ..., title='AWS Credentials Configuration'
+    model_config = ConfigDict(
+        extra='forbid',
     )
-    supportsMetadataExtraction: Optional[
-        connectionBasicType.SupportsMetadataExtraction
-    ] = Field(None, title='Supports Metadata Extraction')
+    type: Annotated[
+        Optional[KinesisType],
+        Field(KinesisType.Kinesis, description='Service Type', title='Service Type'),
+    ]
+    awsConfig: Annotated[
+        awsCredentials.AWSCredentials, Field(title='AWS Credentials Configuration')
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py

@@ -1,13 +1,16 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/pulsarConnection.json
-#   timestamp: 2024-07-23T12:02:47+00:00
+#   timestamp: 2024-08-05T15:12:43+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .. import connectionBasicType
 
@@ -17,12 +20,14 @@ class PulsarType(Enum):
 
 
 class PulsarConnection(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    type: Optional[PulsarType] = Field(
-        PulsarType.Pulsar, description='Service Type', title='Service Type'
+    model_config = ConfigDict(
+        extra='forbid',
     )
-    supportsMetadataExtraction: Optional[
-        connectionBasicType.SupportsMetadataExtraction
-    ] = Field(None, title='Supports Metadata Extraction')
+    type: Annotated[
+        Optional[PulsarType],
+        Field(PulsarType.Pulsar, description='Service Type', title='Service Type'),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]