openmetadata-ingestion 1.6.7.0__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/generated/schema/entity/services/connections/database/sasConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sasConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,8 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
+
 
 class SasType(Enum):
     SAS = 'SAS'
@@ -60,3 +62,27 @@ class SASConnection(BaseModel):
             title='Custom Filter for dataflows',
         ),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]

metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/singleStoreConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -87,6 +88,30 @@ class SingleStoreConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py

@@ -1,20 +1,31 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/snowflakeConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^information_schema$']
+
+
+class DatabaseFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^snowflake$']
+
+
 class SnowflakeType(Enum):
     Snowflake = 'Snowflake'
 
@@ -108,11 +119,19 @@ class SnowflakeConnection(BaseModel):
     includeTransientTables: Annotated[
         Optional[bool],
         Field(
-            True,
+            False,
             description='Optional configuration for ingestion of TRANSIENT tables, By default, it will skip the TRANSIENT tables.',
             title='Include Transient Tables',
         ),
     ]
+    includeStreams: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Optional configuration for ingestion of streams, By default, it will skip the streams.',
+            title='Include Streams',
+        ),
+    ]
     clientSessionKeepAlive: Annotated[
         Optional[bool],
         Field(
@@ -129,10 +148,38 @@ class SnowflakeConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[DatabaseFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),
     ]
+    supportsIncrementalMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsIncrementalMetadataExtraction],
+        Field(None, title='Supports Incremental Metadata Extraction'),
+    ]
     supportsUsageExtraction: Optional[
         connectionBasicType.SupportsUsageExtraction
     ] = None
@@ -164,3 +211,11 @@ class SnowflakeConnection(BaseModel):
         Optional[connectionBasicType.SupportsDataDiff],
         Field(None, title='Supports Data Diff Extraction.'),
     ]
+    creditCost: Annotated[
+        Optional[float],
+        Field(
+            3.3,
+            description='Cost of credit for the Snowflake account.',
+            title='Cost of Credit',
+        ),
+    ]

metadata/generated/schema/entity/services/connections/database/sqliteConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sqliteConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -87,6 +88,30 @@ class SQLiteConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/synapseConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/synapseConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -139,6 +140,30 @@ class SynapseConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/teradataConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/teradataConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -111,6 +112,30 @@ class TeradataConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/trinoConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/trinoConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 from .common import azureConfig, basicAuth, jwtAuth, noConfigAuthenticationTypes
 
@@ -102,6 +103,30 @@ class TrinoConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py

@@ -1,20 +1,31 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/unityCatalogConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^information_schema$']
+
+
+class DatabaseFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^system$']
+
+
 class DatabricksType(Enum):
     UnityCatalog = 'UnityCatalog'
 
@@ -90,6 +101,30 @@ class UnityCatalogConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[DatabaseFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsUsageExtraction: Optional[
         connectionBasicType.SupportsUsageExtraction
     ] = None

metadata/generated/schema/entity/services/connections/database/verticaConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/verticaConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -72,6 +73,30 @@ class VerticaConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/messaging/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00

metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/customMessagingConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -39,3 +40,11 @@ class CustomMessagingConnection(BaseModel):
         Optional[connectionBasicType.ConnectionOptions],
         Field(None, title='Connection Options'),
     ]
+    topicFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch topics that matches the pattern.',
+            title='Default Topic Filter Pattern',
+        ),
+    ]

metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/kafkaConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 from . import saslMechanismType
 
@@ -131,6 +132,14 @@ class KafkaConnection(BaseModel):
             title='Schema Registry SSL',
         ),
     ]
+    topicFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch topics that matches the pattern.',
+            title='Default Topic Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/kinesisConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.credentials import awsCredentials
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -31,6 +32,14 @@ class KinesisConnection(BaseModel):
     awsConfig: Annotated[
         awsCredentials.AWSCredentials, Field(title='AWS Credentials Configuration')
     ]
+    topicFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch topics that matches the pattern.',
+            title='Default Topic Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/pulsarConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -27,6 +28,14 @@ class PulsarConnection(BaseModel):
         Optional[PulsarType],
         Field(PulsarType.Pulsar, description='Service Type', title='Service Type'),
     ]
+    topicFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch topics that matches the pattern.',
+            title='Default Topic Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/redpandaConnection.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 from . import saslMechanismType
 
@@ -106,6 +107,14 @@ class RedpandaConnection(BaseModel):
             title='Schema Registry Config',
         ),
     ]
+    topicFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch topics that matches the pattern.',
+            title='Default Topic Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/messaging/saslMechanismType.json
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/metadata/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-03-28T07:59:21+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00