openmetadata-ingestion 1.6.7.1__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/mongoDBConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -72,6 +73,30 @@ class MongoDBConnection(BaseModel):
             title='Database Name',
         ),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/mssqlConnection.py

@@ -1,20 +1,36 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/mssqlConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = [
+        '^db_.*',
+        '^guest$',
+        '^information_schema$',
+        '^sys$',
+    ]
+
+
+class DatabaseFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^msdb$', '^model$', '^tempdb$']
+
+
 class MssqlType(Enum):
     Mssql = 'Mssql'
 
@@ -84,6 +100,30 @@ class MssqlConnection(BaseModel):
             title='Ingest All Databases',
         ),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[DatabaseFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     connectionOptions: Annotated[
         Optional[connectionBasicType.ConnectionOptions],
         Field(None, title='Connection Options'),

metadata/generated/schema/entity/services/connections/database/mysqlConnection.py

@@ -1,11 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/mysqlConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
@@ -13,10 +13,16 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 from .common import azureConfig, basicAuth, iamAuthConfig
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^information_schema$', '^performance_schema$']
+
+
 class MySQLType(Enum):
     Mysql = 'Mysql'
 
@@ -94,6 +100,30 @@ class MysqlConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/oracleConnection.py

@@ -1,20 +1,26 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/oracleConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^sys$', '^ctxsys$', '^dbsnmp$', '^outln$']
+
+
 class OracleType(Enum):
     Oracle = 'Oracle'
 
@@ -119,6 +125,30 @@ class OracleConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/pinotDBConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -85,6 +86,30 @@ class PinotDBConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/postgresConnection.py

@@ -1,11 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/postgresConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
@@ -13,10 +13,21 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 from .common import azureConfig, basicAuth, iamAuthConfig
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^information_schema$']
+
+
+class DatabaseFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^template1$', '^template0$']
+
+
 class PostgresType(Enum):
     Postgres = 'Postgres'
 
@@ -84,6 +95,30 @@ class PostgresConnection(BaseModel):
             title='Ingest All Databases',
         ),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[DatabaseFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     sslMode: Optional[verifySSLConfig.SslMode] = verifySSLConfig.SslMode.disable
     sslConfig: Optional[verifySSLConfig.SslConfig] = None
     classificationName: Annotated[

metadata/generated/schema/entity/services/connections/database/prestoConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/prestoConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -91,6 +92,30 @@ class PrestoConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/redshiftConnection.py

@@ -1,11 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/redshiftConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
@@ -13,9 +13,20 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
+class SchemaFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^information_schema$']
+
+
+class DatabaseFilterPattern(BaseModel):
+    includes: Optional[List[str]] = []
+    excludes: Optional[List[str]] = ['^template1$']
+
+
 class RedshiftType(Enum):
     Redshift = 'Redshift'
 
@@ -72,6 +83,30 @@ class RedshiftConnection(BaseModel):
             title='Ingest All Databases',
         ),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[SchemaFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[DatabaseFilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     sslMode: Optional[verifySSLConfig.SslMode] = verifySSLConfig.SslMode.disable
     sslConfig: Optional[verifySSLConfig.SslConfig] = None
     connectionOptions: Annotated[
@@ -86,6 +121,10 @@ class RedshiftConnection(BaseModel):
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),
     ]
+    supportsIncrementalMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsIncrementalMetadataExtraction],
+        Field(None, title='Supports Incremental Metadata Extraction'),
+    ]
     supportsUsageExtraction: Optional[
         connectionBasicType.SupportsUsageExtraction
     ] = None

metadata/generated/schema/entity/services/connections/database/salesforceConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/salesforceConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -97,6 +98,30 @@ class SalesforceConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/sapErpConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sapErpConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
 from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -78,6 +79,30 @@ class SapErpConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00

metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sapHana/sapHanaHDBConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sapHana/sapHanaSQLConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sapHanaConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 from .sapHana import sapHanaHDBConnection, sapHanaSQLConnection
 
@@ -58,6 +59,30 @@ class SapHanaConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/database/sasConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/database/sasConnection.json
-#   timestamp: 2025-04-02T12:35:09+00:00
+#   timestamp: 2025-04-02T05:35:35+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,8 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
+
 
 class SasType(Enum):
     SAS = 'SAS'
@@ -60,3 +62,27 @@ class SASConnection(BaseModel):
             title='Custom Filter for dataflows',
         ),
     ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude schemas that matches the pattern.',
+            title='Default Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude tables that matches the pattern.',
+            title='Default Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only include/exclude databases that matches the pattern.',
+            title='Default Database Filter Pattern',
+        ),
+    ]