openmetadata-ingestion 1.6.2.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/pipeline/sparkConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,9 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
+from .. import connectionBasicType
+
 
 class SparkType(Enum):
     Spark = 'Spark'
@@ -25,3 +28,15 @@ class SparkConnection(BaseModel):
         Optional[SparkType],
         Field(SparkType.Spark, description='Service Type', title='Service Type'),
     ]
+    pipelineFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex exclude pipelines.',
+            title='Default Pipeline Filter Pattern',
+        ),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/pipeline/splineConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -40,6 +41,14 @@ class SplineConnection(BaseModel):
             None, description='Spline UI Host & Port.', title='Spline UI Host & Port'
         ),
     ]
+    pipelineFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex exclude pipelines.',
+            title='Default Pipeline Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/pipeline/ssisConnection.py

@@ -0,0 +1,56 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/connections/pipeline/ssisConnection.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional, Union
+
+from pydantic import ConfigDict, Field, RootModel
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from .. import connectionBasicType
+from ..database import mssqlConnection
+from ..storage import s3Connection
+
+
+class SSISType(Enum):
+    SSIS = 'SSIS'
+
+
+class LocalProjectsPath(RootModel[str]):
+    root: Annotated[
+        str,
+        Field(description='Path leading to your projects', title='Local Projects Path'),
+    ]
+
+
+class SSISConnection(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Annotated[
+        Optional[SSISType],
+        Field(SSISType.SSIS, description='Service Type', title='Service Type'),
+    ]
+    databaseConnection: Annotated[
+        mssqlConnection.MssqlConnection,
+        Field(
+            description='Underlying database connection',
+            title='Metadata Database Connection',
+        ),
+    ]
+    packageConnection: Annotated[
+        Union[s3Connection.S3Connection, LocalProjectsPath],
+        Field(
+            description='Underlying storage connection',
+            title='SSIS Projects Location (Local Path or S3 Bucket)',
+        ),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/pipeline/stitchConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -37,6 +38,14 @@ class StitchConnection(BaseModel):
         CustomSecretStr,
         Field(description='Token to connect to Stitch api doc', title='JWT Token'),
     ]
+    pipelineFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex exclude pipelines.',
+            title='Default Pipeline Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py

@@ -0,0 +1,51 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/connections/pipeline/wherescapeConnection.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import AnyUrl, ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from .. import connectionBasicType
+from ..database import mssqlConnection
+
+
+class WherescapeType(Enum):
+    Wherescape = 'Wherescape'
+
+
+class WherescapeConnection(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Annotated[
+        Optional[WherescapeType],
+        Field(
+            WherescapeType.Wherescape, description='Service Type', title='Service Type'
+        ),
+    ]
+    hostPort: Annotated[
+        Optional[AnyUrl],
+        Field(
+            None,
+            description='Pipeline Service Management/UI URI.',
+            title='Host And Port',
+        ),
+    ]
+    databaseConnection: Annotated[
+        mssqlConnection.MssqlConnection,
+        Field(
+            description='Underlying database connection',
+            title='Metadata Database Connection',
+        ),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/search/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/services/connections/search/customSearchConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/search/customSearchConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -21,7 +22,7 @@ class CustomSearchType(Enum):
 
 class CustomSearchConnection(BaseModel):
     model_config = ConfigDict(
-        extra='forbid',
+        extra='allow',
     )
     type: Annotated[
         CustomSearchType,
@@ -39,3 +40,15 @@ class CustomSearchConnection(BaseModel):
         Optional[connectionBasicType.ConnectionOptions],
         Field(None, title='Connection Options'),
     ]
+    searchIndexFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch search indexes that matches the pattern.',
+            title='Default Search Index Filter Pattern',
+        ),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/search/elasticSearch/apiAuth.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/search/elasticSearch/basicAuth.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/search/elasticSearchConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 from ..common import sslConfig
 from .elasticSearch import apiAuth, basicAuth
@@ -62,6 +63,14 @@ class ElasticsearchConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    searchIndexFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch search indexes that matches the pattern.',
+            title='Default Search Index Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/search/openSearchConnection.py

@@ -1,71 +1,79 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/search/openSearchConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import Optional
+from typing import Optional, Union
 
-from pydantic import ConfigDict, Field
+from pydantic import AnyUrl, ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....security.credentials import awsCredentials
+from .....security.ssl import verifySSLConfig
+from .....type import filterPattern
 from .. import connectionBasicType
+from ..common import sslConfig
+from .elasticSearch import basicAuth
 
 
 class OpenSearchType(Enum):
     OpenSearch = 'OpenSearch'
 
 
-class OpensearchConnection(BaseModel):
+class OpenSearchConnection(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
     )
     type: Annotated[
         Optional[OpenSearchType],
         Field(
-            OpenSearchType.OpenSearch, description='Service Type', title='Service Type'
+            OpenSearchType.OpenSearch,
+            description='OpenSearch Type',
+            title='OpenSearch Type',
         ),
     ]
     hostPort: Annotated[
-        str,
+        Optional[AnyUrl],
         Field(
+            None,
             description='Host and port of the OpenSearch service.',
             title='Host and Port',
         ),
     ]
-    scheme: Annotated[
-        Optional[str], Field(None, description='Http/Https connection scheme')
-    ]
-    username: Annotated[
-        Optional[str], Field(None, description='OpenSearch Username for Login')
-    ]
-    password: Annotated[
-        Optional[str], Field(None, description='OpenSearch Password for Login')
-    ]
-    truststorePath: Annotated[Optional[str], Field(None, description='Truststore Path')]
-    truststorePassword: Annotated[
-        Optional[str], Field(None, description='Truststore Password')
+    authType: Annotated[
+        Optional[Union[basicAuth.BasicAuthentication, awsCredentials.AWSCredentials]],
+        Field(
+            None,
+            description='Choose Auth Config Type.',
+            title='Auth Configuration Type',
+        ),
     ]
+    verifySSL: Optional[verifySSLConfig.VerifySSL] = verifySSLConfig.VerifySSL.no_ssl
+    sslConfig: Annotated[Optional[sslConfig.SslConfig], Field(None, title='SSL Config')]
     connectionTimeoutSecs: Annotated[
-        Optional[int], Field(5, description='Connection Timeout in Seconds')
-    ]
-    socketTimeoutSecs: Annotated[
-        Optional[int], Field(60, description='Socket Timeout in Seconds')
-    ]
-    keepAliveTimeoutSecs: Annotated[
-        Optional[int], Field(None, description='Keep Alive Timeout in Seconds')
-    ]
-    connectionOptions: Annotated[
-        Optional[connectionBasicType.ConnectionOptions],
-        Field(None, title='Connection Options'),
+        Optional[int],
+        Field(
+            30,
+            description='Connection Timeout in Seconds',
+            title='Connection Timeout in Seconds',
+        ),
     ]
     connectionArguments: Annotated[
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    searchIndexFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch search indexes that matches the pattern.',
+            title='Default Search Index Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/security/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/services/connections/security/ranger/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/services/connections/security/ranger/basicAuth.py

@@ -0,0 +1,26 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/connections/security/ranger/basicAuth.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel, CustomSecretStr
+
+
+class RangerBasicAuth(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    username: Annotated[
+        str,
+        Field(description='Ranger user to authenticate to the API.', title='Username'),
+    ]
+    password: Annotated[
+        CustomSecretStr,
+        Field(
+            description='Ranger password to authenticate to the API.', title='Password'
+        ),
+    ]

metadata/generated/schema/entity/services/connections/security/rangerConnection.py

@@ -0,0 +1,39 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/connections/security/rangerConnection.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import AnyUrl, ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from .ranger import basicAuth
+
+
+class RangerType(Enum):
+    Ranger = 'Ranger'
+
+
+class RangerConnection(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Annotated[
+        Optional[RangerType],
+        Field(RangerType.Ranger, description='Service Type', title='Service Type'),
+    ]
+    hostPort: Annotated[
+        AnyUrl, Field(description='Apache Ranger Admin URL.', title='Host and Port')
+    ]
+    authType: Annotated[
+        basicAuth.RangerBasicAuth,
+        Field(
+            description='Authentication type to connect to Apache Ranger.',
+            title='Authentication Type',
+        ),
+    ]

metadata/generated/schema/entity/services/connections/serviceConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/serviceConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -15,11 +15,13 @@ from .. import (
     apiService,
     dashboardService,
     databaseService,
+    driveService,
     messagingService,
     metadataService,
     mlmodelService,
     pipelineService,
     searchService,
+    securityService,
     storageService,
 )
 
@@ -36,6 +38,8 @@ class ServiceConnection(
             mlmodelService.MlModelConnection,
             storageService.StorageConnection,
             searchService.SearchConnection,
+            securityService.SecurityConnection,
+            driveService.DriveConnection,
         ]
     ]
 ):
@@ -50,6 +54,8 @@ class ServiceConnection(
             mlmodelService.MlModelConnection,
             storageService.StorageConnection,
             searchService.SearchConnection,
+            securityService.SecurityConnection,
+            driveService.DriveConnection,
         ],
         Field(description='Supported services'),
     ]

metadata/generated/schema/entity/services/connections/storage/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/services/connections/storage/adlsConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/storage/adlsConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.credentials import azureCredentials
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -40,6 +41,14 @@ class AdlsConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    containerFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch containers that matches the pattern.',
+            title='Default Storage container Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),

metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/storage/customStorageConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -21,7 +22,7 @@ class CustomStorageType(Enum):
 
 class CustomStorageConnection(BaseModel):
     model_config = ConfigDict(
-        extra='forbid',
+        extra='allow',
     )
     type: Annotated[
         CustomStorageType,
@@ -39,3 +40,15 @@ class CustomStorageConnection(BaseModel):
         Optional[connectionBasicType.ConnectionOptions],
         Field(None, title='Connection Options'),
     ]
+    containerFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch containers that matches the pattern.',
+            title='Default Storage container Filter Pattern',
+        ),
+    ]
+    supportsMetadataExtraction: Annotated[
+        Optional[connectionBasicType.SupportsMetadataExtraction],
+        Field(None, title='Supports Metadata Extraction'),
+    ]

metadata/generated/schema/entity/services/connections/storage/gcsConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/storage/gcsConnection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.credentials import gcpCredentials
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -46,6 +47,14 @@ class GcsConnection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    containerFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch containers that matches the pattern.',
+            title='Default Storage container Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),