openmetadata-ingestion 1.6.2.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/generated/schema/entity/services/connections/storage/s3Connection.py

@@ -1,18 +1,19 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/storage/s3Connection.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import List, Optional
 
-from pydantic import ConfigDict, Field
+from pydantic import AnyUrl, ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
 from .....security.credentials import awsCredentials
+from .....type import filterPattern
 from .. import connectionBasicType
 
 
@@ -45,7 +46,23 @@ class S3Connection(BaseModel):
         Optional[connectionBasicType.ConnectionArguments],
         Field(None, title='Connection Arguments'),
     ]
+    containerFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            {'includes': [], 'excludes': ['_SUCCESS']},
+            description='Regex to only fetch containers that matches the pattern.',
+            title='Default Storage container Filter Pattern',
+        ),
+    ]
     supportsMetadataExtraction: Annotated[
         Optional[connectionBasicType.SupportsMetadataExtraction],
         Field(None, title='Supports Metadata Extraction'),
     ]
+    consoleEndpointURL: Annotated[
+        Optional[AnyUrl],
+        Field(
+            None,
+            description='Console EndPoint URL for S3-compatible services',
+            title='Console Endpoint URL',
+        ),
+    ]

metadata/generated/schema/entity/services/connections/testConnectionDefinition.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/testConnectionDefinition.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -11,7 +11,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
-from ....type import basic, entityHistory, entityReference, entityReferenceList
+from ....type import basic, entityHistory, entityReferenceList
 
 
 class TestConnectionStep(BaseModel):
@@ -97,16 +97,20 @@ class TestConnectionDefinition(BaseModel):
         Optional[entityHistory.ChangeDescription],
         Field(None, description='Change that lead to this version of the entity.'),
     ]
+    incrementalChangeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
     deleted: Annotated[
         Optional[bool],
         Field(
             False, description='When `true` indicates the entity has been soft deleted.'
         ),
     ]
-    domain: Annotated[
-        Optional[entityReference.EntityReference],
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
         Field(
             None,
-            description='Domain the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
+            description='Domains the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
         ),
     ]

metadata/generated/schema/entity/services/connections/testConnectionResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/testConnectionResult.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/dashboardService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/dashboardService.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -17,6 +17,8 @@ from .connections import testConnectionResult
 from .connections.dashboard import (
     customDashboardConnection,
     domoDashboardConnection,
+    grafanaConnection,
+    hexConnection,
     lightdashConnection,
     lookerConnection,
     metabaseConnection,
@@ -31,6 +33,7 @@ from .connections.dashboard import (
     sigmaConnection,
     supersetConnection,
     tableauConnection,
+    thoughtSpotConnection,
 )
 
 
@@ -51,6 +54,9 @@ class DashboardServiceType(Enum):
     MicroStrategy = 'MicroStrategy'
     QlikCloud = 'QlikCloud'
     Sigma = 'Sigma'
+    ThoughtSpot = 'ThoughtSpot'
+    Grafana = 'Grafana'
+    Hex = 'Hex'
 
 
 class DashboardConnection(BaseModel):
@@ -75,6 +81,9 @@ class DashboardConnection(BaseModel):
             microStrategyConnection.MicroStrategyConnection,
             qlikCloudConnection.QlikCloudConnection,
             sigmaConnection.SigmaConnection,
+            thoughtSpotConnection.ThoughtSpotConnection,
+            grafanaConnection.GrafanaConnection,
+            hexConnection.HexConnection,
         ]
     ] = None
 
@@ -152,6 +161,10 @@ class DashboardService(BaseModel):
         Optional[entityHistory.ChangeDescription],
         Field(None, description='Change that lead to this version of the entity.'),
     ]
+    incrementalChangeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
     deleted: Annotated[
         Optional[bool],
         Field(
@@ -162,7 +175,18 @@ class DashboardService(BaseModel):
         Optional[entityReferenceList.EntityReferenceList],
         Field(None, description='List of data products this entity is part of.'),
     ]
-    domain: Annotated[
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Domains the Dashboard service belongs to.'),
+    ]
+    followers: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Followers of this entity.'),
+    ]
+    ingestionRunner: Annotated[
         Optional[entityReference.EntityReference],
-        Field(None, description='Domain the Dashboard service belongs to.'),
+        Field(
+            None,
+            description='The ingestion agent responsible for executing the ingestion pipeline.',
+        ),
     ]

metadata/generated/schema/entity/services/databaseService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/databaseService.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -19,7 +19,9 @@ from .connections.database import (
     azureSQLConnection,
     bigQueryConnection,
     bigTableConnection,
+    cassandraConnection,
     clickhouseConnection,
+    cockroachConnection,
     couchbaseConnection,
     customDatabaseConnection,
     databricksConnection,
@@ -30,6 +32,7 @@ from .connections.database import (
     dorisConnection,
     druidConnection,
     dynamoDBConnection,
+    epicConnection,
     exasolConnection,
     glueConnection,
     greenplumConnection,
@@ -49,11 +52,14 @@ from .connections.database import (
     sapErpConnection,
     sapHanaConnection,
     sasConnection,
+    serviceNowConnection,
     singleStoreConnection,
     snowflakeConnection,
     sqliteConnection,
+    ssasConnection,
     synapseConnection,
     teradataConnection,
+    timescaleConnection,
     trinoConnection,
     unityCatalogConnection,
     verticaConnection,
@@ -67,6 +73,7 @@ class DatabaseServiceType(Enum):
     Redshift = 'Redshift'
     Snowflake = 'Snowflake'
     Postgres = 'Postgres'
+    Timescale = 'Timescale'
     Mssql = 'Mssql'
     Oracle = 'Oracle'
     Athena = 'Athena'
@@ -95,6 +102,7 @@ class DatabaseServiceType(Enum):
     Dbt = 'Dbt'
     SapHana = 'SapHana'
     MongoDB = 'MongoDB'
+    Cassandra = 'Cassandra'
     Couchbase = 'Couchbase'
     Greenplum = 'Greenplum'
     Doris = 'Doris'
@@ -105,6 +113,10 @@ class DatabaseServiceType(Enum):
     SapErp = 'SapErp'
     Synapse = 'Synapse'
     Exasol = 'Exasol'
+    Cockroach = 'Cockroach'
+    SSAS = 'SSAS'
+    Epic = 'Epic'
+    ServiceNow = 'ServiceNow'
 
 
 class DatabaseConnection(BaseModel):
@@ -132,6 +144,7 @@ class DatabaseConnection(BaseModel):
             sqliteConnection.SQLiteConnection,
             oracleConnection.OracleConnection,
             postgresConnection.PostgresConnection,
+            timescaleConnection.TimescaleConnection,
             prestoConnection.PrestoConnection,
             redshiftConnection.RedshiftConnection,
             salesforceConnection.SalesforceConnection,
@@ -145,6 +158,7 @@ class DatabaseConnection(BaseModel):
             customDatabaseConnection.CustomDatabaseConnection,
             sapHanaConnection.SapHanaConnection,
             mongoDBConnection.MongoDBConnection,
+            cassandraConnection.CassandraConnection,
             couchbaseConnection.CouchbaseConnection,
             greenplumConnection.GreenplumConnection,
             dorisConnection.DorisConnection,
@@ -155,6 +169,10 @@ class DatabaseConnection(BaseModel):
             sapErpConnection.SapErpConnection,
             synapseConnection.SynapseConnection,
             exasolConnection.ExasolConnection,
+            cockroachConnection.CockroachConnection,
+            ssasConnection.SSASConnection,
+            epicConnection.EpicConnection,
+            serviceNowConnection.ServiceNowConnection,
         ]
     ] = None
 
@@ -230,10 +248,18 @@ class DatabaseService(BaseModel):
             description='Link to the resource corresponding to this database service.',
         ),
     ]
+    followers: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Followers of this entity.'),
+    ]
     changeDescription: Annotated[
         Optional[entityHistory.ChangeDescription],
         Field(None, description='Change that lead to this version of the entity.'),
     ]
+    incrementalChangeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
     deleted: Annotated[
         Optional[bool],
         Field(
@@ -244,7 +270,14 @@ class DatabaseService(BaseModel):
         Optional[entityReferenceList.EntityReferenceList],
         Field(None, description='List of data products this entity is part of.'),
     ]
-    domain: Annotated[
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Domains the Database service belongs to.'),
+    ]
+    ingestionRunner: Annotated[
         Optional[entityReference.EntityReference],
-        Field(None, description='Domain the Database service belongs to.'),
+        Field(
+            None,
+            description='The ingestion agent responsible for executing the ingestion pipeline.',
+        ),
     ]

metadata/generated/schema/entity/services/driveService.py

@@ -0,0 +1,143 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/driveService.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ...type import basic, entityHistory, entityReference, entityReferenceList, tagLabel
+from .connections import testConnectionResult
+from .connections.drive import (
+    customDriveConnection,
+    googleDriveConnection,
+    sharePointConnection,
+)
+
+
+class DriveServiceType(Enum):
+    GoogleDrive = 'GoogleDrive'
+    SharePoint = 'SharePoint'
+    CustomDrive = 'CustomDrive'
+
+
+class DriveConnection(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    config: Optional[
+        Union[
+            googleDriveConnection.GoogleDriveConnection,
+            sharePointConnection.SharePointConnection,
+            customDriveConnection.CustomDriveConnection,
+        ]
+    ] = None
+
+
+class DriveService(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: Annotated[
+        basic.Uuid,
+        Field(description='Unique identifier of this drive service instance.'),
+    ]
+    name: Annotated[
+        basic.EntityName, Field(description='Name that identifies this drive service.')
+    ]
+    fullyQualifiedName: Annotated[
+        Optional[basic.FullyQualifiedEntityName],
+        Field(None, description='FullyQualifiedName same as `name`.'),
+    ]
+    displayName: Annotated[
+        Optional[str],
+        Field(None, description='Display Name that identifies this drive service.'),
+    ]
+    serviceType: Annotated[
+        DriveServiceType,
+        Field(description='Type of drive service such as Google Drive...'),
+    ]
+    description: Annotated[
+        Optional[basic.Markdown],
+        Field(None, description='Description of a drive service instance.'),
+    ]
+    connection: Optional[DriveConnection] = None
+    pipelines: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(
+            None,
+            description='References to pipelines deployed for this drive service to extract metadata, usage, lineage etc..',
+        ),
+    ]
+    testConnectionResult: Annotated[
+        Optional[testConnectionResult.TestConnectionResult],
+        Field(None, description='Last test connection results for this service'),
+    ]
+    tags: Annotated[
+        Optional[List[tagLabel.TagLabel]],
+        Field([], description='Tags for this drive Service.'),
+    ]
+    version: Annotated[
+        Optional[entityHistory.EntityVersion],
+        Field(None, description='Metadata version of the entity.'),
+    ]
+    updatedAt: Annotated[
+        Optional[basic.Timestamp],
+        Field(
+            None,
+            description='Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.',
+        ),
+    ]
+    updatedBy: Annotated[
+        Optional[str], Field(None, description='User who made the update.')
+    ]
+    href: Annotated[
+        Optional[basic.Href],
+        Field(
+            None,
+            description='Link to the resource corresponding to this drive service.',
+        ),
+    ]
+    owners: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Owners of this drive service.'),
+    ]
+    changeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
+    incrementalChangeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
+    deleted: Annotated[
+        Optional[bool],
+        Field(
+            False, description='When `true` indicates the entity has been soft deleted.'
+        ),
+    ]
+    dataProducts: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='List of data products this entity is part of.'),
+    ]
+    followers: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Followers of this entity.'),
+    ]
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Domains the Drive service belongs to.'),
+    ]
+    ingestionRunner: Annotated[
+        Optional[entityReference.EntityReference],
+        Field(
+            None,
+            description='The ingestion agent responsible for executing the ingestion pipeline.',
+        ),
+    ]

metadata/generated/schema/entity/services/ingestionPipelines/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/ingestionPipelines/ingestionPipeline.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -134,6 +134,16 @@ class PipelineStatus(BaseModel):
             description='Ingestion Pipeline summary status. Informed at the end of the execution.',
         ),
     ]
+    config: Annotated[
+        Optional[basic.Map],
+        Field(
+            None, description='Pipeline configuration for this particular execution.'
+        ),
+    ]
+    metadata: Annotated[
+        Optional[basic.Map],
+        Field(None, description='Metadata for the pipeline status.'),
+    ]
 
 
 class IngestionPipeline(BaseModel):
@@ -188,6 +198,13 @@ class IngestionPipeline(BaseModel):
             description='Set the logging level for the workflow.',
         ),
     ]
+    raiseOnError: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Control if we want to flag the workflow as failed if we encounter any processing errors.',
+        ),
+    ]
     deployed: Annotated[
         Optional[bool],
         Field(
@@ -202,6 +219,13 @@ class IngestionPipeline(BaseModel):
             description='True if the pipeline is ready to be run in the next schedule. False if it is paused.',
         ),
     ]
+    enableStreamableLogs: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Enable real-time log streaming to the OpenMetadata server. When enabled, ingestion logs will be automatically shipped to the server's configured log storage backend (S3 or compatible).",
+        ),
+    ]
     href: Annotated[
         Optional[basic.Href],
         Field(None, description='Link to this ingestion pipeline resource.'),
@@ -224,6 +248,14 @@ class IngestionPipeline(BaseModel):
         Optional[entityHistory.ChangeDescription],
         Field(None, description='Change that led to this version of the entity.'),
     ]
+    incrementalChangeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
+    followers: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Followers of this entity.'),
+    ]
     deleted: Annotated[
         Optional[bool],
         Field(
@@ -231,11 +263,11 @@ class IngestionPipeline(BaseModel):
         ),
     ]
     provider: Optional[basic.ProviderType] = basic.ProviderType.user
-    domain: Annotated[
-        Optional[entityReference.EntityReference],
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
         Field(
             None,
-            description='Domain the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
+            description='Domains the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
         ),
     ]
     applicationType: Annotated[
@@ -245,3 +277,17 @@ class IngestionPipeline(BaseModel):
             description="Type of the application when pipelineType is 'application'.",
         ),
     ]
+    ingestionRunner: Annotated[
+        Optional[entityReference.EntityReference],
+        Field(
+            None,
+            description='The ingestion agent responsible for executing the ingestion pipeline.',
+        ),
+    ]
+    processingEngine: Annotated[
+        Optional[entityReference.EntityReference],
+        Field(
+            None,
+            description='The processing engine responsible for executing the ingestion pipeline logic.',
+        ),
+    ]

metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/ingestionPipelines/pipelineServiceClientResponse.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py

@@ -0,0 +1,47 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/ingestionPipelines/reverseIngestionResponse.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ....type import basic
+
+
+class ReverseIngestionOperationResult(BaseModel):
+    id: Annotated[basic.Uuid, Field(description='The id of the operation')]
+    success: Annotated[
+        bool, Field(description='Whether the specific operation was successful')
+    ]
+    message: Annotated[
+        Optional[str], Field(None, description='Error message in case of failure')
+    ]
+
+
+class ReverseIngestionResponse(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    serviceId: Annotated[
+        basic.Uuid, Field(description='The id of the service to be modified')
+    ]
+    success: Annotated[
+        Optional[bool],
+        Field(
+            None,
+            description='Whether the workflow was successful. Failure indicates a critical failure such as connection issues.',
+        ),
+    ]
+    message: Annotated[
+        Optional[str], Field(None, description='Error message in case of failure')
+    ]
+    results: Annotated[
+        List[ReverseIngestionOperationResult],
+        Field(description='List of operations to be performed on the service'),
+    ]

metadata/generated/schema/entity/services/ingestionPipelines/status.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/ingestionPipelines/status.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/messagingService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/messagingService.json
-#   timestamp: 2025-01-10T14:18:34+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -128,6 +128,10 @@ class MessagingService(BaseModel):
         Optional[entityHistory.ChangeDescription],
         Field(None, description='Change that lead to this version of the entity.'),
     ]
+    incrementalChangeDescription: Annotated[
+        Optional[entityHistory.ChangeDescription],
+        Field(None, description='Change that lead to this version of the entity.'),
+    ]
     deleted: Annotated[
         Optional[bool],
         Field(
@@ -138,7 +142,18 @@ class MessagingService(BaseModel):
         Optional[entityReferenceList.EntityReferenceList],
         Field(None, description='List of data products this entity is part of.'),
     ]
-    domain: Annotated[
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Domains the Messaging service belongs to.'),
+    ]
+    followers: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Followers of this entity.'),
+    ]
+    ingestionRunner: Annotated[
         Optional[entityReference.EntityReference],
-        Field(None, description='Domain the Messaging service belongs to.'),
+        Field(
+            None,
+            description='The ingestion agent responsible for executing the ingestion pipeline.',
+        ),
     ]