openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

metadata/generated/schema/entity/services/connections/storage/s3Connection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/storage/s3Connection.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/testConnectionDefinition.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/testConnectionDefinition.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/connections/testConnectionResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/connections/testConnectionResult.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/dashboardService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/dashboardService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -18,6 +18,7 @@ from .connections.dashboard import (
     lookerConnection,
     metabaseConnection,
     modeConnection,
+    mstrConnection,
     powerBIConnection,
     qlikSenseConnection,
     quickSightConnection,
@@ -40,6 +41,7 @@ class DashboardServiceType(Enum):
     QuickSight = 'QuickSight'
     QlikSense = 'QlikSense'
     Lightdash = 'Lightdash'
+    Mstr = 'Mstr'
 
 
 class DashboardConnection(BaseModel):
@@ -60,6 +62,7 @@ class DashboardConnection(BaseModel):
             quickSightConnection.QuickSightConnection,
             qlikSenseConnection.QlikSenseConnection,
             lightdashConnection.LightdashConnection,
+            mstrConnection.MstrConnection,
         ]
     ] = None
 

metadata/generated/schema/entity/services/databaseService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/databaseService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -15,6 +15,7 @@ from .connections.database import (
     athenaConnection,
     azureSQLConnection,
     bigQueryConnection,
+    bigTableConnection,
     clickhouseConnection,
     couchbaseConnection,
     customDatabaseConnection,
@@ -23,11 +24,13 @@ from .connections.database import (
     db2Connection,
     deltaLakeConnection,
     domoDatabaseConnection,
+    dorisConnection,
     druidConnection,
     dynamoDBConnection,
     glueConnection,
     greenplumConnection,
     hiveConnection,
+    icebergConnection,
     impalaConnection,
     mariaDBConnection,
     mongoDBConnection,
@@ -40,16 +43,19 @@ from .connections.database import (
     redshiftConnection,
     salesforceConnection,
     sapHanaConnection,
+    sasConnection,
     singleStoreConnection,
     snowflakeConnection,
     sqliteConnection,
     trinoConnection,
+    unityCatalogConnection,
     verticaConnection,
 )
 
 
 class DatabaseServiceType(Enum):
     BigQuery = 'BigQuery'
+    BigTable = 'BigTable'
     Mysql = 'Mysql'
     Redshift = 'Redshift'
     Snowflake = 'Snowflake'
@@ -84,6 +90,10 @@ class DatabaseServiceType(Enum):
     MongoDB = 'MongoDB'
     Couchbase = 'Couchbase'
     Greenplum = 'Greenplum'
+    Doris = 'Doris'
+    UnityCatalog = 'UnityCatalog'
+    SAS = 'SAS'
+    Iceberg = 'Iceberg'
 
 
 class DatabaseConnection(BaseModel):
@@ -93,6 +103,7 @@ class DatabaseConnection(BaseModel):
     config: Optional[
         Union[
             bigQueryConnection.BigQueryConnection,
+            bigTableConnection.BigTableConnection,
             athenaConnection.AthenaConnection,
             azureSQLConnection.AzureSQLConnection,
             clickhouseConnection.ClickhouseConnection,
@@ -125,6 +136,10 @@ class DatabaseConnection(BaseModel):
             mongoDBConnection.MongoDBConnection,
             couchbaseConnection.CouchbaseConnection,
             greenplumConnection.GreenplumConnection,
+            dorisConnection.DorisConnection,
+            unityCatalogConnection.UnityCatalogConnection,
+            sasConnection.SASConnection,
+            icebergConnection.IcebergConnection,
         ]
     ] = None
 

metadata/generated/schema/entity/services/ingestionPipelines/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00

metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/ingestionPipelines/ingestionPipeline.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,7 @@ from pydantic import BaseModel, Extra, Field
 from ....metadataIngestion import workflow
 from ....type import basic, entityHistory, entityReference
 from ..connections.metadata import openMetadataConnection
+from . import status
 
 
 class PipelineType(Enum):
@@ -23,7 +24,7 @@ class PipelineType(Enum):
     dataInsight = 'dataInsight'
     elasticSearchReindex = 'elasticSearchReindex'
     dbt = 'dbt'
-    Application = 'Application'
+    application = 'application'
 
 
 class PipelineState(Enum):
@@ -34,26 +35,6 @@ class PipelineState(Enum):
     partialSuccess = 'partialSuccess'
 
 
-class PipelineStatus(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    runId: Optional[str] = Field(None, description='Pipeline unique run ID.')
-    pipelineState: Optional[PipelineState] = Field(
-        None, description='Pipeline status denotes if its failed or succeeded.'
-    )
-    startDate: Optional[basic.Timestamp] = Field(
-        None, description='startDate of the pipeline run for this particular execution.'
-    )
-    timestamp: Optional[basic.Timestamp] = Field(
-        None,
-        description='executionDate of the pipeline run for this particular execution.',
-    )
-    endDate: Optional[basic.Timestamp] = Field(
-        None, description='endDate of the pipeline run for this particular execution.'
-    )
-
-
 class AirflowConfig(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -99,6 +80,30 @@ class AirflowConfig(BaseModel):
     )
 
 
+class PipelineStatus(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    runId: Optional[str] = Field(None, description='Pipeline unique run ID.')
+    pipelineState: Optional[PipelineState] = Field(
+        None, description='Pipeline status denotes if its failed or succeeded.'
+    )
+    startDate: Optional[basic.Timestamp] = Field(
+        None, description='startDate of the pipeline run for this particular execution.'
+    )
+    timestamp: Optional[basic.Timestamp] = Field(
+        None,
+        description='executionDate of the pipeline run for this particular execution.',
+    )
+    endDate: Optional[basic.Timestamp] = Field(
+        None, description='endDate of the pipeline run for this particular execution.'
+    )
+    status: Optional[status.IngestionStatus] = Field(
+        None,
+        description='Ingestion Pipeline summary status. Informed at the end of the execution.',
+    )
+
+
 class IngestionPipeline(BaseModel):
     class Config:
         extra = Extra.forbid

metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/ingestionPipelines/pipelineServiceClientResponse.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/ingestionPipelines/status.py

@@ -0,0 +1,56 @@
+# generated by datamodel-codegen:
+#   filename:  entity/services/ingestionPipelines/status.json
+#   timestamp: 2024-03-25T11:44:26+00:00
+
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+from pydantic import BaseModel, Extra, Field
+
+
+class IngestionStatusModel(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    __root__: Any = Field(
+        ..., description='Ingestion detailed status', title='IngestionStatusModel'
+    )
+
+
+class StackTraceError(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: str = Field(..., description='Name of the asset with the error')
+    error: str = Field(..., description='Error being handled')
+    stackTrace: Optional[str] = Field(None, description='Exception stack trace')
+
+
+class StepSummary(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    name: str = Field(..., description='Step name')
+    records: Optional[int] = Field(
+        0, description='Number of successfully processed records.'
+    )
+    updated_records: Optional[int] = Field(
+        0, description='Number of successfully updated records.'
+    )
+    warnings: Optional[int] = Field(
+        0, description='Number of records raising warnings.'
+    )
+    errors: Optional[int] = Field(0, description='Number of records with errors.')
+    filtered: Optional[int] = Field(0, description='Number of filtered records.')
+    failures: Optional[List[StackTraceError]] = Field(
+        None, description='Sample of errors encountered in the step'
+    )
+
+
+class IngestionStatus(BaseModel):
+    __root__: List[StepSummary] = Field(
+        ...,
+        description='Summary for each step of the ingestion pipeline',
+        title='IngestionStatus',
+    )

metadata/generated/schema/entity/services/messagingService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/messagingService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/metadataService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/metadataService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/mlmodelService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/mlmodelService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/pipelineService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/pipelineService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -21,6 +21,7 @@ from .connections.pipeline import (
     fivetranConnection,
     gluePipelineConnection,
     nifiConnection,
+    sparkConnection,
     splineConnection,
 )
 
@@ -36,6 +37,7 @@ class PipelineServiceType(Enum):
     CustomPipeline = 'CustomPipeline'
     DatabricksPipeline = 'DatabricksPipeline'
     Spline = 'Spline'
+    Spark = 'Spark'
 
 
 class PipelineConnection(BaseModel):
@@ -54,6 +56,7 @@ class PipelineConnection(BaseModel):
             customPipelineConnection.CustomPipelineConnection,
             databricksPipelineConnection.DatabricksPipelineConnection,
             splineConnection.SplineConnection,
+            sparkConnection.SparkConnection,
         ]
     ] = None
 

metadata/generated/schema/entity/services/searchService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/searchService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/serviceType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/serviceType.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/services/storageService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/storageService.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -11,12 +11,18 @@ from pydantic import BaseModel, Extra, Field
 
 from ...type import basic, entityHistory, entityReference, entityReferenceList, tagLabel
 from .connections import testConnectionResult
-from .connections.storage import adlsConection, customStorageConnection, s3Connection
+from .connections.storage import (
+    adlsConnection,
+    customStorageConnection,
+    gcsConnection,
+    s3Connection,
+)
 
 
 class StorageServiceType(Enum):
     S3 = 'S3'
     ADLS = 'ADLS'
+    GCS = 'GCS'
     CustomStorage = 'CustomStorage'
 
 
@@ -27,7 +33,8 @@ class StorageConnection(BaseModel):
     config: Optional[
         Union[
             s3Connection.S3Connection,
-            adlsConection.AdlsConnection,
+            adlsConnection.AdlsConnection,
+            gcsConnection.GcsConnection,
             customStorageConnection.CustomStorageConnection,
         ]
     ] = None

metadata/generated/schema/entity/teams/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00

metadata/generated/schema/entity/teams/persona.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/persona.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/role.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/role.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -11,16 +11,12 @@ from pydantic import BaseModel, Extra, Field
 from ...type import basic, entityHistory, entityReferenceList
 
 
-class RoleName(BaseModel):
-    __root__: basic.EntityName = Field(..., description='A unique name for the role.')
-
-
 class Role(BaseModel):
     class Config:
         extra = Extra.forbid
 
     id: basic.Uuid
-    name: RoleName
+    name: basic.EntityName
     fullyQualifiedName: Optional[basic.FullyQualifiedEntityName] = Field(
         None, description='FullyQualifiedName same as `name`.'
     )

metadata/generated/schema/entity/teams/team.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/team.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/teamHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/teamHierarchy.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/user.py

@@ -1,25 +1,18 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/user.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Optional, Union
 
-from pydantic import BaseModel, Extra, Field, constr
+from pydantic import BaseModel, Extra, Field
 
 from ...auth import basicAuth, jwtAuth, ssoAuth
 from ...type import basic, entityHistory, entityReference, entityReferenceList, profile
 
 
-class EntityName(BaseModel):
-    __root__: constr(regex=r'^(?u)[\w\-.]+$', min_length=1, max_length=64) = Field(
-        ...,
-        description='Login name of the user, typically the user ID from an identity provider. Example - uid from LDAP.',
-    )
-
-
 class AuthType(Enum):
     JWT = 'JWT'
     SSO = 'SSO'
@@ -47,7 +40,7 @@ class User(BaseModel):
     id: basic.Uuid = Field(
         ..., description='Unique identifier that identifies a user entity instance.'
     )
-    name: EntityName = Field(
+    name: basic.EntityName = Field(
         ...,
         description='A unique name of the user, typically the user ID from an identity provider. Example - uid from LDAP.',
     )

metadata/generated/schema/entity/type.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/type.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 
@@ -9,11 +9,11 @@ from typing import List, Optional
 
 from pydantic import BaseModel, Extra, Field, constr
 
-from ..type import basic, entityHistory, entityReference
+from ..type import basic, customProperty, entityHistory
 
 
 class EntityName(BaseModel):
-    __root__: constr(regex=r'^(?u)[\w]+$') = Field(
+    __root__: constr(regex=r'(?u)^[\w]+$') = Field(
         ...,
         description='Name of the property or entity types. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.',
     )
@@ -24,21 +24,6 @@ class Category(Enum):
     entity = 'entity'
 
 
-class CustomProperty(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    name: EntityName = Field(
-        ...,
-        description='Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.',
-    )
-    description: basic.Markdown
-    propertyType: entityReference.EntityReference = Field(
-        ...,
-        description='Reference to a property type. Only property types are allowed and entity types are not allowed as custom properties to extend an existing entity',
-    )
-
-
 class Type(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -66,7 +51,7 @@ class Type(BaseModel):
         alias='schema',
         description='JSON schema encoded as string that defines the type. This will be used to validate the type values.',
     )
-    customProperties: Optional[List[CustomProperty]] = Field(
+    customProperties: Optional[List[customProperty.CustomProperty]] = Field(
         None,
         description='Custom properties added to extend the entity. Only available for entity type',
     )

metadata/generated/schema/entity/utils/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00

metadata/generated/schema/entity/utils/entitiesCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/utils/entitiesCount.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/utils/servicesCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/utils/servicesCount.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/utils/supersetApiConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/utils/supersetApiConnection.json
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00

metadata/generated/schema/events/alertMetrics.py

@@ -0,0 +1,27 @@
+# generated by datamodel-codegen:
+#   filename:  events/alertMetrics.json
+#   timestamp: 2024-03-25T11:44:26+00:00
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel, Extra, Field
+
+from ..type import basic
+
+
+class AlertMetrics(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    totalEvents: Optional[int] = Field(None, description='Total number of events.')
+    successEvents: Optional[int] = Field(
+        None, description='Number of events that were successfully processed.'
+    )
+    failedEvents: Optional[int] = Field(
+        None, description='Number of events that failed to be processed.'
+    )
+    timestamp: Optional[basic.Timestamp] = Field(
+        None, description='Update time of the job status.'
+    )

metadata/generated/schema/events/api/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-01-05T10:55:05+00:00
+#   timestamp: 2024-03-25T11:44:26+00:00