openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/generated/schema/entity/data/pipeline.py

@@ -1,11 +1,11 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/pipeline.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
 from enum import Enum
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import AnyUrl, ConfigDict, Field
 from typing_extensions import Annotated
@@ -19,6 +19,7 @@ from ...type import (
     entityReference,
     entityReferenceList,
     lifeCycle,
+    status,
     tagLabel,
     usageDetails,
     votes,
@@ -38,6 +39,82 @@ class PipelineState(Enum):
     Inactive = 'Inactive'
 
 
+class TransformationType(Enum):
+    filter = 'filter'
+    aggregate = 'aggregate'
+    join = 'join'
+    pivot = 'pivot'
+    custom = 'custom'
+    extract = 'extract'
+    load = 'load'
+
+
+class DatasetUsage(BaseModel):
+    datasetFQN: Annotated[str, Field(description='Fully qualified name of the dataset')]
+    columns: Annotated[
+        Optional[List[str]], Field(None, description='Specific columns used')
+    ]
+    rowCount: Annotated[Optional[int], Field(None, description='Number of rows read')]
+    sizeBytes: Annotated[Optional[int], Field(None, description='Data size in bytes')]
+    filter: Annotated[
+        Optional[str], Field(None, description='Filter applied when reading')
+    ]
+    version: Annotated[
+        Optional[str], Field(None, description='Dataset version/snapshot used')
+    ]
+
+
+class Mode(Enum):
+    overwrite = 'overwrite'
+    append = 'append'
+    merge = 'merge'
+    upsert = 'upsert'
+
+
+class DatasetGeneration(BaseModel):
+    datasetFQN: Annotated[str, Field(description='Fully qualified name of the dataset')]
+    columns: Annotated[
+        Optional[List[str]], Field(None, description='Columns written/updated')
+    ]
+    rowCount: Annotated[
+        Optional[int], Field(None, description='Number of rows written')
+    ]
+    sizeBytes: Annotated[Optional[int], Field(None, description='Data size in bytes')]
+    mode: Annotated[Optional[Mode], Field(None, description='Write mode')]
+    partitions: Annotated[
+        Optional[List[str]], Field(None, description='Partitions written')
+    ]
+    version: Annotated[
+        Optional[str], Field(None, description='Dataset version created')
+    ]
+
+
+class ExecutionMetrics(BaseModel):
+    totalRowsProcessed: Optional[int] = None
+    totalBytesProcessed: Optional[int] = None
+    cpuTime: Annotated[Optional[float], Field(None, description='CPU time in seconds')]
+    memoryUsed: Annotated[
+        Optional[int], Field(None, description='Peak memory usage in bytes')
+    ]
+    cost: Annotated[
+        Optional[float], Field(None, description='Execution cost if applicable')
+    ]
+
+
+class TaskMetrics(BaseModel):
+    rowsRead: Optional[int] = None
+    rowsWritten: Optional[int] = None
+    rowsFiltered: Optional[int] = None
+    executionTimeMs: Optional[int] = None
+
+
+class ExecutionError(BaseModel):
+    errorType: Optional[str] = None
+    errorMessage: Optional[str] = None
+    stackTrace: Optional[str] = None
+    failedTask: Optional[str] = None
+
+
 class TaskStatus(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
@@ -53,6 +130,23 @@ class TaskStatus(BaseModel):
         Optional[basic.Timestamp], Field(None, description='Task end time')
     ]
     logLink: Annotated[Optional[AnyUrl], Field(None, description='Task end time')]
+    taskId: Annotated[
+        Optional[str], Field(None, description='Task ID from orchestrator')
+    ]
+    transformationType: Annotated[
+        Optional[TransformationType], Field(None, description='Type of transformation')
+    ]
+    transformationLogic: Annotated[
+        Optional[str], Field(None, description='SQL or code executed')
+    ]
+    inputs: Annotated[
+        Optional[List[DatasetUsage]], Field(None, description='Task-specific inputs')
+    ]
+    outputs: Annotated[
+        Optional[List[DatasetGeneration]],
+        Field(None, description='Task-specific outputs'),
+    ]
+    metrics: Optional[TaskMetrics] = None
 
 
 class PipelineStatus(BaseModel):
@@ -69,6 +163,42 @@ class PipelineStatus(BaseModel):
         Optional[List[TaskStatus]],
         Field(None, description='Series of task executions and its status.'),
     ]
+    executionId: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='External execution ID from the orchestrator (e.g., Airflow run_id)',
+        ),
+    ]
+    endTime: Annotated[
+        Optional[basic.Timestamp], Field(None, description='Execution end time')
+    ]
+    version: Annotated[
+        Optional[str], Field(None, description='Pipeline version that was executed')
+    ]
+    parameters: Annotated[
+        Optional[Dict[str, Any]],
+        Field(None, description='Execution parameters/configuration'),
+    ]
+    executedBy: Annotated[
+        Optional[entityReference.EntityReference],
+        Field(None, description='User or service that triggered the execution'),
+    ]
+    inputs: Annotated[
+        Optional[List[DatasetUsage]],
+        Field(None, description='Input datasets consumed by this execution'),
+    ]
+    outputs: Annotated[
+        Optional[List[DatasetGeneration]],
+        Field(None, description='Output datasets generated by this execution'),
+    ]
+    metrics: Annotated[
+        Optional[ExecutionMetrics], Field(None, description='Execution metrics')
+    ]
+    error: Annotated[
+        Optional[ExecutionError],
+        Field(None, description='Error details if execution failed'),
+    ]
 
 
 class Task(BaseModel):
@@ -281,3 +411,7 @@ class Pipeline(BaseModel):
             None, description='Source hash of the entity', max_length=32, min_length=1
         ),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(status.EntityStatus.Unprocessed, description='Status of the Pipeline.'),
+    ]

metadata/generated/schema/entity/data/query.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/query.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -16,6 +16,7 @@ from ...type import (
     entityHistory,
     entityReference,
     entityReferenceList,
+    status,
     tagLabel,
     votes,
 )
@@ -145,3 +146,7 @@ class Query(BaseModel):
             description='Domains the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
         ),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(status.EntityStatus.Unprocessed, description='Status of the Query.'),
+    ]

metadata/generated/schema/entity/data/queryCostRecord.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/queryCostRecord.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/data/queryCostSearchResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/queryCostSearchResult.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/data/report.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/report.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -16,6 +16,7 @@ from ...type import (
     entityHistory,
     entityReference,
     entityReferenceList,
+    status,
     usageDetails,
     votes,
 )
@@ -104,3 +105,7 @@ class Report(BaseModel):
             description='Domains the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
         ),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(status.EntityStatus.Unprocessed, description='Status of the Report.'),
+    ]

metadata/generated/schema/entity/data/searchIndex.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/searchIndex.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -19,6 +19,7 @@ from ...type import (
     entityReference,
     entityReferenceList,
     lifeCycle,
+    status,
     tagLabel,
     votes,
 )
@@ -269,6 +270,12 @@ class SearchIndex(BaseModel):
             None, description='Source hash of the entity', max_length=32, min_length=1
         ),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(
+            status.EntityStatus.Unprocessed, description='Status of the SearchIndex.'
+        ),
+    ]
 
 
 SearchIndexField.model_rebuild()

metadata/generated/schema/entity/data/spreadsheet.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/spreadsheet.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -19,6 +19,7 @@ from ...type import (
     entityReference,
     entityReferenceList,
     lifeCycle,
+    status,
     tagLabel,
     usageDetails,
     votes,
@@ -69,10 +70,9 @@ class Spreadsheet(BaseModel):
         Field(None, description='Description of the spreadsheet instance.'),
     ]
     service: Annotated[
-        Optional[entityReference.EntityReference],
+        entityReference.EntityReference,
         Field(
-            None,
-            description='Link to the drive service where this spreadsheet is hosted.',
+            description='Link to the drive service where this spreadsheet is hosted.'
         ),
     ]
     serviceType: Annotated[
@@ -207,3 +207,9 @@ class Spreadsheet(BaseModel):
         Optional[votes.Votes], Field(None, description='Votes on the entity.')
     ]
     certification: Optional[assetCertification.AssetCertification] = None
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(
+            status.EntityStatus.Unprocessed, description='Status of the Spreadsheet.'
+        ),
+    ]

metadata/generated/schema/entity/data/storedProcedure.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/storedProcedure.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -19,6 +19,7 @@ from ...type import (
     entityReference,
     entityReferenceList,
     lifeCycle,
+    status,
     tagLabel,
     votes,
 )
@@ -195,3 +196,10 @@ class StoredProcedure(BaseModel):
         Optional[bool],
         Field(False, description='Processed lineage for the stored procedure'),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(
+            status.EntityStatus.Unprocessed,
+            description='Status of the StoredProcedure.',
+        ),
+    ]

metadata/generated/schema/entity/data/table.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/table.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -20,6 +20,7 @@ from ...type import (
     entityReference,
     entityReferenceList,
     lifeCycle,
+    status,
     tagLabel,
     usageDetails,
     votes,
@@ -27,6 +28,48 @@ from ...type import (
 from ..services import databaseService
 
 
+class CompressionType(Enum):
+    AUTOMATIC = 'AUTOMATIC'
+    MANUAL = 'MANUAL'
+    POLICY_BASED = 'POLICY_BASED'
+
+
+class CompressionStrategy(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    compressionType: Annotated[
+        Optional[CompressionType],
+        Field(
+            None,
+            description='Type of compression: AUTOMATIC (Snowflake/BigQuery), MANUAL (Redshift), POLICY_BASED (TimescaleDB)',
+        ),
+    ]
+    segmentColumns: Annotated[
+        Optional[List[str]],
+        Field(
+            None,
+            description='Columns used for segmenting/partitioning compressed data (TimescaleDB segment-by, ClickHouse partition key)',
+        ),
+    ]
+    orderColumns: Annotated[
+        Optional[List[str]],
+        Field(
+            None,
+            description='Columns defining sort order within compressed blocks (TimescaleDB order-by, ClickHouse order by)',
+        ),
+    ]
+    compressionLevel: Annotated[
+        Optional[int],
+        Field(
+            None,
+            description='Compression level (1-9) for codecs that support it',
+            ge=1,
+            le=9,
+        ),
+    ]
+
+
 class ProfileSampleType(Enum):
     PERCENTAGE = 'PERCENTAGE'
     ROWS = 'ROWS'
@@ -138,6 +181,7 @@ class DataType(Enum):
     MEASURE = 'MEASURE'
     KPI = 'KPI'
     HEIRARCHY = 'HEIRARCHY'
+    HIERARCHYID = 'HIERARCHYID'
 
 
 class Constraint(Enum):
@@ -300,6 +344,24 @@ class Histogram(BaseModel):
     ]
 
 
+class CardinalityDistribution(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    categories: Annotated[
+        Optional[List[str]],
+        Field(None, description="List of category names including 'Others'."),
+    ]
+    counts: Annotated[
+        Optional[List[int]],
+        Field(None, description='List of counts corresponding to each category.'),
+    ]
+    percentages: Annotated[
+        Optional[List[float]],
+        Field(None, description='List of percentages corresponding to each category.'),
+    ]
+
+
 class ColumnProfile(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
@@ -407,6 +469,13 @@ class ColumnProfile(BaseModel):
         Optional[List[CustomMetricProfile]],
         Field(None, description='Custom Metrics profile list bound to a column.'),
     ]
+    cardinalityDistribution: Annotated[
+        Optional[CardinalityDistribution],
+        Field(
+            None,
+            description="Cardinality distribution showing top categories with an 'Others' bucket.",
+        ),
+    ]
 
 
 class DmlOperationType(Enum):
@@ -745,6 +814,13 @@ class DataModel(BaseModel):
     resourceType: Annotated[
         Optional[str], Field(None, description='Resource Type of the model.')
     ]
+    dbtSourceProject: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description="The DBT project name that served as the source for ingesting this table's metadata and lineage information.",
+        ),
+    ]
     description: Annotated[
         Optional[basic.Markdown],
         Field(None, description='Description of the Table from the model.'),
@@ -947,6 +1023,27 @@ class Table(BaseModel):
             description='Retention period of the data in the table. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. When not set, the retention period is inherited from the parent database schema, if it exists.',
         ),
     ]
+    compressionEnabled: Annotated[
+        Optional[bool],
+        Field(
+            None,
+            description='Indicates whether compression is enabled on this table. Applicable for databases that support compression like Snowflake, TimescaleDB, ClickHouse, BigQuery, Redshift, etc.',
+        ),
+    ]
+    compressionCodec: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='Compression algorithm/codec used. Examples: LZ4, ZSTD, Snappy, Gorilla, Delta, etc. Database-specific values allowed.',
+        ),
+    ]
+    compressionStrategy: Annotated[
+        Optional[CompressionStrategy],
+        Field(
+            None,
+            description='Compression strategy configuration including segment/partition/order keys',
+        ),
+    ]
     extension: Annotated[
         Optional[basic.EntityExtension],
         Field(
@@ -993,6 +1090,10 @@ class Table(BaseModel):
         Optional[List[basic.SqlQuery]],
         Field(None, description='List of queries that are used to create this table.'),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(status.EntityStatus.Unprocessed, description='Status of the Table.'),
+    ]
 
 
 Column.model_rebuild()

metadata/generated/schema/entity/data/topic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/topic.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -20,6 +20,7 @@ from ...type import (
     entityReferenceList,
     lifeCycle,
     schema,
+    status,
     tagLabel,
     votes,
 )
@@ -215,3 +216,7 @@ class Topic(BaseModel):
             None, description='Source hash of the entity', max_length=32, min_length=1
         ),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(status.EntityStatus.Unprocessed, description='Status of the Topic.'),
+    ]

metadata/generated/schema/entity/data/worksheet.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/data/worksheet.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -18,6 +18,7 @@ from ...type import (
     entityReference,
     entityReferenceList,
     lifeCycle,
+    status,
     tagLabel,
     usageDetails,
     votes,
@@ -166,3 +167,7 @@ class Worksheet(BaseModel):
         Optional[usageDetails.UsageDetails],
         Field(None, description='Latest usage information for this worksheet.'),
     ]
+    entityStatus: Annotated[
+        Optional[status.EntityStatus],
+        Field(status.EntityStatus.Unprocessed, description='Status of the Worksheet.'),
+    ]

metadata/generated/schema/entity/datacontract/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/datacontract/dataContractResult.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/datacontract/dataContractResult.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/datacontract/qualityValidation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/datacontract/qualityValidation.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/datacontract/schemaValidation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/datacontract/schemaValidation.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/datacontract/semanticsValidation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/datacontract/semanticsValidation.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/datacontract/slaValidation.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/datacontract/slaValidation.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/docStore/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/entity/docStore/document.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/docStore/document.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/domains/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00