openmetadata-ingestion 1.5.13.0__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py

@@ -0,0 +1,30 @@
+# generated by datamodel-codegen:
+#   filename:  governance/workflows/elements/nodes/startEvent/startEvent.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ......type import basic
+
+
+class StartEventDefinition(BaseModel):
+    type: Optional[str] = 'startEvent'
+    subType: Optional[str] = 'startEvent'
+    name: Annotated[
+        Optional[basic.EntityName],
+        Field(None, description='Name that identifies this Node.'),
+    ]
+    displayName: Annotated[
+        Optional[str],
+        Field(None, description='Display Name that identifies this Node.'),
+    ]
+    description: Annotated[
+        Optional[basic.Markdown], Field(None, description='Description of the Node.')
+    ]

metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2024-11-29T19:36:27+00:00

metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py

@@ -0,0 +1,57 @@
+# generated by datamodel-codegen:
+#   filename:  governance/workflows/elements/nodes/userTask/userApprovalTask.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ......type import basic, entityReference
+
+
+class Assignees(BaseModel):
+    addReviewers: Annotated[
+        Optional[bool],
+        Field(False, description='Add the Reviewers to the assignees List.'),
+    ]
+    extraAssignees: Annotated[
+        Optional[List[entityReference.EntityReference]],
+        Field(None, description='Manually add Specific Assignees.'),
+    ]
+
+
+class Config(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    assignees: Annotated[
+        Assignees, Field(description='People/Teams assigned to the Task.')
+    ]
+
+
+class UserApprovalTaskDefinition(BaseModel):
+    type: Optional[str] = 'userTask'
+    subType: Optional[str] = 'userApprovalTask'
+    name: Annotated[
+        Optional[basic.EntityName],
+        Field(None, description='Name that identifies this Node.'),
+    ]
+    displayName: Annotated[
+        Optional[str],
+        Field(None, description='Display Name that identifies this Node.'),
+    ]
+    description: Annotated[
+        Optional[basic.Markdown], Field(None, description='Description of the Node.')
+    ]
+    config: Optional[Config] = None
+    input: Annotated[
+        Optional[List[str]], Field(['relatedEntity'], max_length=1, min_length=1)
+    ]
+    output: Annotated[
+        Optional[List[str]], Field(['result', 'resolvedBy'], max_length=2, min_length=2)
+    ]

metadata/generated/schema/governance/workflows/elements/triggers/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2024-11-29T19:36:27+00:00

metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py

@@ -0,0 +1,43 @@
+# generated by datamodel-codegen:
+#   filename:  governance/workflows/elements/triggers/eventBasedEntityTrigger.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+
+class Event(Enum):
+    Created = 'Created'
+    Updated = 'Updated'
+
+
+class Config(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    entityType: Annotated[
+        str, Field(description='Entity Type for which it should be triggered.')
+    ]
+    events: List[Event]
+    exclude: Annotated[
+        Optional[List[str]],
+        Field(None, description='Exclude events that only modify given attributes.'),
+    ]
+
+
+class EventBasedEntityTriggerDefinition(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Optional[str] = 'eventBasedEntityTrigger'
+    config: Optional[Config] = None
+    output: Annotated[
+        Optional[List[str]], Field(['relatedEntity'], max_length=1, min_length=1)
+    ]

metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py

@@ -0,0 +1,44 @@
+# generated by datamodel-codegen:
+#   filename:  governance/workflows/elements/triggers/periodicBatchEntityTrigger.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from .....entity.applications import app
+
+
+class Config(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    schedule: Annotated[
+        app.AppSchedule,
+        Field(description='Defines the schedule of the Periodic Trigger.'),
+    ]
+    entityType: Annotated[
+        str, Field(description='Entity Type for which it should be triggered.')
+    ]
+    filters: Annotated[
+        str, Field(description='Search Filters to filter down the entities fetched.')
+    ]
+    batchSize: Annotated[
+        Optional[int], Field(500, description='Number of Entities to process at once.')
+    ]
+
+
+class PeriodicBatchEntityTriggerDefinition(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Optional[str] = 'periodicBatchEntityTrigger'
+    config: Optional[Config] = None
+    output: Annotated[
+        Optional[List[str]], Field(['relatedEntity'], max_length=1, min_length=1)
+    ]

metadata/generated/schema/{entity/data/metrics.py → governance/workflows/workflowDefinition.py}

@@ -1,56 +1,54 @@
 # generated by datamodel-codegen:
-#   filename:  entity/data/metrics.json
-#   timestamp: 2024-11-28T18:44:30+00:00
+#   filename:  governance/workflows/workflowDefinition.json
+#   timestamp: 2024-11-29T19:36:27+00:00
 
 from __future__ import annotations
 
-from typing import List, Optional
+from enum import Enum
+from typing import Any, List, Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
-from ...type import (
-    basic,
-    entityHistory,
-    entityReference,
-    entityReferenceList,
-    tagLabel,
-    usageDetails,
-    votes,
-)
+from ...type import basic, entityHistory, entityReferenceList
+from .elements import edge
 
 
-class Metrics(BaseModel):
+class Type(Enum):
+    eventBasedEntityWorkflow = 'eventBasedEntityWorkflow'
+    periodicBatchEntityWorkflow = 'periodicBatchEntityWorkflow'
+
+
+class WorkflowDefinition(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
     )
     id: Annotated[
-        basic.Uuid,
-        Field(description='Unique identifier that identifies this metrics instance.'),
+        Optional[basic.Uuid],
+        Field(None, description='Unique identifier of this workflow definition.'),
     ]
     name: Annotated[
         basic.EntityName,
-        Field(description='Name that identifies this metrics instance uniquely.'),
+        Field(description='Name that identifies this workflow definition.'),
     ]
-    fullyQualifiedName: Annotated[
-        Optional[basic.FullyQualifiedEntityName],
+    displayName: Annotated[
+        Optional[str],
         Field(
-            None,
-            description="A unique name that identifies a metric in the format 'ServiceName.MetricName'.",
+            None, description='Display Name that identifies this workflow definition.'
         ),
     ]
-    displayName: Annotated[
-        Optional[str],
-        Field(None, description='Display Name that identifies this metric.'),
+    fullyQualifiedName: Annotated[
+        Optional[basic.FullyQualifiedEntityName],
+        Field(None, description='FullyQualifiedName same as `name`.'),
     ]
     description: Annotated[
-        Optional[basic.Markdown],
-        Field(
-            None,
-            description='Description of metrics instance, what it is, and how to use it.',
-        ),
+        basic.Markdown, Field(description='Description of the workflow definition.')
+    ]
+    owners: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
+        Field(None, description='Owners of this workflow definition.'),
     ]
     version: Annotated[
         Optional[entityHistory.EntityVersion],
@@ -70,22 +68,6 @@ class Metrics(BaseModel):
         Optional[basic.Href],
         Field(None, description='Link to the resource corresponding to this entity.'),
     ]
-    owners: Annotated[
-        Optional[entityReferenceList.EntityReferenceList],
-        Field(None, description='Owners of this metrics.'),
-    ]
-    tags: Annotated[
-        Optional[List[tagLabel.TagLabel]],
-        Field(None, description='Tags for this chart.'),
-    ]
-    service: Annotated[
-        entityReference.EntityReference,
-        Field(description='Link to service where this metrics is hosted in.'),
-    ]
-    usageSummary: Annotated[
-        Optional[usageDetails.UsageDetails],
-        Field(None, description='Latest usage information for this database.'),
-    ]
     changeDescription: Annotated[
         Optional[entityHistory.ChangeDescription],
         Field(None, description='Change that lead to this version of the entity.'),
@@ -96,10 +78,15 @@ class Metrics(BaseModel):
             False, description='When `true` indicates the entity has been soft deleted.'
         ),
     ]
-    domain: Annotated[
-        Optional[entityReference.EntityReference],
-        Field(None, description='Domain the Metrics belongs to.'),
+    type: Optional[Type] = None
+    trigger: Annotated[Optional[Any], Field(None, description='Workflow Trigger.')]
+    nodes: Annotated[
+        Optional[Any], Field(None, description='List of nodes used on the workflow.')
     ]
-    votes: Annotated[
-        Optional[votes.Votes], Field(None, description='Votes on the entity.')
+    edges: Annotated[
+        Optional[List[edge.EdgeDefinition]],
+        Field(
+            None,
+            description='List of edges that connect the workflow elements and guide its flow.',
+        ),
     ]

metadata/generated/schema/governance/workflows/workflowInstance.py

@@ -0,0 +1,50 @@
+# generated by datamodel-codegen:
+#   filename:  governance/workflows/workflowInstance.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ...type import basic
+
+
+class WorkflowInstance(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: Annotated[
+        Optional[basic.Uuid],
+        Field(None, description='Unique identifier of this workflow instance state.'),
+    ]
+    workflowDefinitionId: Annotated[
+        Optional[basic.Uuid], Field(None, description='Workflow Definition Id.')
+    ]
+    startedAt: Annotated[
+        Optional[basic.Timestamp],
+        Field(None, description='Timestamp on which the workflow instance started.'),
+    ]
+    endedAt: Annotated[
+        Optional[basic.Timestamp],
+        Field(None, description='Timestamp on which the workflow instance ended.'),
+    ]
+    variables: Optional[Dict[str, Any]] = None
+    timestamp: Annotated[
+        Optional[basic.Timestamp],
+        Field(
+            None,
+            description='Timestamp on which the workflow instance state was created.',
+        ),
+    ]
+    exception: Annotated[
+        Optional[bool],
+        Field(
+            None,
+            description="If the Workflow Instance has errors, 'True'. Else, 'False'.",
+        ),
+    ]

metadata/generated/schema/governance/workflows/workflowInstanceState.py

@@ -0,0 +1,73 @@
+# generated by datamodel-codegen:
+#   filename:  governance/workflows/workflowInstanceState.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ...type import basic
+
+
+class Stage(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    name: Optional[basic.EntityName] = None
+    startedAt: Annotated[
+        Optional[basic.Timestamp],
+        Field(
+            None, description='Timestamp on which the workflow instance stage started.'
+        ),
+    ]
+    endedAt: Annotated[
+        Optional[basic.Timestamp],
+        Field(
+            None, description='Timestamp on which the workflow instance stage ended.'
+        ),
+    ]
+    tasks: Optional[List[basic.Uuid]] = []
+    variables: Optional[Dict[str, Any]] = None
+
+
+class WorkflowInstanceState(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: Annotated[
+        Optional[basic.Uuid],
+        Field(None, description='Unique identifier of this workflow instance state.'),
+    ]
+    workflowInstanceId: Annotated[
+        Optional[basic.Uuid], Field(None, description='Workflow Instance ID.')
+    ]
+    workflowInstanceExecutionId: Annotated[
+        Optional[basic.Uuid],
+        Field(
+            None,
+            description='One WorkflowInstance might execute a flow multiple times. This ID groups together the States of one of those flows.',
+        ),
+    ]
+    workflowDefinitionId: Annotated[
+        Optional[basic.Uuid], Field(None, description='Workflow Definition Reference.')
+    ]
+    stage: Optional[Stage] = None
+    timestamp: Annotated[
+        Optional[basic.Timestamp],
+        Field(
+            None,
+            description='Timestamp on which the workflow instance state was created.',
+        ),
+    ]
+    exception: Annotated[
+        Optional[bool],
+        Field(
+            None,
+            description="If the Workflow Instance has errors, 'True'. Else, 'False'.",
+        ),
+    ]

metadata/generated/schema/metadataIngestion/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-11-28T18:44:30+00:00
+#   timestamp: 2024-11-29T19:36:27+00:00

metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py

@@ -0,0 +1,53 @@
+# generated by datamodel-codegen:
+#   filename:  metadataIngestion/apiServiceMetadataPipeline.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ..type import filterPattern
+
+
+class ApiMetadataConfigType(Enum):
+    ApiMetadata = 'ApiMetadata'
+
+
+class ApiServiceMetadataPipeline(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Annotated[
+        Optional[ApiMetadataConfigType],
+        Field(ApiMetadataConfigType.ApiMetadata, description='Pipeline type'),
+    ]
+    apiCollectionFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch api collections with names matching the pattern.',
+            title='API Collection Filter Pattern',
+        ),
+    ]
+    markDeletedApiCollections: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to soft delete api collections in OpenMetadata if the source collections are deleted. Also, if the collection is deleted, all the associated entities like endpoints, etc., with that collection will be deleted',
+            title='Mark Deleted Api Collection',
+        ),
+    ]
+    overrideMetadata: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description="Set the 'Override Metadata' toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source. If the toggle is set to true, the metadata fetched from the source will override the existing metadata in the OpenMetadata server. If the toggle is set to false, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. This is applicable for fields like description, tags, owner and displayName",
+            title='Override Metadata',
+        ),
+    ]

metadata/generated/schema/metadataIngestion/application.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/application.json
-#   timestamp: 2024-11-28T18:44:30+00:00
+#   timestamp: 2024-11-29T19:36:27+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/applicationPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/applicationPipeline.json
-#   timestamp: 2024-11-28T18:44:30+00:00
+#   timestamp: 2024-11-29T19:36:27+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dashboardServiceMetadataPipeline.json
-#   timestamp: 2024-11-28T18:44:30+00:00
+#   timestamp: 2024-11-29T19:36:27+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dataInsightPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dataInsightPipeline.json
-#   timestamp: 2024-11-28T18:44:30+00:00
+#   timestamp: 2024-11-29T19:36:27+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py

@@ -0,0 +1,127 @@
+# generated by datamodel-codegen:
+#   filename:  metadataIngestion/databaseServiceAutoClassificationPipeline.json
+#   timestamp: 2024-11-29T19:36:27+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from ..entity.data import table
+from ..type import filterPattern
+
+
+class AutoClassificationConfigType(Enum):
+    AutoClassification = 'AutoClassification'
+
+
+class DatabaseServiceAutoClassificationPipeline(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    type: Annotated[
+        Optional[AutoClassificationConfigType],
+        Field(
+            AutoClassificationConfigType.AutoClassification, description='Pipeline type'
+        ),
+    ]
+    classificationFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only compute metrics for table that matches the given tag, tiers, gloassary pattern.',
+            title='Classification Filter Pattern',
+        ),
+    ]
+    schemaFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch tables or databases that matches the pattern.',
+            title='Schema Filter Pattern',
+        ),
+    ]
+    tableFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex exclude tables or databases that matches the pattern.',
+            title='Table Filter Pattern',
+        ),
+    ]
+    databaseFilterPattern: Annotated[
+        Optional[filterPattern.FilterPattern],
+        Field(
+            None,
+            description='Regex to only fetch databases that matches the pattern.',
+            title='Database Filter Pattern',
+        ),
+    ]
+    includeViews: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Optional configuration to turn off fetching metadata for views.',
+            title='Include Views',
+        ),
+    ]
+    useFqnForFiltering: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Regex will be applied on fully qualified name (e.g service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name)',
+            title='Use FQN For Filtering',
+        ),
+    ]
+    storeSampleData: Annotated[
+        Optional[bool],
+        Field(
+            True,
+            description='Option to turn on/off storing sample data. If enabled, we will ingest sample data for each table.',
+            title='Store Sample Data',
+        ),
+    ]
+    enableAutoClassification: Annotated[
+        Optional[bool],
+        Field(
+            False,
+            description='Optional configuration to automatically tag columns that might contain sensitive information',
+            title='Enable Auto Classification',
+        ),
+    ]
+    confidence: Annotated[
+        Optional[float],
+        Field(
+            80,
+            description='Set the Confidence value for which you want the column to be tagged as PII. Confidence value ranges from 0 to 100. A higher number will yield less false positives but more false negatives. A lower number will yield more false positives but less false negatives.',
+            title='Auto Classification Inference Confidence Level',
+        ),
+    ]
+    profileSampleType: Annotated[
+        Optional[table.ProfileSampleType],
+        Field(table.ProfileSampleType.PERCENTAGE, title='Profile Sample Type'),
+    ]
+    profileSample: Annotated[
+        Optional[float],
+        Field(
+            None,
+            description='Percentage of data or no. of rows used to compute the profiler metrics and run data quality tests',
+            title='Profile Sample',
+        ),
+    ]
+    samplingMethodType: Annotated[
+        Optional[table.SamplingMethodType], Field(None, title='Sampling Method Type')
+    ]
+    sampleDataCount: Annotated[
+        Optional[int],
+        Field(
+            50,
+            description="Number of sample rows to ingest when 'Generate Sample Data' is enabled",
+            title='Sample Data Rows Count',
+        ),
+    ]