openmetadata-ingestion 1.3.3.0rc2__py3-none-any.whl → 1.4.0.0__py3-none-any.whl

metadata/generated/schema/entity/services/storageService.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/services/storageService.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00

metadata/generated/schema/entity/teams/persona.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/persona.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/role.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/role.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/team.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/team.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/teamHierarchy.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/teamHierarchy.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/teams/user.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/teams/user.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 
@@ -68,7 +68,7 @@ class User(BaseModel):
     )
     timezone: Optional[str] = Field(None, description='Timezone of the user.')
     isBot: Optional[bool] = Field(
-        None, description='When true indicates a special type of user called Bot.'
+        False, description='When true indicates a special type of user called Bot.'
     )
     isAdmin: Optional[bool] = Field(
         None,

metadata/generated/schema/entity/type.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/type.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/utils/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00

metadata/generated/schema/entity/utils/entitiesCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/utils/entitiesCount.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/utils/servicesCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/utils/servicesCount.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/entity/utils/supersetApiConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  entity/utils/supersetApiConnection.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00

metadata/generated/schema/events/alertMetrics.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/alertMetrics.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/api/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00

metadata/generated/schema/events/api/createEventSubscription.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/api/createEventSubscription.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/emailAlertConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/emailAlertConfig.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/eventFilterRule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/eventFilterRule.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 
@@ -23,6 +23,11 @@ class Effect(Enum):
     exclude = 'exclude'
 
 
+class PrefixCondition(Enum):
+    AND = 'AND'
+    OR = 'OR'
+
+
 class EventFilterRule(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -45,3 +50,7 @@ class EventFilterRule(BaseModel):
         None, description='Arguments to the Condition.'
     )
     inputType: Optional[InputType] = None
+    prefixCondition: Optional[PrefixCondition] = Field(
+        PrefixCondition.AND,
+        description='Prefix Condition to be applied to the Condition.',
+    )

metadata/generated/schema/events/eventSubscription.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/eventSubscription.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 
@@ -28,6 +28,10 @@ class ArgumentsInput(BaseModel):
 
     name: Optional[str] = Field(None, description='Name of the filter')
     effect: Optional[eventFilterRule.Effect] = eventFilterRule.Effect.include
+    prefixCondition: Optional[eventFilterRule.PrefixCondition] = Field(
+        eventFilterRule.PrefixCondition.AND,
+        description='Prefix Condition for the filter.',
+    )
     arguments: Optional[List[Argument]] = Field(None, description='Arguments List')
 
 
@@ -66,7 +70,7 @@ class SubscriptionCategory(Enum):
 
 
 class SubscriptionType(Enum):
-    Generic = 'Generic'
+    Webhook = 'Webhook'
     Slack = 'Slack'
     MsTeams = 'MsTeams'
     GChat = 'GChat'

metadata/generated/schema/events/eventSubscriptionOffset.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/eventSubscriptionOffset.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/failedEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/failedEvent.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/filterResourceDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/filterResourceDescriptor.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/events/subscriptionResourceDescriptor.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  events/subscriptionResourceDescriptor.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00

metadata/generated/schema/metadataIngestion/application.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/application.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/applicationPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/applicationPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dashboardServiceMetadataPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 
@@ -12,6 +12,14 @@ from pydantic import BaseModel, Extra, Field
 from ..type import filterPattern
 
 
+class LineageInformation(BaseModel):
+    dbServiceNames: Optional[List[str]] = Field(
+        None,
+        description='List of Database Service Names for creation of lineage',
+        title='Database Service Names List',
+    )
+
+
 class DashboardMetadataConfigType(Enum):
     DashboardMetadata = 'DashboardMetadata'
 
@@ -23,6 +31,11 @@ class DashboardServiceMetadataPipeline(BaseModel):
     type: Optional[DashboardMetadataConfigType] = Field(
         DashboardMetadataConfigType.DashboardMetadata, description='Pipeline type'
     )
+    lineageInformation: Optional[LineageInformation] = Field(
+        None,
+        description='Details required to generate Lineage',
+        title='Lineage Information',
+    )
     dashboardFilterPattern: Optional[filterPattern.FilterPattern] = Field(
         None,
         description='Regex to exclude or include dashboards that matches the pattern.',
@@ -43,11 +56,6 @@ class DashboardServiceMetadataPipeline(BaseModel):
         description='Regex to exclude or include projects that matches the pattern.',
         title='Project Filter Pattern',
     )
-    dbServiceNames: Optional[List[str]] = Field(
-        None,
-        description='List of Database Service Names for creation of lineage',
-        title='Database Service Names List',
-    )
     includeOwners: Optional[bool] = Field(
         False,
         description='Enabling a flag will replace the current owner with a new owner from the source during metadata ingestion, if the current owner is null. It is recommended to keep the flag enabled to obtain the owner information during the first metadata ingestion.',
@@ -73,3 +81,8 @@ class DashboardServiceMetadataPipeline(BaseModel):
         description='Optional configuration to toggle the ingestion of data models.',
         title='Include Data Models',
     )
+    includeDraftDashboard: Optional[bool] = Field(
+        True,
+        description='Optional Configuration to include/exclude draft dashboards. By default it will include draft dashboards',
+        title='Include Draft Dashboards',
+    )

metadata/generated/schema/metadataIngestion/dataInsightPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dataInsightPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/databaseServiceMetadataPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 
@@ -16,6 +16,27 @@ class DatabaseMetadataConfigType(Enum):
     DatabaseMetadata = 'DatabaseMetadata'
 
 
+class Incremental(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    enabled: bool = Field(
+        ...,
+        description='If True, enables Metadata Extraction to be incremental',
+        title='Enabled',
+    )
+    lookbackDays: Optional[int] = Field(
+        7,
+        description='Number os days to search back for a successful pipeline run. The timestamp of the last found successful pipeline run will be used as a base to search for updated entities.',
+        title='Successful Pipeline Run Lookback Days',
+    )
+    safetyMarginDays: Optional[int] = Field(
+        1,
+        description='Number of days to add to the last successful pipeline run timestamp to search for updated entities.',
+        title='Safety Margin Days',
+    )
+
+
 class DatabaseServiceMetadataPipeline(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -58,6 +79,11 @@ class DatabaseServiceMetadataPipeline(BaseModel):
         description='Optional configuration to toggle the Stored Procedures ingestion.',
         title='Include Stored Procedures',
     )
+    includeDDL: Optional[bool] = Field(
+        True,
+        description='Optional configuration to toggle the DDL Statements ingestion.',
+        title='Include DDL Statements',
+    )
     queryLogDuration: Optional[int] = Field(
         1,
         description='Configuration to tune how far we want to look back in query logs to process Stored Procedures results.',
@@ -88,3 +114,13 @@ class DatabaseServiceMetadataPipeline(BaseModel):
         description='Regex to only fetch databases that matches the pattern.',
         title='Database Filter Pattern',
     )
+    threads: Optional[int] = Field(
+        1,
+        description='Number of Threads to use in order to paralellize Table ingestion.',
+        title='Number of Threads',
+    )
+    incremental: Optional[Incremental] = Field(
+        None,
+        description='Use incremental Metadata extraction after the first execution. This is commonly done by getting the changes from Audit tables on the supporting databases.',
+        title='Incremental Metadata Extraction Configuration',
+    )

metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/databaseServiceProfilerPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 
@@ -44,6 +44,21 @@ class DatabaseServiceProfilerPipeline(BaseModel):
         description='Optional configuration to turn off fetching metadata for views.',
         title='Include Views',
     )
+    useFqnForFiltering: Optional[bool] = Field(
+        False,
+        description='Regex will be applied on fully qualified name (e.g service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name)',
+        title='Use FQN For Filtering',
+    )
+    generateSampleData: Optional[bool] = Field(
+        True,
+        description='Option to turn on/off generating sample data. If enabled, profiler will ingest sample data for each table.',
+        title='Generate Sample Data',
+    )
+    computeMetrics: Optional[bool] = Field(
+        True,
+        description='Option to turn on/off computing profiler metrics.',
+        title='Compute Metrics',
+    )
     processPiiSensitive: Optional[bool] = Field(
         False,
         description='Optional configuration to automatically tag columns that might contain sensitive information',
@@ -51,30 +66,20 @@ class DatabaseServiceProfilerPipeline(BaseModel):
     )
     confidence: Optional[float] = Field(
         80,
-        description='Set the Confidence value for which you want the column to be marked',
-        title='Confidence',
-    )
-    profileSample: Optional[float] = Field(
-        None,
-        description='Percentage of data or no. of rows we want to execute the profiler and tests on',
-        title='Profile Sample',
+        description='Set the Confidence value for which you want the column to be tagged as PII. Confidence value ranges from 0 to 100. A higher number will yield less false positives but more false negatives. A lower number will yield more false positives but less false negatives.',
+        title='PII Inference Confidence Level',
     )
     profileSampleType: Optional[table.ProfileSampleType] = Field(
         table.ProfileSampleType.PERCENTAGE, title='Profile Sample Type'
     )
-    generateSampleData: Optional[bool] = Field(
-        True,
-        description='Option to turn on/off generating sample data.',
-        title='Generate Sample Data',
-    )
-    computeMetrics: Optional[bool] = Field(
-        True,
-        description='Option to turn on/off computing profiler metrics.',
-        title='Compute Metrics',
+    profileSample: Optional[float] = Field(
+        None,
+        description='Percentage of data or no. of rows used to compute the profiler metrics and run data quality tests',
+        title='Profile Sample',
     )
     sampleDataCount: Optional[int] = Field(
         50,
-        description='Number of row of sample data to be generated',
+        description="Number of sample rows to ingest when 'Generate Sample Data' is enabled",
         title='Sample Data Rows Count',
     )
     threadCount: Optional[float] = Field(
@@ -85,8 +90,3 @@ class DatabaseServiceProfilerPipeline(BaseModel):
     timeoutSeconds: Optional[int] = Field(
         43200, description='Profiler Timeout in Seconds', title='Timeout (in sec.)'
     )
-    useFqnForFiltering: Optional[bool] = Field(
-        False,
-        description='Regex will be applied on fully qualified name (e.g service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name)',
-        title='Use FQN For Filtering',
-    )

metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/databaseServiceQueryLineagePipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/databaseServiceQueryUsagePipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00

metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtAzureConfig.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtBucketDetails.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtCloudConfig.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtGCSConfig.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtHttpConfig.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtLocalConfig.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/dbtconfig/dbtS3Config.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/messagingServiceMetadataPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/metadataToElasticSearchPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  metadataIngestion/mlmodelServiceMetadataPipeline.json
-#   timestamp: 2024-04-16T13:31:19+00:00
+#   timestamp: 2024-05-21T12:21:21+00:00
 
 from __future__ import annotations