openmetadata-ingestion 1.7.0.1__py3-none-any.whl → 1.7.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (780) hide show
  1. metadata/applications/example.py +74 -0
  2. metadata/cli/classify.py +2 -4
  3. metadata/cli/common.py +26 -0
  4. metadata/cli/dataquality.py +2 -4
  5. metadata/cli/ingest.py +2 -4
  6. metadata/cli/profile.py +2 -4
  7. metadata/cli/usage.py +2 -4
  8. metadata/data_quality/source/test_suite.py +11 -1
  9. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +14 -2
  10. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  11. metadata/data_quality/validations/models.py +3 -0
  12. metadata/data_quality/validations/runtime_param_setter/base_diff_params_setter.py +120 -0
  13. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +61 -47
  14. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +24 -0
  15. metadata/examples/workflows/tableau.yaml +0 -3
  16. metadata/generated/schema/analytics/__init__.py +1 -1
  17. metadata/generated/schema/analytics/basic.py +1 -1
  18. metadata/generated/schema/analytics/reportData.py +1 -1
  19. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  20. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  21. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  22. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  23. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  24. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  25. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  26. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  27. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  28. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  29. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  30. metadata/generated/schema/api/__init__.py +1 -1
  31. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  32. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  33. metadata/generated/schema/api/analytics/__init__.py +1 -1
  34. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  35. metadata/generated/schema/api/automations/__init__.py +1 -1
  36. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  37. metadata/generated/schema/api/bulkAssets.py +1 -1
  38. metadata/generated/schema/api/classification/__init__.py +1 -1
  39. metadata/generated/schema/api/classification/createClassification.py +1 -1
  40. metadata/generated/schema/api/classification/createTag.py +1 -1
  41. metadata/generated/schema/api/classification/loadTags.py +1 -1
  42. metadata/generated/schema/api/createBot.py +1 -1
  43. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  44. metadata/generated/schema/api/createType.py +1 -1
  45. metadata/generated/schema/api/data/__init__.py +1 -1
  46. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  47. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  48. metadata/generated/schema/api/data/createChart.py +1 -1
  49. metadata/generated/schema/api/data/createContainer.py +1 -1
  50. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  51. metadata/generated/schema/api/data/createDashboard.py +1 -1
  52. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  53. metadata/generated/schema/api/data/createDatabase.py +1 -1
  54. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  55. metadata/generated/schema/api/data/createGlossary.py +1 -1
  56. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  57. metadata/generated/schema/api/data/createMetric.py +1 -1
  58. metadata/generated/schema/api/data/createMlModel.py +1 -1
  59. metadata/generated/schema/api/data/createPipeline.py +1 -1
  60. metadata/generated/schema/api/data/createQuery.py +1 -1
  61. metadata/generated/schema/api/data/createQueryCostRecord.py +1 -1
  62. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  63. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  64. metadata/generated/schema/api/data/createTable.py +1 -1
  65. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  66. metadata/generated/schema/api/data/createTopic.py +1 -1
  67. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  68. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  69. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  70. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  71. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  72. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  73. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  74. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  75. metadata/generated/schema/api/docStore/__init__.py +1 -1
  76. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  77. metadata/generated/schema/api/domains/__init__.py +1 -1
  78. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  79. metadata/generated/schema/api/domains/createDomain.py +1 -1
  80. metadata/generated/schema/api/feed/__init__.py +1 -1
  81. metadata/generated/schema/api/feed/closeTask.py +1 -1
  82. metadata/generated/schema/api/feed/createPost.py +1 -1
  83. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  84. metadata/generated/schema/api/feed/createThread.py +1 -1
  85. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  86. metadata/generated/schema/api/feed/threadCount.py +1 -1
  87. metadata/generated/schema/api/governance/__init__.py +1 -1
  88. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  89. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  90. metadata/generated/schema/api/lineage/__init__.py +1 -1
  91. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  92. metadata/generated/schema/api/lineage/esLineageData.py +1 -1
  93. metadata/generated/schema/api/lineage/lineageDirection.py +1 -1
  94. metadata/generated/schema/api/lineage/nodeInformation.py +1 -1
  95. metadata/generated/schema/api/lineage/searchLineageRequest.py +1 -1
  96. metadata/generated/schema/api/lineage/searchLineageResult.py +1 -1
  97. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  98. metadata/generated/schema/api/policies/__init__.py +1 -1
  99. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  100. metadata/generated/schema/api/search/__init__.py +1 -1
  101. metadata/generated/schema/api/search/previewSearchRequest.py +1 -1
  102. metadata/generated/schema/api/services/__init__.py +1 -1
  103. metadata/generated/schema/api/services/createApiService.py +1 -1
  104. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  105. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  106. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  107. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  108. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  109. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  110. metadata/generated/schema/api/services/createSearchService.py +1 -1
  111. metadata/generated/schema/api/services/createStorageService.py +1 -1
  112. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  113. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +8 -1
  114. metadata/generated/schema/api/setOwner.py +1 -1
  115. metadata/generated/schema/api/teams/__init__.py +1 -1
  116. metadata/generated/schema/api/teams/createPersona.py +1 -1
  117. metadata/generated/schema/api/teams/createRole.py +1 -1
  118. metadata/generated/schema/api/teams/createTeam.py +1 -1
  119. metadata/generated/schema/api/teams/createUser.py +1 -1
  120. metadata/generated/schema/api/tests/__init__.py +1 -1
  121. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  122. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  123. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  124. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  125. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  126. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  127. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  128. metadata/generated/schema/api/voteRequest.py +1 -1
  129. metadata/generated/schema/auth/__init__.py +1 -1
  130. metadata/generated/schema/auth/basicAuth.py +1 -1
  131. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  132. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  133. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  134. metadata/generated/schema/auth/emailRequest.py +1 -1
  135. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  136. metadata/generated/schema/auth/generateToken.py +1 -1
  137. metadata/generated/schema/auth/jwtAuth.py +1 -1
  138. metadata/generated/schema/auth/loginRequest.py +1 -1
  139. metadata/generated/schema/auth/logoutRequest.py +1 -1
  140. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  141. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  142. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  143. metadata/generated/schema/auth/refreshToken.py +1 -1
  144. metadata/generated/schema/auth/registrationRequest.py +1 -1
  145. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  146. metadata/generated/schema/auth/revokeToken.py +1 -1
  147. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  148. metadata/generated/schema/auth/ssoAuth.py +1 -1
  149. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  150. metadata/generated/schema/configuration/__init__.py +1 -1
  151. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  152. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  153. metadata/generated/schema/configuration/authConfig.py +1 -1
  154. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  155. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  156. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  157. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  158. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  159. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  160. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  161. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  162. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  163. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  164. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  165. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  166. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  167. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  168. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  169. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  170. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  171. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  172. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  173. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +3 -2
  175. metadata/generated/schema/configuration/opertionalConfiguration.py +1 -1
  176. metadata/generated/schema/configuration/opsConfig.py +1 -1
  177. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  178. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  179. metadata/generated/schema/configuration/searchSettings.py +1 -1
  180. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  181. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  182. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  183. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  184. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  185. metadata/generated/schema/configuration/workflowSettings.py +3 -3
  186. metadata/generated/schema/dataInsight/__init__.py +1 -1
  187. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  188. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  189. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  190. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  191. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  192. metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
  193. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  194. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  195. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  196. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  197. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  198. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  199. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  200. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  201. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  202. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  203. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  204. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  205. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  206. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  207. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  208. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  209. metadata/generated/schema/email/__init__.py +1 -1
  210. metadata/generated/schema/email/emailRequest.py +1 -1
  211. metadata/generated/schema/email/emailTemplate.py +1 -1
  212. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  213. metadata/generated/schema/email/smtpSettings.py +1 -1
  214. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  215. metadata/generated/schema/entity/__init__.py +1 -1
  216. metadata/generated/schema/entity/applications/__init__.py +1 -1
  217. metadata/generated/schema/entity/applications/app.py +4 -3
  218. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  219. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  220. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  221. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +10 -4
  222. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  223. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  224. metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
  225. metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +1 -1
  226. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  227. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  228. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  229. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  230. metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +1 -1
  231. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  232. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  233. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  234. metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
  235. metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +1 -1
  236. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  237. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  238. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  240. metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +1 -1
  241. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  242. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  243. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  244. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  245. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  246. metadata/generated/schema/entity/applications/configuration/internal/autoPilotAppConfig.py +1 -1
  247. metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +1 -1
  248. metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +1 -1
  249. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +3 -1
  250. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  251. metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +1 -1
  252. metadata/generated/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.py +18 -0
  253. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  254. metadata/generated/schema/entity/applications/configuration/private/__init__.py +1 -1
  255. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  256. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  257. metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +1 -1
  258. metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +1 -1
  259. metadata/generated/schema/entity/applications/configuration/private/limits.py +1 -1
  260. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  261. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  262. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  263. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  264. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +5 -2
  265. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -2
  266. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  267. metadata/generated/schema/entity/automations/__init__.py +1 -1
  268. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  269. metadata/generated/schema/entity/automations/workflow.py +1 -1
  270. metadata/generated/schema/entity/bot.py +1 -1
  271. metadata/generated/schema/entity/classification/__init__.py +1 -1
  272. metadata/generated/schema/entity/classification/classification.py +1 -1
  273. metadata/generated/schema/entity/classification/tag.py +1 -1
  274. metadata/generated/schema/entity/data/__init__.py +1 -1
  275. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  276. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  277. metadata/generated/schema/entity/data/chart.py +1 -1
  278. metadata/generated/schema/entity/data/container.py +1 -1
  279. metadata/generated/schema/entity/data/dashboard.py +1 -1
  280. metadata/generated/schema/entity/data/dashboardDataModel.py +2 -1
  281. metadata/generated/schema/entity/data/database.py +5 -1
  282. metadata/generated/schema/entity/data/databaseSchema.py +5 -1
  283. metadata/generated/schema/entity/data/glossary.py +1 -1
  284. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  285. metadata/generated/schema/entity/data/metric.py +1 -1
  286. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  287. metadata/generated/schema/entity/data/pipeline.py +1 -1
  288. metadata/generated/schema/entity/data/query.py +1 -1
  289. metadata/generated/schema/entity/data/queryCostRecord.py +1 -1
  290. metadata/generated/schema/entity/data/queryCostSearchResult.py +1 -1
  291. metadata/generated/schema/entity/data/report.py +1 -1
  292. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  293. metadata/generated/schema/entity/data/storedProcedure.py +1 -1
  294. metadata/generated/schema/entity/data/table.py +1 -1
  295. metadata/generated/schema/entity/data/topic.py +1 -1
  296. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  297. metadata/generated/schema/entity/docStore/document.py +1 -1
  298. metadata/generated/schema/entity/domains/__init__.py +1 -1
  299. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  300. metadata/generated/schema/entity/domains/domain.py +1 -1
  301. metadata/generated/schema/entity/events/__init__.py +1 -1
  302. metadata/generated/schema/entity/events/webhook.py +1 -1
  303. metadata/generated/schema/entity/feed/__init__.py +1 -1
  304. metadata/generated/schema/entity/feed/assets.py +1 -1
  305. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  306. metadata/generated/schema/entity/feed/description.py +1 -1
  307. metadata/generated/schema/entity/feed/domain.py +1 -1
  308. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  309. metadata/generated/schema/entity/feed/owner.py +1 -1
  310. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  311. metadata/generated/schema/entity/feed/tag.py +1 -1
  312. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  313. metadata/generated/schema/entity/feed/thread.py +1 -1
  314. metadata/generated/schema/entity/policies/__init__.py +1 -1
  315. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  316. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  317. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  318. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  319. metadata/generated/schema/entity/policies/filters.py +1 -1
  320. metadata/generated/schema/entity/policies/policy.py +1 -1
  321. metadata/generated/schema/entity/services/__init__.py +1 -1
  322. metadata/generated/schema/entity/services/apiService.py +5 -1
  323. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  324. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  325. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  326. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  327. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  328. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  329. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  330. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  331. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  332. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  333. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  334. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  335. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  336. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  337. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  340. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  341. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  342. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  343. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  344. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  345. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  346. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  347. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  352. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +2 -11
  353. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +1 -1
  360. metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +1 -1
  361. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  367. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  372. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  380. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
  389. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  391. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  395. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  396. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +9 -1
  403. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  410. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  411. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  416. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  422. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  423. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  425. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  427. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  428. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  429. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  430. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  431. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  432. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  435. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  436. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  437. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  438. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  439. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  440. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  441. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  442. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  443. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  444. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  446. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  449. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
  451. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  455. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  456. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
  458. metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
  459. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
  460. metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
  461. metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
  462. metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
  463. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  464. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  466. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  467. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py +1 -1
  469. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  470. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  471. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  472. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  473. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  474. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  475. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  476. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  477. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  478. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  479. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  480. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  481. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  482. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  483. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  484. metadata/generated/schema/entity/services/dashboardService.py +5 -1
  485. metadata/generated/schema/entity/services/databaseService.py +5 -1
  486. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  487. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +12 -1
  488. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  489. metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +1 -1
  490. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  491. metadata/generated/schema/entity/services/messagingService.py +5 -1
  492. metadata/generated/schema/entity/services/metadataService.py +5 -1
  493. metadata/generated/schema/entity/services/mlmodelService.py +5 -1
  494. metadata/generated/schema/entity/services/pipelineService.py +5 -1
  495. metadata/generated/schema/entity/services/searchService.py +5 -1
  496. metadata/generated/schema/entity/services/serviceType.py +1 -1
  497. metadata/generated/schema/entity/services/storageService.py +5 -1
  498. metadata/generated/schema/entity/teams/__init__.py +1 -1
  499. metadata/generated/schema/entity/teams/persona.py +1 -1
  500. metadata/generated/schema/entity/teams/role.py +1 -1
  501. metadata/generated/schema/entity/teams/team.py +1 -1
  502. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  503. metadata/generated/schema/entity/teams/user.py +1 -1
  504. metadata/generated/schema/entity/type.py +1 -1
  505. metadata/generated/schema/entity/utils/__init__.py +1 -1
  506. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  507. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  508. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  509. metadata/generated/schema/events/__init__.py +1 -1
  510. metadata/generated/schema/events/alertMetrics.py +1 -1
  511. metadata/generated/schema/events/api/__init__.py +1 -1
  512. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  513. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  514. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  515. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  516. metadata/generated/schema/events/api/typedEvent.py +1 -1
  517. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  518. metadata/generated/schema/events/eventFilterRule.py +1 -1
  519. metadata/generated/schema/events/eventSubscription.py +1 -1
  520. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  521. metadata/generated/schema/events/failedEvent.py +1 -1
  522. metadata/generated/schema/events/failedEventResponse.py +1 -1
  523. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  524. metadata/generated/schema/events/statusContext.py +1 -1
  525. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  526. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  527. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  528. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  529. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  530. metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
  531. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
  532. metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
  533. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  534. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  535. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
  536. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +1 -1
  537. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +1 -1
  538. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
  539. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
  540. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  541. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
  542. metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +1 -1
  543. metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +1 -1
  544. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  545. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
  546. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  547. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
  548. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  549. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
  550. metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +1 -1
  551. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +1 -1
  552. metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
  553. metadata/generated/schema/governance/workflows/workflowInstance.py +1 -1
  554. metadata/generated/schema/governance/workflows/workflowInstanceState.py +1 -1
  555. metadata/generated/schema/jobs/__init__.py +1 -1
  556. metadata/generated/schema/jobs/backgroundJob.py +1 -1
  557. metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
  558. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  559. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  560. metadata/generated/schema/metadataIngestion/application.py +1 -1
  561. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  562. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  563. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  564. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -1
  565. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
  566. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -1
  567. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  568. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  569. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  570. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  571. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  572. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  573. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  574. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  575. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  576. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  577. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  578. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  579. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  580. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  581. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  582. metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +1 -1
  583. metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +1 -1
  584. metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +1 -1
  585. metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +1 -1
  586. metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +1 -1
  587. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  588. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  589. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  590. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  591. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  592. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  593. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  594. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  595. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  596. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  597. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  598. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  599. metadata/generated/schema/metadataIngestion/workflow.py +8 -1
  600. metadata/generated/schema/monitoring/__init__.py +1 -1
  601. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  602. metadata/generated/schema/search/__init__.py +1 -1
  603. metadata/generated/schema/search/aggregationRequest.py +1 -1
  604. metadata/generated/schema/search/searchRequest.py +1 -1
  605. metadata/generated/schema/security/__init__.py +1 -1
  606. metadata/generated/schema/security/client/__init__.py +1 -1
  607. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  608. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  609. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  610. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  611. metadata/generated/schema/security/client/oidcClientConfig.py +4 -1
  612. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  613. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  614. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  615. metadata/generated/schema/security/credentials/__init__.py +1 -1
  616. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  617. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  618. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  619. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  620. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  621. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  622. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  623. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  624. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  625. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  626. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  627. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  628. metadata/generated/schema/security/sasl/__init__.py +1 -1
  629. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  630. metadata/generated/schema/security/secrets/__init__.py +1 -1
  631. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  632. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  633. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  634. metadata/generated/schema/security/securityConfiguration.py +1 -1
  635. metadata/generated/schema/security/ssl/__init__.py +1 -1
  636. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  637. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  638. metadata/generated/schema/settings/__init__.py +1 -1
  639. metadata/generated/schema/settings/settings.py +1 -1
  640. metadata/generated/schema/system/__init__.py +1 -1
  641. metadata/generated/schema/system/entityError.py +1 -1
  642. metadata/generated/schema/system/eventPublisherJob.py +17 -5
  643. metadata/generated/schema/system/indexingError.py +1 -1
  644. metadata/generated/schema/system/limitsResponse.py +1 -1
  645. metadata/generated/schema/system/ui/__init__.py +1 -1
  646. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  647. metadata/generated/schema/system/ui/navigationItem.py +1 -1
  648. metadata/generated/schema/system/ui/page.py +1 -1
  649. metadata/generated/schema/system/ui/tab.py +1 -1
  650. metadata/generated/schema/system/ui/uiCustomization.py +1 -1
  651. metadata/generated/schema/system/validationResponse.py +1 -1
  652. metadata/generated/schema/tests/__init__.py +1 -1
  653. metadata/generated/schema/tests/assigned.py +1 -1
  654. metadata/generated/schema/tests/basic.py +20 -20
  655. metadata/generated/schema/tests/customMetric.py +1 -1
  656. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  657. metadata/generated/schema/tests/resolved.py +1 -1
  658. metadata/generated/schema/tests/testCase.py +1 -1
  659. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  660. metadata/generated/schema/tests/testDefinition.py +1 -1
  661. metadata/generated/schema/tests/testSuite.py +1 -1
  662. metadata/generated/schema/type/__init__.py +1 -1
  663. metadata/generated/schema/type/apiSchema.py +1 -1
  664. metadata/generated/schema/type/assetCertification.py +1 -1
  665. metadata/generated/schema/type/auditLog.py +1 -1
  666. metadata/generated/schema/type/basic.py +1 -1
  667. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  668. metadata/generated/schema/type/changeEvent.py +1 -1
  669. metadata/generated/schema/type/changeEventType.py +1 -1
  670. metadata/generated/schema/type/changeSummaryMap.py +1 -1
  671. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  672. metadata/generated/schema/type/csvDocumentation.py +1 -1
  673. metadata/generated/schema/type/csvErrorType.py +1 -1
  674. metadata/generated/schema/type/csvFile.py +1 -1
  675. metadata/generated/schema/type/csvImportResult.py +1 -1
  676. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  677. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  678. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  679. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  680. metadata/generated/schema/type/customProperty.py +1 -1
  681. metadata/generated/schema/type/dailyCount.py +1 -1
  682. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  683. metadata/generated/schema/type/entityHierarchy.py +1 -1
  684. metadata/generated/schema/type/entityHistory.py +1 -1
  685. metadata/generated/schema/type/entityLineage.py +1 -1
  686. metadata/generated/schema/type/entityReference.py +1 -1
  687. metadata/generated/schema/type/entityReferenceList.py +1 -1
  688. metadata/generated/schema/type/entityRelationship.py +1 -1
  689. metadata/generated/schema/type/entityUsage.py +1 -1
  690. metadata/generated/schema/type/filterPattern.py +1 -1
  691. metadata/generated/schema/type/function.py +1 -1
  692. metadata/generated/schema/type/include.py +1 -1
  693. metadata/generated/schema/type/jdbcConnection.py +1 -1
  694. metadata/generated/schema/type/lifeCycle.py +1 -1
  695. metadata/generated/schema/type/paging.py +1 -1
  696. metadata/generated/schema/type/profile.py +1 -1
  697. metadata/generated/schema/type/queryParserData.py +1 -1
  698. metadata/generated/schema/type/reaction.py +1 -1
  699. metadata/generated/schema/type/schedule.py +1 -1
  700. metadata/generated/schema/type/schema.py +1 -1
  701. metadata/generated/schema/type/tableQuery.py +1 -1
  702. metadata/generated/schema/type/tableUsageCount.py +1 -1
  703. metadata/generated/schema/type/tagLabel.py +2 -1
  704. metadata/generated/schema/type/usageDetails.py +1 -1
  705. metadata/generated/schema/type/usageRequest.py +1 -1
  706. metadata/generated/schema/type/votes.py +1 -1
  707. metadata/ingestion/api/topology_runner.py +5 -1
  708. metadata/ingestion/models/patch_request.py +71 -3
  709. metadata/ingestion/ometa/mixins/es_mixin.py +11 -5
  710. metadata/ingestion/source/api/rest/metadata.py +15 -2
  711. metadata/ingestion/source/dashboard/powerbi/metadata.py +131 -39
  712. metadata/ingestion/source/dashboard/powerbi/models.py +23 -1
  713. metadata/ingestion/source/dashboard/tableau/client.py +152 -171
  714. metadata/ingestion/source/dashboard/tableau/connection.py +23 -48
  715. metadata/ingestion/source/dashboard/tableau/metadata.py +73 -99
  716. metadata/ingestion/source/dashboard/tableau/models.py +8 -18
  717. metadata/ingestion/source/dashboard/tableau/queries.py +2 -2
  718. metadata/ingestion/source/database/athena/metadata.py +26 -0
  719. metadata/ingestion/source/database/bigquery/connection.py +8 -3
  720. metadata/ingestion/source/database/bigquery/helper.py +8 -6
  721. metadata/ingestion/source/database/bigquery/metadata.py +13 -5
  722. metadata/ingestion/source/database/dbt/metadata.py +30 -17
  723. metadata/ingestion/source/database/life_cycle_query_mixin.py +9 -0
  724. metadata/ingestion/source/database/mysql/connection.py +11 -3
  725. metadata/ingestion/source/database/mysql/lineage.py +4 -4
  726. metadata/ingestion/source/database/mysql/queries.py +29 -0
  727. metadata/ingestion/source/database/mysql/query_parser.py +31 -0
  728. metadata/ingestion/source/database/oracle/queries.py +2 -2
  729. metadata/ingestion/source/database/postgres/metadata.py +3 -1
  730. metadata/ingestion/source/database/postgres/queries.py +7 -0
  731. metadata/ingestion/source/database/postgres/utils.py +28 -19
  732. metadata/ingestion/source/database/snowflake/data_diff/__init__.py +0 -0
  733. metadata/ingestion/source/database/snowflake/data_diff/data_diff.py +37 -0
  734. metadata/ingestion/source/database/snowflake/metadata.py +14 -0
  735. metadata/ingestion/source/database/snowflake/queries.py +11 -7
  736. metadata/ingestion/source/database/snowflake/service_spec.py +4 -0
  737. metadata/ingestion/source/database/snowflake/utils.py +32 -4
  738. metadata/ingestion/source/database/vertica/queries.py +5 -20
  739. metadata/ingestion/source/pipeline/airbyte/constants.py +29 -0
  740. metadata/ingestion/source/pipeline/airbyte/metadata.py +67 -26
  741. metadata/ingestion/source/pipeline/airbyte/utils.py +99 -0
  742. metadata/ingestion/source/pipeline/openlineage/models.py +3 -2
  743. metadata/ingestion/source/pipeline/pipeline_service.py +2 -3
  744. metadata/ingestion/source/storage/s3/metadata.py +7 -8
  745. metadata/pii/algorithms/classifiers.py +180 -0
  746. metadata/pii/algorithms/column_patterns.py +61 -0
  747. metadata/pii/algorithms/feature_extraction.py +154 -0
  748. metadata/pii/algorithms/preprocessing.py +62 -0
  749. metadata/pii/algorithms/presidio_patches.py +45 -0
  750. metadata/pii/algorithms/presidio_utils.py +119 -0
  751. metadata/pii/algorithms/tags.py +111 -0
  752. metadata/pii/algorithms/utils.py +38 -0
  753. metadata/pii/base_processor.py +125 -0
  754. metadata/pii/constants.py +8 -0
  755. metadata/pii/processor.py +47 -138
  756. metadata/profiler/interface/sqlalchemy/profiler_interface.py +66 -36
  757. metadata/profiler/processor/runner.py +29 -6
  758. metadata/profiler/source/database/mssql/profiler_source.py +86 -0
  759. metadata/profiler/source/fetcher/profiler_source_factory.py +13 -0
  760. metadata/readers/dataframe/json.py +5 -1
  761. metadata/readers/dataframe/parquet.py +10 -2
  762. metadata/readers/dataframe/reader_factory.py +8 -0
  763. metadata/sampler/processor.py +1 -1
  764. metadata/sampler/sampler_interface.py +3 -0
  765. metadata/sampler/sqlalchemy/bigquery/sampler.py +2 -63
  766. metadata/sampler/sqlalchemy/sampler.py +32 -40
  767. metadata/utils/datalake/datalake_utils.py +9 -3
  768. metadata/utils/fqn.py +4 -4
  769. metadata/utils/service_spec/default.py +4 -0
  770. metadata/utils/service_spec/service_spec.py +1 -0
  771. metadata/utils/sqa_utils.py +15 -0
  772. metadata/workflow/base.py +8 -1
  773. metadata/workflow/profiler.py +9 -9
  774. metadata/workflow/workflow_status_mixin.py +1 -7
  775. {openmetadata_ingestion-1.7.0.1.dist-info → openmetadata_ingestion-1.7.1.1.dist-info}/METADATA +427 -421
  776. {openmetadata_ingestion-1.7.0.1.dist-info → openmetadata_ingestion-1.7.1.1.dist-info}/RECORD +780 -762
  777. {openmetadata_ingestion-1.7.0.1.dist-info → openmetadata_ingestion-1.7.1.1.dist-info}/LICENSE +0 -0
  778. {openmetadata_ingestion-1.7.0.1.dist-info → openmetadata_ingestion-1.7.1.1.dist-info}/WHEEL +0 -0
  779. {openmetadata_ingestion-1.7.0.1.dist-info → openmetadata_ingestion-1.7.1.1.dist-info}/entry_points.txt +0 -0
  780. {openmetadata_ingestion-1.7.0.1.dist-info → openmetadata_ingestion-1.7.1.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,74 @@
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+ """
12
+ Example external application
13
+ """
14
+ from time import sleep
15
+ from typing import Any
16
+
17
+ from metadata.generated.schema.entity.applications.configuration.internal.helloPipelinesConfiguration import (
18
+ HelloPipelinesAppConfiguration,
19
+ )
20
+ from metadata.generated.schema.metadataIngestion.application import (
21
+ OpenMetadataApplicationConfig,
22
+ )
23
+ from metadata.ingestion.ometa.ometa_api import OpenMetadata
24
+ from metadata.utils.logger import app_logger
25
+ from metadata.workflow.application import AppRunner, InvalidAppConfiguration
26
+
27
+ logger = app_logger()
28
+
29
+
30
+ class HelloPipelines(AppRunner):
31
+ """
32
+ Example external application that sleeps for a given time and then echoes a message.
33
+ You can execute it with `metadata app -c <path-to-yaml>`
34
+ with a YAML file like:
35
+
36
+ sourcePythonClass: metadata.applications.example.HelloPipelines
37
+ appConfig:
38
+ type: HelloPipelines
39
+ sleep: 5
40
+ echo: this will be echoed
41
+ workflowConfig:
42
+ loggerLevel: INFO
43
+ openMetadataServerConfig:
44
+ hostPort: http://localhost:8585/api
45
+ authProvider: openmetadata
46
+ securityConfig:
47
+ jwtToken: "..."
48
+ """
49
+
50
+ def __init__(
51
+ self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata[Any, Any]
52
+ ):
53
+ super().__init__(config, metadata) # pyright: ignore [reportUnknownMemberType]
54
+ try:
55
+ self.app_config: HelloPipelinesAppConfiguration = (
56
+ HelloPipelinesAppConfiguration.model_validate(self.app_config)
57
+ )
58
+ except Exception as e:
59
+ raise InvalidAppConfiguration(
60
+ f"Hello pipelines received invalid configuration: {e}"
61
+ )
62
+
63
+ @property
64
+ def name(self) -> str:
65
+ return "HelloPipelines"
66
+
67
+ def run(self) -> None:
68
+ logger.info(f"sleeping for {self.app_config.sleep}")
69
+ sleep(self.app_config.sleep)
70
+ logger.info("echoing")
71
+ logger.info(self.app_config.echo)
72
+
73
+ def close(self) -> None:
74
+ """Nothing to close"""
metadata/cli/classify.py CHANGED
@@ -16,6 +16,7 @@ import sys
16
16
  import traceback
17
17
  from pathlib import Path
18
18
 
19
+ from metadata.cli.common import execute_workflow
19
20
  from metadata.config.common import load_config_file
20
21
  from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
21
22
  PipelineType,
@@ -46,7 +47,4 @@ def run_classification(config_path: Path) -> None:
46
47
  )
47
48
  sys.exit(1)
48
49
 
49
- workflow.execute()
50
- workflow.stop()
51
- workflow.print_status()
52
- workflow.raise_from_status()
50
+ execute_workflow(workflow=workflow, config_dict=config_dict)
metadata/cli/common.py ADDED
@@ -0,0 +1,26 @@
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Handle workflow execution
14
+ """
15
+ from typing import Any, Dict
16
+
17
+ from metadata.workflow.base import BaseWorkflow
18
+
19
+
20
+ def execute_workflow(workflow: BaseWorkflow, config_dict: Dict[str, Any]) -> None:
21
+ """Execute the workflow and raise if needed"""
22
+ workflow.execute()
23
+ workflow.print_status()
24
+ workflow.stop()
25
+ if config_dict.get("workflowConfig", {}).get("raiseOnError", True):
26
+ workflow.raise_from_status()
@@ -16,6 +16,7 @@ import sys
16
16
  import traceback
17
17
  from pathlib import Path
18
18
 
19
+ from metadata.cli.common import execute_workflow
19
20
  from metadata.config.common import load_config_file
20
21
  from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
21
22
  PipelineType,
@@ -48,7 +49,4 @@ def run_test(config_path: Path) -> None:
48
49
  )
49
50
  sys.exit(1)
50
51
 
51
- workflow.execute()
52
- workflow.stop()
53
- workflow.print_status()
54
- workflow.raise_from_status()
52
+ execute_workflow(workflow=workflow, config_dict=workflow_config_dict)
metadata/cli/ingest.py CHANGED
@@ -16,6 +16,7 @@ import sys
16
16
  import traceback
17
17
  from pathlib import Path
18
18
 
19
+ from metadata.cli.common import execute_workflow
19
20
  from metadata.config.common import load_config_file
20
21
  from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
21
22
  PipelineType,
@@ -46,7 +47,4 @@ def run_ingest(config_path: Path) -> None:
46
47
  )
47
48
  sys.exit(1)
48
49
 
49
- workflow.execute()
50
- workflow.stop()
51
- workflow.print_status()
52
- workflow.raise_from_status()
50
+ execute_workflow(workflow=workflow, config_dict=config_dict)
metadata/cli/profile.py CHANGED
@@ -16,6 +16,7 @@ import sys
16
16
  import traceback
17
17
  from pathlib import Path
18
18
 
19
+ from metadata.cli.common import execute_workflow
19
20
  from metadata.config.common import load_config_file
20
21
  from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
21
22
  PipelineType,
@@ -48,7 +49,4 @@ def run_profiler(config_path: Path) -> None:
48
49
  )
49
50
  sys.exit(1)
50
51
 
51
- workflow.execute()
52
- workflow.stop()
53
- workflow.print_status()
54
- workflow.raise_from_status()
52
+ execute_workflow(workflow=workflow, config_dict=workflow_config_dict)
metadata/cli/usage.py CHANGED
@@ -16,6 +16,7 @@ import sys
16
16
  import traceback
17
17
  from pathlib import Path
18
18
 
19
+ from metadata.cli.common import execute_workflow
19
20
  from metadata.config.common import load_config_file
20
21
  from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
21
22
  PipelineType,
@@ -44,7 +45,4 @@ def run_usage(config_path: Path) -> None:
44
45
  WorkflowInitErrorHandler.print_init_error(exc, config_dict, PipelineType.usage)
45
46
  sys.exit(1)
46
47
 
47
- workflow.execute()
48
- workflow.stop()
49
- workflow.print_status()
50
- workflow.raise_from_status()
48
+ execute_workflow(workflow=workflow, config_dict=config_dict)
@@ -136,7 +136,17 @@ class TestSuiteSource(Source):
136
136
  f"Service with name `{service_name}` does not have a connection. "
137
137
  "If the connection is not stored in OpenMetadata, please provide it in the YAML file."
138
138
  )
139
- self.service_connection_map[service_name] = service.connection
139
+
140
+ # TODO: Clean after https://github.com/open-metadata/OpenMetadata/issues/21259
141
+ # We are forcing the secret evaluation to "ignore" null secrets down the line
142
+ # Remove this when the issue above is fixed and empty secrets migrated
143
+ source_config_class = type(service.connection)
144
+ dumped_config = service.connection.model_dump()
145
+ service_connection_clean = source_config_class.model_validate(
146
+ dumped_config
147
+ )
148
+
149
+ self.service_connection_map[service_name] = service_connection_clean
140
150
 
141
151
  except Exception as exc:
142
152
  logger.debug(traceback.format_exc())
@@ -15,7 +15,7 @@ Validator for column values to be unique test case
15
15
 
16
16
  from typing import Optional
17
17
 
18
- from sqlalchemy import Column, inspect
18
+ from sqlalchemy import Column, inspect, literal_column
19
19
  from sqlalchemy.exc import SQLAlchemyError
20
20
 
21
21
  from metadata.data_quality.validations.column.base.columnValuesToBeUnique import (
@@ -25,6 +25,7 @@ from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
25
25
  SQAValidatorMixin,
26
26
  )
27
27
  from metadata.profiler.metrics.registry import Metrics
28
+ from metadata.profiler.orm.registry import Dialects
28
29
 
29
30
 
30
31
  class ColumnValuesToBeUniqueValidator(
@@ -57,7 +58,18 @@ class ColumnValuesToBeUniqueValidator(
57
58
  ) # type: ignore
58
59
 
59
60
  try:
60
- self.value = dict(self.runner.dispatch_query_select_first(count, unique_count.scalar_subquery().label("uniqueCount"))) # type: ignore
61
+ if self.runner.dialect == Dialects.Oracle:
62
+ query_group_by_ = [literal_column("2")]
63
+ else:
64
+ query_group_by_ = None
65
+
66
+ self.value = dict(
67
+ self.runner.dispatch_query_select_first(
68
+ count,
69
+ unique_count.scalar_subquery().label("uniqueCount"),
70
+ query_group_by_=query_group_by_,
71
+ )
72
+ ) # type: ignore
61
73
  res = self.value.get(Metrics.COUNT.name)
62
74
  except Exception as exc:
63
75
  raise SQLAlchemyError(exc)
@@ -76,7 +76,7 @@ class ColumnValuesToMatchRegexValidator(
76
76
  res = dict(
77
77
  self.runner.dispatch_query_select_first(
78
78
  Metrics.COUNT(column).fn(),
79
- regex_count,
79
+ regex_count_fn,
80
80
  )
81
81
  )
82
82
 
@@ -8,6 +8,7 @@ from metadata.generated.schema.entity.data.table import Column, TableProfilerCon
8
8
  from metadata.generated.schema.entity.services.databaseService import (
9
9
  DatabaseServiceType,
10
10
  )
11
+ from metadata.ingestion.models.custom_pydantic import CustomSecretStr
11
12
 
12
13
 
13
14
  class TableParameter(BaseModel):
@@ -15,6 +16,8 @@ class TableParameter(BaseModel):
15
16
  path: str
16
17
  columns: List[Column]
17
18
  database_service_type: DatabaseServiceType
19
+ privateKey: Optional[CustomSecretStr]
20
+ passPhrase: Optional[CustomSecretStr]
18
21
 
19
22
 
20
23
  class TableDiffRuntimeParameters(BaseModel):
@@ -0,0 +1,120 @@
1
+ """Base class for param setter logic for table data diff"""
2
+
3
+ from typing import List, Optional, Set
4
+ from urllib.parse import urlparse
5
+
6
+ from metadata.data_quality.validations.models import Column, TableParameter
7
+ from metadata.generated.schema.entity.data.table import Table
8
+ from metadata.generated.schema.entity.services.databaseService import DatabaseService
9
+ from metadata.ingestion.source.connections import get_connection
10
+ from metadata.profiler.orm.registry import Dialects
11
+ from metadata.utils import fqn
12
+ from metadata.utils.collections import CaseInsensitiveList
13
+
14
+
15
+ class BaseTableParameter:
16
+ """Base table parameter setter for the table diff test."""
17
+
18
+ def get(
19
+ self,
20
+ service: DatabaseService,
21
+ entity: Table,
22
+ key_columns,
23
+ extra_columns,
24
+ case_sensitive_columns,
25
+ service_url: Optional[str],
26
+ ) -> TableParameter:
27
+ """Getter table parameter for the table diff test.
28
+
29
+ Returns:
30
+ TableParameter
31
+ """
32
+ return TableParameter(
33
+ database_service_type=service.serviceType,
34
+ path=self.get_data_diff_table_path(entity.fullyQualifiedName.root),
35
+ serviceUrl=self.get_data_diff_url(
36
+ service,
37
+ entity.fullyQualifiedName.root,
38
+ override_url=service_url,
39
+ ),
40
+ columns=self.filter_relevant_columns(
41
+ entity.columns,
42
+ key_columns,
43
+ extra_columns,
44
+ case_sensitive=case_sensitive_columns,
45
+ ),
46
+ privateKey=None,
47
+ passPhrase=None,
48
+ )
49
+
50
+ @staticmethod
51
+ def get_data_diff_table_path(table_fqn: str) -> str:
52
+ """Get the data diff table path.
53
+
54
+ Args:
55
+ table_fqn (str): The fully qualified name of the table
56
+
57
+ Returns:
58
+ str
59
+ """
60
+ _, _, schema, table = fqn.split(table_fqn)
61
+ return fqn._build( # pylint: disable=protected-access
62
+ "___SERVICE___", "__DATABASE__", schema, table
63
+ ).replace("___SERVICE___.__DATABASE__.", "")
64
+
65
+ @staticmethod
66
+ def get_data_diff_url(
67
+ db_service: DatabaseService, table_fqn, override_url: Optional[str] = None
68
+ ) -> str:
69
+ """Get the url for the data diff service.
70
+
71
+ Args:
72
+ db_service (DatabaseService): The database service entity
73
+ table_fqn (str): The fully qualified name of the table
74
+ override_url (Optional[str], optional): Override the url. Defaults to None.
75
+
76
+ Returns:
77
+ str: The url for the data diff service
78
+ """
79
+ source_url = (
80
+ str(get_connection(db_service.connection.config).url)
81
+ if not override_url
82
+ else override_url
83
+ )
84
+ url = urlparse(source_url)
85
+ # remove the driver name from the url because table-diff doesn't support it
86
+ kwargs = {"scheme": url.scheme.split("+")[0]}
87
+ service, database, schema, table = fqn.split( # pylint: disable=unused-variable
88
+ table_fqn
89
+ )
90
+ # path needs to include the database AND schema in some of the connectors
91
+ if hasattr(db_service.connection.config, "supportsDatabase"):
92
+ kwargs["path"] = f"/{database}"
93
+ if kwargs["scheme"] in {Dialects.MSSQL, Dialects.Snowflake, Dialects.Trino}:
94
+ kwargs["path"] = f"/{database}/{schema}"
95
+ return url._replace(**kwargs).geturl()
96
+
97
+ @staticmethod
98
+ def filter_relevant_columns(
99
+ columns: List[Column],
100
+ key_columns: Set[str],
101
+ extra_columns: Set[str],
102
+ case_sensitive: bool,
103
+ ) -> List[Column]:
104
+ """Filter relevant columns.
105
+
106
+ Args:
107
+ columns (List[Column]): list of columns
108
+ key_columns (Set[str]): set of key columns
109
+ extra_columns (Set[str]): set of extra columns
110
+ case_sensitive (bool): case sensitive flag
111
+
112
+ Returns:
113
+ List[Column]
114
+ """
115
+ validated_columns = (
116
+ [*key_columns, *extra_columns]
117
+ if case_sensitive
118
+ else CaseInsensitiveList([*key_columns, *extra_columns])
119
+ )
120
+ return [c for c in columns if c.name.root in validated_columns]
@@ -14,21 +14,32 @@ from typing import List, Optional, Set
14
14
  from urllib.parse import urlparse
15
15
 
16
16
  from metadata.data_quality.validations import utils
17
- from metadata.data_quality.validations.models import (
18
- Column,
19
- TableDiffRuntimeParameters,
20
- TableParameter,
21
- )
17
+ from metadata.data_quality.validations.models import Column, TableDiffRuntimeParameters
22
18
  from metadata.data_quality.validations.runtime_param_setter.param_setter import (
23
19
  RuntimeParameterSetter,
24
20
  )
25
21
  from metadata.generated.schema.entity.data.table import Constraint, Table
26
22
  from metadata.generated.schema.entity.services.databaseService import DatabaseService
23
+ from metadata.generated.schema.entity.services.serviceType import ServiceType
27
24
  from metadata.generated.schema.tests.testCase import TestCase
28
25
  from metadata.ingestion.source.connections import get_connection
29
26
  from metadata.profiler.orm.registry import Dialects
30
27
  from metadata.utils import fqn
31
28
  from metadata.utils.collections import CaseInsensitiveList
29
+ from metadata.utils.importer import get_module_dir, import_from_module
30
+
31
+
32
+ def get_for_source(
33
+ service_type: ServiceType, source_type: str, from_: str = "ingestion"
34
+ ):
35
+ return import_from_module(
36
+ "metadata.{}.source.{}.{}.{}.ServiceSpec".format( # pylint: disable=C0209
37
+ from_,
38
+ service_type.name.lower(),
39
+ get_module_dir(source_type),
40
+ "service_spec",
41
+ )
42
+ )
32
43
 
33
44
 
34
45
  class TableDiffParamsSetter(RuntimeParameterSetter):
@@ -51,67 +62,70 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
51
62
  }
52
63
 
53
64
  def get_parameters(self, test_case) -> TableDiffRuntimeParameters:
65
+ # Using the specs class method causes circular import as TestSuiteInterface
66
+ # imports RuntimeParameterSetter
67
+ cls_path = get_for_source(
68
+ ServiceType.Database,
69
+ source_type=self.service_connection_config.type.value.lower(),
70
+ ).data_diff
71
+ cls = import_from_module(cls_path)()
72
+
73
+ service1: DatabaseService = self.ometa_client.get_by_id(
74
+ DatabaseService, self.table_entity.service.id, nullable=False
75
+ )
76
+
54
77
  service1_url = (
55
78
  str(get_connection(self.service_connection_config).url)
56
79
  if self.service_connection_config
57
80
  else None
58
81
  )
59
- service1: DatabaseService = self.ometa_client.get_by_id(
60
- DatabaseService, self.table_entity.service.id, nullable=False
61
- )
82
+
62
83
  table2_fqn = self.get_parameter(test_case, "table2")
63
- case_sensitive_columns: bool = utils.get_bool_test_case_param(
64
- test_case.parameterValues, "caseSensitiveColumns"
65
- )
66
84
  if table2_fqn is None:
67
85
  raise ValueError("table2 not set")
68
86
  table2: Table = self.ometa_client.get_by_name(
69
87
  Table, fqn=table2_fqn, nullable=False
70
88
  )
71
- service2_url = (
72
- service1_url if table2.service == self.table_entity.service else None
73
- )
74
89
  service2: DatabaseService = self.ometa_client.get_by_id(
75
90
  DatabaseService, table2.service.id, nullable=False
76
91
  )
92
+ service2_url = (
93
+ self.get_parameter(test_case, "service2Url") or service1_url
94
+ if table2.service == self.table_entity.service
95
+ else None
96
+ )
97
+
77
98
  key_columns = self.get_key_columns(test_case)
78
- extra_columns = self.get_extra_columns(
79
- key_columns, test_case, self.table_entity.columns, table2.columns
99
+ extra_columns = (
100
+ self.get_extra_columns(
101
+ key_columns, test_case, self.table_entity.columns, table2.columns
102
+ )
103
+ or set()
80
104
  )
105
+ case_sensitive_columns: bool = (
106
+ utils.get_bool_test_case_param(
107
+ test_case.parameterValues, "caseSensitiveColumns"
108
+ )
109
+ or False
110
+ )
111
+
81
112
  return TableDiffRuntimeParameters(
82
113
  table_profile_config=self.table_entity.tableProfilerConfig,
83
- table1=TableParameter(
84
- database_service_type=service1.serviceType,
85
- path=self.get_data_diff_table_path(
86
- self.table_entity.fullyQualifiedName.root
87
- ),
88
- serviceUrl=self.get_data_diff_url(
89
- service1,
90
- self.table_entity.fullyQualifiedName.root,
91
- override_url=service1_url,
92
- ),
93
- columns=self.filter_relevant_columns(
94
- self.table_entity.columns,
95
- key_columns,
96
- extra_columns,
97
- case_sensitive=case_sensitive_columns,
98
- ),
114
+ table1=cls.get(
115
+ service1,
116
+ self.table_entity,
117
+ key_columns,
118
+ extra_columns,
119
+ case_sensitive_columns,
120
+ service1_url,
99
121
  ),
100
- table2=TableParameter(
101
- database_service_type=service2.serviceType,
102
- path=self.get_data_diff_table_path(table2_fqn),
103
- serviceUrl=self.get_data_diff_url(
104
- service2,
105
- table2_fqn,
106
- override_url=self.get_parameter(test_case, "service2Url")
107
- or service2_url,
108
- ),
109
- columns=self.filter_relevant_columns(
110
- table2.columns,
111
- key_columns,
112
- extra_columns,
113
- case_sensitive=case_sensitive_columns,
114
- ),
122
+ table2=cls.get(
123
+ service2,
124
+ table2,
125
+ key_columns,
126
+ extra_columns,
127
+ case_sensitive_columns,
128
+ service2_url,
115
129
  ),
116
130
  keyColumns=list(key_columns),
117
131
  extraColumns=list(extra_columns),
@@ -265,6 +265,12 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
265
265
  self.runtime_params.keyColumns,
266
266
  extra_columns=self.runtime_params.extraColumns,
267
267
  case_sensitive=self.get_case_sensitive(),
268
+ key_content=self.runtime_params.table1.privateKey.get_secret_value()
269
+ if self.runtime_params.table1.privateKey
270
+ else None,
271
+ private_key_passphrase=self.runtime_params.table1.passPhrase.get_secret_value()
272
+ if self.runtime_params.table1.passPhrase
273
+ else None,
268
274
  ).with_schema()
269
275
  table2 = data_diff.connect_to_table(
270
276
  self.runtime_params.table2.serviceUrl,
@@ -272,6 +278,12 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
272
278
  self.runtime_params.keyColumns,
273
279
  extra_columns=self.runtime_params.extraColumns,
274
280
  case_sensitive=self.get_case_sensitive(),
281
+ key_content=self.runtime_params.table2.privateKey.get_secret_value()
282
+ if self.runtime_params.table2.privateKey
283
+ else None,
284
+ private_key_passphrase=self.runtime_params.table2.passPhrase.get_secret_value()
285
+ if self.runtime_params.table2.passPhrase
286
+ else None,
275
287
  ).with_schema()
276
288
  result = []
277
289
  for column in table1.key_columns + table1.extra_columns:
@@ -332,6 +344,12 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
332
344
  self.runtime_params.keyColumns, # type: ignore
333
345
  case_sensitive=self.get_case_sensitive(),
334
346
  where=left_where,
347
+ key_content=self.runtime_params.table1.privateKey.get_secret_value()
348
+ if self.runtime_params.table1.privateKey
349
+ else None,
350
+ private_key_passphrase=self.runtime_params.table1.passPhrase.get_secret_value()
351
+ if self.runtime_params.table1.passPhrase
352
+ else None,
335
353
  )
336
354
  table2 = data_diff.connect_to_table(
337
355
  self.runtime_params.table2.serviceUrl,
@@ -339,6 +357,12 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
339
357
  self.runtime_params.keyColumns, # type: ignore
340
358
  case_sensitive=self.get_case_sensitive(),
341
359
  where=right_where,
360
+ key_content=self.runtime_params.table1.privateKey.get_secret_value()
361
+ if self.runtime_params.table1.privateKey
362
+ else None,
363
+ private_key_passphrase=self.runtime_params.table1.passPhrase.get_secret_value()
364
+ if self.runtime_params.table1.passPhrase
365
+ else None,
342
366
  )
343
367
  data_diff_kwargs = {
344
368
  "key_columns": self.runtime_params.keyColumns,
@@ -13,11 +13,8 @@ source:
13
13
  # authType:
14
14
  # personalAccessTokenName: personal_access_token_name
15
15
  # personalAccessTokenSecret: personal_access_token_secret
16
- env: tableau_prod
17
16
  hostPort: http://localhost
18
17
  siteName: site_name
19
- siteUrl: site_url
20
- apiVersion: api_version
21
18
  paginationLimit: 10
22
19
  sourceConfig:
23
20
  config:
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/basic.json
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportData.json
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/aggregatedCostAnalysisReportData.json
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/entityReportData.json
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/rawCostAnalysisReportData.json
3
- # timestamp: 2025-05-15T07:28:23+00:00
3
+ # timestamp: 2025-05-28T05:11:14+00:00
4
4
 
5
5
  from __future__ import annotations
6
6