openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (826) hide show
  1. airflow_provider_openmetadata/lineage/runner.py +44 -5
  2. airflow_provider_openmetadata/lineage/status.py +1 -1
  3. metadata/antlr/split_listener.py +1 -4
  4. metadata/applications/auto_tagger.py +212 -0
  5. metadata/cli/app.py +47 -0
  6. metadata/cli/db_dump.py +6 -3
  7. metadata/clients/aws_client.py +4 -0
  8. metadata/clients/domo_client.py +0 -2
  9. metadata/cmd.py +24 -13
  10. metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
  11. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
  12. metadata/data_insight/source/metadata.py +11 -4
  13. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
  14. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
  15. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
  16. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
  17. metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
  18. metadata/data_quality/processor/test_case_runner.py +17 -8
  19. metadata/data_quality/source/test_suite.py +8 -2
  20. metadata/data_quality/validations/base_test_handler.py +21 -17
  21. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
  22. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
  23. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
  24. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
  25. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
  26. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
  27. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
  28. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
  29. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
  30. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
  31. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
  32. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
  33. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
  34. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
  35. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
  36. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
  37. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
  38. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
  39. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
  40. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
  41. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
  42. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
  43. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
  44. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
  45. metadata/examples/workflows/databricks.yaml +0 -1
  46. metadata/examples/workflows/mongodb.yaml +4 -2
  47. metadata/examples/workflows/mstr.yaml +24 -0
  48. metadata/examples/workflows/sas.yaml +28 -0
  49. metadata/examples/workflows/unity_catalog.yaml +27 -0
  50. metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
  51. metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
  52. metadata/generated/antlr/EntityLinkLexer.py +391 -292
  53. metadata/generated/antlr/EntityLinkListener.py +12 -12
  54. metadata/generated/antlr/EntityLinkParser.py +228 -103
  55. metadata/generated/schema/analytics/__init__.py +1 -1
  56. metadata/generated/schema/analytics/basic.py +1 -1
  57. metadata/generated/schema/analytics/reportData.py +1 -1
  58. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  59. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  60. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  61. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  62. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  63. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  64. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  65. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  66. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  67. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  68. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  69. metadata/generated/schema/api/__init__.py +1 -1
  70. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  71. metadata/generated/schema/api/analytics/__init__.py +1 -1
  72. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  73. metadata/generated/schema/api/automations/__init__.py +1 -1
  74. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  75. metadata/generated/schema/api/bulkAssets.py +1 -1
  76. metadata/generated/schema/api/classification/__init__.py +1 -1
  77. metadata/generated/schema/api/classification/createClassification.py +1 -1
  78. metadata/generated/schema/api/classification/createTag.py +1 -1
  79. metadata/generated/schema/api/classification/loadTags.py +1 -1
  80. metadata/generated/schema/api/createBot.py +2 -3
  81. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  82. metadata/generated/schema/api/createType.py +1 -1
  83. metadata/generated/schema/api/data/__init__.py +1 -1
  84. metadata/generated/schema/api/data/createChart.py +1 -1
  85. metadata/generated/schema/api/data/createContainer.py +2 -2
  86. metadata/generated/schema/api/data/createCustomProperty.py +2 -3
  87. metadata/generated/schema/api/data/createDashboard.py +1 -1
  88. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  89. metadata/generated/schema/api/data/createDatabase.py +2 -3
  90. metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
  91. metadata/generated/schema/api/data/createGlossary.py +2 -3
  92. metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
  93. metadata/generated/schema/api/data/createMlModel.py +1 -1
  94. metadata/generated/schema/api/data/createPipeline.py +2 -2
  95. metadata/generated/schema/api/data/createQuery.py +2 -2
  96. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  97. metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
  98. metadata/generated/schema/api/data/createTable.py +2 -2
  99. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  100. metadata/generated/schema/api/data/createTopic.py +1 -1
  101. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  102. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  103. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  104. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  105. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  106. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  107. metadata/generated/schema/api/docStore/__init__.py +1 -1
  108. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  109. metadata/generated/schema/api/domains/__init__.py +1 -1
  110. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  111. metadata/generated/schema/api/domains/createDomain.py +1 -1
  112. metadata/generated/schema/api/feed/__init__.py +1 -1
  113. metadata/generated/schema/api/feed/closeTask.py +8 -1
  114. metadata/generated/schema/api/feed/createPost.py +1 -1
  115. metadata/generated/schema/api/feed/createSuggestion.py +30 -0
  116. metadata/generated/schema/api/feed/createThread.py +1 -1
  117. metadata/generated/schema/api/feed/resolveTask.py +12 -1
  118. metadata/generated/schema/api/feed/threadCount.py +18 -14
  119. metadata/generated/schema/api/lineage/__init__.py +1 -1
  120. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  121. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  122. metadata/generated/schema/api/policies/__init__.py +1 -1
  123. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  124. metadata/generated/schema/api/services/__init__.py +1 -1
  125. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  126. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  127. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  128. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  129. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  130. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  131. metadata/generated/schema/api/services/createSearchService.py +1 -1
  132. metadata/generated/schema/api/services/createStorageService.py +1 -1
  133. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  134. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  135. metadata/generated/schema/api/setOwner.py +1 -1
  136. metadata/generated/schema/api/teams/__init__.py +1 -1
  137. metadata/generated/schema/api/teams/createPersona.py +1 -1
  138. metadata/generated/schema/api/teams/createRole.py +2 -3
  139. metadata/generated/schema/api/teams/createTeam.py +1 -1
  140. metadata/generated/schema/api/teams/createUser.py +2 -2
  141. metadata/generated/schema/api/tests/__init__.py +1 -1
  142. metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
  143. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  144. metadata/generated/schema/api/tests/createTestCase.py +4 -1
  145. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
  146. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  147. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  148. metadata/generated/schema/api/voteRequest.py +1 -1
  149. metadata/generated/schema/auth/__init__.py +1 -1
  150. metadata/generated/schema/auth/basicAuth.py +1 -1
  151. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  152. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  153. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  154. metadata/generated/schema/auth/emailRequest.py +1 -1
  155. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  156. metadata/generated/schema/auth/generateToken.py +1 -1
  157. metadata/generated/schema/auth/jwtAuth.py +1 -1
  158. metadata/generated/schema/auth/loginRequest.py +1 -1
  159. metadata/generated/schema/auth/logoutRequest.py +1 -1
  160. metadata/generated/schema/auth/passwordResetRequest.py +3 -3
  161. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  162. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  163. metadata/generated/schema/auth/refreshToken.py +1 -1
  164. metadata/generated/schema/auth/registrationRequest.py +2 -2
  165. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  166. metadata/generated/schema/auth/revokeToken.py +1 -1
  167. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  168. metadata/generated/schema/auth/ssoAuth.py +1 -1
  169. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  170. metadata/generated/schema/configuration/__init__.py +1 -1
  171. metadata/generated/schema/configuration/appsPrivateConfiguration.py +28 -0
  172. metadata/generated/schema/configuration/authConfig.py +1 -1
  173. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  175. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  176. metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
  177. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  178. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  179. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  180. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  181. metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
  182. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  183. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  184. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  185. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  186. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  187. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  188. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  189. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  190. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  194. metadata/generated/schema/dataInsight/__init__.py +1 -1
  195. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  196. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  197. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  198. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  199. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  200. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  201. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  202. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  203. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  204. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  205. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  206. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  207. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  208. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  209. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
  210. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
  211. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
  212. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
  213. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
  214. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
  215. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  216. metadata/generated/schema/email/__init__.py +1 -1
  217. metadata/generated/schema/email/emailRequest.py +1 -1
  218. metadata/generated/schema/email/smtpSettings.py +1 -1
  219. metadata/generated/schema/entity/__init__.py +1 -1
  220. metadata/generated/schema/entity/applications/__init__.py +1 -1
  221. metadata/generated/schema/entity/applications/app.py +18 -4
  222. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  223. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  224. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
  225. metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
  226. metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
  227. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
  228. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
  229. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
  230. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
  231. metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +13 -3
  232. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
  233. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
  234. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  235. metadata/generated/schema/entity/applications/jobStatus.py +3 -3
  236. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  237. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  238. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
  239. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
  240. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  241. metadata/generated/schema/entity/automations/__init__.py +1 -1
  242. metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
  243. metadata/generated/schema/entity/automations/workflow.py +1 -1
  244. metadata/generated/schema/entity/bot.py +2 -3
  245. metadata/generated/schema/entity/classification/__init__.py +1 -1
  246. metadata/generated/schema/entity/classification/classification.py +1 -1
  247. metadata/generated/schema/entity/classification/tag.py +2 -2
  248. metadata/generated/schema/entity/data/__init__.py +1 -1
  249. metadata/generated/schema/entity/data/chart.py +1 -1
  250. metadata/generated/schema/entity/data/container.py +4 -9
  251. metadata/generated/schema/entity/data/dashboard.py +1 -1
  252. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  253. metadata/generated/schema/entity/data/database.py +4 -8
  254. metadata/generated/schema/entity/data/databaseSchema.py +2 -8
  255. metadata/generated/schema/entity/data/glossary.py +1 -1
  256. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  257. metadata/generated/schema/entity/data/metrics.py +1 -1
  258. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  259. metadata/generated/schema/entity/data/pipeline.py +6 -9
  260. metadata/generated/schema/entity/data/query.py +2 -2
  261. metadata/generated/schema/entity/data/report.py +1 -1
  262. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  263. metadata/generated/schema/entity/data/storedProcedure.py +3 -9
  264. metadata/generated/schema/entity/data/table.py +18 -12
  265. metadata/generated/schema/entity/data/topic.py +1 -1
  266. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  267. metadata/generated/schema/entity/docStore/document.py +1 -1
  268. metadata/generated/schema/entity/domains/__init__.py +1 -1
  269. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  270. metadata/generated/schema/entity/domains/domain.py +1 -1
  271. metadata/generated/schema/entity/events/__init__.py +1 -1
  272. metadata/generated/schema/entity/events/webhook.py +6 -3
  273. metadata/generated/schema/entity/feed/__init__.py +1 -1
  274. metadata/generated/schema/entity/feed/suggestion.py +61 -0
  275. metadata/generated/schema/entity/feed/thread.py +6 -1
  276. metadata/generated/schema/entity/policies/__init__.py +1 -1
  277. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  278. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  279. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  280. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  281. metadata/generated/schema/entity/policies/filters.py +2 -5
  282. metadata/generated/schema/entity/policies/policy.py +1 -1
  283. metadata/generated/schema/entity/services/__init__.py +1 -1
  284. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  285. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  286. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  287. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  288. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  289. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  290. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  291. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  292. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  293. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  294. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  295. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  296. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  297. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
  298. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  299. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
  300. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  301. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  302. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  304. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  305. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
  306. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +2 -2
  307. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
  308. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
  309. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  310. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  311. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  312. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  313. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
  314. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  315. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
  316. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  319. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  320. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
  321. metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
  322. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  323. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
  325. metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
  326. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
  329. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
  330. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
  331. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
  332. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
  333. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
  334. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
  335. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
  336. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
  337. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
  338. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
  339. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
  340. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
  341. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
  342. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +2 -2
  343. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
  344. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +2 -2
  345. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +2 -2
  346. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
  347. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
  348. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +2 -2
  350. metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
  351. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
  352. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
  353. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
  354. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
  355. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
  356. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
  357. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  358. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  359. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  360. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  361. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  364. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  365. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  366. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  367. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  368. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  369. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
  370. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  371. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  372. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  374. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  375. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  376. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
  378. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
  387. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  389. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  391. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  392. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  393. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  397. metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
  398. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  402. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  403. metadata/generated/schema/entity/services/dashboardService.py +4 -1
  404. metadata/generated/schema/entity/services/databaseService.py +13 -1
  405. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  406. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
  407. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  408. metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
  409. metadata/generated/schema/entity/services/messagingService.py +1 -1
  410. metadata/generated/schema/entity/services/metadataService.py +1 -1
  411. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  412. metadata/generated/schema/entity/services/pipelineService.py +4 -1
  413. metadata/generated/schema/entity/services/searchService.py +1 -1
  414. metadata/generated/schema/entity/services/serviceType.py +1 -1
  415. metadata/generated/schema/entity/services/storageService.py +10 -3
  416. metadata/generated/schema/entity/teams/__init__.py +1 -1
  417. metadata/generated/schema/entity/teams/persona.py +1 -1
  418. metadata/generated/schema/entity/teams/role.py +2 -6
  419. metadata/generated/schema/entity/teams/team.py +1 -1
  420. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  421. metadata/generated/schema/entity/teams/user.py +3 -10
  422. metadata/generated/schema/entity/type.py +2 -2
  423. metadata/generated/schema/entity/utils/__init__.py +1 -1
  424. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  425. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  426. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  427. metadata/generated/schema/events/__init__.py +1 -1
  428. metadata/generated/schema/events/alertMetrics.py +27 -0
  429. metadata/generated/schema/events/api/__init__.py +1 -1
  430. metadata/generated/schema/events/api/createEventSubscription.py +14 -12
  431. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  432. metadata/generated/schema/events/eventFilterRule.py +13 -2
  433. metadata/generated/schema/events/eventSubscription.py +89 -49
  434. metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
  435. metadata/generated/schema/events/failedEvent.py +26 -0
  436. metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
  437. metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
  438. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  439. metadata/generated/schema/metadataIngestion/application.py +40 -0
  440. metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
  441. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  442. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  443. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
  444. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +6 -1
  445. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  446. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  447. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  448. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  449. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
  450. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  451. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  452. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
  453. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  454. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  455. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
  456. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  457. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  458. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  459. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  460. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  461. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  462. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  463. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  464. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  465. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
  466. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
  467. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  468. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  469. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  470. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
  471. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  472. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  473. metadata/generated/schema/monitoring/__init__.py +1 -1
  474. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  475. metadata/generated/schema/security/__init__.py +1 -1
  476. metadata/generated/schema/security/client/__init__.py +1 -1
  477. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  478. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  479. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  480. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  481. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  482. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  483. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  484. metadata/generated/schema/security/credentials/__init__.py +1 -1
  485. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  486. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  487. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  488. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  489. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  490. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  491. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  492. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  493. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  494. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  495. metadata/generated/schema/security/secrets/__init__.py +1 -1
  496. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  497. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
  498. metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -2
  499. metadata/generated/schema/security/securityConfiguration.py +1 -1
  500. metadata/generated/schema/security/ssl/__init__.py +1 -1
  501. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  502. metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
  503. metadata/generated/schema/settings/__init__.py +1 -1
  504. metadata/generated/schema/settings/settings.py +1 -1
  505. metadata/generated/schema/system/__init__.py +1 -1
  506. metadata/generated/schema/system/entityError.py +17 -0
  507. metadata/generated/schema/system/eventPublisherJob.py +5 -19
  508. metadata/generated/schema/system/indexingError.py +34 -0
  509. metadata/generated/schema/system/ui/__init__.py +1 -1
  510. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  511. metadata/generated/schema/system/ui/page.py +1 -1
  512. metadata/generated/schema/tests/__init__.py +1 -1
  513. metadata/generated/schema/tests/assigned.py +18 -0
  514. metadata/generated/schema/tests/basic.py +17 -39
  515. metadata/generated/schema/tests/customMetric.py +4 -2
  516. metadata/generated/schema/tests/resolved.py +35 -0
  517. metadata/generated/schema/tests/testCase.py +11 -2
  518. metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
  519. metadata/generated/schema/tests/testDefinition.py +5 -1
  520. metadata/generated/schema/tests/testSuite.py +1 -1
  521. metadata/generated/schema/type/__init__.py +1 -1
  522. metadata/generated/schema/type/auditLog.py +1 -1
  523. metadata/generated/schema/type/basic.py +5 -5
  524. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  525. metadata/generated/schema/type/changeEvent.py +5 -75
  526. metadata/generated/schema/type/changeEventType.py +29 -0
  527. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  528. metadata/generated/schema/type/csvDocumentation.py +1 -1
  529. metadata/generated/schema/type/csvErrorType.py +1 -1
  530. metadata/generated/schema/type/csvFile.py +1 -1
  531. metadata/generated/schema/type/csvImportResult.py +1 -1
  532. metadata/generated/schema/type/dailyCount.py +1 -1
  533. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  534. metadata/generated/schema/type/entityHistory.py +1 -1
  535. metadata/generated/schema/type/entityLineage.py +2 -1
  536. metadata/generated/schema/type/entityReference.py +5 -1
  537. metadata/generated/schema/type/entityReferenceList.py +1 -1
  538. metadata/generated/schema/type/entityRelationship.py +1 -1
  539. metadata/generated/schema/type/entityUsage.py +1 -1
  540. metadata/generated/schema/type/filterPattern.py +1 -1
  541. metadata/generated/schema/type/function.py +7 -6
  542. metadata/generated/schema/type/include.py +1 -1
  543. metadata/generated/schema/type/jdbcConnection.py +1 -1
  544. metadata/generated/schema/type/lifeCycle.py +1 -1
  545. metadata/generated/schema/type/paging.py +1 -1
  546. metadata/generated/schema/type/profile.py +1 -1
  547. metadata/generated/schema/type/queryParserData.py +2 -2
  548. metadata/generated/schema/type/reaction.py +1 -1
  549. metadata/generated/schema/type/schedule.py +1 -1
  550. metadata/generated/schema/type/schema.py +1 -1
  551. metadata/generated/schema/type/tableQuery.py +2 -2
  552. metadata/generated/schema/type/tableUsageCount.py +1 -1
  553. metadata/generated/schema/type/tagLabel.py +1 -1
  554. metadata/generated/schema/type/usageDetails.py +1 -1
  555. metadata/generated/schema/type/usageRequest.py +1 -1
  556. metadata/generated/schema/type/votes.py +1 -1
  557. metadata/great_expectations/action.py +3 -2
  558. metadata/ingestion/api/delete.py +5 -2
  559. metadata/ingestion/api/models.py +4 -10
  560. metadata/ingestion/api/parser.py +3 -37
  561. metadata/ingestion/api/status.py +15 -4
  562. metadata/ingestion/api/step.py +39 -6
  563. metadata/ingestion/api/steps.py +20 -0
  564. metadata/ingestion/api/topology_runner.py +114 -168
  565. metadata/ingestion/bulksink/metadata_usage.py +11 -5
  566. metadata/ingestion/lineage/models.py +4 -0
  567. metadata/ingestion/lineage/parser.py +4 -5
  568. metadata/ingestion/lineage/sql_lineage.py +9 -17
  569. metadata/ingestion/models/custom_pydantic.py +4 -2
  570. metadata/ingestion/models/lf_tags_model.py +33 -0
  571. metadata/ingestion/models/patch_request.py +108 -2
  572. metadata/ingestion/models/tests_data.py +9 -0
  573. metadata/ingestion/models/topology.py +177 -60
  574. metadata/ingestion/ometa/auth_provider.py +0 -349
  575. metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
  576. metadata/ingestion/ometa/mixins/patch_mixin.py +17 -68
  577. metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
  578. metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
  579. metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
  580. metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
  581. metadata/ingestion/ometa/ometa_api.py +18 -25
  582. metadata/ingestion/ometa/routes.py +7 -0
  583. metadata/ingestion/processor/query_parser.py +9 -2
  584. metadata/ingestion/sink/metadata_rest.py +25 -6
  585. metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
  586. metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
  587. metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
  588. metadata/ingestion/source/dashboard/looker/metadata.py +38 -25
  589. metadata/ingestion/source/dashboard/metabase/metadata.py +23 -6
  590. metadata/ingestion/source/dashboard/mode/client.py +10 -23
  591. metadata/ingestion/source/dashboard/mode/connection.py +6 -1
  592. metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
  593. metadata/ingestion/source/dashboard/mstr/client.py +208 -0
  594. metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
  595. metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
  596. metadata/ingestion/source/dashboard/mstr/models.py +144 -0
  597. metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
  598. metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
  599. metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
  600. metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
  601. metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
  602. metadata/ingestion/source/dashboard/superset/client.py +16 -9
  603. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  604. metadata/ingestion/source/dashboard/superset/db_source.py +14 -10
  605. metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
  606. metadata/ingestion/source/dashboard/superset/queries.py +1 -1
  607. metadata/ingestion/source/dashboard/tableau/client.py +91 -11
  608. metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
  609. metadata/ingestion/source/dashboard/tableau/metadata.py +58 -70
  610. metadata/ingestion/source/dashboard/tableau/models.py +0 -8
  611. metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
  612. metadata/ingestion/source/database/athena/client.py +80 -0
  613. metadata/ingestion/source/database/athena/connection.py +7 -0
  614. metadata/ingestion/source/database/athena/metadata.py +161 -19
  615. metadata/ingestion/source/database/azuresql/metadata.py +0 -1
  616. metadata/ingestion/source/database/bigquery/connection.py +24 -3
  617. metadata/ingestion/source/database/bigquery/metadata.py +39 -27
  618. metadata/ingestion/source/database/bigquery/queries.py +11 -4
  619. metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
  620. metadata/ingestion/source/database/bigquery/usage.py +1 -3
  621. metadata/ingestion/source/database/column_helpers.py +0 -10
  622. metadata/ingestion/source/database/column_type_parser.py +11 -5
  623. metadata/ingestion/source/database/common_db_source.py +8 -2
  624. metadata/ingestion/source/database/common_nosql_source.py +8 -4
  625. metadata/ingestion/source/database/database_service.py +89 -7
  626. metadata/ingestion/source/database/databricks/client.py +5 -10
  627. metadata/ingestion/source/database/databricks/connection.py +16 -55
  628. metadata/ingestion/source/database/databricks/lineage.py +29 -26
  629. metadata/ingestion/source/database/databricks/metadata.py +448 -11
  630. metadata/ingestion/source/database/databricks/queries.py +28 -0
  631. metadata/ingestion/source/database/databricks/query_parser.py +5 -1
  632. metadata/ingestion/source/database/databricks/usage.py +2 -2
  633. metadata/ingestion/source/database/datalake/connection.py +31 -4
  634. metadata/ingestion/source/database/datalake/metadata.py +113 -27
  635. metadata/ingestion/source/database/dbt/dbt_config.py +5 -0
  636. metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
  637. metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
  638. metadata/ingestion/source/database/dbt/metadata.py +11 -25
  639. metadata/ingestion/source/database/deltalake/metadata.py +6 -3
  640. metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
  641. metadata/ingestion/source/database/doris/connection.py +72 -0
  642. metadata/ingestion/source/database/doris/metadata.py +315 -0
  643. metadata/ingestion/source/database/doris/queries.py +54 -0
  644. metadata/ingestion/source/database/doris/utils.py +64 -0
  645. metadata/ingestion/source/database/extended_sample_data.py +532 -0
  646. metadata/ingestion/source/database/glue/metadata.py +8 -5
  647. metadata/ingestion/source/database/hive/connection.py +0 -2
  648. metadata/ingestion/source/database/hive/utils.py +3 -0
  649. metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
  650. metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
  651. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
  652. metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
  653. metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
  654. metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
  655. metadata/ingestion/source/database/iceberg/connection.py +68 -0
  656. metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
  657. metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
  658. metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
  659. metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
  660. metadata/ingestion/source/database/iceberg/helper.py +124 -0
  661. metadata/ingestion/source/database/iceberg/metadata.py +311 -0
  662. metadata/ingestion/source/database/iceberg/models.py +66 -0
  663. metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
  664. metadata/ingestion/source/database/mongodb/connection.py +1 -5
  665. metadata/ingestion/source/database/mssql/lineage.py +2 -0
  666. metadata/ingestion/source/database/mssql/metadata.py +108 -4
  667. metadata/ingestion/source/database/mssql/models.py +30 -0
  668. metadata/ingestion/source/database/mssql/queries.py +179 -1
  669. metadata/ingestion/source/database/mssql/utils.py +207 -4
  670. metadata/ingestion/source/database/mysql/metadata.py +0 -2
  671. metadata/ingestion/source/database/oracle/metadata.py +108 -2
  672. metadata/ingestion/source/database/oracle/models.py +30 -0
  673. metadata/ingestion/source/database/oracle/queries.py +98 -17
  674. metadata/ingestion/source/database/oracle/utils.py +0 -1
  675. metadata/ingestion/source/database/postgres/lineage.py +32 -14
  676. metadata/ingestion/source/database/postgres/metadata.py +15 -7
  677. metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
  678. metadata/ingestion/source/database/postgres/queries.py +4 -2
  679. metadata/ingestion/source/database/postgres/query_parser.py +4 -72
  680. metadata/ingestion/source/database/postgres/usage.py +41 -0
  681. metadata/ingestion/source/database/postgres/utils.py +34 -0
  682. metadata/ingestion/source/database/query_parser_source.py +8 -2
  683. metadata/ingestion/source/database/redshift/metadata.py +14 -4
  684. metadata/ingestion/source/database/redshift/queries.py +10 -4
  685. metadata/ingestion/source/database/redshift/query_parser.py +16 -0
  686. metadata/ingestion/source/database/redshift/usage.py +0 -2
  687. metadata/ingestion/source/database/salesforce/metadata.py +32 -3
  688. metadata/ingestion/source/database/sample_data.py +120 -6
  689. metadata/ingestion/source/database/sas/client.py +184 -0
  690. metadata/ingestion/source/database/sas/connection.py +47 -0
  691. metadata/ingestion/source/database/sas/extension_attr.py +103 -0
  692. metadata/ingestion/source/database/sas/metadata.py +914 -0
  693. metadata/ingestion/source/database/snowflake/metadata.py +8 -51
  694. metadata/ingestion/source/database/snowflake/models.py +6 -1
  695. metadata/ingestion/source/database/snowflake/queries.py +0 -4
  696. metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
  697. metadata/ingestion/source/database/snowflake/utils.py +2 -6
  698. metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
  699. metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
  700. metadata/ingestion/source/database/unitycatalog/client.py +87 -0
  701. metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
  702. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
  703. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
  704. metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
  705. metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
  706. metadata/ingestion/source/database/usage_source.py +3 -2
  707. metadata/ingestion/source/messaging/common_broker_source.py +6 -4
  708. metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
  709. metadata/ingestion/source/messaging/messaging_service.py +6 -2
  710. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  711. metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
  712. metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
  713. metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
  714. metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
  715. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
  716. metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
  717. metadata/ingestion/source/pipeline/airflow/models.py +5 -4
  718. metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
  719. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +12 -9
  720. metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
  721. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
  722. metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
  723. metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
  724. metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
  725. metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
  726. metadata/ingestion/source/search/search_service.py +6 -2
  727. metadata/ingestion/source/storage/s3/metadata.py +22 -17
  728. metadata/ingestion/source/storage/storage_service.py +53 -11
  729. metadata/ingestion/stage/table_usage.py +9 -2
  730. metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
  731. metadata/parsers/protobuf_parser.py +29 -11
  732. metadata/pii/processor.py +9 -2
  733. metadata/profiler/api/models.py +19 -1
  734. metadata/profiler/interface/pandas/profiler_interface.py +59 -18
  735. metadata/profiler/interface/profiler_interface.py +13 -2
  736. metadata/profiler/interface/profiler_interface_factory.py +49 -14
  737. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
  738. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
  739. metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
  740. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
  741. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
  742. metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
  743. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  744. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
  745. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  746. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
  747. metadata/profiler/metrics/composed/null_ratio.py +1 -1
  748. metadata/profiler/metrics/hybrid/histogram.py +1 -0
  749. metadata/profiler/metrics/static/max.py +4 -1
  750. metadata/profiler/metrics/static/min.py +4 -1
  751. metadata/profiler/metrics/system/queries/snowflake.py +89 -17
  752. metadata/profiler/metrics/system/system.py +62 -20
  753. metadata/profiler/orm/functions/length.py +1 -0
  754. metadata/profiler/orm/functions/median.py +7 -0
  755. metadata/profiler/orm/functions/table_metric_computer.py +462 -0
  756. metadata/profiler/orm/registry.py +1 -0
  757. metadata/profiler/processor/core.py +116 -52
  758. metadata/profiler/processor/default.py +14 -3
  759. metadata/profiler/processor/handle_partition.py +2 -2
  760. metadata/profiler/processor/processor.py +9 -4
  761. metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
  762. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
  763. metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
  764. metadata/profiler/source/base/profiler_source.py +5 -2
  765. metadata/profiler/source/bigquery/type_mapper.py +0 -1
  766. metadata/profiler/source/mariadb/functions/median.py +20 -0
  767. metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
  768. metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
  769. metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
  770. metadata/profiler/source/metadata.py +43 -10
  771. metadata/profiler/source/metadata_ext.py +16 -50
  772. metadata/py.typed +0 -0
  773. metadata/readers/dataframe/json.py +5 -4
  774. metadata/readers/file/api_reader.py +0 -1
  775. metadata/utils/constants.py +5 -0
  776. metadata/utils/datalake/datalake_utils.py +363 -129
  777. metadata/utils/entity_link.py +26 -6
  778. metadata/utils/fqn.py +20 -0
  779. metadata/utils/helpers.py +55 -9
  780. metadata/utils/importer.py +2 -3
  781. metadata/utils/life_cycle_utils.py +4 -4
  782. metadata/utils/logger.py +13 -2
  783. metadata/utils/partition.py +10 -5
  784. metadata/utils/secrets/client/loader.py +0 -1
  785. metadata/utils/secrets/noop_secrets_manager.py +4 -3
  786. metadata/utils/secrets/secrets_manager_factory.py +3 -4
  787. metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
  788. metadata/utils/sqlalchemy_utils.py +21 -0
  789. metadata/utils/storage_metadata_config.py +42 -1
  790. metadata/utils/tag_utils.py +5 -2
  791. metadata/workflow/application.py +154 -0
  792. metadata/workflow/application_output_handler.py +34 -0
  793. metadata/workflow/base.py +84 -153
  794. metadata/workflow/data_insight.py +8 -7
  795. metadata/workflow/data_quality.py +3 -2
  796. metadata/workflow/ingestion.py +203 -0
  797. metadata/workflow/metadata.py +2 -3
  798. metadata/workflow/output_handler.py +204 -0
  799. metadata/workflow/profiler.py +2 -2
  800. metadata/workflow/usage.py +3 -4
  801. metadata/workflow/workflow_output_handler.py +15 -255
  802. metadata/workflow/workflow_status_mixin.py +44 -52
  803. openmetadata_ingestion-1.3.0.0.dist-info/METADATA +749 -0
  804. {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/RECORD +812 -725
  805. {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/WHEEL +1 -1
  806. metadata/generated/schema/entity/applications/appConfig.py +0 -21
  807. metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
  808. metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
  809. metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
  810. metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
  811. metadata/generated/schema/events/entitySpelFilters.py +0 -19
  812. metadata/ingestion/models/es_documents.py +0 -339
  813. metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
  814. metadata/ingestion/ometa/provider_registry.py +0 -144
  815. metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
  816. metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
  817. metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
  818. metadata/profiler/orm/functions/table_metric_construct.py +0 -365
  819. openmetadata_ingestion-1.2.5.1.dist-info/METADATA +0 -426
  820. /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
  821. /metadata/ingestion/source/database/{databricks/unity_catalog → doris}/__init__.py +0 -0
  822. /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/sas}/__init__.py +0 -0
  823. /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
  824. {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/LICENSE +0 -0
  825. {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/entry_points.txt +0 -0
  826. {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,35 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Interfaces with database for all database engine
14
+ supporting sqlalchemy abstraction layer
15
+ """
16
+
17
+
18
+ from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
19
+ SQATestSuiteInterface,
20
+ )
21
+ from metadata.ingestion.connections.session import create_and_bind_session
22
+ from metadata.ingestion.source.database.databricks.connection import (
23
+ get_connection as databricks_get_connection,
24
+ )
25
+
26
+
27
+ class UnityCatalogTestSuiteInterface(SQATestSuiteInterface):
28
+ def __init__(self, *args, **kwargs):
29
+ super().__init__(*args, **kwargs)
30
+
31
+ def create_session(self):
32
+ self.session = create_and_bind_session(
33
+ databricks_get_connection(self.service_connection_config)
34
+ )
35
+ self.set_catalog(self.session)
@@ -17,14 +17,32 @@ from logging import Logger
17
17
  from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
18
18
  PandasTestSuiteInterface,
19
19
  )
20
+ from metadata.data_quality.interface.sqlalchemy.databricks.test_suite_interface import (
21
+ DatabricksTestSuiteInterface,
22
+ )
23
+ from metadata.data_quality.interface.sqlalchemy.snowflake.test_suite_interface import (
24
+ SnowflakeTestSuiteInterface,
25
+ )
20
26
  from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
21
27
  SQATestSuiteInterface,
22
28
  )
29
+ from metadata.data_quality.interface.sqlalchemy.unity_catalog.test_suite_interface import (
30
+ UnityCatalogTestSuiteInterface,
31
+ )
23
32
  from metadata.data_quality.interface.test_suite_interface import TestSuiteInterface
24
33
  from metadata.generated.schema.entity.data.table import Table
34
+ from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
35
+ DatabricksConnection,
36
+ )
25
37
  from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
26
38
  DatalakeConnection,
27
39
  )
40
+ from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
41
+ SnowflakeConnection,
42
+ )
43
+ from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import (
44
+ UnityCatalogConnection,
45
+ )
28
46
  from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
29
47
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
30
48
  from metadata.utils.logger import test_suite_logger
@@ -51,6 +69,17 @@ class TestSuiteInterfaceFactory:
51
69
  """
52
70
  self._interface_type[interface_type] = interface
53
71
 
72
+ def register_many(self, interface_dict):
73
+ """
74
+ Registers multiple profiler interfaces at once.
75
+
76
+ Args:
77
+ interface_dict: A dictionary mapping connection class names (strings) to their
78
+ corresponding profiler interface classes.
79
+ """
80
+ for interface_type, interface_class in interface_dict.items():
81
+ self.register(interface_type, interface_class)
82
+
54
83
  def create(
55
84
  self,
56
85
  service_connection_config: DatabaseConnection,
@@ -86,3 +115,14 @@ class TestSuiteInterfaceFactory:
86
115
 
87
116
 
88
117
  test_suite_interface_factory = TestSuiteInterfaceFactory()
118
+
119
+
120
+ test_suite_interface = {
121
+ DatabaseConnection.__name__: SQATestSuiteInterface,
122
+ DatalakeConnection.__name__: PandasTestSuiteInterface,
123
+ SnowflakeConnection.__name__: SnowflakeTestSuiteInterface,
124
+ UnityCatalogConnection.__name__: UnityCatalogTestSuiteInterface,
125
+ DatabricksConnection.__name__: DatabricksTestSuiteInterface,
126
+ }
127
+
128
+ test_suite_interface_factory.register_many(test_suite_interface)
@@ -28,6 +28,10 @@ from metadata.data_quality.runner.test_suite_source_factory import (
28
28
  test_suite_source_factory,
29
29
  )
30
30
  from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
31
+ from metadata.generated.schema.entity.data.table import Table
32
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
33
+ StackTraceError,
34
+ )
31
35
  from metadata.generated.schema.metadataIngestion.workflow import (
32
36
  OpenMetadataWorkflowConfig,
33
37
  )
@@ -35,7 +39,7 @@ from metadata.generated.schema.tests.testCase import TestCase
35
39
  from metadata.generated.schema.tests.testDefinition import TestDefinition, TestPlatform
36
40
  from metadata.generated.schema.tests.testSuite import TestSuite
37
41
  from metadata.generated.schema.type.basic import EntityLink, FullyQualifiedEntityName
38
- from metadata.ingestion.api.models import Either, StackTraceError
42
+ from metadata.ingestion.api.models import Either
39
43
  from metadata.ingestion.api.parser import parse_workflow_config_gracefully
40
44
  from metadata.ingestion.api.step import Step
41
45
  from metadata.ingestion.api.steps import Processor
@@ -50,7 +54,6 @@ class TestCaseRunner(Processor):
50
54
  """Execute the test suite tests and create test cases from the YAML config"""
51
55
 
52
56
  def __init__(self, config: OpenMetadataWorkflowConfig, metadata: OpenMetadata):
53
-
54
57
  super().__init__()
55
58
 
56
59
  self.config = config
@@ -62,6 +65,10 @@ class TestCaseRunner(Processor):
62
65
  )
63
66
  )
64
67
 
68
+ @property
69
+ def name(self) -> str:
70
+ return "Data Quality"
71
+
65
72
  def _run(self, record: TableAndTests) -> Either:
66
73
  # First, create the executable test suite if it does not exist yet
67
74
  # This could happen if the process is executed from YAML and not the UI
@@ -193,8 +200,9 @@ class TestCaseRunner(Processor):
193
200
  ),
194
201
  entityLink=EntityLink(
195
202
  __root__=entity_link.get_entity_link(
196
- table_fqn,
197
- test_case_to_create.columnName,
203
+ Table,
204
+ fqn=table_fqn,
205
+ column_name=test_case_to_create.columnName,
198
206
  )
199
207
  ),
200
208
  testSuite=test_suite_fqn,
@@ -215,7 +223,7 @@ class TestCaseRunner(Processor):
215
223
  StackTraceError(
216
224
  name=table_fqn,
217
225
  error=error,
218
- stack_trace=traceback.format_exc(),
226
+ stackTrace=traceback.format_exc(),
219
227
  )
220
228
  )
221
229
 
@@ -245,8 +253,9 @@ class TestCaseRunner(Processor):
245
253
  updated_test_case = self.metadata.patch_test_case_definition(
246
254
  source=test_case,
247
255
  entity_link=entity_link.get_entity_link(
248
- table_fqn,
249
- test_case_definition.columnName,
256
+ Table,
257
+ fqn=table_fqn,
258
+ column_name=test_case_definition.columnName,
250
259
  ),
251
260
  test_case_parameter_values=test_case_definition.parameterValues,
252
261
  )
@@ -293,7 +302,7 @@ class TestCaseRunner(Processor):
293
302
  StackTraceError(
294
303
  name=test_case.name.__root__,
295
304
  error=error,
296
- stack_trace=traceback.format_exc(),
305
+ stackTrace=traceback.format_exc(),
297
306
  )
298
307
  )
299
308
  return None
@@ -19,6 +19,9 @@ from typing import Iterable, List, Optional, cast
19
19
  from metadata.data_quality.api.models import TableAndTests
20
20
  from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
21
21
  from metadata.generated.schema.entity.data.table import Table
22
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
23
+ StackTraceError,
24
+ )
22
25
  from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
23
26
  TestSuitePipeline,
24
27
  )
@@ -27,7 +30,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
27
30
  )
28
31
  from metadata.generated.schema.tests.testCase import TestCase
29
32
  from metadata.generated.schema.tests.testSuite import TestSuite
30
- from metadata.ingestion.api.models import Either, StackTraceError
33
+ from metadata.ingestion.api.models import Either
31
34
  from metadata.ingestion.api.parser import parse_workflow_config_gracefully
32
35
  from metadata.ingestion.api.step import Step
33
36
  from metadata.ingestion.api.steps import Source
@@ -57,6 +60,10 @@ class TestSuiteSource(Source):
57
60
 
58
61
  self.test_connection()
59
62
 
63
+ @property
64
+ def name(self) -> str:
65
+ return "OpenMetadata"
66
+
60
67
  def _get_table_entity(self) -> Optional[Table]:
61
68
  """given an entity fqn return the table entity
62
69
 
@@ -142,7 +149,6 @@ class TestSuiteSource(Source):
142
149
  )
143
150
 
144
151
  else:
145
-
146
152
  test_suite_cases = self._get_test_cases_from_test_suite(table.testSuite)
147
153
 
148
154
  yield Either(
@@ -21,14 +21,11 @@ from datetime import datetime
21
21
  from typing import Callable, List, Optional, TypeVar, Union
22
22
 
23
23
  from metadata.generated.schema.tests.basic import (
24
- TestCaseFailureStatus,
25
- TestCaseFailureStatusType,
26
24
  TestCaseResult,
27
25
  TestCaseStatus,
28
26
  TestResultValue,
29
27
  )
30
28
  from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
31
- from metadata.generated.schema.type.basic import Timestamp
32
29
  from metadata.profiler.processor.runner import QueryRunner
33
30
 
34
31
  T = TypeVar("T", bound=Callable)
@@ -88,12 +85,15 @@ class BaseTestValidator(ABC):
88
85
  pre_processed_value = pre_processor(value)
89
86
  return type_(pre_processed_value)
90
87
 
91
- def get_test_case_result_object(
88
+ def get_test_case_result_object( # pylint: disable=too-many-arguments
92
89
  self,
93
90
  execution_date: Union[datetime, float],
94
91
  status: TestCaseStatus,
95
92
  result: str,
96
93
  test_result_value: List[TestResultValue],
94
+ row_count: Optional[int] = None,
95
+ failed_rows: Optional[int] = None,
96
+ passed_rows: Optional[int] = None,
97
97
  ) -> TestCaseResult:
98
98
  """Returns a TestCaseResult object with the given args
99
99
 
@@ -105,26 +105,30 @@ class BaseTestValidator(ABC):
105
105
  Returns:
106
106
  TestCaseResult:
107
107
  """
108
- if status == TestCaseStatus.Failed:
109
- test_case_failure_status = TestCaseFailureStatus(
110
- testCaseFailureStatusType=TestCaseFailureStatusType.New,
111
- testCaseFailureReason=None,
112
- testCaseFailureComment=None,
113
- updatedAt=Timestamp(__root__=int(datetime.utcnow().timestamp() * 1000)),
114
- updatedBy=None,
115
- )
116
- else:
117
- test_case_failure_status = None
118
-
119
- return TestCaseResult(
108
+ test_case_result = TestCaseResult(
120
109
  timestamp=execution_date, # type: ignore
121
110
  testCaseStatus=status,
122
111
  result=result,
123
112
  testResultValue=test_result_value,
124
113
  sampleData=None,
125
- testCaseFailureStatus=test_case_failure_status,
126
114
  )
127
115
 
116
+ if (row_count is not None) and (
117
+ # we'll need at least one of these to be not None to compute the other
118
+ (failed_rows is not None)
119
+ or (passed_rows is not None)
120
+ ):
121
+ passed_rows = passed_rows if passed_rows is not None else (row_count - failed_rows) # type: ignore
122
+ failed_rows = (
123
+ failed_rows if failed_rows is not None else (row_count - passed_rows)
124
+ )
125
+ test_case_result.passedRows = passed_rows
126
+ test_case_result.failedRows = failed_rows
127
+ test_case_result.passedRowsPercentage = (passed_rows / row_count) * 100
128
+ test_case_result.failedRowsPercentage = (failed_rows / row_count) * 100 # type: ignore
129
+
130
+ return test_case_result
131
+
128
132
  def format_column_list(self, status: TestCaseStatus, cols: List):
129
133
  """Format column list based on the test status
130
134
 
@@ -15,7 +15,7 @@ Validator for column value length to be between test case
15
15
 
16
16
  import traceback
17
17
  from abc import abstractmethod
18
- from typing import Union
18
+ from typing import Tuple, Union
19
19
 
20
20
  from sqlalchemy import Column
21
21
 
@@ -65,6 +65,13 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
65
65
  min_bound = self.get_min_bound("minLength")
66
66
  max_bound = self.get_max_bound("maxLength")
67
67
 
68
+ if self.test_case.computePassedFailedRowCount:
69
+ row_count, failed_rows = self.compute_row_count(
70
+ column, min_bound, max_bound
71
+ )
72
+ else:
73
+ row_count, failed_rows = None, None
74
+
68
75
  return self.get_test_case_result_object(
69
76
  self.execution_date,
70
77
  self.get_test_case_status(min_bound <= min_res and max_bound >= max_res),
@@ -74,6 +81,8 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
74
81
  TestResultValue(name=MIN, value=str(min_res)),
75
82
  TestResultValue(name=MAX, value=str(max_res)),
76
83
  ],
84
+ row_count=row_count,
85
+ failed_rows=failed_rows,
77
86
  )
78
87
 
79
88
  @abstractmethod
@@ -83,3 +92,31 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
83
92
  @abstractmethod
84
93
  def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
85
94
  raise NotImplementedError
95
+
96
+ @abstractmethod
97
+ def compute_row_count(
98
+ self, column: Union[SQALikeColumn, Column], min_bound, max_bound
99
+ ):
100
+ """Compute row count for the given column
101
+
102
+ Args:
103
+ column (Union[SQALikeColumn, Column]): column to compute row count for
104
+ min_bound (_type_): min bound to filter out rows within the bound
105
+ max_bound (_type_): max bound to filter out rows within the bound
106
+
107
+ Raises:
108
+ NotImplementedError:
109
+ """
110
+ raise NotImplementedError
111
+
112
+ def get_row_count(self, min_bound, max_bound) -> Tuple[int, int]:
113
+ """Get row count
114
+
115
+ Args:
116
+ min_bound (_type_): min bound to filter out rows within the bound
117
+ max_bound (_type_): max bound to filter out rows within the bound
118
+
119
+ Returns:
120
+ Tuple[int, int]:
121
+ """
122
+ return self.compute_row_count(self._get_column_name(), min_bound, max_bound)
@@ -16,7 +16,7 @@ Validator for column values to be between test case
16
16
  import traceback
17
17
  from abc import abstractmethod
18
18
  from datetime import date, datetime, time
19
- from typing import Union
19
+ from typing import Tuple, Union
20
20
 
21
21
  from sqlalchemy import Column
22
22
 
@@ -100,6 +100,11 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
100
100
  pre_processor=convert_timestamp if is_date_time(column.type) else None,
101
101
  )
102
102
 
103
+ if self.test_case.computePassedFailedRowCount:
104
+ row_count, failed_rows = self.get_row_count(min_bound, max_bound)
105
+ else:
106
+ row_count, failed_rows = None, None
107
+
103
108
  return self.get_test_case_result_object(
104
109
  self.execution_date,
105
110
  self.get_test_case_status(min_res >= min_bound and max_res <= max_bound),
@@ -108,6 +113,8 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
108
113
  TestResultValue(name=MIN, value=str(min_res)),
109
114
  TestResultValue(name=MAX, value=str(max_res)),
110
115
  ],
116
+ row_count=row_count,
117
+ failed_rows=failed_rows,
111
118
  )
112
119
 
113
120
  @abstractmethod
@@ -117,3 +124,31 @@ class BaseColumnValuesToBeBetweenValidator(BaseTestValidator):
117
124
  @abstractmethod
118
125
  def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
119
126
  raise NotImplementedError
127
+
128
+ @abstractmethod
129
+ def compute_row_count(
130
+ self, column: Union[SQALikeColumn, Column], min_bound, max_bound
131
+ ):
132
+ """Compute row count for the given column
133
+
134
+ Args:
135
+ column (Union[SQALikeColumn, Column]): column to compute row count for
136
+ min_bound (_type_): min bound to filter out rows within the bound
137
+ max_bound (_type_): max bound to filter out rows within the bound
138
+
139
+ Raises:
140
+ NotImplementedError:
141
+ """
142
+ raise NotImplementedError
143
+
144
+ def get_row_count(self, min_bound, max_bound) -> Tuple[int, int]:
145
+ """Get row count
146
+
147
+ Args:
148
+ min_bound (_type_): min bound to filter out rows within the bound
149
+ max_bound (_type_): max bound to filter out rows within the bound
150
+
151
+ Returns:
152
+ Tuple[int, int]:
153
+ """
154
+ return self.compute_row_count(self._get_column_name(), min_bound, max_bound)
@@ -64,11 +64,18 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
64
64
  [TestResultValue(name=ALLOWED_VALUE_COUNT, value=None)],
65
65
  )
66
66
 
67
+ if self.test_case.computePassedFailedRowCount:
68
+ row_count = self.get_row_count()
69
+ else:
70
+ row_count = None
71
+
67
72
  return self.get_test_case_result_object(
68
73
  self.execution_date,
69
74
  self.get_test_case_status(res >= 1),
70
75
  f"Found countInSet={res}.",
71
76
  [TestResultValue(name=ALLOWED_VALUE_COUNT, value=str(res))],
77
+ row_count=row_count,
78
+ passed_rows=res,
72
79
  )
73
80
 
74
81
  @abstractmethod
@@ -80,3 +87,23 @@ class BaseColumnValuesToBeInSetValidator(BaseTestValidator):
80
87
  self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
81
88
  ):
82
89
  raise NotImplementedError
90
+
91
+ @abstractmethod
92
+ def compute_row_count(self, column: Union[SQALikeColumn, Column]):
93
+ """Compute row count for the given column
94
+
95
+ Args:
96
+ column (Union[SQALikeColumn, Column]): column to compute row count for
97
+
98
+ Raises:
99
+ NotImplementedError:
100
+ """
101
+ raise NotImplementedError
102
+
103
+ def get_row_count(self) -> int:
104
+ """Get row count
105
+
106
+ Returns:
107
+ Tuple[int, int]:
108
+ """
109
+ return self.compute_row_count(self._get_column_name())
@@ -70,11 +70,18 @@ class BaseColumnValuesToBeNotInSetValidator(BaseTestValidator):
70
70
  [TestResultValue(name=COUNT_FORBIDDEN_VALUES, value=None)],
71
71
  )
72
72
 
73
+ if self.test_case.computePassedFailedRowCount:
74
+ row_count = self.get_row_count()
75
+ else:
76
+ row_count = None
77
+
73
78
  return self.get_test_case_result_object(
74
79
  self.execution_date,
75
80
  self.get_test_case_status(res == 0),
76
81
  f"Found countInSet={res}. It should be 0",
77
82
  [TestResultValue(name=COUNT_FORBIDDEN_VALUES, value=str(res))],
83
+ row_count=row_count,
84
+ failed_rows=res,
78
85
  )
79
86
 
80
87
  @abstractmethod
@@ -86,3 +93,23 @@ class BaseColumnValuesToBeNotInSetValidator(BaseTestValidator):
86
93
  self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
87
94
  ):
88
95
  raise NotImplementedError
96
+
97
+ @abstractmethod
98
+ def compute_row_count(self, column: Union[SQALikeColumn, Column]):
99
+ """Compute row count for the given column
100
+
101
+ Args:
102
+ column (Union[SQALikeColumn, Column]): column to compute row count for
103
+
104
+ Raises:
105
+ NotImplementedError:
106
+ """
107
+ raise NotImplementedError
108
+
109
+ def get_row_count(self) -> int:
110
+ """Get row count
111
+
112
+ Returns:
113
+ Tuple[int, int]:
114
+ """
115
+ return self.compute_row_count(self._get_column_name())
@@ -57,11 +57,18 @@ class BaseColumnValuesToBeNotNullValidator(BaseTestValidator):
57
57
  [TestResultValue(name=NULL_COUNT, value=None)],
58
58
  )
59
59
 
60
+ if self.test_case.computePassedFailedRowCount:
61
+ row_count = self.get_row_count()
62
+ else:
63
+ row_count = None
64
+
60
65
  return self.get_test_case_result_object(
61
66
  self.execution_date,
62
67
  self.get_test_case_status(res == 0),
63
68
  f"Found nullCount={res}. It should be 0",
64
69
  [TestResultValue(name=NULL_COUNT, value=str(res))],
70
+ row_count=row_count,
71
+ failed_rows=res,
65
72
  )
66
73
 
67
74
  @abstractmethod
@@ -71,3 +78,23 @@ class BaseColumnValuesToBeNotNullValidator(BaseTestValidator):
71
78
  @abstractmethod
72
79
  def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
73
80
  raise NotImplementedError
81
+
82
+ @abstractmethod
83
+ def compute_row_count(self, column: Union[SQALikeColumn, Column]):
84
+ """Compute row count for the given column
85
+
86
+ Args:
87
+ column (Union[SQALikeColumn, Column]): column to compute row count for
88
+
89
+ Raises:
90
+ NotImplementedError:
91
+ """
92
+ raise NotImplementedError
93
+
94
+ def get_row_count(self) -> int:
95
+ """Get row count
96
+
97
+ Returns:
98
+ Tuple[int, int]:
99
+ """
100
+ return self.compute_row_count(self._get_column_name())
@@ -71,6 +71,8 @@ class BaseColumnValuesToBeUniqueValidator(BaseTestValidator):
71
71
  TestResultValue(name=VALUE_COUNT, value=str(count)),
72
72
  TestResultValue(name=UNIQUE_COUNT, value=str(unique_count)),
73
73
  ],
74
+ row_count=count,
75
+ passed_rows=unique_count,
74
76
  )
75
77
 
76
78
  @abstractmethod
@@ -79,8 +81,21 @@ class BaseColumnValuesToBeUniqueValidator(BaseTestValidator):
79
81
 
80
82
  @abstractmethod
81
83
  def _run_results(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
84
+ """Compute row count for the given column
85
+
86
+ Args:
87
+ column (Union[SQALikeColumn, Column]): column to compute row count for
88
+
89
+ Raises:
90
+ NotImplementedError:
91
+ """
82
92
  raise NotImplementedError
83
93
 
84
94
  @abstractmethod
85
95
  def _get_unique_count(self, metric: Metrics, column: Union[SQALikeColumn, Column]):
96
+ """Get row count
97
+
98
+ Returns:
99
+ Tuple[int, int]:
100
+ """
86
101
  raise NotImplementedError
@@ -65,11 +65,18 @@ class BaseColumnValuesToMatchRegexValidator(BaseTestValidator):
65
65
  [TestResultValue(name=LIKE_COUNT, value=None)],
66
66
  )
67
67
 
68
+ if self.test_case.computePassedFailedRowCount:
69
+ row_count = self.get_row_count()
70
+ else:
71
+ row_count = None
72
+
68
73
  return self.get_test_case_result_object(
69
74
  self.execution_date,
70
75
  self.get_test_case_status(count == match_count),
71
76
  f"Found {match_count} value(s) matching regex pattern vs {count} value(s) in the column.",
72
77
  [TestResultValue(name=LIKE_COUNT, value=str(match_count))],
78
+ row_count=row_count,
79
+ passed_rows=match_count,
73
80
  )
74
81
 
75
82
  @abstractmethod
@@ -81,3 +88,23 @@ class BaseColumnValuesToMatchRegexValidator(BaseTestValidator):
81
88
  self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
82
89
  ):
83
90
  raise NotImplementedError
91
+
92
+ @abstractmethod
93
+ def compute_row_count(self, column: Union[SQALikeColumn, Column]):
94
+ """Compute row count for the given column
95
+
96
+ Args:
97
+ column (Union[SQALikeColumn, Column]): column to compute row count for
98
+
99
+ Raises:
100
+ NotImplementedError:
101
+ """
102
+ raise NotImplementedError
103
+
104
+ def get_row_count(self) -> int:
105
+ """Get row count
106
+
107
+ Returns:
108
+ Tuple[int, int]:
109
+ """
110
+ return self.compute_row_count(self._get_column_name())
@@ -64,12 +64,19 @@ class BaseColumnValuesToNotMatchRegexValidator(BaseTestValidator):
64
64
  [TestResultValue(name=NOT_LIKE_COUNT, value=None)],
65
65
  )
66
66
 
67
+ if self.test_case.computePassedFailedRowCount:
68
+ row_count = self.get_row_count()
69
+ else:
70
+ row_count = None
71
+
67
72
  return self.get_test_case_result_object(
68
73
  self.execution_date,
69
74
  self.get_test_case_status(not not_match_count),
70
75
  f"Found {not_match_count} value(s) matching the forbidden regex pattern vs "
71
76
  f"{not_match_count} value(s) in the column.",
72
77
  [TestResultValue(name=NOT_LIKE_COUNT, value=str(not_match_count))],
78
+ row_count=row_count,
79
+ failed_rows=not_match_count,
73
80
  )
74
81
 
75
82
  @abstractmethod
@@ -81,3 +88,23 @@ class BaseColumnValuesToNotMatchRegexValidator(BaseTestValidator):
81
88
  self, metric: Metrics, column: Union[SQALikeColumn, Column], **kwargs
82
89
  ):
83
90
  raise NotImplementedError
91
+
92
+ @abstractmethod
93
+ def compute_row_count(self, column: Union[SQALikeColumn, Column]):
94
+ """Compute row count for the given column
95
+
96
+ Args:
97
+ column (Union[SQALikeColumn, Column]): column to compute row count for
98
+
99
+ Raises:
100
+ NotImplementedError:
101
+ """
102
+ raise NotImplementedError
103
+
104
+ def get_row_count(self) -> int:
105
+ """Get row count
106
+
107
+ Returns:
108
+ Tuple[int, int]:
109
+ """
110
+ return self.compute_row_count(self._get_column_name())