openmetadata-ingestion 1.5.13.0__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1009) hide show
  1. _openmetadata_testutils/dict.py +23 -0
  2. _openmetadata_testutils/pydantic/test_utils.py +22 -2
  3. _openmetadata_testutils/pytest_openmetadata/plugin.py +86 -0
  4. _openmetadata_testutils/pytest_openmetadata/test_utils.py +99 -0
  5. metadata/cli/app.py +1 -0
  6. metadata/cli/classify.py +52 -0
  7. metadata/cli/dataquality.py +4 -1
  8. metadata/cli/ingest.py +2 -1
  9. metadata/cli/lineage.py +2 -1
  10. metadata/cli/profile.py +4 -1
  11. metadata/cli/usage.py +2 -1
  12. metadata/cmd.py +9 -0
  13. metadata/config/common.py +3 -1
  14. metadata/data_quality/api/models.py +2 -1
  15. metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +45 -31
  16. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -24
  17. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -49
  18. metadata/data_quality/interface/test_suite_interface.py +36 -52
  19. metadata/data_quality/processor/test_case_runner.py +9 -9
  20. metadata/data_quality/runner/base_test_suite_source.py +52 -26
  21. metadata/data_quality/source/test_suite.py +1 -1
  22. metadata/data_quality/validations/base_test_handler.py +13 -8
  23. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +4 -2
  24. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
  25. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  26. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  27. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  28. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  29. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  30. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  31. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  32. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
  33. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
  34. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  35. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  36. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  37. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
  38. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  39. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  40. metadata/data_quality/validations/models.py +6 -1
  41. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
  42. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
  43. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +26 -16
  44. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +192 -17
  45. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
  46. metadata/examples/workflows/bigquery_classifier.yaml +56 -0
  47. metadata/examples/workflows/bigquery_profiler.yaml +1 -2
  48. metadata/examples/workflows/datafactory.yaml +29 -0
  49. metadata/examples/workflows/db2_profiler.yaml +1 -2
  50. metadata/examples/workflows/dbtcloud.yaml +2 -1
  51. metadata/examples/workflows/matillion.yaml +31 -0
  52. metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
  53. metadata/examples/workflows/mysql_profiler.yaml +0 -1
  54. metadata/examples/workflows/redshift_classifier.yaml +38 -0
  55. metadata/examples/workflows/redshift_profiler.yaml +2 -3
  56. metadata/examples/workflows/rest.yaml +20 -0
  57. metadata/examples/workflows/sigma.yaml +25 -0
  58. metadata/examples/workflows/trino_profiler.yaml +20 -0
  59. metadata/generated/antlr/EntityLinkLexer.py +70 -99
  60. metadata/generated/schema/analytics/__init__.py +1 -1
  61. metadata/generated/schema/analytics/basic.py +1 -1
  62. metadata/generated/schema/analytics/reportData.py +1 -1
  63. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  64. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  65. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  66. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  67. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  68. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  69. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  70. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  71. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  72. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  73. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  74. metadata/generated/schema/api/__init__.py +1 -1
  75. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  76. metadata/generated/schema/api/addTagToAssetsRequest.py +43 -0
  77. metadata/generated/schema/api/analytics/__init__.py +1 -1
  78. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  79. metadata/generated/schema/api/automations/__init__.py +1 -1
  80. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  81. metadata/generated/schema/api/bulkAssets.py +1 -1
  82. metadata/generated/schema/api/classification/__init__.py +1 -1
  83. metadata/generated/schema/api/classification/createClassification.py +1 -1
  84. metadata/generated/schema/api/classification/createTag.py +1 -1
  85. metadata/generated/schema/api/classification/loadTags.py +1 -1
  86. metadata/generated/schema/api/createBot.py +1 -1
  87. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  88. metadata/generated/schema/api/createType.py +1 -1
  89. metadata/generated/schema/api/data/__init__.py +1 -1
  90. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  91. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  92. metadata/generated/schema/api/data/createChart.py +1 -1
  93. metadata/generated/schema/api/data/createContainer.py +1 -1
  94. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  95. metadata/generated/schema/api/data/createDashboard.py +1 -1
  96. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  97. metadata/generated/schema/api/data/createDatabase.py +1 -1
  98. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  99. metadata/generated/schema/api/data/createGlossary.py +1 -1
  100. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  101. metadata/generated/schema/api/data/createMetric.py +83 -0
  102. metadata/generated/schema/api/data/createMlModel.py +1 -1
  103. metadata/generated/schema/api/data/createPipeline.py +1 -1
  104. metadata/generated/schema/api/data/createQuery.py +1 -1
  105. metadata/generated/schema/api/data/createSearchIndex.py +8 -1
  106. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  107. metadata/generated/schema/api/data/createTable.py +1 -1
  108. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  109. metadata/generated/schema/api/data/createTopic.py +1 -1
  110. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  111. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  112. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  113. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  114. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  115. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  116. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  117. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  118. metadata/generated/schema/api/docStore/__init__.py +1 -1
  119. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  120. metadata/generated/schema/api/domains/__init__.py +1 -1
  121. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  122. metadata/generated/schema/api/domains/createDomain.py +1 -1
  123. metadata/generated/schema/api/feed/__init__.py +1 -1
  124. metadata/generated/schema/api/feed/closeTask.py +1 -1
  125. metadata/generated/schema/api/feed/createPost.py +1 -1
  126. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  127. metadata/generated/schema/api/feed/createThread.py +1 -1
  128. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  129. metadata/generated/schema/api/feed/threadCount.py +1 -1
  130. metadata/generated/schema/api/governance/__init__.py +3 -0
  131. metadata/generated/schema/api/governance/createWorkflowDefinition.py +83 -0
  132. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +22 -0
  133. metadata/generated/schema/api/lineage/__init__.py +1 -1
  134. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  135. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  136. metadata/generated/schema/api/policies/__init__.py +1 -1
  137. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  138. metadata/generated/schema/api/services/__init__.py +1 -1
  139. metadata/generated/schema/api/services/createApiService.py +2 -2
  140. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  141. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  142. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  143. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  144. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  145. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  146. metadata/generated/schema/api/services/createSearchService.py +1 -1
  147. metadata/generated/schema/api/services/createStorageService.py +1 -1
  148. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  149. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  150. metadata/generated/schema/api/setOwner.py +1 -1
  151. metadata/generated/schema/api/teams/__init__.py +1 -1
  152. metadata/generated/schema/api/teams/createPersona.py +1 -1
  153. metadata/generated/schema/api/teams/createRole.py +1 -1
  154. metadata/generated/schema/api/teams/createTeam.py +1 -1
  155. metadata/generated/schema/api/teams/createUser.py +1 -1
  156. metadata/generated/schema/api/tests/__init__.py +1 -1
  157. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  158. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  159. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  160. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  161. metadata/generated/schema/api/tests/createTestCaseResult.py +71 -0
  162. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  163. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  164. metadata/generated/schema/api/voteRequest.py +1 -1
  165. metadata/generated/schema/auth/__init__.py +1 -1
  166. metadata/generated/schema/auth/basicAuth.py +1 -1
  167. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  168. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  169. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  170. metadata/generated/schema/auth/emailRequest.py +1 -1
  171. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  172. metadata/generated/schema/auth/generateToken.py +1 -1
  173. metadata/generated/schema/auth/jwtAuth.py +1 -1
  174. metadata/generated/schema/auth/loginRequest.py +1 -1
  175. metadata/generated/schema/auth/logoutRequest.py +1 -1
  176. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  177. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  178. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  179. metadata/generated/schema/auth/refreshToken.py +1 -1
  180. metadata/generated/schema/auth/registrationRequest.py +1 -1
  181. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  182. metadata/generated/schema/auth/revokeToken.py +1 -1
  183. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  184. metadata/generated/schema/auth/ssoAuth.py +1 -1
  185. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  186. metadata/generated/schema/configuration/__init__.py +1 -1
  187. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  188. metadata/generated/schema/configuration/assetCertificationSettings.py +22 -0
  189. metadata/generated/schema/configuration/authConfig.py +1 -1
  190. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  195. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  196. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  197. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  198. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  199. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  200. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  201. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  202. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  203. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  204. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  205. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  206. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  207. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  208. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  209. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  210. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  211. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  212. metadata/generated/schema/configuration/searchSettings.py +1 -1
  213. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
  214. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  215. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  216. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  217. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  218. metadata/generated/schema/dataInsight/__init__.py +1 -1
  219. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  220. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  221. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +9 -1
  222. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  223. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  224. metadata/generated/schema/dataInsight/custom/lineChart.py +23 -8
  225. metadata/generated/schema/dataInsight/custom/summaryCard.py +20 -9
  226. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  227. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  228. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  229. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  230. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  231. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  232. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  233. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  234. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  235. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  236. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  237. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  238. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  239. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  240. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  241. metadata/generated/schema/email/__init__.py +1 -1
  242. metadata/generated/schema/email/emailRequest.py +1 -1
  243. metadata/generated/schema/email/emailTemplate.py +1 -1
  244. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  245. metadata/generated/schema/email/smtpSettings.py +1 -1
  246. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  247. metadata/generated/schema/entity/__init__.py +1 -1
  248. metadata/generated/schema/entity/applications/__init__.py +1 -1
  249. metadata/generated/schema/entity/applications/app.py +9 -1
  250. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  251. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  252. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  253. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
  254. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  255. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  256. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  257. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  258. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  259. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  260. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  261. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +54 -2
  262. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  263. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  264. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  265. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  266. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  267. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  268. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  269. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  270. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
  271. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  272. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  273. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  274. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  275. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  276. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  277. metadata/generated/schema/entity/applications/createAppRequest.py +8 -1
  278. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  279. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  280. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  281. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +8 -1
  282. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +8 -1
  283. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  284. metadata/generated/schema/entity/automations/__init__.py +1 -1
  285. metadata/generated/schema/entity/automations/testServiceConnection.py +3 -1
  286. metadata/generated/schema/entity/automations/workflow.py +1 -1
  287. metadata/generated/schema/entity/bot.py +1 -1
  288. metadata/generated/schema/entity/classification/__init__.py +1 -1
  289. metadata/generated/schema/entity/classification/classification.py +2 -2
  290. metadata/generated/schema/entity/classification/tag.py +2 -2
  291. metadata/generated/schema/entity/data/__init__.py +1 -1
  292. metadata/generated/schema/entity/data/apiCollection.py +4 -2
  293. metadata/generated/schema/entity/data/apiEndpoint.py +4 -2
  294. metadata/generated/schema/entity/data/chart.py +4 -3
  295. metadata/generated/schema/entity/data/container.py +4 -2
  296. metadata/generated/schema/entity/data/dashboard.py +4 -2
  297. metadata/generated/schema/entity/data/dashboardDataModel.py +5 -2
  298. metadata/generated/schema/entity/data/database.py +7 -7
  299. metadata/generated/schema/entity/data/databaseSchema.py +5 -2
  300. metadata/generated/schema/entity/data/glossary.py +2 -2
  301. metadata/generated/schema/entity/data/glossaryTerm.py +3 -2
  302. metadata/generated/schema/entity/data/metric.py +196 -0
  303. metadata/generated/schema/entity/data/mlmodel.py +4 -2
  304. metadata/generated/schema/entity/data/pipeline.py +4 -3
  305. metadata/generated/schema/entity/data/query.py +3 -5
  306. metadata/generated/schema/entity/data/report.py +1 -1
  307. metadata/generated/schema/entity/data/searchIndex.py +16 -2
  308. metadata/generated/schema/entity/data/storedProcedure.py +4 -2
  309. metadata/generated/schema/entity/data/table.py +36 -3
  310. metadata/generated/schema/entity/data/topic.py +4 -3
  311. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  312. metadata/generated/schema/entity/docStore/document.py +1 -1
  313. metadata/generated/schema/entity/domains/__init__.py +1 -1
  314. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  315. metadata/generated/schema/entity/domains/domain.py +1 -1
  316. metadata/generated/schema/entity/events/__init__.py +1 -1
  317. metadata/generated/schema/entity/events/webhook.py +19 -2
  318. metadata/generated/schema/entity/feed/__init__.py +1 -1
  319. metadata/generated/schema/entity/feed/assets.py +1 -1
  320. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  321. metadata/generated/schema/entity/feed/description.py +1 -1
  322. metadata/generated/schema/entity/feed/domain.py +1 -1
  323. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  324. metadata/generated/schema/entity/feed/owner.py +1 -1
  325. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  326. metadata/generated/schema/entity/feed/tag.py +1 -1
  327. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  328. metadata/generated/schema/entity/feed/thread.py +1 -1
  329. metadata/generated/schema/entity/policies/__init__.py +1 -1
  330. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  331. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +4 -1
  332. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  333. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  334. metadata/generated/schema/entity/policies/filters.py +1 -1
  335. metadata/generated/schema/entity/policies/policy.py +1 -1
  336. metadata/generated/schema/entity/services/__init__.py +1 -1
  337. metadata/generated/schema/entity/services/apiService.py +7 -7
  338. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  339. metadata/generated/schema/entity/services/connections/{apiService → api}/__init__.py +1 -1
  340. metadata/generated/schema/entity/services/connections/{apiService → api}/restConnection.py +15 -7
  341. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  342. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  343. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  344. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  345. metadata/generated/schema/entity/services/connections/connectionBasicType.py +29 -1
  346. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  347. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  352. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
  353. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  356. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  357. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  358. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  359. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  360. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  361. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  365. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +41 -0
  366. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  367. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +5 -1
  370. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +9 -1
  372. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +11 -0
  380. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/database/db2Connection.py +4 -1
  389. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  391. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  392. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  393. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/database/druidConnection.py +4 -1
  396. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +84 -0
  398. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +4 -1
  400. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +4 -1
  401. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +4 -1
  410. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +4 -1
  411. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +3 -2
  412. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -1
  413. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +11 -7
  414. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +5 -1
  415. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +4 -1
  416. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +5 -1
  417. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +9 -1
  419. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  422. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  423. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  424. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +5 -1
  425. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +4 -1
  427. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +9 -1
  428. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +4 -1
  429. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +4 -1
  431. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +7 -2
  432. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  435. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  436. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  437. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  438. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  439. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  440. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  441. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  442. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  443. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  444. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  446. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  449. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  451. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  455. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  456. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  458. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  459. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  460. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  461. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +69 -0
  462. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
  463. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  464. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  466. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  467. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +65 -0
  469. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  470. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  471. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  472. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  473. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +43 -0
  474. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  475. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  476. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  477. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  478. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  479. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  480. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  481. metadata/generated/schema/entity/services/connections/serviceConnection.py +3 -3
  482. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  483. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  484. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  485. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  486. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  487. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  488. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  489. metadata/generated/schema/entity/services/dashboardService.py +8 -5
  490. metadata/generated/schema/entity/services/databaseService.py +5 -2
  491. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  492. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
  493. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  494. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  495. metadata/generated/schema/entity/services/messagingService.py +2 -2
  496. metadata/generated/schema/entity/services/metadataService.py +2 -2
  497. metadata/generated/schema/entity/services/mlmodelService.py +2 -2
  498. metadata/generated/schema/entity/services/pipelineService.py +11 -2
  499. metadata/generated/schema/entity/services/searchService.py +2 -2
  500. metadata/generated/schema/entity/services/serviceType.py +2 -2
  501. metadata/generated/schema/entity/services/storageService.py +2 -2
  502. metadata/generated/schema/entity/teams/__init__.py +1 -1
  503. metadata/generated/schema/entity/teams/persona.py +1 -1
  504. metadata/generated/schema/entity/teams/role.py +1 -1
  505. metadata/generated/schema/entity/teams/team.py +1 -1
  506. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  507. metadata/generated/schema/entity/teams/user.py +1 -1
  508. metadata/generated/schema/entity/type.py +1 -1
  509. metadata/generated/schema/entity/utils/__init__.py +1 -1
  510. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  511. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  512. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  513. metadata/generated/schema/events/__init__.py +1 -1
  514. metadata/generated/schema/events/alertMetrics.py +1 -1
  515. metadata/generated/schema/events/api/__init__.py +1 -1
  516. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  517. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +69 -0
  518. metadata/generated/schema/events/api/eventsRecord.py +37 -0
  519. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -4
  520. metadata/generated/schema/events/api/typedEvent.py +46 -0
  521. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  522. metadata/generated/schema/events/eventFilterRule.py +1 -1
  523. metadata/generated/schema/events/eventSubscription.py +15 -46
  524. metadata/generated/schema/events/eventSubscriptionOffset.py +7 -2
  525. metadata/generated/schema/events/failedEvent.py +1 -1
  526. metadata/generated/schema/events/failedEventResponse.py +38 -0
  527. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  528. metadata/generated/schema/events/statusContext.py +43 -0
  529. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  530. metadata/generated/schema/events/subscriptionStatus.py +78 -0
  531. metadata/generated/schema/events/testDestinationStatus.py +88 -0
  532. metadata/generated/schema/governance/workflows/__init__.py +3 -0
  533. metadata/generated/schema/governance/workflows/elements/__init__.py +5 -0
  534. metadata/generated/schema/governance/workflows/elements/edge.py +29 -0
  535. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +16 -0
  536. metadata/generated/schema/governance/workflows/elements/nodeType.py +14 -0
  537. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +3 -0
  538. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +5 -0
  539. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +44 -0
  540. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +49 -0
  541. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +42 -0
  542. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +3 -0
  543. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +30 -0
  544. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +3 -0
  545. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +30 -0
  546. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +3 -0
  547. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +57 -0
  548. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +3 -0
  549. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +43 -0
  550. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +44 -0
  551. metadata/generated/schema/{entity/data/metrics.py → governance/workflows/workflowDefinition.py} +36 -49
  552. metadata/generated/schema/governance/workflows/workflowInstance.py +50 -0
  553. metadata/generated/schema/governance/workflows/workflowInstanceState.py +73 -0
  554. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  555. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +53 -0
  556. metadata/generated/schema/metadataIngestion/application.py +1 -1
  557. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  558. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  560. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +127 -0
  561. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -9
  562. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +28 -17
  563. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +42 -1
  564. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  565. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  566. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  567. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  568. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  569. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  570. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  571. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
  572. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
  573. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  574. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  575. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  576. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  577. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  578. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
  579. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  580. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  581. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  582. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  583. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  584. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  585. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  586. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  587. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  588. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  589. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +4 -1
  590. metadata/generated/schema/metadataIngestion/workflow.py +15 -1
  591. metadata/generated/schema/monitoring/__init__.py +1 -1
  592. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  593. metadata/generated/schema/security/__init__.py +1 -1
  594. metadata/generated/schema/security/client/__init__.py +1 -1
  595. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  596. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  597. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  598. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  599. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  600. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  601. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  602. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  603. metadata/generated/schema/security/credentials/__init__.py +1 -1
  604. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  605. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  606. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  607. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  608. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  609. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  610. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  611. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  612. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  613. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  614. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  615. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  616. metadata/generated/schema/security/sasl/__init__.py +1 -1
  617. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  618. metadata/generated/schema/security/secrets/__init__.py +1 -1
  619. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  620. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  621. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  622. metadata/generated/schema/security/securityConfiguration.py +1 -1
  623. metadata/generated/schema/security/ssl/__init__.py +1 -1
  624. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  625. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  626. metadata/generated/schema/settings/__init__.py +1 -1
  627. metadata/generated/schema/settings/settings.py +7 -1
  628. metadata/generated/schema/system/__init__.py +1 -1
  629. metadata/generated/schema/system/entityError.py +1 -1
  630. metadata/generated/schema/system/eventPublisherJob.py +2 -1
  631. metadata/generated/schema/system/indexingError.py +1 -1
  632. metadata/generated/schema/system/limitsResponse.py +1 -1
  633. metadata/generated/schema/system/ui/__init__.py +1 -1
  634. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  635. metadata/generated/schema/system/ui/page.py +1 -1
  636. metadata/generated/schema/system/validationResponse.py +1 -1
  637. metadata/generated/schema/tests/__init__.py +1 -1
  638. metadata/generated/schema/tests/assigned.py +1 -1
  639. metadata/generated/schema/tests/basic.py +19 -4
  640. metadata/generated/schema/tests/customMetric.py +1 -1
  641. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  642. metadata/generated/schema/tests/resolved.py +1 -1
  643. metadata/generated/schema/tests/testCase.py +6 -2
  644. metadata/generated/schema/tests/testCaseResolutionStatus.py +17 -2
  645. metadata/generated/schema/tests/testDefinition.py +1 -1
  646. metadata/generated/schema/tests/testSuite.py +9 -2
  647. metadata/generated/schema/type/__init__.py +1 -1
  648. metadata/generated/schema/type/apiSchema.py +1 -1
  649. metadata/generated/schema/type/assetCertification.py +26 -0
  650. metadata/generated/schema/type/auditLog.py +1 -1
  651. metadata/generated/schema/type/basic.py +1 -1
  652. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  653. metadata/generated/schema/type/changeEvent.py +1 -1
  654. metadata/generated/schema/type/changeEventType.py +1 -1
  655. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  656. metadata/generated/schema/type/csvDocumentation.py +1 -1
  657. metadata/generated/schema/type/csvErrorType.py +1 -1
  658. metadata/generated/schema/type/csvFile.py +1 -1
  659. metadata/generated/schema/type/csvImportResult.py +1 -1
  660. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  661. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  662. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  663. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  664. metadata/generated/schema/type/customProperty.py +8 -1
  665. metadata/generated/schema/type/dailyCount.py +1 -1
  666. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  667. metadata/generated/schema/type/entityHierarchy.py +1 -1
  668. metadata/generated/schema/type/entityHistory.py +1 -1
  669. metadata/generated/schema/type/entityLineage.py +1 -1
  670. metadata/generated/schema/type/entityReference.py +1 -1
  671. metadata/generated/schema/type/entityReferenceList.py +1 -1
  672. metadata/generated/schema/type/entityRelationship.py +1 -1
  673. metadata/generated/schema/type/entityUsage.py +1 -1
  674. metadata/generated/schema/type/filterPattern.py +1 -1
  675. metadata/generated/schema/type/function.py +1 -1
  676. metadata/generated/schema/type/include.py +1 -1
  677. metadata/generated/schema/type/jdbcConnection.py +1 -1
  678. metadata/generated/schema/type/lifeCycle.py +1 -1
  679. metadata/generated/schema/type/paging.py +1 -1
  680. metadata/generated/schema/type/profile.py +1 -1
  681. metadata/generated/schema/type/queryParserData.py +1 -1
  682. metadata/generated/schema/type/reaction.py +1 -1
  683. metadata/generated/schema/type/schedule.py +1 -1
  684. metadata/generated/schema/type/schema.py +1 -1
  685. metadata/generated/schema/type/tableQuery.py +1 -1
  686. metadata/generated/schema/type/tableUsageCount.py +1 -1
  687. metadata/generated/schema/type/tagLabel.py +1 -1
  688. metadata/generated/schema/type/usageDetails.py +1 -1
  689. metadata/generated/schema/type/usageRequest.py +1 -1
  690. metadata/generated/schema/type/votes.py +1 -1
  691. metadata/ingestion/api/models.py +9 -5
  692. metadata/ingestion/api/parser.py +14 -0
  693. metadata/ingestion/lineage/parser.py +4 -6
  694. metadata/ingestion/lineage/sql_lineage.py +29 -185
  695. metadata/ingestion/models/custom_basemodel_validation.py +80 -0
  696. metadata/ingestion/models/custom_pydantic.py +33 -1
  697. metadata/ingestion/models/custom_types.py +2 -0
  698. metadata/ingestion/models/entity_interface.py +71 -0
  699. metadata/ingestion/models/patch_request.py +0 -1
  700. metadata/ingestion/ometa/mixins/es_mixin.py +66 -8
  701. metadata/ingestion/ometa/models.py +1 -0
  702. metadata/ingestion/ometa/ometa_api.py +6 -3
  703. metadata/ingestion/ometa/routes.py +4 -2
  704. metadata/ingestion/processor/query_parser.py +2 -2
  705. metadata/ingestion/sink/metadata_rest.py +23 -18
  706. metadata/ingestion/source/api/api_service.py +212 -0
  707. metadata/ingestion/source/api/rest/connection.py +93 -0
  708. metadata/ingestion/source/api/rest/metadata.py +278 -0
  709. metadata/ingestion/source/api/rest/models.py +41 -0
  710. metadata/ingestion/source/api/rest/service_spec.py +4 -0
  711. metadata/ingestion/source/dashboard/domodashboard/service_spec.py +6 -0
  712. metadata/ingestion/source/dashboard/lightdash/service_spec.py +4 -0
  713. metadata/ingestion/source/dashboard/looker/metadata.py +33 -4
  714. metadata/ingestion/source/dashboard/looker/service_spec.py +4 -0
  715. metadata/ingestion/source/dashboard/metabase/service_spec.py +4 -0
  716. metadata/ingestion/source/dashboard/microstrategy/__init__.py +0 -0
  717. metadata/ingestion/source/dashboard/{mstr → microstrategy}/client.py +27 -32
  718. metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +7 -10
  719. metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +18 -14
  720. metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +1 -1
  721. metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
  722. metadata/ingestion/source/dashboard/mode/service_spec.py +4 -0
  723. metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
  724. metadata/ingestion/source/dashboard/powerbi/service_spec.py +4 -0
  725. metadata/ingestion/source/dashboard/qlikcloud/service_spec.py +4 -0
  726. metadata/ingestion/source/dashboard/qliksense/service_spec.py +4 -0
  727. metadata/ingestion/source/dashboard/quicksight/service_spec.py +4 -0
  728. metadata/ingestion/source/dashboard/redash/service_spec.py +4 -0
  729. metadata/ingestion/source/dashboard/sigma/__init__.py +0 -0
  730. metadata/ingestion/source/dashboard/sigma/client.py +186 -0
  731. metadata/ingestion/source/dashboard/sigma/connection.py +67 -0
  732. metadata/ingestion/source/dashboard/sigma/metadata.py +331 -0
  733. metadata/ingestion/source/dashboard/sigma/models.py +97 -0
  734. metadata/ingestion/source/dashboard/sigma/service_spec.py +4 -0
  735. metadata/ingestion/source/dashboard/superset/service_spec.py +4 -0
  736. metadata/ingestion/source/dashboard/tableau/service_spec.py +4 -0
  737. metadata/ingestion/source/database/athena/service_spec.py +10 -0
  738. metadata/ingestion/source/database/azuresql/service_spec.py +12 -0
  739. metadata/ingestion/source/database/bigquery/connection.py +28 -2
  740. metadata/ingestion/source/database/bigquery/lineage.py +29 -2
  741. metadata/ingestion/source/database/bigquery/metadata.py +1 -25
  742. metadata/ingestion/source/database/bigquery/profiler/__init__.py +0 -0
  743. metadata/ingestion/source/database/bigquery/profiler/profiler.py +30 -0
  744. metadata/ingestion/source/database/bigquery/profiler/system.py +165 -0
  745. metadata/ingestion/source/database/bigquery/queries.py +69 -1
  746. metadata/ingestion/source/database/bigquery/service_spec.py +16 -0
  747. metadata/ingestion/source/database/bigtable/service_spec.py +4 -0
  748. metadata/ingestion/source/database/clickhouse/service_spec.py +12 -0
  749. metadata/ingestion/source/database/common_db_source.py +150 -147
  750. metadata/ingestion/source/database/common_nosql_source.py +0 -14
  751. metadata/ingestion/source/database/couchbase/metadata.py +62 -2
  752. metadata/{data_quality/builders/sqa_validator_builder.py → ingestion/source/database/couchbase/models.py} +15 -9
  753. metadata/ingestion/source/database/couchbase/queries.py +6 -1
  754. metadata/ingestion/source/database/couchbase/service_spec.py +4 -0
  755. metadata/ingestion/source/database/database_service.py +15 -22
  756. metadata/ingestion/source/database/databricks/metadata.py +12 -1
  757. metadata/ingestion/source/database/databricks/service_spec.py +20 -0
  758. metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
  759. metadata/ingestion/source/database/datalake/clients/base.py +10 -1
  760. metadata/ingestion/source/database/datalake/clients/gcs.py +10 -6
  761. metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
  762. metadata/ingestion/source/database/datalake/metadata.py +2 -13
  763. metadata/ingestion/source/database/datalake/service_spec.py +16 -0
  764. metadata/ingestion/source/database/db2/lineage.py +46 -0
  765. metadata/ingestion/source/database/db2/service_spec.py +9 -0
  766. metadata/ingestion/source/database/dbt/constants.py +4 -0
  767. metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
  768. metadata/ingestion/source/database/dbt/dbt_service.py +8 -1
  769. metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
  770. metadata/ingestion/source/database/dbt/metadata.py +73 -7
  771. metadata/ingestion/source/database/dbt/models.py +2 -0
  772. metadata/ingestion/source/database/dbt/service_spec.py +4 -0
  773. metadata/ingestion/source/database/deltalake/metadata.py +1 -12
  774. metadata/ingestion/source/database/deltalake/service_spec.py +4 -0
  775. metadata/ingestion/source/database/domodatabase/metadata.py +1 -12
  776. metadata/ingestion/source/database/domodatabase/service_spec.py +4 -0
  777. metadata/ingestion/source/database/doris/metadata.py +1 -1
  778. metadata/ingestion/source/database/doris/service_spec.py +4 -0
  779. metadata/ingestion/source/database/druid/lineage.py +46 -0
  780. metadata/ingestion/source/database/druid/service_spec.py +4 -0
  781. metadata/ingestion/source/database/dynamodb/service_spec.py +10 -0
  782. metadata/ingestion/source/database/exasol/__init__.py +0 -0
  783. metadata/ingestion/source/database/exasol/connection.py +87 -0
  784. metadata/ingestion/source/database/exasol/metadata.py +27 -0
  785. metadata/ingestion/source/database/exasol/service_spec.py +4 -0
  786. metadata/ingestion/source/database/glue/metadata.py +49 -19
  787. metadata/ingestion/source/database/glue/models.py +8 -0
  788. metadata/ingestion/source/database/glue/service_spec.py +4 -0
  789. metadata/ingestion/source/database/greenplum/lineage.py +46 -0
  790. metadata/ingestion/source/database/greenplum/service_spec.py +4 -0
  791. metadata/ingestion/source/database/hive/lineage.py +46 -0
  792. metadata/ingestion/source/database/hive/service_spec.py +4 -0
  793. metadata/ingestion/source/database/iceberg/metadata.py +1 -16
  794. metadata/ingestion/source/database/iceberg/service_spec.py +4 -0
  795. metadata/ingestion/source/database/impala/lineage.py +46 -0
  796. metadata/ingestion/source/database/impala/service_spec.py +4 -0
  797. metadata/ingestion/source/database/life_cycle_query_mixin.py +3 -5
  798. metadata/ingestion/source/database/lineage_source.py +124 -32
  799. metadata/ingestion/source/database/mariadb/lineage.py +46 -0
  800. metadata/ingestion/source/database/mariadb/service_spec.py +4 -0
  801. metadata/ingestion/source/database/mongodb/service_spec.py +10 -0
  802. metadata/ingestion/source/database/mssql/lineage.py +51 -2
  803. metadata/ingestion/source/database/mssql/metadata.py +2 -44
  804. metadata/ingestion/source/database/mssql/service_spec.py +10 -0
  805. metadata/ingestion/source/database/mysql/lineage.py +0 -2
  806. metadata/ingestion/source/database/mysql/queries.py +0 -2
  807. metadata/ingestion/source/database/mysql/service_spec.py +10 -0
  808. metadata/ingestion/source/database/mysql/usage.py +0 -2
  809. metadata/ingestion/source/database/oracle/lineage.py +28 -1
  810. metadata/ingestion/source/database/oracle/metadata.py +2 -23
  811. metadata/ingestion/source/database/oracle/service_spec.py +10 -0
  812. metadata/ingestion/source/database/pinotdb/lineage.py +46 -0
  813. metadata/ingestion/source/database/pinotdb/service_spec.py +4 -0
  814. metadata/ingestion/source/database/postgres/metadata.py +4 -13
  815. metadata/ingestion/source/database/postgres/service_spec.py +10 -0
  816. metadata/ingestion/source/database/postgres/utils.py +26 -0
  817. metadata/ingestion/source/database/presto/service_spec.py +4 -0
  818. metadata/ingestion/source/database/query/service_spec.py +9 -0
  819. metadata/ingestion/source/database/query_parser_source.py +1 -0
  820. metadata/ingestion/source/database/redshift/connection.py +9 -2
  821. metadata/ingestion/source/database/redshift/lineage.py +27 -3
  822. metadata/ingestion/source/database/redshift/metadata.py +2 -27
  823. metadata/ingestion/source/database/redshift/profiler/__init__.py +0 -0
  824. metadata/ingestion/source/database/redshift/profiler/profiler.py +14 -0
  825. metadata/ingestion/source/database/redshift/profiler/system.py +148 -0
  826. metadata/ingestion/source/database/redshift/queries.py +99 -8
  827. metadata/ingestion/source/database/redshift/service_spec.py +14 -0
  828. metadata/ingestion/source/database/redshift/utils.py +3 -3
  829. metadata/ingestion/source/database/salesforce/metadata.py +1 -12
  830. metadata/ingestion/source/database/salesforce/service_spec.py +4 -0
  831. metadata/ingestion/source/database/sample_data.py +129 -34
  832. metadata/ingestion/source/database/saperp/service_spec.py +4 -0
  833. metadata/ingestion/source/database/saphana/service_spec.py +7 -0
  834. metadata/ingestion/source/database/sas/metadata.py +1 -10
  835. metadata/ingestion/source/database/sas/service_spec.py +4 -0
  836. metadata/ingestion/source/database/singlestore/lineage.py +46 -0
  837. metadata/ingestion/source/database/singlestore/service_spec.py +9 -0
  838. metadata/ingestion/source/database/snowflake/connection.py +6 -1
  839. metadata/ingestion/source/database/snowflake/lineage.py +31 -2
  840. metadata/ingestion/source/database/snowflake/metadata.py +20 -62
  841. metadata/ingestion/source/database/snowflake/models.py +50 -2
  842. metadata/ingestion/source/database/snowflake/profiler/__init__.py +0 -0
  843. metadata/ingestion/source/database/snowflake/profiler/profiler.py +26 -0
  844. metadata/ingestion/source/database/snowflake/profiler/system.py +436 -0
  845. metadata/ingestion/source/database/snowflake/queries.py +28 -22
  846. metadata/ingestion/source/database/snowflake/service_spec.py +20 -0
  847. metadata/ingestion/source/database/sql_column_handler.py +26 -0
  848. metadata/ingestion/source/database/sqlite/lineage.py +46 -0
  849. metadata/ingestion/source/database/sqlite/service_spec.py +4 -0
  850. metadata/ingestion/source/database/stored_procedures_mixin.py +78 -52
  851. metadata/ingestion/source/database/teradata/lineage.py +46 -0
  852. metadata/ingestion/source/database/teradata/service_spec.py +4 -0
  853. metadata/ingestion/source/database/trino/connection.py +60 -23
  854. metadata/ingestion/source/database/trino/profiler/__init__.py +0 -0
  855. metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py +167 -0
  856. metadata/ingestion/source/database/trino/service_spec.py +16 -0
  857. metadata/ingestion/source/database/unitycatalog/metadata.py +3 -24
  858. metadata/ingestion/source/database/unitycatalog/service_spec.py +22 -0
  859. metadata/ingestion/source/database/usage_source.py +4 -1
  860. metadata/ingestion/source/database/vertica/service_spec.py +10 -0
  861. metadata/ingestion/source/messaging/kafka/service_spec.py +4 -0
  862. metadata/ingestion/source/messaging/kinesis/service_spec.py +4 -0
  863. metadata/ingestion/source/messaging/redpanda/service_spec.py +4 -0
  864. metadata/ingestion/source/metadata/alationsink/service_spec.py +4 -0
  865. metadata/ingestion/source/metadata/amundsen/service_spec.py +4 -0
  866. metadata/ingestion/source/metadata/atlas/service_spec.py +4 -0
  867. metadata/ingestion/source/mlmodel/mlflow/service_spec.py +4 -0
  868. metadata/ingestion/source/mlmodel/sagemaker/service_spec.py +4 -0
  869. metadata/ingestion/source/pipeline/airbyte/service_spec.py +4 -0
  870. metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
  871. metadata/ingestion/source/pipeline/airflow/metadata.py +71 -27
  872. metadata/ingestion/source/pipeline/airflow/service_spec.py +4 -0
  873. metadata/ingestion/source/pipeline/dagster/metadata.py +3 -1
  874. metadata/ingestion/source/pipeline/dagster/service_spec.py +4 -0
  875. metadata/ingestion/source/pipeline/databrickspipeline/service_spec.py +6 -0
  876. metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
  877. metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
  878. metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
  879. metadata/ingestion/source/pipeline/dbtcloud/service_spec.py +4 -0
  880. metadata/ingestion/source/pipeline/domopipeline/service_spec.py +4 -0
  881. metadata/ingestion/source/pipeline/fivetran/service_spec.py +4 -0
  882. metadata/ingestion/source/pipeline/flink/service_spec.py +4 -0
  883. metadata/ingestion/source/pipeline/gluepipeline/service_spec.py +4 -0
  884. metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
  885. metadata/ingestion/source/pipeline/kafkaconnect/service_spec.py +4 -0
  886. metadata/ingestion/source/pipeline/nifi/service_spec.py +4 -0
  887. metadata/ingestion/source/pipeline/openlineage/service_spec.py +4 -0
  888. metadata/ingestion/source/pipeline/pipeline_service.py +3 -1
  889. metadata/ingestion/source/pipeline/spline/service_spec.py +4 -0
  890. metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
  891. metadata/ingestion/source/search/elasticsearch/service_spec.py +4 -0
  892. metadata/ingestion/source/search/search_service.py +44 -1
  893. metadata/ingestion/source/storage/gcs/metadata.py +2 -2
  894. metadata/ingestion/source/storage/gcs/service_spec.py +4 -0
  895. metadata/ingestion/source/storage/s3/service_spec.py +4 -0
  896. metadata/mixins/pandas/pandas_mixin.py +2 -31
  897. metadata/mixins/sqalchemy/sqa_mixin.py +18 -17
  898. metadata/parsers/avro_parser.py +51 -16
  899. metadata/parsers/json_schema_parser.py +45 -16
  900. metadata/pii/processor.py +10 -9
  901. metadata/pii/scanners/ner_scanner.py +2 -1
  902. metadata/profiler/api/models.py +4 -75
  903. metadata/profiler/config.py +39 -0
  904. metadata/profiler/interface/nosql/profiler_interface.py +1 -26
  905. metadata/profiler/interface/pandas/profiler_interface.py +37 -77
  906. metadata/profiler/interface/profiler_interface.py +25 -279
  907. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
  908. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
  909. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
  910. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +7 -5
  911. metadata/profiler/interface/sqlalchemy/profiler_interface.py +73 -125
  912. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +5 -5
  913. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
  914. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +145 -0
  915. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +9 -4
  916. metadata/profiler/metrics/system/system.py +69 -374
  917. metadata/profiler/orm/converter/base.py +23 -13
  918. metadata/profiler/orm/converter/bigquery/converter.py +1 -1
  919. metadata/profiler/orm/functions/count.py +9 -0
  920. metadata/profiler/orm/functions/md5.py +39 -0
  921. metadata/{data_quality/builders/pandas_validator_builder.py → profiler/orm/functions/substr.py} +13 -10
  922. metadata/profiler/orm/functions/table_metric_computer.py +6 -5
  923. metadata/profiler/orm/functions/unique_count.py +6 -0
  924. metadata/profiler/orm/registry.py +27 -2
  925. metadata/profiler/processor/core.py +13 -62
  926. metadata/profiler/processor/handle_partition.py +0 -48
  927. metadata/profiler/processor/processor.py +1 -1
  928. metadata/profiler/processor/runner.py +111 -35
  929. metadata/profiler/processor/sample_data_handler.py +7 -4
  930. metadata/profiler/source/database/base/profiler_source.py +215 -0
  931. metadata/profiler/source/{bigquery → database/bigquery}/profiler_source.py +4 -4
  932. metadata/profiler/source/{databricks → database/databricks}/profiler_source.py +6 -7
  933. metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/first_quartile.py +1 -1
  934. metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/median.py +1 -1
  935. metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/third_quartile.py +1 -1
  936. metadata/profiler/source/{single_store → database/single_store}/metrics/window/first_quartile.py +3 -1
  937. metadata/profiler/source/{single_store → database/single_store}/metrics/window/median.py +3 -1
  938. metadata/profiler/source/{single_store → database/single_store}/metrics/window/third_quartile.py +3 -1
  939. metadata/profiler/source/fetcher/config.py +44 -0
  940. metadata/profiler/source/fetcher/entity_fetcher.py +62 -0
  941. metadata/profiler/source/fetcher/fetcher_strategy.py +299 -0
  942. metadata/profiler/source/fetcher/profiler_source_factory.py +89 -0
  943. metadata/profiler/source/metadata.py +16 -201
  944. metadata/profiler/source/metadata_ext.py +23 -18
  945. metadata/profiler/source/model.py +29 -0
  946. metadata/sampler/config.py +237 -0
  947. metadata/sampler/models.py +106 -0
  948. metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
  949. metadata/sampler/pandas/sampler.py +239 -0
  950. metadata/{utils → sampler}/partition.py +66 -51
  951. metadata/sampler/processor.py +158 -0
  952. metadata/sampler/sampler_interface.py +251 -0
  953. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
  954. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
  955. metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
  956. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +90 -98
  957. metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
  958. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
  959. metadata/utils/class_helper.py +5 -0
  960. metadata/utils/collaborative_super.py +28 -0
  961. metadata/utils/constants.py +65 -0
  962. metadata/utils/constraints.py +65 -0
  963. metadata/utils/dict.py +8 -0
  964. metadata/utils/dispatch.py +1 -53
  965. metadata/utils/entity_link.py +8 -2
  966. metadata/utils/entity_utils.py +93 -0
  967. metadata/utils/filters.py +15 -0
  968. metadata/utils/fqn.py +15 -0
  969. metadata/utils/helpers.py +3 -2
  970. metadata/utils/importer.py +12 -24
  971. metadata/utils/logger.py +31 -1
  972. metadata/utils/lru_cache.py +76 -15
  973. metadata/utils/profiler_utils.py +43 -9
  974. metadata/utils/service_spec/__init__.py +5 -0
  975. metadata/utils/service_spec/default.py +21 -0
  976. metadata/utils/service_spec/service_spec.py +146 -0
  977. metadata/utils/ssl_manager.py +33 -0
  978. metadata/utils/time_utils.py +33 -5
  979. metadata/workflow/application.py +7 -36
  980. metadata/workflow/base.py +47 -17
  981. metadata/workflow/classification.py +53 -0
  982. metadata/workflow/ingestion.py +6 -38
  983. metadata/workflow/profiler.py +1 -17
  984. metadata/workflow/workflow_output_handler.py +11 -8
  985. metadata/workflow/workflow_status_mixin.py +0 -2
  986. openmetadata_ingestion-1.6.0.0rc2.dist-info/LICENSE +114 -0
  987. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/METADATA +473 -548
  988. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/RECORD +995 -809
  989. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/entry_points.txt +3 -0
  990. metadata/data_quality/interface/test_suite_interface_factory.py +0 -128
  991. metadata/data_quality/runner/test_suite_source_factory.py +0 -38
  992. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
  993. metadata/profiler/interface/profiler_interface_factory.py +0 -107
  994. metadata/profiler/metrics/system/queries/bigquery.py +0 -54
  995. metadata/profiler/metrics/system/queries/redshift.py +0 -103
  996. metadata/profiler/metrics/system/queries/snowflake.py +0 -191
  997. metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
  998. metadata/profiler/processor/sampler/sampler_factory.py +0 -100
  999. metadata/profiler/processor/sampler/sampler_interface.py +0 -74
  1000. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -60
  1001. metadata/profiler/source/base/profiler_source.py +0 -299
  1002. metadata/profiler/source/profiler_source_factory.py +0 -54
  1003. openmetadata_ingestion-1.5.13.0.dist-info/LICENSE +0 -201
  1004. {metadata/ingestion/source/dashboard/mstr → _openmetadata_testutils/pytest_openmetadata}/__init__.py +0 -0
  1005. /metadata/profiler/source/{bigquery → database/bigquery}/type_mapper.py +0 -0
  1006. /metadata/profiler/source/{mariadb → database/mariadb}/functions/median.py +0 -0
  1007. /metadata/profiler/source/{single_store → database/single_store}/functions/median.py +0 -0
  1008. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/WHEEL +0 -0
  1009. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -16,10 +16,10 @@ supporting sqlalchemy abstraction layer
16
16
  from typing import List
17
17
 
18
18
  from pyhive.sqlalchemy_hive import HiveCompiler
19
- from sqlalchemy import Column, inspect
19
+ from sqlalchemy import Column
20
20
 
21
21
  from metadata.generated.schema.entity.data.table import Column as OMColumn
22
- from metadata.generated.schema.entity.data.table import ColumnName, DataType, TableData
22
+ from metadata.generated.schema.entity.data.table import ColumnName, DataType
23
23
  from metadata.generated.schema.entity.services.databaseService import (
24
24
  DatabaseServiceType,
25
25
  )
@@ -97,18 +97,3 @@ class DatabricksProfilerInterface(SQAProfilerInterface):
97
97
  )
98
98
  columns.append(col)
99
99
  return columns
100
-
101
- def fetch_sample_data(self, table, columns) -> TableData:
102
- """Fetch sample data from database
103
-
104
- Args:
105
- table: ORM declarative table
106
-
107
- Returns:
108
- TableData: sample table data
109
- """
110
- sampler = self._get_sampler(
111
- table=table,
112
- )
113
-
114
- return sampler.fetch_sample_data(list(inspect(self.table).c))
@@ -32,7 +32,7 @@ class DB2ProfilerInterface(SQAProfilerInterface):
32
32
  # pylint: disable=protected-access
33
33
  if exc.orig and "overflow" in exc.orig._message:
34
34
  logger.info(
35
- f"Computing metrics without sum for {runner.table.__tablename__}.{column.name}"
35
+ f"Computing metrics without sum for {runner.table_name}.{column.name}"
36
36
  )
37
37
  return self._compute_static_metrics_wo_sum(metrics, runner, session, column)
38
38
  return None
@@ -24,11 +24,13 @@ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
24
24
  )
25
25
  from metadata.profiler.metrics.registry import Metrics
26
26
  from metadata.profiler.processor.runner import QueryRunner
27
- from metadata.profiler.source.mariadb.metrics.window.first_quartile import (
27
+ from metadata.profiler.source.database.mariadb.metrics.window.first_quartile import (
28
28
  MariaDBFirstQuartile,
29
29
  )
30
- from metadata.profiler.source.mariadb.metrics.window.median import MariaDBMedian
31
- from metadata.profiler.source.mariadb.metrics.window.third_quartile import (
30
+ from metadata.profiler.source.database.mariadb.metrics.window.median import (
31
+ MariaDBMedian,
32
+ )
33
+ from metadata.profiler.source.database.mariadb.metrics.window.third_quartile import (
32
34
  MariaDBThirdQuartile,
33
35
  )
34
36
  from metadata.utils.logger import profiler_interface_registry_logger
@@ -75,11 +77,11 @@ class MariaDBProfilerInterface(SQAProfilerInterface):
75
77
  return dict(row)
76
78
  except ProgrammingError:
77
79
  logger.info(
78
- f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to overflow"
80
+ f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
79
81
  )
80
82
  return None
81
83
 
82
84
  except Exception as exc:
83
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
85
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
84
86
  handle_query_exception(msg, exc, session)
85
87
  return None
@@ -21,28 +21,41 @@ import threading
21
21
  import traceback
22
22
  from collections import defaultdict
23
23
  from datetime import datetime
24
- from typing import Any, Dict, List, Optional
24
+ from typing import Any, Dict, List, Optional, Type, Union
25
25
 
26
26
  from sqlalchemy import Column, inspect, text
27
27
  from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
28
28
  from sqlalchemy.orm import scoped_session
29
29
 
30
- from metadata.generated.schema.entity.data.table import CustomMetricProfile, TableData
30
+ from metadata.generated.schema.entity.data.table import (
31
+ CustomMetricProfile,
32
+ SystemProfile,
33
+ Table,
34
+ )
35
+ from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
36
+ DatalakeConnection,
37
+ )
38
+ from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
39
+ from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
40
+ DatabaseServiceProfilerPipeline,
41
+ )
31
42
  from metadata.generated.schema.tests.customMetric import CustomMetric
32
43
  from metadata.ingestion.connections.session import create_and_bind_thread_safe_session
44
+ from metadata.ingestion.ometa.ometa_api import OpenMetadata
33
45
  from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
34
46
  from metadata.profiler.api.models import ThreadPoolMetrics
35
47
  from metadata.profiler.interface.profiler_interface import ProfilerInterface
36
- from metadata.profiler.metrics.core import MetricTypes
48
+ from metadata.profiler.metrics.core import HybridMetric, MetricTypes
37
49
  from metadata.profiler.metrics.registry import Metrics
38
50
  from metadata.profiler.metrics.static.mean import Mean
39
51
  from metadata.profiler.metrics.static.stddev import StdDev
40
52
  from metadata.profiler.metrics.static.sum import Sum
53
+ from metadata.profiler.metrics.system.system import System, SystemMetricsComputer
41
54
  from metadata.profiler.orm.functions.table_metric_computer import TableMetricComputer
42
55
  from metadata.profiler.orm.registry import Dialects
43
56
  from metadata.profiler.processor.metric_filter import MetricFilter
44
57
  from metadata.profiler.processor.runner import QueryRunner
45
- from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
58
+ from metadata.sampler.sampler_interface import SamplerInterface
46
59
  from metadata.utils.custom_thread_pool import CustomThreadPoolExecutor
47
60
  from metadata.utils.helpers import is_safe_sql_query
48
61
  from metadata.utils.logger import profiler_interface_registry_logger
@@ -70,41 +83,40 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
70
83
  """
71
84
 
72
85
  # pylint: disable=too-many-arguments
73
-
74
86
  def __init__(
75
87
  self,
76
- service_connection_config,
77
- ometa_client,
78
- entity,
79
- storage_config,
80
- profile_sample_config,
81
- source_config,
82
- sample_query,
83
- table_partition_config,
88
+ service_connection_config: Union[DatabaseConnection, DatalakeConnection],
89
+ ometa_client: OpenMetadata,
90
+ entity: Table,
91
+ source_config: DatabaseServiceProfilerPipeline,
92
+ sampler: SamplerInterface,
84
93
  thread_count: int = 5,
85
94
  timeout_seconds: int = 43200,
86
- sqa_metadata=None,
87
- sample_data_count: Optional[int] = SAMPLE_DATA_DEFAULT_COUNT,
88
95
  **kwargs,
89
96
  ):
90
97
  """Instantiate SQA Interface object"""
98
+ self.session_factory = None
99
+ self.session = None
91
100
 
92
101
  super().__init__(
93
- service_connection_config,
94
- ometa_client,
95
- entity,
96
- storage_config,
97
- profile_sample_config,
98
- source_config,
99
- sample_query,
100
- table_partition_config,
101
- thread_count,
102
- timeout_seconds,
103
- sample_data_count,
102
+ service_connection_config=service_connection_config,
103
+ ometa_client=ometa_client,
104
+ entity=entity,
105
+ source_config=source_config,
106
+ sampler=sampler,
107
+ thread_count=thread_count,
108
+ timeout_seconds=timeout_seconds,
104
109
  )
105
110
 
106
- self._table = self._convert_table_to_orm_object(sqa_metadata)
111
+ self._table = self.sampler.raw_dataset
107
112
  self.create_session()
113
+ self.system_metrics_computer = self.initialize_system_metrics_computer()
114
+
115
+ def initialize_system_metrics_computer(self) -> SystemMetricsComputer:
116
+ """Initialize system metrics computer. Override this if you want to use a metric source with
117
+ state or other dependencies.
118
+ """
119
+ return SystemMetricsComputer()
108
120
 
109
121
  def create_session(self):
110
122
  self.session_factory = self._session_factory()
@@ -114,25 +126,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
114
126
  def table(self):
115
127
  return self._table
116
128
 
117
- def _get_sampler(self, **kwargs):
118
- """get sampler object"""
119
- from metadata.profiler.processor.sampler.sampler_factory import ( # pylint: disable=import-outside-toplevel
120
- sampler_factory_,
121
- )
122
-
123
- session = kwargs.get("session")
124
- table = kwargs["table"]
125
-
126
- return sampler_factory_.create(
127
- self.service_connection_config.__class__.__name__,
128
- client=session or self.session,
129
- table=table,
130
- profile_sample_config=self.profile_sample_config,
131
- partition_details=self.partition_details,
132
- profile_sample_query=self.profile_query,
133
- sample_data_count=self.sample_data_count,
134
- )
135
-
136
129
  def _session_factory(self) -> scoped_session:
137
130
  """Create thread safe session that will be automatically
138
131
  garbage collected once the application thread ends
@@ -163,7 +156,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
163
156
  )
164
157
  return dict(row)
165
158
  except Exception as exc:
166
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
159
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
167
160
  handle_query_exception(msg, exc, session)
168
161
  return None
169
162
 
@@ -174,7 +167,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
174
167
  session,
175
168
  *args,
176
169
  **kwargs,
177
- ):
170
+ ) -> Optional[Dict[str, Any]]:
178
171
  """Given a list of metrics, compute the given results
179
172
  and returns the values
180
173
 
@@ -201,7 +194,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
201
194
  except Exception as exc:
202
195
  logger.debug(traceback.format_exc())
203
196
  logger.warning(
204
- f"Error trying to compute profile for {runner.table.__tablename__}: {exc}" # type: ignore
197
+ f"Error trying to compute profile for {runner.table_name}: {exc}" # type: ignore
205
198
  )
206
199
  session.rollback()
207
200
  raise RuntimeError(exc)
@@ -238,7 +231,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
238
231
  runner, column, exc, session, metrics
239
232
  )
240
233
  except Exception as exc:
241
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
234
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
242
235
  handle_query_exception(msg, exc, session)
243
236
  return None
244
237
 
@@ -281,10 +274,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
281
274
  runner._session.get_bind().dialect.name
282
275
  != Dialects.Druid
283
276
  ):
284
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
277
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
285
278
  handle_query_exception(msg, exc, session)
286
279
  except Exception as exc:
287
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
280
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
288
281
  handle_query_exception(msg, exc, session)
289
282
  return None
290
283
 
@@ -317,10 +310,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
317
310
  return dict(row)
318
311
  except ProgrammingError as exc:
319
312
  logger.info(
320
- f"Skipping metrics for {runner.table.__tablename__}.{column.name} due to {exc}"
313
+ f"Skipping metrics for {runner.table_name}.{column.name} due to {exc}"
321
314
  )
322
315
  except Exception as exc:
323
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
316
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
324
317
  handle_query_exception(msg, exc, session)
325
318
  return None
326
319
 
@@ -354,7 +347,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
354
347
  )
355
348
 
356
349
  except Exception as exc:
357
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{metric.columnName}: {exc}"
350
+ msg = f"Error trying to compute profile for {runner.table_name}.{metric.columnName}: {exc}"
358
351
  logger.debug(traceback.format_exc())
359
352
  logger.warning(msg)
360
353
  if custom_metrics:
@@ -363,12 +356,11 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
363
356
 
364
357
  def _compute_system_metrics(
365
358
  self,
366
- metrics: Metrics,
359
+ metrics: Type[System],
367
360
  runner: QueryRunner,
368
- session,
369
361
  *args,
370
362
  **kwargs,
371
- ):
363
+ ) -> List[SystemProfile]:
372
364
  """Get system metric for tables
373
365
 
374
366
  Args:
@@ -379,44 +371,21 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
379
371
  Returns:
380
372
  dictionnary of results
381
373
  """
382
- try:
383
- rows = metrics().sql(session, conn_config=self.service_connection_config)
384
- return rows
385
- except Exception as exc:
386
- msg = f"Error trying to compute profile for {runner.table.__tablename__}: {exc}"
387
- handle_query_exception(msg, exc, session)
388
- return None
374
+ logger.debug(f"Computing system metrics for {runner.table_name}")
375
+ return self.system_metrics_computer.get_system_metrics(table=runner.dataset)
389
376
 
390
- def _create_thread_safe_sampler(
391
- self,
392
- session,
393
- table,
394
- ):
395
- """Create thread safe runner"""
396
- if not hasattr(thread_local, "sampler"):
397
- thread_local.sampler = self._get_sampler(
398
- table=table,
399
- session=session,
400
- )
401
- return thread_local.sampler
402
-
403
- def _create_thread_safe_runner(
404
- self,
405
- session,
406
- table,
407
- sample,
408
- ):
377
+ def _create_thread_safe_runner(self, session, column=None):
409
378
  """Create thread safe runner"""
410
379
  if not hasattr(thread_local, "runner"):
411
380
  thread_local.runner = QueryRunner(
412
381
  session=session,
413
- table=table,
414
- sample=sample,
415
- partition_details=self.partition_details,
416
- profile_sample_query=self.profile_query,
382
+ dataset=self.sampler.get_dataset(column=column),
383
+ raw_dataset=self.sampler.raw_dataset,
384
+ partition_details=self.sampler.partition_details,
385
+ profile_sample_query=self.sampler.sample_query,
417
386
  )
418
387
  return thread_local.runner
419
- thread_local.runner._sample = sample # pylint: disable=protected-access
388
+ thread_local.runner.dataset = self.sampler.get_dataset(column=column)
420
389
  return thread_local.runner
421
390
 
422
391
  def compute_metrics_in_thread(
@@ -431,31 +400,19 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
431
400
  with Session() as session:
432
401
  self.set_session_tag(session)
433
402
  self.set_catalog(session)
434
- sampler = self._create_thread_safe_sampler(
435
- session,
436
- metric_func.table,
437
- )
438
- sample = sampler.random_sample(metric_func.column)
439
- runner = self._create_thread_safe_runner(
440
- session,
441
- metric_func.table,
442
- sample,
443
- )
403
+ runner = self._create_thread_safe_runner(session, metric_func.column)
444
404
  row = None
445
-
446
405
  try:
447
406
  row = self._get_metric_fn[metric_func.metric_type.value](
448
407
  metric_func.metrics,
449
408
  runner=runner,
450
409
  session=session,
451
410
  column=metric_func.column,
452
- sample=sample,
411
+ sample=runner.dataset,
453
412
  )
454
- if row and isinstance(row, dict):
413
+ if isinstance(row, dict):
455
414
  row = self._validate_nulls(row)
456
-
457
- # System metrics return a list of dictionaries, with UPDATE, INSERT or DELETE ops results
458
- if row and metric_func.metric_type == MetricTypes.System:
415
+ if isinstance(row, list):
459
416
  row = [
460
417
  self._validate_nulls(r) if isinstance(r, dict) else r
461
418
  for r in row
@@ -537,24 +494,12 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
537
494
  logger.debug(traceback.format_exc())
538
495
  logger.error(f"Operation was cancelled due to TimeoutError - {exc}")
539
496
  raise concurrent.futures.TimeoutError
497
+ except KeyboardInterrupt:
498
+ pool.shutdown39(wait=True, cancel_futures=True)
499
+ raise
540
500
 
541
501
  return profile_results
542
502
 
543
- def fetch_sample_data(self, table, columns) -> TableData:
544
- """Fetch sample data from database
545
-
546
- Args:
547
- table: ORM declarative table
548
-
549
- Returns:
550
- TableData: sample table data
551
- """
552
- sampler = self._get_sampler(
553
- table=table,
554
- )
555
-
556
- return sampler.fetch_sample_data(columns)
557
-
558
503
  def get_composed_metrics(
559
504
  self, column: Column, metric: Metrics, column_results: Dict
560
505
  ):
@@ -576,21 +521,24 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
576
521
  return None
577
522
 
578
523
  def get_hybrid_metrics(
579
- self, column: Column, metric: Metrics, column_results: Dict, **kwargs
524
+ self,
525
+ column: Column,
526
+ metric: Type[HybridMetric],
527
+ column_results: Dict[str, Any],
580
528
  ):
581
529
  """Given a list of metrics, compute the given results
582
530
  and returns the values
583
531
 
584
532
  Args:
585
533
  column: the column to compute the metrics against
586
- metrics: list of metrics to compute
534
+ metric: metric to compute
535
+ column_results: results of the column
587
536
  Returns:
588
537
  dictionnary of results
589
538
  """
590
- sampler = self._get_sampler(table=kwargs.get("table"))
591
- sample = sampler.random_sample(column)
539
+ dataset = self.sampler.get_dataset(column=column)
592
540
  try:
593
- return metric(column).fn(sample, column_results, self.session)
541
+ return metric(column).fn(dataset, column_results, self.session)
594
542
  except Exception as exc:
595
543
  logger.debug(traceback.format_exc())
596
544
  logger.warning(f"Unexpected exception computing metrics: {exc}")
@@ -24,13 +24,13 @@ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
24
24
  )
25
25
  from metadata.profiler.metrics.registry import Metrics
26
26
  from metadata.profiler.processor.runner import QueryRunner
27
- from metadata.profiler.source.single_store.metrics.window.first_quartile import (
27
+ from metadata.profiler.source.database.single_store.metrics.window.first_quartile import (
28
28
  SingleStoreFirstQuartile,
29
29
  )
30
- from metadata.profiler.source.single_store.metrics.window.median import (
30
+ from metadata.profiler.source.database.single_store.metrics.window.median import (
31
31
  SingleStoreMedian,
32
32
  )
33
- from metadata.profiler.source.single_store.metrics.window.third_quartile import (
33
+ from metadata.profiler.source.database.single_store.metrics.window.third_quartile import (
34
34
  SingleStoreThirdQuartile,
35
35
  )
36
36
  from metadata.utils.logger import profiler_interface_registry_logger
@@ -76,11 +76,11 @@ class SingleStoreProfilerInterface(SQAProfilerInterface):
76
76
  return dict(row)
77
77
  except ProgrammingError:
78
78
  logger.info(
79
- f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to overflow"
79
+ f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
80
80
  )
81
81
  return None
82
82
 
83
83
  except Exception as exc:
84
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
84
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
85
85
  handle_query_exception(msg, exc, session)
86
86
  return None
@@ -41,7 +41,7 @@ class SnowflakeProfilerInterface(SQAProfilerInterface):
41
41
  session.bind.dialect.name
42
42
  ):
43
43
  logger.info(
44
- f"Computing metrics without sum for {runner.table.__tablename__}.{column.name}"
44
+ f"Computing metrics without sum for {runner.table_name}.{column.name}"
45
45
  )
46
46
  return self._compute_static_metrics_wo_sum(metrics, runner, session, column)
47
47
  return None
@@ -0,0 +1,145 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Interfaces with database for all database engine
14
+ supporting sqlalchemy abstraction layer
15
+ """
16
+
17
+ import threading
18
+ from typing import Any, Dict, List, Set
19
+
20
+ from more_itertools import partition
21
+ from sqlalchemy import Column
22
+
23
+ from metadata.generated.schema.entity.data.table import Table
24
+ from metadata.mixins.sqalchemy.sqa_mixin import Root
25
+ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
26
+ SQAProfilerInterface,
27
+ )
28
+ from metadata.profiler.metrics.core import Metric
29
+ from metadata.profiler.metrics.registry import Metrics
30
+ from metadata.profiler.processor.runner import QueryRunner
31
+ from metadata.utils.logger import profiler_interface_registry_logger
32
+
33
+ logger = profiler_interface_registry_logger()
34
+ thread_local = threading.local()
35
+
36
+
37
+ class StoredStatisticsSource(Root):
38
+ def get_statistics_metrics(self) -> Set[Metrics]:
39
+ """Statistic metrics that are found in system tables. Different for each database."""
40
+ return set()
41
+
42
+ def get_column_statistics(
43
+ self, metric: List[Metrics], schema: str, table_name: Table, column: str
44
+ ) -> dict:
45
+ raise NotImplementedError(
46
+ "You used a connector that does not support using statistics tables."
47
+ )
48
+
49
+ def get_table_statistics(
50
+ self, metric: List[Metrics], schema: str, table_name: Table
51
+ ) -> dict:
52
+ raise NotImplementedError(
53
+ "You used a connector that does not support using statistics tables."
54
+ )
55
+
56
+
57
+ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
58
+ """
59
+ Interface to interact with registry supporting
60
+ sqlalchemy.
61
+ """
62
+
63
+ def __init__(self, *args, **kwargs):
64
+ """Collaborative constructor"""
65
+ super().__init__(*args, **kwargs)
66
+
67
+ def _compute_static_metrics(
68
+ self,
69
+ metrics: List[Metrics],
70
+ runner: QueryRunner,
71
+ column,
72
+ session,
73
+ *args,
74
+ **kwargs,
75
+ ) -> dict:
76
+ result = {}
77
+ if self.source_config.useStatistics:
78
+ metrics, stat_metrics = map(
79
+ list,
80
+ partition(self.is_statistic_metric, metrics),
81
+ )
82
+ schema = runner.schema_name
83
+ table_name = runner.table_name
84
+ logger.debug(
85
+ "Getting statistics for column: %s.%s.%s",
86
+ schema,
87
+ table_name,
88
+ column.name,
89
+ )
90
+ result.update(
91
+ super().get_column_statistics(
92
+ stat_metrics, schema, table_name, column.name
93
+ )
94
+ )
95
+ result.update(
96
+ super()._compute_static_metrics(
97
+ metrics,
98
+ runner,
99
+ column,
100
+ session,
101
+ *args,
102
+ **kwargs,
103
+ )
104
+ )
105
+ return result
106
+
107
+ def _compute_table_metrics(
108
+ self,
109
+ metrics: List[Metric],
110
+ runner: QueryRunner,
111
+ session,
112
+ *args,
113
+ **kwargs,
114
+ ) -> Dict[str, Any]:
115
+ result = {}
116
+ if self.source_config.useStatistics:
117
+ metrics, stat_metrics = map(
118
+ list,
119
+ partition(self.is_statistic_metric, metrics),
120
+ )
121
+ schema = runner.schema_name
122
+ table_name = runner.table_name
123
+ logger.debug("Geting statistics for table: %s.%s", schema, table_name)
124
+ result.update(
125
+ super().get_table_statistics(stat_metrics, schema, table_name)
126
+ )
127
+ super_table_metrics = super()._compute_table_metrics(
128
+ metrics,
129
+ runner,
130
+ session,
131
+ *args,
132
+ **kwargs,
133
+ )
134
+ if super_table_metrics is not None:
135
+ result.update(super_table_metrics)
136
+ return result
137
+
138
+ def get_hybrid_metrics(self, column: Column, metric: Metric, column_results: Dict):
139
+ # this metrics might have been computed in a previous step
140
+ return column_results.get(metric.name()) or super().get_hybrid_metrics(
141
+ column, metric, column_results
142
+ )
143
+
144
+ def is_statistic_metric(self, metric: Metric) -> bool:
145
+ return metric.name() in {m.name for m in super().get_statistics_metrics()}
@@ -19,10 +19,15 @@ from typing import List
19
19
  from sqlalchemy import func
20
20
  from sqlalchemy.exc import ProgrammingError
21
21
 
22
+ from metadata.ingestion.source.database.trino.profiler.system_tables_profiler import (
23
+ TrinoStoredStatisticsSource,
24
+ )
22
25
  from metadata.profiler.interface.sqlalchemy.profiler_interface import (
23
- SQAProfilerInterface,
24
26
  handle_query_exception,
25
27
  )
28
+ from metadata.profiler.interface.sqlalchemy.stored_statistics_profiler import (
29
+ ProfilerWithStatistics,
30
+ )
26
31
  from metadata.profiler.metrics.registry import Metrics
27
32
  from metadata.profiler.orm.registry import FLOAT_SET
28
33
  from metadata.profiler.processor.runner import QueryRunner
@@ -31,7 +36,7 @@ from metadata.utils.logger import profiler_interface_registry_logger
31
36
  logger = profiler_interface_registry_logger()
32
37
 
33
38
 
34
- class TrinoProfilerInterface(SQAProfilerInterface):
39
+ class TrinoProfilerInterface(ProfilerWithStatistics, TrinoStoredStatisticsSource):
35
40
  """
36
41
  Interface to interact with registry supporting
37
42
  sqlalchemy.
@@ -71,11 +76,11 @@ class TrinoProfilerInterface(SQAProfilerInterface):
71
76
  return dict(row)
72
77
  except ProgrammingError as err:
73
78
  logger.info(
74
- f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to {err}"
79
+ f"Skipping window metrics for {runner.table_name}.{column.name} due to {err}"
75
80
  )
76
81
  return None
77
82
 
78
83
  except Exception as exc:
79
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
84
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
80
85
  handle_query_exception(msg, exc, session)
81
86
  return None