openmetadata-ingestion 1.2.5.2__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (826) hide show
  1. airflow_provider_openmetadata/lineage/runner.py +44 -5
  2. airflow_provider_openmetadata/lineage/status.py +1 -1
  3. metadata/antlr/split_listener.py +1 -4
  4. metadata/applications/auto_tagger.py +212 -0
  5. metadata/cli/app.py +47 -0
  6. metadata/cli/db_dump.py +6 -3
  7. metadata/clients/aws_client.py +4 -0
  8. metadata/clients/domo_client.py +0 -2
  9. metadata/cmd.py +24 -13
  10. metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
  11. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
  12. metadata/data_insight/source/metadata.py +11 -4
  13. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
  14. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
  15. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
  16. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
  17. metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
  18. metadata/data_quality/processor/test_case_runner.py +17 -8
  19. metadata/data_quality/source/test_suite.py +8 -2
  20. metadata/data_quality/validations/base_test_handler.py +21 -17
  21. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
  22. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
  23. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
  24. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
  25. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
  26. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
  27. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
  28. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
  29. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
  30. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
  31. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
  32. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
  33. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
  34. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
  35. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
  36. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
  37. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
  38. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
  39. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
  40. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
  41. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
  42. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
  43. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
  44. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
  45. metadata/examples/workflows/databricks.yaml +0 -1
  46. metadata/examples/workflows/mongodb.yaml +4 -2
  47. metadata/examples/workflows/mstr.yaml +24 -0
  48. metadata/examples/workflows/sas.yaml +28 -0
  49. metadata/examples/workflows/unity_catalog.yaml +27 -0
  50. metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
  51. metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
  52. metadata/generated/antlr/EntityLinkLexer.py +391 -292
  53. metadata/generated/antlr/EntityLinkListener.py +12 -12
  54. metadata/generated/antlr/EntityLinkParser.py +228 -103
  55. metadata/generated/schema/analytics/__init__.py +1 -1
  56. metadata/generated/schema/analytics/basic.py +1 -1
  57. metadata/generated/schema/analytics/reportData.py +1 -1
  58. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  59. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  60. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  61. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  62. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  63. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  64. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  65. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  66. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  67. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  68. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  69. metadata/generated/schema/api/__init__.py +1 -1
  70. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  71. metadata/generated/schema/api/analytics/__init__.py +1 -1
  72. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  73. metadata/generated/schema/api/automations/__init__.py +1 -1
  74. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  75. metadata/generated/schema/api/bulkAssets.py +1 -1
  76. metadata/generated/schema/api/classification/__init__.py +1 -1
  77. metadata/generated/schema/api/classification/createClassification.py +1 -1
  78. metadata/generated/schema/api/classification/createTag.py +1 -1
  79. metadata/generated/schema/api/classification/loadTags.py +1 -1
  80. metadata/generated/schema/api/createBot.py +2 -3
  81. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  82. metadata/generated/schema/api/createType.py +1 -1
  83. metadata/generated/schema/api/data/__init__.py +1 -1
  84. metadata/generated/schema/api/data/createChart.py +1 -1
  85. metadata/generated/schema/api/data/createContainer.py +2 -2
  86. metadata/generated/schema/api/data/createCustomProperty.py +2 -3
  87. metadata/generated/schema/api/data/createDashboard.py +1 -1
  88. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  89. metadata/generated/schema/api/data/createDatabase.py +2 -3
  90. metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
  91. metadata/generated/schema/api/data/createGlossary.py +2 -3
  92. metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
  93. metadata/generated/schema/api/data/createMlModel.py +1 -1
  94. metadata/generated/schema/api/data/createPipeline.py +2 -2
  95. metadata/generated/schema/api/data/createQuery.py +2 -2
  96. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  97. metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
  98. metadata/generated/schema/api/data/createTable.py +2 -2
  99. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  100. metadata/generated/schema/api/data/createTopic.py +1 -1
  101. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  102. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  103. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  104. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  105. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  106. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  107. metadata/generated/schema/api/docStore/__init__.py +1 -1
  108. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  109. metadata/generated/schema/api/domains/__init__.py +1 -1
  110. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  111. metadata/generated/schema/api/domains/createDomain.py +1 -1
  112. metadata/generated/schema/api/feed/__init__.py +1 -1
  113. metadata/generated/schema/api/feed/closeTask.py +8 -1
  114. metadata/generated/schema/api/feed/createPost.py +1 -1
  115. metadata/generated/schema/api/feed/createSuggestion.py +30 -0
  116. metadata/generated/schema/api/feed/createThread.py +1 -1
  117. metadata/generated/schema/api/feed/resolveTask.py +12 -1
  118. metadata/generated/schema/api/feed/threadCount.py +18 -14
  119. metadata/generated/schema/api/lineage/__init__.py +1 -1
  120. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  121. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  122. metadata/generated/schema/api/policies/__init__.py +1 -1
  123. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  124. metadata/generated/schema/api/services/__init__.py +1 -1
  125. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  126. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  127. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  128. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  129. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  130. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  131. metadata/generated/schema/api/services/createSearchService.py +1 -1
  132. metadata/generated/schema/api/services/createStorageService.py +1 -1
  133. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  134. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  135. metadata/generated/schema/api/setOwner.py +1 -1
  136. metadata/generated/schema/api/teams/__init__.py +1 -1
  137. metadata/generated/schema/api/teams/createPersona.py +1 -1
  138. metadata/generated/schema/api/teams/createRole.py +2 -3
  139. metadata/generated/schema/api/teams/createTeam.py +1 -1
  140. metadata/generated/schema/api/teams/createUser.py +2 -2
  141. metadata/generated/schema/api/tests/__init__.py +1 -1
  142. metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
  143. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  144. metadata/generated/schema/api/tests/createTestCase.py +4 -1
  145. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
  146. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  147. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  148. metadata/generated/schema/api/voteRequest.py +1 -1
  149. metadata/generated/schema/auth/__init__.py +1 -1
  150. metadata/generated/schema/auth/basicAuth.py +1 -1
  151. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  152. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  153. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  154. metadata/generated/schema/auth/emailRequest.py +1 -1
  155. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  156. metadata/generated/schema/auth/generateToken.py +1 -1
  157. metadata/generated/schema/auth/jwtAuth.py +1 -1
  158. metadata/generated/schema/auth/loginRequest.py +1 -1
  159. metadata/generated/schema/auth/logoutRequest.py +1 -1
  160. metadata/generated/schema/auth/passwordResetRequest.py +3 -3
  161. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  162. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  163. metadata/generated/schema/auth/refreshToken.py +1 -1
  164. metadata/generated/schema/auth/registrationRequest.py +2 -2
  165. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  166. metadata/generated/schema/auth/revokeToken.py +1 -1
  167. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  168. metadata/generated/schema/auth/ssoAuth.py +1 -1
  169. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  170. metadata/generated/schema/configuration/__init__.py +1 -1
  171. metadata/generated/schema/configuration/appsPrivateConfiguration.py +28 -0
  172. metadata/generated/schema/configuration/authConfig.py +1 -1
  173. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  175. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  176. metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
  177. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  178. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  179. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  180. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  181. metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
  182. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  183. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  184. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  185. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  186. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  187. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  188. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  189. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  190. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  194. metadata/generated/schema/dataInsight/__init__.py +1 -1
  195. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  196. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  197. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  198. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  199. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  200. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  201. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  202. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  203. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  204. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  205. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  206. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  207. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  208. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  209. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
  210. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
  211. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
  212. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
  213. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
  214. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
  215. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  216. metadata/generated/schema/email/__init__.py +1 -1
  217. metadata/generated/schema/email/emailRequest.py +1 -1
  218. metadata/generated/schema/email/smtpSettings.py +1 -1
  219. metadata/generated/schema/entity/__init__.py +1 -1
  220. metadata/generated/schema/entity/applications/__init__.py +1 -1
  221. metadata/generated/schema/entity/applications/app.py +18 -4
  222. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  223. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  224. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
  225. metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
  226. metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
  227. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
  228. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
  229. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
  230. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
  231. metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +13 -3
  232. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
  233. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
  234. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  235. metadata/generated/schema/entity/applications/jobStatus.py +3 -3
  236. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  237. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  238. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
  239. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
  240. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  241. metadata/generated/schema/entity/automations/__init__.py +1 -1
  242. metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
  243. metadata/generated/schema/entity/automations/workflow.py +1 -1
  244. metadata/generated/schema/entity/bot.py +2 -3
  245. metadata/generated/schema/entity/classification/__init__.py +1 -1
  246. metadata/generated/schema/entity/classification/classification.py +1 -1
  247. metadata/generated/schema/entity/classification/tag.py +2 -2
  248. metadata/generated/schema/entity/data/__init__.py +1 -1
  249. metadata/generated/schema/entity/data/chart.py +1 -1
  250. metadata/generated/schema/entity/data/container.py +4 -9
  251. metadata/generated/schema/entity/data/dashboard.py +1 -1
  252. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  253. metadata/generated/schema/entity/data/database.py +4 -8
  254. metadata/generated/schema/entity/data/databaseSchema.py +2 -8
  255. metadata/generated/schema/entity/data/glossary.py +1 -1
  256. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  257. metadata/generated/schema/entity/data/metrics.py +1 -1
  258. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  259. metadata/generated/schema/entity/data/pipeline.py +6 -9
  260. metadata/generated/schema/entity/data/query.py +2 -2
  261. metadata/generated/schema/entity/data/report.py +1 -1
  262. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  263. metadata/generated/schema/entity/data/storedProcedure.py +3 -9
  264. metadata/generated/schema/entity/data/table.py +18 -12
  265. metadata/generated/schema/entity/data/topic.py +1 -1
  266. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  267. metadata/generated/schema/entity/docStore/document.py +1 -1
  268. metadata/generated/schema/entity/domains/__init__.py +1 -1
  269. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  270. metadata/generated/schema/entity/domains/domain.py +1 -1
  271. metadata/generated/schema/entity/events/__init__.py +1 -1
  272. metadata/generated/schema/entity/events/webhook.py +6 -3
  273. metadata/generated/schema/entity/feed/__init__.py +1 -1
  274. metadata/generated/schema/entity/feed/suggestion.py +61 -0
  275. metadata/generated/schema/entity/feed/thread.py +6 -1
  276. metadata/generated/schema/entity/policies/__init__.py +1 -1
  277. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  278. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  279. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  280. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  281. metadata/generated/schema/entity/policies/filters.py +2 -5
  282. metadata/generated/schema/entity/policies/policy.py +1 -1
  283. metadata/generated/schema/entity/services/__init__.py +1 -1
  284. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  285. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  286. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  287. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  288. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  289. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  290. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  291. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  292. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  293. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  294. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  295. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  296. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  297. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
  298. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  299. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
  300. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  301. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  302. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  304. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  305. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
  306. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +2 -2
  307. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
  308. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
  309. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  310. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  311. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  312. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  313. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
  314. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  315. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
  316. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  319. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  320. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
  321. metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
  322. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  323. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
  325. metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
  326. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
  329. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
  330. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
  331. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
  332. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
  333. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
  334. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
  335. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
  336. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
  337. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
  338. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
  339. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
  340. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
  341. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
  342. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +2 -2
  343. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
  344. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +2 -2
  345. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +2 -2
  346. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
  347. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
  348. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +2 -2
  350. metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
  351. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
  352. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
  353. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
  354. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
  355. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
  356. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
  357. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  358. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  359. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  360. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  361. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  364. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  365. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  366. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  367. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  368. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  369. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
  370. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  371. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  372. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  374. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  375. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  376. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
  378. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
  387. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  389. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  391. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  392. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  393. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  397. metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
  398. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  402. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  403. metadata/generated/schema/entity/services/dashboardService.py +4 -1
  404. metadata/generated/schema/entity/services/databaseService.py +13 -1
  405. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  406. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
  407. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  408. metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
  409. metadata/generated/schema/entity/services/messagingService.py +1 -1
  410. metadata/generated/schema/entity/services/metadataService.py +1 -1
  411. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  412. metadata/generated/schema/entity/services/pipelineService.py +4 -1
  413. metadata/generated/schema/entity/services/searchService.py +1 -1
  414. metadata/generated/schema/entity/services/serviceType.py +1 -1
  415. metadata/generated/schema/entity/services/storageService.py +10 -3
  416. metadata/generated/schema/entity/teams/__init__.py +1 -1
  417. metadata/generated/schema/entity/teams/persona.py +1 -1
  418. metadata/generated/schema/entity/teams/role.py +2 -6
  419. metadata/generated/schema/entity/teams/team.py +1 -1
  420. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  421. metadata/generated/schema/entity/teams/user.py +3 -10
  422. metadata/generated/schema/entity/type.py +2 -2
  423. metadata/generated/schema/entity/utils/__init__.py +1 -1
  424. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  425. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  426. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  427. metadata/generated/schema/events/__init__.py +1 -1
  428. metadata/generated/schema/events/alertMetrics.py +27 -0
  429. metadata/generated/schema/events/api/__init__.py +1 -1
  430. metadata/generated/schema/events/api/createEventSubscription.py +14 -12
  431. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  432. metadata/generated/schema/events/eventFilterRule.py +13 -2
  433. metadata/generated/schema/events/eventSubscription.py +89 -49
  434. metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
  435. metadata/generated/schema/events/failedEvent.py +26 -0
  436. metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
  437. metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
  438. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  439. metadata/generated/schema/metadataIngestion/application.py +40 -0
  440. metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
  441. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  442. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  443. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
  444. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +6 -1
  445. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  446. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  447. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  448. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  449. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
  450. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  451. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  452. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
  453. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  454. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  455. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
  456. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  457. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  458. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  459. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  460. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  461. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  462. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  463. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  464. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  465. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
  466. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
  467. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  468. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  469. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  470. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
  471. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  472. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  473. metadata/generated/schema/monitoring/__init__.py +1 -1
  474. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  475. metadata/generated/schema/security/__init__.py +1 -1
  476. metadata/generated/schema/security/client/__init__.py +1 -1
  477. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  478. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  479. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  480. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  481. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  482. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  483. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  484. metadata/generated/schema/security/credentials/__init__.py +1 -1
  485. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  486. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  487. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  488. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  489. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  490. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  491. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  492. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  493. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  494. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  495. metadata/generated/schema/security/secrets/__init__.py +1 -1
  496. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  497. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
  498. metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -2
  499. metadata/generated/schema/security/securityConfiguration.py +1 -1
  500. metadata/generated/schema/security/ssl/__init__.py +1 -1
  501. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  502. metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
  503. metadata/generated/schema/settings/__init__.py +1 -1
  504. metadata/generated/schema/settings/settings.py +1 -1
  505. metadata/generated/schema/system/__init__.py +1 -1
  506. metadata/generated/schema/system/entityError.py +17 -0
  507. metadata/generated/schema/system/eventPublisherJob.py +5 -19
  508. metadata/generated/schema/system/indexingError.py +34 -0
  509. metadata/generated/schema/system/ui/__init__.py +1 -1
  510. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  511. metadata/generated/schema/system/ui/page.py +1 -1
  512. metadata/generated/schema/tests/__init__.py +1 -1
  513. metadata/generated/schema/tests/assigned.py +18 -0
  514. metadata/generated/schema/tests/basic.py +17 -39
  515. metadata/generated/schema/tests/customMetric.py +4 -2
  516. metadata/generated/schema/tests/resolved.py +35 -0
  517. metadata/generated/schema/tests/testCase.py +11 -2
  518. metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
  519. metadata/generated/schema/tests/testDefinition.py +5 -1
  520. metadata/generated/schema/tests/testSuite.py +1 -1
  521. metadata/generated/schema/type/__init__.py +1 -1
  522. metadata/generated/schema/type/auditLog.py +1 -1
  523. metadata/generated/schema/type/basic.py +5 -5
  524. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  525. metadata/generated/schema/type/changeEvent.py +5 -75
  526. metadata/generated/schema/type/changeEventType.py +29 -0
  527. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  528. metadata/generated/schema/type/csvDocumentation.py +1 -1
  529. metadata/generated/schema/type/csvErrorType.py +1 -1
  530. metadata/generated/schema/type/csvFile.py +1 -1
  531. metadata/generated/schema/type/csvImportResult.py +1 -1
  532. metadata/generated/schema/type/dailyCount.py +1 -1
  533. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  534. metadata/generated/schema/type/entityHistory.py +1 -1
  535. metadata/generated/schema/type/entityLineage.py +2 -1
  536. metadata/generated/schema/type/entityReference.py +5 -1
  537. metadata/generated/schema/type/entityReferenceList.py +1 -1
  538. metadata/generated/schema/type/entityRelationship.py +1 -1
  539. metadata/generated/schema/type/entityUsage.py +1 -1
  540. metadata/generated/schema/type/filterPattern.py +1 -1
  541. metadata/generated/schema/type/function.py +7 -6
  542. metadata/generated/schema/type/include.py +1 -1
  543. metadata/generated/schema/type/jdbcConnection.py +1 -1
  544. metadata/generated/schema/type/lifeCycle.py +1 -1
  545. metadata/generated/schema/type/paging.py +1 -1
  546. metadata/generated/schema/type/profile.py +1 -1
  547. metadata/generated/schema/type/queryParserData.py +2 -2
  548. metadata/generated/schema/type/reaction.py +1 -1
  549. metadata/generated/schema/type/schedule.py +1 -1
  550. metadata/generated/schema/type/schema.py +1 -1
  551. metadata/generated/schema/type/tableQuery.py +2 -2
  552. metadata/generated/schema/type/tableUsageCount.py +1 -1
  553. metadata/generated/schema/type/tagLabel.py +1 -1
  554. metadata/generated/schema/type/usageDetails.py +1 -1
  555. metadata/generated/schema/type/usageRequest.py +1 -1
  556. metadata/generated/schema/type/votes.py +1 -1
  557. metadata/great_expectations/action.py +3 -2
  558. metadata/ingestion/api/delete.py +5 -2
  559. metadata/ingestion/api/models.py +4 -10
  560. metadata/ingestion/api/parser.py +3 -37
  561. metadata/ingestion/api/status.py +15 -4
  562. metadata/ingestion/api/step.py +39 -6
  563. metadata/ingestion/api/steps.py +20 -0
  564. metadata/ingestion/api/topology_runner.py +114 -168
  565. metadata/ingestion/bulksink/metadata_usage.py +11 -5
  566. metadata/ingestion/lineage/models.py +4 -0
  567. metadata/ingestion/lineage/parser.py +4 -5
  568. metadata/ingestion/lineage/sql_lineage.py +9 -17
  569. metadata/ingestion/models/custom_pydantic.py +4 -2
  570. metadata/ingestion/models/lf_tags_model.py +33 -0
  571. metadata/ingestion/models/patch_request.py +108 -2
  572. metadata/ingestion/models/tests_data.py +9 -0
  573. metadata/ingestion/models/topology.py +177 -60
  574. metadata/ingestion/ometa/auth_provider.py +0 -349
  575. metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
  576. metadata/ingestion/ometa/mixins/patch_mixin.py +17 -68
  577. metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
  578. metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
  579. metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
  580. metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
  581. metadata/ingestion/ometa/ometa_api.py +18 -25
  582. metadata/ingestion/ometa/routes.py +7 -0
  583. metadata/ingestion/processor/query_parser.py +9 -2
  584. metadata/ingestion/sink/metadata_rest.py +25 -6
  585. metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
  586. metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
  587. metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
  588. metadata/ingestion/source/dashboard/looker/metadata.py +38 -25
  589. metadata/ingestion/source/dashboard/metabase/metadata.py +23 -6
  590. metadata/ingestion/source/dashboard/mode/client.py +10 -23
  591. metadata/ingestion/source/dashboard/mode/connection.py +6 -1
  592. metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
  593. metadata/ingestion/source/dashboard/mstr/client.py +208 -0
  594. metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
  595. metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
  596. metadata/ingestion/source/dashboard/mstr/models.py +144 -0
  597. metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
  598. metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
  599. metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
  600. metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
  601. metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
  602. metadata/ingestion/source/dashboard/superset/client.py +16 -9
  603. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  604. metadata/ingestion/source/dashboard/superset/db_source.py +14 -10
  605. metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
  606. metadata/ingestion/source/dashboard/superset/queries.py +1 -1
  607. metadata/ingestion/source/dashboard/tableau/client.py +91 -11
  608. metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
  609. metadata/ingestion/source/dashboard/tableau/metadata.py +58 -70
  610. metadata/ingestion/source/dashboard/tableau/models.py +0 -8
  611. metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
  612. metadata/ingestion/source/database/athena/client.py +80 -0
  613. metadata/ingestion/source/database/athena/connection.py +7 -0
  614. metadata/ingestion/source/database/athena/metadata.py +161 -19
  615. metadata/ingestion/source/database/azuresql/metadata.py +0 -1
  616. metadata/ingestion/source/database/bigquery/connection.py +24 -3
  617. metadata/ingestion/source/database/bigquery/metadata.py +39 -27
  618. metadata/ingestion/source/database/bigquery/queries.py +11 -4
  619. metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
  620. metadata/ingestion/source/database/bigquery/usage.py +1 -3
  621. metadata/ingestion/source/database/column_helpers.py +0 -10
  622. metadata/ingestion/source/database/column_type_parser.py +11 -5
  623. metadata/ingestion/source/database/common_db_source.py +8 -2
  624. metadata/ingestion/source/database/common_nosql_source.py +8 -4
  625. metadata/ingestion/source/database/database_service.py +89 -7
  626. metadata/ingestion/source/database/databricks/client.py +5 -10
  627. metadata/ingestion/source/database/databricks/connection.py +16 -55
  628. metadata/ingestion/source/database/databricks/lineage.py +29 -26
  629. metadata/ingestion/source/database/databricks/metadata.py +448 -11
  630. metadata/ingestion/source/database/databricks/queries.py +28 -0
  631. metadata/ingestion/source/database/databricks/query_parser.py +5 -1
  632. metadata/ingestion/source/database/databricks/usage.py +2 -2
  633. metadata/ingestion/source/database/datalake/connection.py +31 -4
  634. metadata/ingestion/source/database/datalake/metadata.py +113 -27
  635. metadata/ingestion/source/database/dbt/dbt_config.py +5 -0
  636. metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
  637. metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
  638. metadata/ingestion/source/database/dbt/metadata.py +11 -25
  639. metadata/ingestion/source/database/deltalake/metadata.py +6 -3
  640. metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
  641. metadata/ingestion/source/database/doris/connection.py +72 -0
  642. metadata/ingestion/source/database/doris/metadata.py +315 -0
  643. metadata/ingestion/source/database/doris/queries.py +54 -0
  644. metadata/ingestion/source/database/doris/utils.py +64 -0
  645. metadata/ingestion/source/database/extended_sample_data.py +532 -0
  646. metadata/ingestion/source/database/glue/metadata.py +8 -5
  647. metadata/ingestion/source/database/hive/connection.py +0 -2
  648. metadata/ingestion/source/database/hive/utils.py +3 -0
  649. metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
  650. metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
  651. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
  652. metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
  653. metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
  654. metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
  655. metadata/ingestion/source/database/iceberg/connection.py +68 -0
  656. metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
  657. metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
  658. metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
  659. metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
  660. metadata/ingestion/source/database/iceberg/helper.py +124 -0
  661. metadata/ingestion/source/database/iceberg/metadata.py +311 -0
  662. metadata/ingestion/source/database/iceberg/models.py +66 -0
  663. metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
  664. metadata/ingestion/source/database/mongodb/connection.py +1 -5
  665. metadata/ingestion/source/database/mssql/lineage.py +2 -0
  666. metadata/ingestion/source/database/mssql/metadata.py +108 -4
  667. metadata/ingestion/source/database/mssql/models.py +30 -0
  668. metadata/ingestion/source/database/mssql/queries.py +179 -1
  669. metadata/ingestion/source/database/mssql/utils.py +207 -4
  670. metadata/ingestion/source/database/mysql/metadata.py +0 -2
  671. metadata/ingestion/source/database/oracle/metadata.py +108 -2
  672. metadata/ingestion/source/database/oracle/models.py +30 -0
  673. metadata/ingestion/source/database/oracle/queries.py +98 -17
  674. metadata/ingestion/source/database/oracle/utils.py +0 -1
  675. metadata/ingestion/source/database/postgres/lineage.py +32 -14
  676. metadata/ingestion/source/database/postgres/metadata.py +15 -7
  677. metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
  678. metadata/ingestion/source/database/postgres/queries.py +4 -2
  679. metadata/ingestion/source/database/postgres/query_parser.py +4 -72
  680. metadata/ingestion/source/database/postgres/usage.py +41 -0
  681. metadata/ingestion/source/database/postgres/utils.py +34 -0
  682. metadata/ingestion/source/database/query_parser_source.py +8 -2
  683. metadata/ingestion/source/database/redshift/metadata.py +14 -4
  684. metadata/ingestion/source/database/redshift/queries.py +10 -4
  685. metadata/ingestion/source/database/redshift/query_parser.py +16 -0
  686. metadata/ingestion/source/database/redshift/usage.py +0 -2
  687. metadata/ingestion/source/database/salesforce/metadata.py +32 -3
  688. metadata/ingestion/source/database/sample_data.py +120 -6
  689. metadata/ingestion/source/database/sas/client.py +184 -0
  690. metadata/ingestion/source/database/sas/connection.py +47 -0
  691. metadata/ingestion/source/database/sas/extension_attr.py +103 -0
  692. metadata/ingestion/source/database/sas/metadata.py +914 -0
  693. metadata/ingestion/source/database/snowflake/metadata.py +8 -51
  694. metadata/ingestion/source/database/snowflake/models.py +6 -1
  695. metadata/ingestion/source/database/snowflake/queries.py +0 -4
  696. metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
  697. metadata/ingestion/source/database/snowflake/utils.py +2 -6
  698. metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
  699. metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
  700. metadata/ingestion/source/database/unitycatalog/client.py +87 -0
  701. metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
  702. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
  703. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
  704. metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
  705. metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
  706. metadata/ingestion/source/database/usage_source.py +3 -2
  707. metadata/ingestion/source/messaging/common_broker_source.py +6 -4
  708. metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
  709. metadata/ingestion/source/messaging/messaging_service.py +6 -2
  710. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  711. metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
  712. metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
  713. metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
  714. metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
  715. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
  716. metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
  717. metadata/ingestion/source/pipeline/airflow/models.py +5 -4
  718. metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
  719. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +12 -9
  720. metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
  721. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
  722. metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
  723. metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
  724. metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
  725. metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
  726. metadata/ingestion/source/search/search_service.py +6 -2
  727. metadata/ingestion/source/storage/s3/metadata.py +22 -17
  728. metadata/ingestion/source/storage/storage_service.py +53 -11
  729. metadata/ingestion/stage/table_usage.py +9 -2
  730. metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
  731. metadata/parsers/protobuf_parser.py +29 -11
  732. metadata/pii/processor.py +9 -2
  733. metadata/profiler/api/models.py +19 -1
  734. metadata/profiler/interface/pandas/profiler_interface.py +59 -18
  735. metadata/profiler/interface/profiler_interface.py +13 -2
  736. metadata/profiler/interface/profiler_interface_factory.py +49 -14
  737. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
  738. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
  739. metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
  740. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
  741. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
  742. metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
  743. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  744. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
  745. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  746. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
  747. metadata/profiler/metrics/composed/null_ratio.py +1 -1
  748. metadata/profiler/metrics/hybrid/histogram.py +1 -0
  749. metadata/profiler/metrics/static/max.py +4 -1
  750. metadata/profiler/metrics/static/min.py +4 -1
  751. metadata/profiler/metrics/system/queries/snowflake.py +89 -17
  752. metadata/profiler/metrics/system/system.py +62 -20
  753. metadata/profiler/orm/functions/length.py +1 -0
  754. metadata/profiler/orm/functions/median.py +7 -0
  755. metadata/profiler/orm/functions/table_metric_computer.py +462 -0
  756. metadata/profiler/orm/registry.py +1 -0
  757. metadata/profiler/processor/core.py +116 -52
  758. metadata/profiler/processor/default.py +14 -3
  759. metadata/profiler/processor/handle_partition.py +2 -2
  760. metadata/profiler/processor/processor.py +9 -4
  761. metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
  762. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
  763. metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
  764. metadata/profiler/source/base/profiler_source.py +5 -2
  765. metadata/profiler/source/bigquery/type_mapper.py +0 -1
  766. metadata/profiler/source/mariadb/functions/median.py +20 -0
  767. metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
  768. metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
  769. metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
  770. metadata/profiler/source/metadata.py +43 -10
  771. metadata/profiler/source/metadata_ext.py +16 -50
  772. metadata/py.typed +0 -0
  773. metadata/readers/dataframe/json.py +5 -4
  774. metadata/readers/file/api_reader.py +0 -1
  775. metadata/utils/constants.py +5 -0
  776. metadata/utils/datalake/datalake_utils.py +363 -129
  777. metadata/utils/entity_link.py +26 -6
  778. metadata/utils/fqn.py +20 -0
  779. metadata/utils/helpers.py +55 -9
  780. metadata/utils/importer.py +2 -3
  781. metadata/utils/life_cycle_utils.py +4 -4
  782. metadata/utils/logger.py +13 -2
  783. metadata/utils/partition.py +10 -5
  784. metadata/utils/secrets/client/loader.py +0 -1
  785. metadata/utils/secrets/noop_secrets_manager.py +4 -3
  786. metadata/utils/secrets/secrets_manager_factory.py +3 -4
  787. metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
  788. metadata/utils/sqlalchemy_utils.py +21 -0
  789. metadata/utils/storage_metadata_config.py +42 -1
  790. metadata/utils/tag_utils.py +5 -2
  791. metadata/workflow/application.py +154 -0
  792. metadata/workflow/application_output_handler.py +34 -0
  793. metadata/workflow/base.py +84 -153
  794. metadata/workflow/data_insight.py +8 -7
  795. metadata/workflow/data_quality.py +3 -2
  796. metadata/workflow/ingestion.py +203 -0
  797. metadata/workflow/metadata.py +2 -3
  798. metadata/workflow/output_handler.py +204 -0
  799. metadata/workflow/profiler.py +2 -2
  800. metadata/workflow/usage.py +3 -4
  801. metadata/workflow/workflow_output_handler.py +15 -255
  802. metadata/workflow/workflow_status_mixin.py +44 -52
  803. openmetadata_ingestion-1.3.0.0.dist-info/METADATA +749 -0
  804. {openmetadata_ingestion-1.2.5.2.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/RECORD +812 -725
  805. {openmetadata_ingestion-1.2.5.2.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/WHEEL +1 -1
  806. metadata/generated/schema/entity/applications/appConfig.py +0 -21
  807. metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
  808. metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
  809. metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
  810. metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
  811. metadata/generated/schema/events/entitySpelFilters.py +0 -19
  812. metadata/ingestion/models/es_documents.py +0 -339
  813. metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
  814. metadata/ingestion/ometa/provider_registry.py +0 -144
  815. metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
  816. metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
  817. metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
  818. metadata/profiler/orm/functions/table_metric_construct.py +0 -365
  819. openmetadata_ingestion-1.2.5.2.dist-info/METADATA +0 -426
  820. /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
  821. /metadata/ingestion/source/database/{databricks/unity_catalog → doris}/__init__.py +0 -0
  822. /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/sas}/__init__.py +0 -0
  823. /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
  824. {openmetadata_ingestion-1.2.5.2.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/LICENSE +0 -0
  825. {openmetadata_ingestion-1.2.5.2.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/entry_points.txt +0 -0
  826. {openmetadata_ingestion-1.2.5.2.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/top_level.txt +0 -0
@@ -13,12 +13,15 @@ Mixin class containing User specific methods
13
13
 
14
14
  To be used by OpenMetadata class
15
15
  """
16
- import traceback
17
16
  from functools import lru_cache
18
- from typing import Optional
17
+ from typing import Optional, Type
19
18
 
19
+ from metadata.generated.schema.entity.teams.team import Team
20
20
  from metadata.generated.schema.entity.teams.user import User
21
+ from metadata.generated.schema.type.entityReference import EntityReference
22
+ from metadata.ingestion.api.common import T
21
23
  from metadata.ingestion.ometa.client import REST
24
+ from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
22
25
  from metadata.utils.elasticsearch import ES_INDEX_MAP
23
26
  from metadata.utils.logger import ometa_logger
24
27
 
@@ -34,42 +37,134 @@ class OMetaUserMixin:
34
37
 
35
38
  client: REST
36
39
 
37
- email_search = (
38
- "/search/query?q=email.keyword:{email}&from={from_}&size={size}&index="
39
- + ES_INDEX_MAP[User.__name__]
40
- )
40
+ @staticmethod
41
+ def email_search_query_es(entity: Type[T]) -> str:
42
+ return (
43
+ "/search/query?q=email.keyword:{email}&from={from_}&size={size}&index="
44
+ + ES_INDEX_MAP[entity.__name__]
45
+ )
41
46
 
42
- @lru_cache(maxsize=None)
43
- def get_user_by_email(
47
+ @staticmethod
48
+ def name_search_query_es(entity: Type[T]) -> str:
49
+ """
50
+ Allow for more flexible lookup following what the UI is doing when searching users.
51
+
52
+ We don't want to stick to `q=name:{name}` since in case a user is named `random.user`
53
+ but looked as `Random User`, we want to find this match.
54
+ """
55
+ return (
56
+ "/search/query?q={name}&from={from_}&size={size}&index="
57
+ + ES_INDEX_MAP[entity.__name__]
58
+ )
59
+
60
+ def _search_by_email(
44
61
  self,
62
+ entity: Type[T],
45
63
  email: Optional[str],
46
64
  from_count: int = 0,
47
65
  size: int = 1,
48
66
  fields: Optional[list] = None,
49
- ) -> Optional[User]:
67
+ ) -> Optional[T]:
50
68
  """
51
- GET user entity by name
69
+ GET user or team entity by mail
52
70
 
53
71
  Args:
54
72
  email: user email to search
55
73
  from_count: records to expect
56
74
  size: number of records
75
+ fields: Optional field list to pass to ES request
57
76
  """
58
77
  if email:
59
- query_string = self.email_search.format(
78
+ query_string = self.email_search_query_es(entity=entity).format(
60
79
  email=email, from_=from_count, size=size
61
80
  )
81
+ return self._get_entity_from_es(
82
+ entity=entity, query_string=query_string, fields=fields
83
+ )
84
+
85
+ return None
86
+
87
+ def _search_by_name(
88
+ self,
89
+ entity: Type[T],
90
+ name: Optional[str],
91
+ from_count: int = 0,
92
+ size: int = 1,
93
+ fields: Optional[list] = None,
94
+ ) -> Optional[T]:
95
+ """
96
+ GET entity by name
97
+
98
+ Args:
99
+ name: user name to search
100
+ from_count: records to expect
101
+ size: number of records
102
+ fields: Optional field list to pass to ES request
103
+ """
104
+ if name:
105
+ query_string = self.name_search_query_es(entity=entity).format(
106
+ name=name, from_=from_count, size=size
107
+ )
108
+ return self._get_entity_from_es(
109
+ entity=entity, query_string=query_string, fields=fields
110
+ )
62
111
 
63
- try:
64
- entity_list = self._search_es_entity(
65
- entity_type=User, query_string=query_string, fields=fields
66
- )
67
- for user in entity_list or []:
68
- return user
69
- except Exception as err:
70
- logger.debug(traceback.format_exc())
71
- logger.warning(
72
- f"Could not get user info from ES for user email {email} due to {err}"
73
- )
112
+ return None
113
+
114
+ @lru_cache(maxsize=None)
115
+ def get_reference_by_email(
116
+ self,
117
+ email: Optional[str],
118
+ from_count: int = 0,
119
+ size: int = 1,
120
+ fields: Optional[list] = None,
121
+ ) -> Optional[EntityReference]:
122
+ """
123
+ Get a User or Team Entity Reference by searching by its mail
124
+ """
125
+ maybe_user = self._search_by_email(
126
+ entity=User, email=email, from_count=from_count, size=size, fields=fields
127
+ )
128
+ if maybe_user:
129
+ return EntityReference(
130
+ id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
131
+ )
132
+
133
+ maybe_team = self._search_by_email(
134
+ entity=Team, email=email, from_count=from_count, size=size, fields=fields
135
+ )
136
+ if maybe_team:
137
+ return EntityReference(
138
+ id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
139
+ )
140
+
141
+ return None
142
+
143
+ @lru_cache(maxsize=None)
144
+ def get_reference_by_name(
145
+ self,
146
+ name: Optional[str],
147
+ from_count: int = 0,
148
+ size: int = 1,
149
+ fields: Optional[list] = None,
150
+ ) -> Optional[EntityReference]:
151
+ """
152
+ Get a User or Team Entity Reference by searching by its name
153
+ """
154
+ maybe_user = self._search_by_name(
155
+ entity=User, name=name, from_count=from_count, size=size, fields=fields
156
+ )
157
+ if maybe_user:
158
+ return EntityReference(
159
+ id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
160
+ )
161
+
162
+ maybe_team = self._search_by_name(
163
+ entity=Team, name=name, from_count=from_count, size=size, fields=fields
164
+ )
165
+ if maybe_team:
166
+ return EntityReference(
167
+ id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
168
+ )
74
169
 
75
170
  return None
@@ -31,7 +31,7 @@ from metadata.generated.schema.type.basic import FullyQualifiedEntityName
31
31
  from metadata.generated.schema.type.entityHistory import EntityVersionHistory
32
32
  from metadata.generated.schema.type.entityReference import EntityReference
33
33
  from metadata.ingestion.models.encoders import show_secrets_encoder
34
- from metadata.ingestion.ometa.auth_provider import AuthenticationProvider
34
+ from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
35
35
  from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
36
36
  from metadata.ingestion.ometa.mixins.custom_property_mixin import (
37
37
  OMetaCustomPropertyMixin,
@@ -39,7 +39,6 @@ from metadata.ingestion.ometa.mixins.custom_property_mixin import (
39
39
  from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
40
40
  from metadata.ingestion.ometa.mixins.data_insight_mixin import DataInsightMixin
41
41
  from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
42
- from metadata.ingestion.ometa.mixins.glossary_mixin import GlossaryMixin
43
42
  from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
44
43
  OMetaIngestionPipelineMixin,
45
44
  )
@@ -51,16 +50,13 @@ from metadata.ingestion.ometa.mixins.role_policy_mixin import OMetaRolePolicyMix
51
50
  from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin
52
51
  from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
53
52
  from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin
53
+ from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin
54
54
  from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin
55
55
  from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin
56
56
  from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin
57
57
  from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
58
58
  from metadata.ingestion.ometa.mixins.version_mixin import OMetaVersionMixin
59
59
  from metadata.ingestion.ometa.models import EntityList
60
- from metadata.ingestion.ometa.provider_registry import (
61
- InvalidAuthProviderException,
62
- auth_provider_registry,
63
- )
64
60
  from metadata.ingestion.ometa.routes import ROUTES
65
61
  from metadata.ingestion.ometa.utils import get_entity_type, model_str
66
62
  from metadata.utils.logger import ometa_logger
@@ -100,7 +96,6 @@ class OpenMetadata(
100
96
  OMetaTableMixin,
101
97
  OMetaTopicMixin,
102
98
  OMetaVersionMixin,
103
- GlossaryMixin,
104
99
  OMetaServiceMixin,
105
100
  ESMixin,
106
101
  OMetaServerMixin,
@@ -114,6 +109,7 @@ class OpenMetadata(
114
109
  OMetaRolePolicyMixin,
115
110
  OMetaSearchIndexMixin,
116
111
  OMetaCustomPropertyMixin,
112
+ OMetaSuggestionsMixin,
117
113
  Generic[T, C],
118
114
  ):
119
115
  """
@@ -125,7 +121,7 @@ class OpenMetadata(
125
121
  """
126
122
 
127
123
  client: REST
128
- _auth_provider: AuthenticationProvider
124
+ _auth_provider: OpenMetadataAuthenticationProvider
129
125
  config: OpenMetadataConnection
130
126
 
131
127
  class_root = ".".join(["metadata", "generated", "schema"])
@@ -146,16 +142,7 @@ class OpenMetadata(
146
142
  config.secretsManagerLoader,
147
143
  ).get_secrets_manager()
148
144
 
149
- # Load the auth provider init from the registry
150
- auth_provider_fn = auth_provider_registry.registry.get(
151
- self.config.authProvider.value
152
- )
153
- if not auth_provider_fn:
154
- raise InvalidAuthProviderException(
155
- f"Cannot find {self.config.authProvider.value} in {auth_provider_registry.registry}"
156
- )
157
-
158
- self._auth_provider = auth_provider_fn(self.config)
145
+ self._auth_provider = OpenMetadataAuthenticationProvider.create(self.config)
159
146
 
160
147
  get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
161
148
 
@@ -261,11 +248,9 @@ class OpenMetadata(
261
248
  )
262
249
  return entity_class
263
250
 
264
- def create_or_update(self, data: C) -> T:
251
+ def _create(self, data: C, method: str) -> T:
265
252
  """
266
- We allow CreateEntity for PUT, so we expect a type C.
267
-
268
- We PUT to the endpoint and return the Entity generated result
253
+ Internal logic to run POST vs. PUT
269
254
  """
270
255
  entity = data.__class__
271
256
  is_create = "create" in data.__class__.__name__.lower()
@@ -277,15 +262,23 @@ class OpenMetadata(
277
262
  raise InvalidEntityException(
278
263
  f"PUT operations need a CreateEntity, not {entity}"
279
264
  )
280
- resp = self.client.put(
281
- self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder)
282
- )
265
+
266
+ fn = getattr(self.client, method)
267
+ resp = fn(self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder))
283
268
  if not resp:
284
269
  raise EmptyPayloadException(
285
270
  f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.json()}"
286
271
  )
287
272
  return entity_class(**resp)
288
273
 
274
+ def create_or_update(self, data: C) -> T:
275
+ """Run a PUT requesting via create request C"""
276
+ return self._create(data=data, method="put")
277
+
278
+ def create(self, data: C) -> T:
279
+ """Run a POST requesting via create request C"""
280
+ return self._create(data=data, method="post")
281
+
289
282
  def get_by_name(
290
283
  self,
291
284
  entity: Type[T],
@@ -50,6 +50,7 @@ from metadata.generated.schema.api.domains.createDataProduct import (
50
50
  CreateDataProductRequest,
51
51
  )
52
52
  from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
53
+ from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest
53
54
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
54
55
  from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest
55
56
  from metadata.generated.schema.api.services.createDashboardService import (
@@ -90,6 +91,7 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
90
91
  from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
91
92
  from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
92
93
  from metadata.generated.schema.entity.automations.workflow import Workflow
94
+ from metadata.generated.schema.entity.bot import Bot
93
95
  from metadata.generated.schema.entity.classification.classification import (
94
96
  Classification,
95
97
  )
@@ -113,6 +115,7 @@ from metadata.generated.schema.entity.data.table import Table
113
115
  from metadata.generated.schema.entity.data.topic import Topic
114
116
  from metadata.generated.schema.entity.domains.dataProduct import DataProduct
115
117
  from metadata.generated.schema.entity.domains.domain import Domain
118
+ from metadata.generated.schema.entity.feed.suggestion import Suggestion
116
119
  from metadata.generated.schema.entity.policies.policy import Policy
117
120
  from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
118
121
  TestConnectionDefinition,
@@ -181,6 +184,7 @@ ROUTES = {
181
184
  User.__name__: "/users",
182
185
  CreateUserRequest.__name__: "/users",
183
186
  AuthenticationMechanism.__name__: "/users/auth-mechanism",
187
+ Bot.__name__: "/bots", # We won't allow bot creation from the client
184
188
  # Roles
185
189
  Role.__name__: "/roles",
186
190
  CreateRoleRequest.__name__: "/roles",
@@ -225,4 +229,7 @@ ROUTES = {
225
229
  CreateDomainRequest.__name__: "/domains",
226
230
  DataProduct.__name__: "/dataProducts",
227
231
  CreateDataProductRequest.__name__: "/dataProducts",
232
+ # Suggestions
233
+ Suggestion.__name__: "/suggestions",
234
+ CreateSuggestionRequest.__name__: "/suggestions",
228
235
  }
@@ -17,10 +17,13 @@ import traceback
17
17
  from typing import Optional
18
18
 
19
19
  from metadata.config.common import ConfigModel
20
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
21
+ StackTraceError,
22
+ )
20
23
  from metadata.generated.schema.type.basic import DateTime
21
24
  from metadata.generated.schema.type.queryParserData import ParsedData, QueryParserData
22
25
  from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
23
- from metadata.ingestion.api.models import Either, StackTraceError
26
+ from metadata.ingestion.api.models import Either
24
27
  from metadata.ingestion.api.steps import Processor
25
28
  from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
26
29
  from metadata.ingestion.lineage.parser import LineageParser
@@ -84,6 +87,10 @@ class QueryParserProcessor(Processor):
84
87
  self.metadata = metadata
85
88
  self.connection_type = connection_type
86
89
 
90
+ @property
91
+ def name(self) -> str:
92
+ return "Query Parser"
93
+
87
94
  @classmethod
88
95
  def create(cls, config_dict: dict, metadata: OpenMetadata, **kwargs):
89
96
  config = ConfigModel.parse_obj(config_dict)
@@ -106,7 +113,7 @@ class QueryParserProcessor(Processor):
106
113
  left=StackTraceError(
107
114
  name="Query",
108
115
  error=f"Error processing query [{table_query.query}]: {exc}",
109
- stack_trace=traceback.format_exc(),
116
+ stackTrace=traceback.format_exc(),
110
117
  )
111
118
  )
112
119
 
@@ -47,6 +47,9 @@ from metadata.generated.schema.entity.teams.team import Team
47
47
  from metadata.generated.schema.entity.teams.user import User
48
48
  from metadata.generated.schema.tests.basic import TestCaseResult
49
49
  from metadata.generated.schema.tests.testCase import TestCase
50
+ from metadata.generated.schema.tests.testCaseResolutionStatus import (
51
+ TestCaseResolutionStatus,
52
+ )
50
53
  from metadata.generated.schema.tests.testSuite import TestSuite
51
54
  from metadata.generated.schema.type.schema import Topic
52
55
  from metadata.ingestion.api.models import Either, Entity, StackTraceError
@@ -60,6 +63,7 @@ from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData
60
63
  from metadata.ingestion.models.patch_request import (
61
64
  ALLOWED_COMMON_PATCH_FIELDS,
62
65
  RESTRICT_UPDATE_LIST,
66
+ PatchedEntity,
63
67
  PatchRequest,
64
68
  )
65
69
  from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
@@ -67,6 +71,7 @@ from metadata.ingestion.models.profile_data import OMetaTableProfileSampleData
67
71
  from metadata.ingestion.models.search_index_data import OMetaIndexSampleData
68
72
  from metadata.ingestion.models.tests_data import (
69
73
  OMetaLogicalTestSuiteSample,
74
+ OMetaTestCaseResolutionStatus,
70
75
  OMetaTestCaseResultsSample,
71
76
  OMetaTestCaseSample,
72
77
  OMetaTestSuiteSample,
@@ -115,6 +120,10 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
115
120
  config = MetadataRestSinkConfig.parse_obj(config_dict)
116
121
  return cls(config, metadata)
117
122
 
123
+ @property
124
+ def name(self) -> str:
125
+ return "OpenMetadata"
126
+
118
127
  @singledispatchmethod
119
128
  def _run_dispatch(self, record: Entity) -> Either[Any]:
120
129
  logger.debug(f"Processing Create request {type(record)}")
@@ -132,14 +141,14 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
132
141
  error = f"Failed to ingest {log} due to api request failure: {err}"
133
142
  return Either(
134
143
  left=StackTraceError(
135
- name=log, error=error, stack_trace=traceback.format_exc()
144
+ name=log, error=error, stackTrace=traceback.format_exc()
136
145
  )
137
146
  )
138
147
  except Exception as exc:
139
148
  error = f"Failed to ingest {log}: {exc}"
140
149
  return Either(
141
150
  left=StackTraceError(
142
- name=log, error=error, stack_trace=traceback.format_exc()
151
+ name=log, error=error, stackTrace=traceback.format_exc()
143
152
  )
144
153
  )
145
154
 
@@ -155,7 +164,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
155
164
  error = f"Failed to ingest {type(entity_request).__name__}"
156
165
  return Either(
157
166
  left=StackTraceError(
158
- name=type(entity_request).__name__, error=error, stack_trace=None
167
+ name=type(entity_request).__name__, error=error, stackTrace=None
159
168
  )
160
169
  )
161
170
 
@@ -171,7 +180,8 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
171
180
  allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
172
181
  restrict_update_fields=RESTRICT_UPDATE_LIST,
173
182
  )
174
- return Either(right=entity)
183
+ patched_entity = PatchedEntity(new_entity=entity) if entity else None
184
+ return Either(right=patched_entity)
175
185
 
176
186
  @_run_dispatch.register
177
187
  def write_custom_properties(self, record: OMetaCustomProperties) -> Either[Dict]:
@@ -200,7 +210,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
200
210
  left=StackTraceError(
201
211
  name="Data Model",
202
212
  error="Sink did not receive a table. We cannot ingest the data model.",
203
- stack_trace=None,
213
+ stackTrace=None,
204
214
  )
205
215
  )
206
216
 
@@ -275,7 +285,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
275
285
  for role in record.roles:
276
286
  try:
277
287
  role_entity = self.metadata.get_by_name(
278
- entity=Role, fqn=str(role.name.__root__.__root__)
288
+ entity=Role, fqn=str(role.name.__root__)
279
289
  )
280
290
  except APIError:
281
291
  role_entity = self._create_role(role)
@@ -411,6 +421,15 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
411
421
  )
412
422
  return Either(right=res)
413
423
 
424
+ @_run_dispatch.register
425
+ def write_test_case_resolution_status(
426
+ self, record: OMetaTestCaseResolutionStatus
427
+ ) -> TestCaseResolutionStatus:
428
+ """For sample data"""
429
+ res = self.metadata.create_test_case_resolution(record.test_case_resolution)
430
+
431
+ return Either(right=res)
432
+
414
433
  @_run_dispatch.register
415
434
  def write_data_insight_sample(
416
435
  self, record: OMetaDataInsightSample
@@ -34,7 +34,6 @@ from metadata.generated.schema.entity.services.dashboardService import (
34
34
  DashboardConnection,
35
35
  DashboardService,
36
36
  )
37
- from metadata.generated.schema.entity.teams.user import User
38
37
  from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import (
39
38
  DashboardServiceMetadataPipeline,
40
39
  )
@@ -55,8 +54,8 @@ from metadata.ingestion.models.patch_request import PatchRequest
55
54
  from metadata.ingestion.models.topology import (
56
55
  NodeStage,
57
56
  ServiceTopology,
57
+ TopologyContext,
58
58
  TopologyNode,
59
- create_source_context,
60
59
  )
61
60
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
62
61
  from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -105,7 +104,7 @@ class DashboardServiceTopology(ServiceTopology):
105
104
  ),
106
105
  NodeStage(
107
106
  type_=OMetaTagAndClassification,
108
- processor="yield_tag",
107
+ processor="yield_bulk_tags",
109
108
  nullable=True,
110
109
  ),
111
110
  ],
@@ -133,14 +132,19 @@ class DashboardServiceTopology(ServiceTopology):
133
132
  dashboard = TopologyNode(
134
133
  producer="get_dashboard",
135
134
  stages=[
135
+ NodeStage(
136
+ type_=OMetaTagAndClassification,
137
+ processor="yield_tags",
138
+ nullable=True,
139
+ ),
136
140
  NodeStage(
137
141
  type_=Chart,
138
142
  context="charts",
139
143
  processor="yield_dashboard_chart",
140
144
  consumer=["dashboard_service"],
141
145
  nullable=True,
142
- cache_all=True,
143
- clear_cache=True,
146
+ store_all_in_context=True,
147
+ clear_context=True,
144
148
  use_cache=True,
145
149
  ),
146
150
  NodeStage(
@@ -149,8 +153,8 @@ class DashboardServiceTopology(ServiceTopology):
149
153
  processor="yield_datamodel",
150
154
  consumer=["dashboard_service"],
151
155
  nullable=True,
152
- cache_all=True,
153
- clear_cache=True,
156
+ store_all_in_context=True,
157
+ clear_context=True,
154
158
  use_cache=True,
155
159
  ),
156
160
  NodeStage(
@@ -160,12 +164,6 @@ class DashboardServiceTopology(ServiceTopology):
160
164
  consumer=["dashboard_service"],
161
165
  use_cache=True,
162
166
  ),
163
- NodeStage(
164
- type_=User,
165
- context="owner",
166
- processor="process_owner",
167
- consumer=["dashboard_service"],
168
- ),
169
167
  NodeStage(
170
168
  type_=AddLineageRequest,
171
169
  processor="yield_dashboard_lineage",
@@ -196,7 +194,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
196
194
  service_connection: DashboardConnection.__fields__["config"].type_
197
195
 
198
196
  topology = DashboardServiceTopology()
199
- context = create_source_context(topology)
197
+ context = TopologyContext.create(topology)
200
198
  dashboard_source_state: Set = set()
201
199
  datamodel_source_state: Set = set()
202
200
 
@@ -218,6 +216,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
218
216
  self.connection_obj = self.client
219
217
  self.test_connection()
220
218
 
219
+ @property
220
+ def name(self) -> str:
221
+ return self.service_connection.type.name
222
+
221
223
  @abstractmethod
222
224
  def yield_dashboard(
223
225
  self, dashboard_details: Any
@@ -333,7 +335,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
333
335
  dashboard_details, db_service_name
334
336
  ) or []
335
337
 
336
- def yield_tag(self, *args, **kwargs) -> Iterable[Either[OMetaTagAndClassification]]:
338
+ def yield_bulk_tags(
339
+ self, *args, **kwargs
340
+ ) -> Iterable[Either[OMetaTagAndClassification]]:
341
+ """
342
+ Method to bulk fetch dashboard tags
343
+ """
344
+
345
+ def yield_tags(
346
+ self, dashboard_details
347
+ ) -> Iterable[Either[OMetaTagAndClassification]]:
337
348
  """
338
349
  Method to fetch dashboard tags
339
350
  """
@@ -386,34 +397,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
386
397
  params={"service": self.context.dashboard_service},
387
398
  )
388
399
 
389
- def process_owner(self, dashboard_details):
400
+ def get_owner_ref( # pylint: disable=unused-argument, useless-return
401
+ self, dashboard_details
402
+ ) -> Optional[EntityReference]:
390
403
  """
391
- Method to process the dashboard onwers
404
+ Method to process the dashboard owners
392
405
  """
393
- try:
394
- if self.source_config.includeOwners:
395
- owner = self.get_owner_details( # pylint: disable=assignment-from-none
396
- dashboard_details=dashboard_details
397
- )
398
- if owner:
399
- dashboard_fqn = fqn.build(
400
- self.metadata,
401
- entity_type=Dashboard,
402
- service_name=self.context.dashboard_service,
403
- dashboard_name=self.context.dashboard,
404
- )
405
- dashboard_entity = self.metadata.get_by_name(
406
- entity=Dashboard, fqn=dashboard_fqn
407
- )
408
- self.metadata.patch_owner(
409
- entity=Dashboard,
410
- source=dashboard_entity,
411
- owner=owner,
412
- force=False,
413
- )
414
- except Exception as exc:
415
- logger.debug(traceback.format_exc())
416
- logger.warning(f"Error processing owner for {dashboard_details}: {exc}")
406
+ logger.debug(
407
+ f"Processing ownership is not supported for {self.service_connection.type.name}"
408
+ )
409
+ return None
417
410
 
418
411
  def register_record(self, dashboard_request: CreateDashboardRequest) -> None:
419
412
  """
@@ -429,7 +422,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
429
422
  self.dashboard_source_state.add(dashboard_fqn)
430
423
 
431
424
  def register_record_datamodel(
432
- self, datamodel_requst: CreateDashboardDataModelRequest
425
+ self, datamodel_request: CreateDashboardDataModelRequest
433
426
  ) -> None:
434
427
  """
435
428
  Mark the datamodel record as scanned and update the datamodel_source_state
@@ -437,27 +430,12 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
437
430
  datamodel_fqn = fqn.build(
438
431
  self.metadata,
439
432
  entity_type=DashboardDataModel,
440
- service_name=datamodel_requst.service.__root__,
441
- data_model_name=datamodel_requst.name.__root__,
433
+ service_name=datamodel_request.service.__root__,
434
+ data_model_name=datamodel_request.name.__root__,
442
435
  )
443
436
 
444
437
  self.datamodel_source_state.add(datamodel_fqn)
445
438
 
446
- def get_owner_details( # pylint: disable=useless-return
447
- self, dashboard_details # pylint: disable=unused-argument
448
- ) -> Optional[EntityReference]:
449
- """Get dashboard owner
450
-
451
- Args:
452
- dashboard_details:
453
- Returns:
454
- Optional[EntityReference]
455
- """
456
- logger.debug(
457
- f"Processing ownership is not supported for {self.service_connection.type.name}"
458
- )
459
- return None
460
-
461
439
  @staticmethod
462
440
  def _get_add_lineage_request(
463
441
  to_entity: Union[Dashboard, DashboardDataModel],
@@ -505,7 +483,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
505
483
  self.context.project_name = ( # pylint: disable=assignment-from-none
506
484
  self.get_project_name(dashboard_details=dashboard_details)
507
485
  )
508
- if self.context.project_name and filter_by_project(
486
+ if filter_by_project(
509
487
  self.source_config.projectFilterPattern,
510
488
  self.context.project_name,
511
489
  ):
@@ -553,7 +531,6 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
553
531
  )
554
532
 
555
533
  def check_database_schema_name(self, database_schema_name: str):
556
-
557
534
  """
558
535
  Check if the input database schema name is equal to "<default>" and return the input name if it is not.
559
536