openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (885) hide show
  1. airflow_provider_openmetadata/lineage/runner.py +44 -5
  2. airflow_provider_openmetadata/lineage/status.py +1 -1
  3. metadata/antlr/split_listener.py +1 -4
  4. metadata/applications/auto_tagger.py +212 -0
  5. metadata/cli/app.py +47 -0
  6. metadata/cli/db_dump.py +7 -3
  7. metadata/clients/aws_client.py +4 -0
  8. metadata/clients/azure_client.py +85 -0
  9. metadata/clients/domo_client.py +0 -2
  10. metadata/cmd.py +24 -13
  11. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +40 -48
  12. metadata/data_insight/processor/reports/data_processor.py +1 -0
  13. metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
  14. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
  15. metadata/data_insight/producer/cost_analysis_producer.py +78 -14
  16. metadata/data_insight/producer/entity_producer.py +1 -1
  17. metadata/data_insight/producer/producer_interface.py +1 -1
  18. metadata/data_insight/producer/web_analytics_producer.py +1 -1
  19. metadata/data_insight/source/metadata.py +21 -5
  20. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
  21. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
  22. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
  23. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
  24. metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
  25. metadata/data_quality/processor/test_case_runner.py +17 -8
  26. metadata/data_quality/source/test_suite.py +10 -4
  27. metadata/data_quality/validations/base_test_handler.py +21 -17
  28. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
  29. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
  30. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
  31. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
  32. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
  33. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
  34. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
  35. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
  36. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
  37. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
  38. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
  39. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
  40. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
  41. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
  42. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
  43. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
  44. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
  45. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
  46. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
  47. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
  48. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
  49. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
  50. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
  51. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
  52. metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +2 -1
  53. metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +2 -1
  54. metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +7 -2
  55. metadata/examples/workflows/bigtable.yaml +32 -0
  56. metadata/examples/workflows/databricks.yaml +0 -1
  57. metadata/examples/workflows/datalake_azure_default.yaml +29 -0
  58. metadata/examples/workflows/mongodb.yaml +4 -2
  59. metadata/examples/workflows/mstr.yaml +24 -0
  60. metadata/examples/workflows/sas.yaml +28 -0
  61. metadata/examples/workflows/unity_catalog.yaml +27 -0
  62. metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
  63. metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
  64. metadata/generated/antlr/EntityLinkLexer.py +425 -292
  65. metadata/generated/antlr/EntityLinkListener.py +12 -12
  66. metadata/generated/antlr/EntityLinkParser.py +228 -103
  67. metadata/generated/schema/analytics/__init__.py +1 -1
  68. metadata/generated/schema/analytics/basic.py +1 -1
  69. metadata/generated/schema/analytics/reportData.py +1 -1
  70. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  71. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  72. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  73. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  74. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  75. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  76. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  77. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  78. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  79. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  80. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  81. metadata/generated/schema/api/__init__.py +1 -1
  82. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  83. metadata/generated/schema/api/analytics/__init__.py +1 -1
  84. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  85. metadata/generated/schema/api/automations/__init__.py +1 -1
  86. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  87. metadata/generated/schema/api/bulkAssets.py +1 -1
  88. metadata/generated/schema/api/classification/__init__.py +1 -1
  89. metadata/generated/schema/api/classification/createClassification.py +1 -1
  90. metadata/generated/schema/api/classification/createTag.py +1 -1
  91. metadata/generated/schema/api/classification/loadTags.py +1 -1
  92. metadata/generated/schema/api/createBot.py +2 -3
  93. metadata/generated/schema/api/createEventPublisherJob.py +2 -2
  94. metadata/generated/schema/api/createType.py +1 -1
  95. metadata/generated/schema/api/data/__init__.py +1 -1
  96. metadata/generated/schema/api/data/createChart.py +1 -1
  97. metadata/generated/schema/api/data/createContainer.py +2 -2
  98. metadata/generated/schema/api/data/createCustomProperty.py +7 -12
  99. metadata/generated/schema/api/data/createDashboard.py +1 -1
  100. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  101. metadata/generated/schema/api/data/createDatabase.py +2 -3
  102. metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
  103. metadata/generated/schema/api/data/createGlossary.py +2 -3
  104. metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
  105. metadata/generated/schema/api/data/createMlModel.py +1 -1
  106. metadata/generated/schema/api/data/createPipeline.py +2 -2
  107. metadata/generated/schema/api/data/createQuery.py +2 -2
  108. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  109. metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
  110. metadata/generated/schema/api/data/createTable.py +2 -2
  111. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  112. metadata/generated/schema/api/data/createTopic.py +1 -1
  113. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  114. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  115. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  116. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  117. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  118. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  119. metadata/generated/schema/api/docStore/__init__.py +1 -1
  120. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  121. metadata/generated/schema/api/domains/__init__.py +1 -1
  122. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  123. metadata/generated/schema/api/domains/createDomain.py +1 -1
  124. metadata/generated/schema/api/feed/__init__.py +1 -1
  125. metadata/generated/schema/api/feed/closeTask.py +8 -1
  126. metadata/generated/schema/api/feed/createPost.py +1 -1
  127. metadata/generated/schema/api/feed/createSuggestion.py +30 -0
  128. metadata/generated/schema/api/feed/createThread.py +5 -1
  129. metadata/generated/schema/api/feed/resolveTask.py +12 -1
  130. metadata/generated/schema/api/feed/threadCount.py +18 -14
  131. metadata/generated/schema/api/lineage/__init__.py +1 -1
  132. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  133. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  134. metadata/generated/schema/api/policies/__init__.py +1 -1
  135. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  136. metadata/generated/schema/api/services/__init__.py +1 -1
  137. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  138. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  139. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  140. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  141. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  142. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  143. metadata/generated/schema/api/services/createSearchService.py +1 -1
  144. metadata/generated/schema/api/services/createStorageService.py +1 -1
  145. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  146. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  147. metadata/generated/schema/api/setOwner.py +1 -1
  148. metadata/generated/schema/api/teams/__init__.py +1 -1
  149. metadata/generated/schema/api/teams/createPersona.py +1 -1
  150. metadata/generated/schema/api/teams/createRole.py +2 -3
  151. metadata/generated/schema/api/teams/createTeam.py +1 -1
  152. metadata/generated/schema/api/teams/createUser.py +2 -2
  153. metadata/generated/schema/api/tests/__init__.py +1 -1
  154. metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
  155. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  156. metadata/generated/schema/api/tests/createTestCase.py +4 -1
  157. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
  158. metadata/generated/schema/api/tests/createTestDefinition.py +2 -1
  159. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  160. metadata/generated/schema/api/voteRequest.py +1 -1
  161. metadata/generated/schema/auth/__init__.py +1 -1
  162. metadata/generated/schema/auth/basicAuth.py +1 -1
  163. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  164. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  165. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  166. metadata/generated/schema/auth/emailRequest.py +1 -1
  167. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  168. metadata/generated/schema/auth/generateToken.py +1 -1
  169. metadata/generated/schema/auth/jwtAuth.py +1 -1
  170. metadata/generated/schema/auth/loginRequest.py +1 -1
  171. metadata/generated/schema/auth/logoutRequest.py +1 -1
  172. metadata/generated/schema/auth/passwordResetRequest.py +3 -3
  173. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  174. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  175. metadata/generated/schema/auth/refreshToken.py +1 -1
  176. metadata/generated/schema/auth/registrationRequest.py +2 -2
  177. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  178. metadata/generated/schema/auth/revokeToken.py +1 -1
  179. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  180. metadata/generated/schema/auth/ssoAuth.py +1 -1
  181. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  182. metadata/generated/schema/configuration/__init__.py +1 -1
  183. metadata/generated/schema/configuration/appsPrivateConfiguration.py +32 -0
  184. metadata/generated/schema/configuration/authConfig.py +1 -1
  185. metadata/generated/schema/configuration/authenticationConfiguration.py +13 -2
  186. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  187. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  188. metadata/generated/schema/configuration/dataQualityConfiguration.py +16 -0
  189. metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
  190. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
  195. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  196. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  197. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  198. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  199. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  200. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  201. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  202. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  203. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  204. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  205. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  206. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  207. metadata/generated/schema/dataInsight/__init__.py +1 -1
  208. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  209. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  210. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  211. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  212. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  213. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  214. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  215. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  216. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  217. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  218. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  219. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  220. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  221. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  222. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
  223. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
  224. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
  225. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
  226. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
  227. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
  228. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  229. metadata/generated/schema/email/__init__.py +1 -1
  230. metadata/generated/schema/email/emailRequest.py +1 -1
  231. metadata/generated/schema/email/smtpSettings.py +1 -1
  232. metadata/generated/schema/entity/__init__.py +1 -1
  233. metadata/generated/schema/entity/applications/__init__.py +1 -1
  234. metadata/generated/schema/entity/applications/app.py +24 -5
  235. metadata/generated/schema/entity/applications/appRunRecord.py +2 -7
  236. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  237. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
  238. metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
  240. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
  241. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
  242. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
  243. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
  244. metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +14 -4
  245. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
  246. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
  247. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  248. metadata/generated/schema/entity/applications/jobStatus.py +3 -3
  249. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  250. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  251. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
  252. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
  253. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  254. metadata/generated/schema/entity/automations/__init__.py +1 -1
  255. metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
  256. metadata/generated/schema/entity/automations/workflow.py +1 -1
  257. metadata/generated/schema/entity/bot.py +2 -3
  258. metadata/generated/schema/entity/classification/__init__.py +1 -1
  259. metadata/generated/schema/entity/classification/classification.py +1 -1
  260. metadata/generated/schema/entity/classification/tag.py +2 -2
  261. metadata/generated/schema/entity/data/__init__.py +1 -1
  262. metadata/generated/schema/entity/data/chart.py +1 -1
  263. metadata/generated/schema/entity/data/container.py +4 -9
  264. metadata/generated/schema/entity/data/dashboard.py +1 -1
  265. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  266. metadata/generated/schema/entity/data/database.py +4 -8
  267. metadata/generated/schema/entity/data/databaseSchema.py +2 -8
  268. metadata/generated/schema/entity/data/glossary.py +1 -1
  269. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  270. metadata/generated/schema/entity/data/metrics.py +1 -1
  271. metadata/generated/schema/entity/data/mlmodel.py +2 -2
  272. metadata/generated/schema/entity/data/pipeline.py +6 -9
  273. metadata/generated/schema/entity/data/query.py +2 -2
  274. metadata/generated/schema/entity/data/report.py +1 -1
  275. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  276. metadata/generated/schema/entity/data/storedProcedure.py +3 -9
  277. metadata/generated/schema/entity/data/table.py +18 -12
  278. metadata/generated/schema/entity/data/topic.py +1 -1
  279. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  280. metadata/generated/schema/entity/docStore/document.py +1 -1
  281. metadata/generated/schema/entity/domains/__init__.py +1 -1
  282. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  283. metadata/generated/schema/entity/domains/domain.py +1 -1
  284. metadata/generated/schema/entity/events/__init__.py +1 -1
  285. metadata/generated/schema/entity/events/webhook.py +6 -3
  286. metadata/generated/schema/entity/feed/__init__.py +1 -1
  287. metadata/generated/schema/entity/feed/suggestion.py +61 -0
  288. metadata/generated/schema/entity/feed/thread.py +17 -1
  289. metadata/generated/schema/entity/policies/__init__.py +1 -1
  290. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  291. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  292. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  293. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  294. metadata/generated/schema/entity/policies/filters.py +2 -5
  295. metadata/generated/schema/entity/policies/policy.py +1 -1
  296. metadata/generated/schema/entity/services/__init__.py +1 -1
  297. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  298. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  299. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  300. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  301. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  302. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  304. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  305. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  306. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  307. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  308. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  309. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  310. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
  311. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  312. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
  313. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  314. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  315. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  316. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
  319. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +35 -2
  320. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
  321. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +41 -0
  322. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
  323. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +20 -0
  325. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  326. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
  329. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  330. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
  331. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  332. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  333. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  334. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  335. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
  336. metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
  337. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
  340. metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
  341. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  342. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  343. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
  344. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
  345. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
  346. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
  347. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
  348. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
  349. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
  350. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
  351. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
  352. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
  353. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
  354. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
  355. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
  356. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
  357. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +8 -4
  358. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
  359. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +9 -7
  360. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +8 -4
  361. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
  362. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
  363. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +3 -0
  365. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +20 -0
  366. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +36 -0
  367. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +7 -41
  368. metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
  369. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
  370. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
  371. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
  372. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
  373. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
  374. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
  375. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  376. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +14 -2
  378. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +14 -2
  381. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  382. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  383. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +2 -2
  387. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
  388. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  389. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  394. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
  396. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
  405. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  407. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  409. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  410. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  411. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  415. metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
  416. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  420. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  421. metadata/generated/schema/entity/services/dashboardService.py +4 -1
  422. metadata/generated/schema/entity/services/databaseService.py +16 -1
  423. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
  425. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  426. metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
  427. metadata/generated/schema/entity/services/messagingService.py +1 -1
  428. metadata/generated/schema/entity/services/metadataService.py +1 -1
  429. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  430. metadata/generated/schema/entity/services/pipelineService.py +4 -1
  431. metadata/generated/schema/entity/services/searchService.py +1 -1
  432. metadata/generated/schema/entity/services/serviceType.py +1 -1
  433. metadata/generated/schema/entity/services/storageService.py +10 -3
  434. metadata/generated/schema/entity/teams/__init__.py +1 -1
  435. metadata/generated/schema/entity/teams/persona.py +1 -1
  436. metadata/generated/schema/entity/teams/role.py +2 -6
  437. metadata/generated/schema/entity/teams/team.py +1 -1
  438. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  439. metadata/generated/schema/entity/teams/user.py +3 -10
  440. metadata/generated/schema/entity/type.py +4 -19
  441. metadata/generated/schema/entity/utils/__init__.py +1 -1
  442. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  443. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  444. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  445. metadata/generated/schema/events/__init__.py +1 -1
  446. metadata/generated/schema/events/alertMetrics.py +27 -0
  447. metadata/generated/schema/events/api/__init__.py +1 -1
  448. metadata/generated/schema/events/api/createEventSubscription.py +14 -12
  449. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  450. metadata/generated/schema/events/eventFilterRule.py +13 -2
  451. metadata/generated/schema/events/eventSubscription.py +91 -49
  452. metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
  453. metadata/generated/schema/events/failedEvent.py +26 -0
  454. metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
  455. metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
  456. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  457. metadata/generated/schema/metadataIngestion/application.py +40 -0
  458. metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
  459. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  460. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  461. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
  462. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +11 -1
  463. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  464. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  465. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  466. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  467. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
  468. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  469. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  470. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
  471. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  472. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  473. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
  474. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  475. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  476. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  477. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  478. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  479. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  480. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  481. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  482. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  483. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
  484. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
  485. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  486. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  487. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  488. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
  489. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  490. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  491. metadata/generated/schema/monitoring/__init__.py +1 -1
  492. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  493. metadata/generated/schema/security/__init__.py +1 -1
  494. metadata/generated/schema/security/client/__init__.py +1 -1
  495. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  496. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  497. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  498. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  499. metadata/generated/schema/security/client/oidcClientConfig.py +46 -0
  500. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  501. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  502. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  503. metadata/generated/schema/security/credentials/__init__.py +1 -1
  504. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  505. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  506. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  507. metadata/generated/schema/security/credentials/azureCredentials.py +21 -9
  508. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  509. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  510. metadata/generated/schema/security/credentials/gcpCredentials.py +7 -3
  511. metadata/generated/schema/security/credentials/gcpExternalAccount.py +37 -0
  512. metadata/generated/schema/security/credentials/gcpValues.py +2 -2
  513. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  514. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  515. metadata/generated/schema/security/secrets/__init__.py +1 -1
  516. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  517. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
  518. metadata/generated/schema/security/secrets/secretsManagerProvider.py +4 -2
  519. metadata/generated/schema/security/securityConfiguration.py +1 -1
  520. metadata/generated/schema/security/ssl/__init__.py +1 -1
  521. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  522. metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
  523. metadata/generated/schema/settings/__init__.py +1 -1
  524. metadata/generated/schema/settings/settings.py +1 -1
  525. metadata/generated/schema/system/__init__.py +1 -1
  526. metadata/generated/schema/system/entityError.py +17 -0
  527. metadata/generated/schema/system/eventPublisherJob.py +5 -19
  528. metadata/generated/schema/system/indexingError.py +34 -0
  529. metadata/generated/schema/system/ui/__init__.py +1 -1
  530. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  531. metadata/generated/schema/system/ui/page.py +1 -1
  532. metadata/generated/schema/system/validationResponse.py +43 -0
  533. metadata/generated/schema/tests/__init__.py +1 -1
  534. metadata/generated/schema/tests/assigned.py +18 -0
  535. metadata/generated/schema/tests/basic.py +17 -39
  536. metadata/generated/schema/tests/customMetric.py +4 -2
  537. metadata/generated/schema/tests/resolved.py +35 -0
  538. metadata/generated/schema/tests/testCase.py +11 -2
  539. metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
  540. metadata/generated/schema/tests/testDefinition.py +6 -1
  541. metadata/generated/schema/tests/testSuite.py +1 -1
  542. metadata/generated/schema/type/__init__.py +1 -1
  543. metadata/generated/schema/type/auditLog.py +1 -1
  544. metadata/generated/schema/type/basic.py +10 -6
  545. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  546. metadata/generated/schema/type/changeEvent.py +5 -75
  547. metadata/generated/schema/type/changeEventType.py +29 -0
  548. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  549. metadata/generated/schema/type/csvDocumentation.py +1 -1
  550. metadata/generated/schema/type/csvErrorType.py +1 -1
  551. metadata/generated/schema/type/csvFile.py +1 -1
  552. metadata/generated/schema/type/csvImportResult.py +1 -1
  553. metadata/generated/schema/type/customProperties/__init__.py +3 -0
  554. metadata/generated/schema/type/customProperties/enumConfig.py +17 -0
  555. metadata/generated/schema/type/customProperty.py +52 -0
  556. metadata/generated/schema/type/dailyCount.py +1 -1
  557. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  558. metadata/generated/schema/type/entityHistory.py +1 -1
  559. metadata/generated/schema/type/entityLineage.py +2 -1
  560. metadata/generated/schema/type/entityReference.py +5 -1
  561. metadata/generated/schema/type/entityReferenceList.py +1 -1
  562. metadata/generated/schema/type/entityRelationship.py +1 -1
  563. metadata/generated/schema/type/entityUsage.py +1 -1
  564. metadata/generated/schema/type/filterPattern.py +1 -1
  565. metadata/generated/schema/type/function.py +7 -6
  566. metadata/generated/schema/type/include.py +1 -1
  567. metadata/generated/schema/type/jdbcConnection.py +1 -1
  568. metadata/generated/schema/type/lifeCycle.py +1 -1
  569. metadata/generated/schema/type/paging.py +1 -1
  570. metadata/generated/schema/type/profile.py +1 -1
  571. metadata/generated/schema/type/queryParserData.py +2 -2
  572. metadata/generated/schema/type/reaction.py +1 -1
  573. metadata/generated/schema/type/schedule.py +1 -1
  574. metadata/generated/schema/type/schema.py +1 -1
  575. metadata/generated/schema/type/tableQuery.py +2 -2
  576. metadata/generated/schema/type/tableUsageCount.py +1 -1
  577. metadata/generated/schema/type/tagLabel.py +1 -1
  578. metadata/generated/schema/type/usageDetails.py +1 -1
  579. metadata/generated/schema/type/usageRequest.py +1 -1
  580. metadata/generated/schema/type/votes.py +1 -1
  581. metadata/great_expectations/action.py +12 -20
  582. metadata/ingestion/api/delete.py +5 -2
  583. metadata/ingestion/api/models.py +4 -10
  584. metadata/ingestion/api/parser.py +3 -37
  585. metadata/ingestion/api/status.py +15 -4
  586. metadata/ingestion/api/step.py +39 -6
  587. metadata/ingestion/api/steps.py +34 -1
  588. metadata/ingestion/api/topology_runner.py +117 -168
  589. metadata/ingestion/bulksink/metadata_usage.py +11 -5
  590. metadata/ingestion/lineage/models.py +4 -0
  591. metadata/ingestion/lineage/parser.py +28 -12
  592. metadata/ingestion/lineage/sql_lineage.py +9 -17
  593. metadata/ingestion/models/custom_properties.py +0 -1
  594. metadata/ingestion/models/custom_pydantic.py +4 -2
  595. metadata/ingestion/models/lf_tags_model.py +33 -0
  596. metadata/ingestion/models/patch_request.py +160 -2
  597. metadata/ingestion/models/tests_data.py +9 -0
  598. metadata/ingestion/models/topology.py +177 -60
  599. metadata/ingestion/ometa/auth_provider.py +0 -349
  600. metadata/ingestion/ometa/client.py +12 -3
  601. metadata/ingestion/ometa/mixins/custom_property_mixin.py +11 -11
  602. metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
  603. metadata/ingestion/ometa/mixins/patch_mixin.py +21 -72
  604. metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
  605. metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
  606. metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
  607. metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
  608. metadata/ingestion/ometa/ometa_api.py +19 -26
  609. metadata/ingestion/ometa/routes.py +7 -0
  610. metadata/ingestion/processor/query_parser.py +9 -2
  611. metadata/ingestion/sink/metadata_rest.py +29 -8
  612. metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
  613. metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
  614. metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
  615. metadata/ingestion/source/dashboard/looker/metadata.py +41 -29
  616. metadata/ingestion/source/dashboard/looker/utils.py +4 -2
  617. metadata/ingestion/source/dashboard/metabase/client.py +4 -0
  618. metadata/ingestion/source/dashboard/metabase/metadata.py +28 -10
  619. metadata/ingestion/source/dashboard/metabase/models.py +2 -2
  620. metadata/ingestion/source/dashboard/mode/client.py +10 -23
  621. metadata/ingestion/source/dashboard/mode/connection.py +6 -1
  622. metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
  623. metadata/ingestion/source/dashboard/mstr/client.py +208 -0
  624. metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
  625. metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
  626. metadata/ingestion/source/dashboard/mstr/models.py +144 -0
  627. metadata/ingestion/source/dashboard/powerbi/client.py +4 -1
  628. metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
  629. metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
  630. metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
  631. metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
  632. metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
  633. metadata/ingestion/source/dashboard/superset/client.py +16 -9
  634. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  635. metadata/ingestion/source/dashboard/superset/db_source.py +14 -11
  636. metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
  637. metadata/ingestion/source/dashboard/superset/queries.py +1 -1
  638. metadata/ingestion/source/dashboard/tableau/client.py +91 -11
  639. metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
  640. metadata/ingestion/source/dashboard/tableau/metadata.py +76 -70
  641. metadata/ingestion/source/dashboard/tableau/models.py +0 -8
  642. metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
  643. metadata/ingestion/source/database/athena/client.py +80 -0
  644. metadata/ingestion/source/database/athena/connection.py +7 -0
  645. metadata/ingestion/source/database/athena/metadata.py +161 -19
  646. metadata/ingestion/source/database/azuresql/connection.py +21 -3
  647. metadata/ingestion/source/database/azuresql/metadata.py +0 -1
  648. metadata/ingestion/source/database/bigquery/connection.py +24 -3
  649. metadata/ingestion/source/database/bigquery/helper.py +68 -1
  650. metadata/ingestion/source/database/bigquery/metadata.py +49 -28
  651. metadata/ingestion/source/database/bigquery/queries.py +33 -4
  652. metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
  653. metadata/ingestion/source/database/bigquery/usage.py +1 -3
  654. metadata/ingestion/source/database/bigtable/client.py +62 -0
  655. metadata/ingestion/source/database/bigtable/connection.py +116 -0
  656. metadata/ingestion/source/database/bigtable/metadata.py +224 -0
  657. metadata/ingestion/source/database/bigtable/models.py +60 -0
  658. metadata/ingestion/source/database/column_helpers.py +0 -10
  659. metadata/ingestion/source/database/column_type_parser.py +11 -5
  660. metadata/ingestion/source/database/common_db_source.py +33 -8
  661. metadata/ingestion/source/database/common_nosql_source.py +27 -6
  662. metadata/ingestion/source/database/database_service.py +89 -7
  663. metadata/ingestion/source/database/databricks/client.py +55 -103
  664. metadata/ingestion/source/database/databricks/connection.py +16 -55
  665. metadata/ingestion/source/database/databricks/lineage.py +29 -26
  666. metadata/ingestion/source/database/databricks/metadata.py +534 -11
  667. metadata/ingestion/source/database/databricks/queries.py +27 -0
  668. metadata/ingestion/source/database/databricks/query_parser.py +5 -1
  669. metadata/ingestion/source/database/databricks/usage.py +3 -3
  670. metadata/ingestion/source/database/datalake/connection.py +33 -18
  671. metadata/ingestion/source/database/datalake/metadata.py +113 -27
  672. metadata/ingestion/source/database/dbt/dbt_config.py +7 -14
  673. metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
  674. metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
  675. metadata/ingestion/source/database/dbt/metadata.py +27 -53
  676. metadata/ingestion/source/database/deltalake/metadata.py +6 -3
  677. metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
  678. metadata/ingestion/source/database/doris/connection.py +72 -0
  679. metadata/ingestion/source/database/doris/metadata.py +315 -0
  680. metadata/ingestion/source/database/doris/queries.py +54 -0
  681. metadata/ingestion/source/database/doris/utils.py +64 -0
  682. metadata/ingestion/source/database/extended_sample_data.py +532 -0
  683. metadata/ingestion/source/database/glue/metadata.py +8 -5
  684. metadata/ingestion/source/database/hive/connection.py +0 -2
  685. metadata/ingestion/source/database/hive/utils.py +3 -0
  686. metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
  687. metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
  688. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
  689. metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
  690. metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
  691. metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
  692. metadata/ingestion/source/database/iceberg/connection.py +68 -0
  693. metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
  694. metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
  695. metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
  696. metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
  697. metadata/ingestion/source/database/iceberg/helper.py +124 -0
  698. metadata/ingestion/source/database/iceberg/metadata.py +311 -0
  699. metadata/ingestion/source/database/iceberg/models.py +66 -0
  700. metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
  701. metadata/ingestion/source/database/mongodb/connection.py +1 -5
  702. metadata/ingestion/source/database/mssql/lineage.py +3 -0
  703. metadata/ingestion/source/database/mssql/metadata.py +108 -4
  704. metadata/ingestion/source/database/mssql/models.py +30 -0
  705. metadata/ingestion/source/database/mssql/queries.py +178 -1
  706. metadata/ingestion/source/database/mssql/usage.py +5 -1
  707. metadata/ingestion/source/database/mssql/utils.py +207 -4
  708. metadata/ingestion/source/database/mysql/connection.py +14 -0
  709. metadata/ingestion/source/database/mysql/metadata.py +0 -2
  710. metadata/ingestion/source/database/oracle/metadata.py +108 -2
  711. metadata/ingestion/source/database/oracle/models.py +30 -0
  712. metadata/ingestion/source/database/oracle/queries.py +99 -18
  713. metadata/ingestion/source/database/oracle/utils.py +0 -1
  714. metadata/ingestion/source/database/postgres/connection.py +15 -0
  715. metadata/ingestion/source/database/postgres/lineage.py +32 -14
  716. metadata/ingestion/source/database/postgres/metadata.py +15 -7
  717. metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
  718. metadata/ingestion/source/database/postgres/queries.py +4 -2
  719. metadata/ingestion/source/database/postgres/query_parser.py +4 -72
  720. metadata/ingestion/source/database/postgres/usage.py +41 -0
  721. metadata/ingestion/source/database/postgres/utils.py +34 -0
  722. metadata/ingestion/source/database/query_parser_source.py +8 -2
  723. metadata/ingestion/source/database/redshift/metadata.py +14 -4
  724. metadata/ingestion/source/database/redshift/queries.py +10 -4
  725. metadata/ingestion/source/database/redshift/query_parser.py +16 -0
  726. metadata/ingestion/source/database/redshift/usage.py +0 -2
  727. metadata/ingestion/source/database/salesforce/metadata.py +32 -3
  728. metadata/ingestion/source/database/sample_data.py +120 -6
  729. metadata/ingestion/source/database/sas/client.py +184 -0
  730. metadata/ingestion/source/database/sas/connection.py +47 -0
  731. metadata/ingestion/source/database/sas/extension_attr.py +103 -0
  732. metadata/ingestion/source/database/sas/metadata.py +914 -0
  733. metadata/ingestion/source/database/snowflake/metadata.py +34 -48
  734. metadata/ingestion/source/database/snowflake/models.py +6 -1
  735. metadata/ingestion/source/database/snowflake/queries.py +6 -4
  736. metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
  737. metadata/ingestion/source/database/snowflake/utils.py +23 -8
  738. metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
  739. metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
  740. metadata/ingestion/source/database/unitycatalog/client.py +87 -0
  741. metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
  742. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
  743. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
  744. metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
  745. metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
  746. metadata/ingestion/source/database/usage_source.py +3 -2
  747. metadata/ingestion/source/messaging/common_broker_source.py +15 -11
  748. metadata/ingestion/source/messaging/kafka/connection.py +45 -4
  749. metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
  750. metadata/ingestion/source/messaging/messaging_service.py +6 -2
  751. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  752. metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
  753. metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
  754. metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
  755. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +20 -8
  756. metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
  757. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
  758. metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
  759. metadata/ingestion/source/pipeline/airflow/models.py +5 -4
  760. metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
  761. metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -5
  762. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +14 -11
  763. metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
  764. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
  765. metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
  766. metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
  767. metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
  768. metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
  769. metadata/ingestion/source/search/search_service.py +6 -2
  770. metadata/ingestion/source/storage/s3/metadata.py +22 -17
  771. metadata/ingestion/source/storage/storage_service.py +53 -11
  772. metadata/ingestion/stage/table_usage.py +9 -2
  773. metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
  774. metadata/parsers/protobuf_parser.py +29 -11
  775. metadata/pii/processor.py +9 -2
  776. metadata/pii/scanners/ner_scanner.py +5 -3
  777. metadata/profiler/api/models.py +19 -1
  778. metadata/profiler/interface/pandas/profiler_interface.py +59 -18
  779. metadata/profiler/interface/profiler_interface.py +14 -4
  780. metadata/profiler/interface/profiler_interface_factory.py +49 -14
  781. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
  782. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
  783. metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
  784. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
  785. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
  786. metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
  787. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  788. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
  789. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  790. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
  791. metadata/profiler/metrics/composed/null_ratio.py +1 -1
  792. metadata/profiler/metrics/hybrid/histogram.py +1 -0
  793. metadata/profiler/metrics/static/max.py +4 -1
  794. metadata/profiler/metrics/static/min.py +4 -1
  795. metadata/profiler/metrics/system/queries/snowflake.py +89 -17
  796. metadata/profiler/metrics/system/system.py +62 -20
  797. metadata/profiler/orm/functions/conn_test.py +1 -0
  798. metadata/profiler/orm/functions/length.py +1 -0
  799. metadata/profiler/orm/functions/median.py +9 -1
  800. metadata/profiler/orm/functions/sum.py +1 -0
  801. metadata/profiler/orm/functions/table_metric_computer.py +462 -0
  802. metadata/profiler/orm/registry.py +2 -0
  803. metadata/profiler/processor/core.py +129 -62
  804. metadata/profiler/processor/default.py +14 -3
  805. metadata/profiler/processor/handle_partition.py +2 -2
  806. metadata/profiler/processor/processor.py +10 -7
  807. metadata/profiler/processor/sample_data_handler.py +6 -2
  808. metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
  809. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
  810. metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
  811. metadata/profiler/source/base/profiler_source.py +5 -2
  812. metadata/profiler/source/bigquery/type_mapper.py +0 -1
  813. metadata/profiler/source/databricks/profiler_source.py +36 -0
  814. metadata/profiler/source/mariadb/functions/median.py +20 -0
  815. metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
  816. metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
  817. metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
  818. metadata/profiler/source/metadata.py +49 -10
  819. metadata/profiler/source/metadata_ext.py +16 -50
  820. metadata/profiler/source/profiler_source_factory.py +8 -0
  821. metadata/py.typed +0 -0
  822. metadata/readers/dataframe/json.py +5 -4
  823. metadata/readers/file/api_reader.py +0 -1
  824. metadata/utils/constants.py +5 -0
  825. metadata/utils/credentials.py +36 -19
  826. metadata/utils/datalake/datalake_utils.py +369 -129
  827. metadata/utils/entity_link.py +26 -6
  828. metadata/utils/execution_time_tracker.py +199 -0
  829. metadata/utils/filters.py +4 -0
  830. metadata/utils/fqn.py +20 -0
  831. metadata/utils/helpers.py +34 -39
  832. metadata/utils/importer.py +2 -3
  833. metadata/utils/life_cycle_utils.py +4 -4
  834. metadata/utils/logger.py +13 -2
  835. metadata/utils/partition.py +10 -5
  836. metadata/utils/secrets/aws_based_secrets_manager.py +67 -4
  837. metadata/utils/secrets/aws_secrets_manager.py +7 -2
  838. metadata/utils/secrets/aws_ssm_secrets_manager.py +7 -2
  839. metadata/utils/secrets/azure_kv_secrets_manager.py +132 -0
  840. metadata/utils/secrets/{noop_secrets_manager.py → db_secrets_manager.py} +4 -3
  841. metadata/utils/secrets/external_secrets_manager.py +25 -3
  842. metadata/utils/secrets/secrets_manager_factory.py +15 -33
  843. metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
  844. metadata/utils/sqlalchemy_utils.py +21 -0
  845. metadata/utils/storage_metadata_config.py +42 -14
  846. metadata/utils/tag_utils.py +5 -2
  847. metadata/workflow/application.py +154 -0
  848. metadata/workflow/application_output_handler.py +34 -0
  849. metadata/workflow/base.py +88 -153
  850. metadata/workflow/data_insight.py +8 -7
  851. metadata/workflow/data_quality.py +3 -2
  852. metadata/workflow/ingestion.py +203 -0
  853. metadata/workflow/metadata.py +2 -3
  854. metadata/workflow/output_handler.py +226 -0
  855. metadata/workflow/profiler.py +2 -2
  856. metadata/workflow/usage.py +3 -4
  857. metadata/workflow/workflow_output_handler.py +15 -255
  858. metadata/workflow/workflow_status_mixin.py +44 -52
  859. openmetadata_ingestion-1.3.2.0rc1.dist-info/METADATA +758 -0
  860. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/RECORD +870 -762
  861. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/WHEEL +1 -1
  862. metadata/generated/schema/entity/applications/appConfig.py +0 -21
  863. metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
  864. metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
  865. metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
  866. metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
  867. metadata/generated/schema/events/entitySpelFilters.py +0 -19
  868. metadata/ingestion/models/es_documents.py +0 -339
  869. metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
  870. metadata/ingestion/ometa/provider_registry.py +0 -144
  871. metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
  872. metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
  873. metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
  874. metadata/profiler/orm/functions/table_metric_construct.py +0 -365
  875. metadata/utils/secrets/client/loader.py +0 -78
  876. openmetadata_ingestion-1.2.4.0.dist-info/METADATA +0 -429
  877. /metadata/examples/workflows/{datalake_azure.yaml → datalake_azure_client_secret.yaml} +0 -0
  878. /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
  879. /metadata/ingestion/source/database/{databricks/unity_catalog → bigtable}/__init__.py +0 -0
  880. /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/doris}/__init__.py +0 -0
  881. /metadata/{utils/secrets/client → ingestion/source/database/sas}/__init__.py +0 -0
  882. /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
  883. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/LICENSE +0 -0
  884. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/entry_points.txt +0 -0
  885. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/top_level.txt +0 -0
@@ -34,7 +34,6 @@ from metadata.generated.schema.entity.services.dashboardService import (
34
34
  DashboardConnection,
35
35
  DashboardService,
36
36
  )
37
- from metadata.generated.schema.entity.teams.user import User
38
37
  from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import (
39
38
  DashboardServiceMetadataPipeline,
40
39
  )
@@ -55,8 +54,8 @@ from metadata.ingestion.models.patch_request import PatchRequest
55
54
  from metadata.ingestion.models.topology import (
56
55
  NodeStage,
57
56
  ServiceTopology,
57
+ TopologyContext,
58
58
  TopologyNode,
59
- create_source_context,
60
59
  )
61
60
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
62
61
  from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -105,7 +104,7 @@ class DashboardServiceTopology(ServiceTopology):
105
104
  ),
106
105
  NodeStage(
107
106
  type_=OMetaTagAndClassification,
108
- processor="yield_tag",
107
+ processor="yield_bulk_tags",
109
108
  nullable=True,
110
109
  ),
111
110
  ],
@@ -133,14 +132,19 @@ class DashboardServiceTopology(ServiceTopology):
133
132
  dashboard = TopologyNode(
134
133
  producer="get_dashboard",
135
134
  stages=[
135
+ NodeStage(
136
+ type_=OMetaTagAndClassification,
137
+ processor="yield_tags",
138
+ nullable=True,
139
+ ),
136
140
  NodeStage(
137
141
  type_=Chart,
138
142
  context="charts",
139
143
  processor="yield_dashboard_chart",
140
144
  consumer=["dashboard_service"],
141
145
  nullable=True,
142
- cache_all=True,
143
- clear_cache=True,
146
+ store_all_in_context=True,
147
+ clear_context=True,
144
148
  use_cache=True,
145
149
  ),
146
150
  NodeStage(
@@ -149,8 +153,8 @@ class DashboardServiceTopology(ServiceTopology):
149
153
  processor="yield_datamodel",
150
154
  consumer=["dashboard_service"],
151
155
  nullable=True,
152
- cache_all=True,
153
- clear_cache=True,
156
+ store_all_in_context=True,
157
+ clear_context=True,
154
158
  use_cache=True,
155
159
  ),
156
160
  NodeStage(
@@ -160,12 +164,6 @@ class DashboardServiceTopology(ServiceTopology):
160
164
  consumer=["dashboard_service"],
161
165
  use_cache=True,
162
166
  ),
163
- NodeStage(
164
- type_=User,
165
- context="owner",
166
- processor="process_owner",
167
- consumer=["dashboard_service"],
168
- ),
169
167
  NodeStage(
170
168
  type_=AddLineageRequest,
171
169
  processor="yield_dashboard_lineage",
@@ -196,7 +194,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
196
194
  service_connection: DashboardConnection.__fields__["config"].type_
197
195
 
198
196
  topology = DashboardServiceTopology()
199
- context = create_source_context(topology)
197
+ context = TopologyContext.create(topology)
200
198
  dashboard_source_state: Set = set()
201
199
  datamodel_source_state: Set = set()
202
200
 
@@ -218,6 +216,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
218
216
  self.connection_obj = self.client
219
217
  self.test_connection()
220
218
 
219
+ @property
220
+ def name(self) -> str:
221
+ return self.service_connection.type.name
222
+
221
223
  @abstractmethod
222
224
  def yield_dashboard(
223
225
  self, dashboard_details: Any
@@ -333,7 +335,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
333
335
  dashboard_details, db_service_name
334
336
  ) or []
335
337
 
336
- def yield_tag(self, *args, **kwargs) -> Iterable[Either[OMetaTagAndClassification]]:
338
+ def yield_bulk_tags(
339
+ self, *args, **kwargs
340
+ ) -> Iterable[Either[OMetaTagAndClassification]]:
341
+ """
342
+ Method to bulk fetch dashboard tags
343
+ """
344
+
345
+ def yield_tags(
346
+ self, dashboard_details
347
+ ) -> Iterable[Either[OMetaTagAndClassification]]:
337
348
  """
338
349
  Method to fetch dashboard tags
339
350
  """
@@ -386,34 +397,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
386
397
  params={"service": self.context.dashboard_service},
387
398
  )
388
399
 
389
- def process_owner(self, dashboard_details):
400
+ def get_owner_ref( # pylint: disable=unused-argument, useless-return
401
+ self, dashboard_details
402
+ ) -> Optional[EntityReference]:
390
403
  """
391
- Method to process the dashboard onwers
404
+ Method to process the dashboard owners
392
405
  """
393
- try:
394
- if self.source_config.includeOwners:
395
- owner = self.get_owner_details( # pylint: disable=assignment-from-none
396
- dashboard_details=dashboard_details
397
- )
398
- if owner:
399
- dashboard_fqn = fqn.build(
400
- self.metadata,
401
- entity_type=Dashboard,
402
- service_name=self.context.dashboard_service,
403
- dashboard_name=self.context.dashboard,
404
- )
405
- dashboard_entity = self.metadata.get_by_name(
406
- entity=Dashboard, fqn=dashboard_fqn
407
- )
408
- self.metadata.patch_owner(
409
- entity=Dashboard,
410
- source=dashboard_entity,
411
- owner=owner,
412
- force=False,
413
- )
414
- except Exception as exc:
415
- logger.debug(traceback.format_exc())
416
- logger.warning(f"Error processing owner for {dashboard_details}: {exc}")
406
+ logger.debug(
407
+ f"Processing ownership is not supported for {self.service_connection.type.name}"
408
+ )
409
+ return None
417
410
 
418
411
  def register_record(self, dashboard_request: CreateDashboardRequest) -> None:
419
412
  """
@@ -429,7 +422,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
429
422
  self.dashboard_source_state.add(dashboard_fqn)
430
423
 
431
424
  def register_record_datamodel(
432
- self, datamodel_requst: CreateDashboardDataModelRequest
425
+ self, datamodel_request: CreateDashboardDataModelRequest
433
426
  ) -> None:
434
427
  """
435
428
  Mark the datamodel record as scanned and update the datamodel_source_state
@@ -437,27 +430,12 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
437
430
  datamodel_fqn = fqn.build(
438
431
  self.metadata,
439
432
  entity_type=DashboardDataModel,
440
- service_name=datamodel_requst.service.__root__,
441
- data_model_name=datamodel_requst.name.__root__,
433
+ service_name=datamodel_request.service.__root__,
434
+ data_model_name=datamodel_request.name.__root__,
442
435
  )
443
436
 
444
437
  self.datamodel_source_state.add(datamodel_fqn)
445
438
 
446
- def get_owner_details( # pylint: disable=useless-return
447
- self, dashboard_details # pylint: disable=unused-argument
448
- ) -> Optional[EntityReference]:
449
- """Get dashboard owner
450
-
451
- Args:
452
- dashboard_details:
453
- Returns:
454
- Optional[EntityReference]
455
- """
456
- logger.debug(
457
- f"Processing ownership is not supported for {self.service_connection.type.name}"
458
- )
459
- return None
460
-
461
439
  @staticmethod
462
440
  def _get_add_lineage_request(
463
441
  to_entity: Union[Dashboard, DashboardDataModel],
@@ -505,7 +483,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
505
483
  self.context.project_name = ( # pylint: disable=assignment-from-none
506
484
  self.get_project_name(dashboard_details=dashboard_details)
507
485
  )
508
- if self.context.project_name and filter_by_project(
486
+ if filter_by_project(
509
487
  self.source_config.projectFilterPattern,
510
488
  self.context.project_name,
511
489
  ):
@@ -553,7 +531,6 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
553
531
  )
554
532
 
555
533
  def check_database_schema_name(self, database_schema_name: str):
556
-
557
534
  """
558
535
  Check if the input database schema name is equal to "<default>" and return the input name if it is not.
559
536
 
@@ -32,11 +32,14 @@ from metadata.generated.schema.entity.services.connections.dashboard.domoDashboa
32
32
  from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
33
33
  OpenMetadataConnection,
34
34
  )
35
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
36
+ StackTraceError,
37
+ )
35
38
  from metadata.generated.schema.metadataIngestion.workflow import (
36
39
  Source as WorkflowSource,
37
40
  )
38
41
  from metadata.generated.schema.type.entityReference import EntityReference
39
- from metadata.ingestion.api.models import Either, StackTraceError
42
+ from metadata.ingestion.api.models import Either
40
43
  from metadata.ingestion.api.steps import InvalidSourceException
41
44
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
42
45
  from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -90,19 +93,14 @@ class DomodashboardSource(DashboardServiceSource):
90
93
  def get_dashboard_details(self, dashboard: DomoDashboardDetails) -> dict:
91
94
  return dashboard
92
95
 
93
- def get_owner_details(
96
+ def get_owner_ref(
94
97
  self, dashboard_details: DomoDashboardDetails
95
98
  ) -> Optional[EntityReference]:
96
- for owner in dashboard_details.owners:
99
+ for owner in dashboard_details.owners or []:
97
100
  try:
98
101
  owner_details = self.client.domo.users_get(owner.id)
99
102
  if owner_details.get("email"):
100
- user = self.metadata.get_user_by_email(owner_details["email"])
101
- if user:
102
- return EntityReference(id=user.id.__root__, type="user")
103
- logger.warning(
104
- f"No user found with email [{owner_details['email']}] in OMD"
105
- )
103
+ return self.metadata.get_reference_by_email(owner_details["email"])
106
104
  except Exception as exc:
107
105
  logger.warning(
108
106
  f"Error while getting details of user {owner.displayName} - {exc}"
@@ -129,9 +127,10 @@ class DomodashboardSource(DashboardServiceSource):
129
127
  service_name=self.context.dashboard_service,
130
128
  chart_name=chart,
131
129
  )
132
- for chart in self.context.charts
130
+ for chart in self.context.charts or []
133
131
  ],
134
132
  service=self.context.dashboard_service,
133
+ owner=self.get_owner_ref(dashboard_details=dashboard_details),
135
134
  )
136
135
  yield Either(right=dashboard_request)
137
136
  self.register_record(dashboard_request=dashboard_request)
@@ -140,7 +139,7 @@ class DomodashboardSource(DashboardServiceSource):
140
139
  left=StackTraceError(
141
140
  name=dashboard_details.name,
142
141
  error=f"Error extracting data from {dashboard_details.name} - {err}",
143
- stack_trace=traceback.format_exc(),
142
+ stackTrace=traceback.format_exc(),
144
143
  )
145
144
  )
146
145
  except ValidationError as err:
@@ -148,7 +147,7 @@ class DomodashboardSource(DashboardServiceSource):
148
147
  left=StackTraceError(
149
148
  name=dashboard_details.name,
150
149
  error=f"Error building pydantic model for {dashboard_details.name} - {err}",
151
- stack_trace=traceback.format_exc(),
150
+ stackTrace=traceback.format_exc(),
152
151
  )
153
152
  )
154
153
  except Exception as err:
@@ -156,7 +155,7 @@ class DomodashboardSource(DashboardServiceSource):
156
155
  left=StackTraceError(
157
156
  name=dashboard_details.name,
158
157
  error=f"Wild error ingesting dashboard {dashboard_details.name} - {err}",
159
- stack_trace=traceback.format_exc(),
158
+ stackTrace=traceback.format_exc(),
160
159
  )
161
160
  )
162
161
 
@@ -230,7 +229,7 @@ class DomodashboardSource(DashboardServiceSource):
230
229
  left=StackTraceError(
231
230
  name=name,
232
231
  error=f"Error creating chart [{name}]: {exc}",
233
- stack_trace=traceback.format_exc(),
232
+ stackTrace=traceback.format_exc(),
234
233
  )
235
234
  )
236
235
 
@@ -114,9 +114,10 @@ class LightdashSource(DashboardServiceSource):
114
114
  service_name=self.context.dashboard_service,
115
115
  chart_name=chart,
116
116
  )
117
- for chart in self.context.charts
117
+ for chart in self.context.charts or []
118
118
  ],
119
119
  service=self.context.dashboard_service,
120
+ owner=self.get_owner_ref(dashboard_details=dashboard_details),
120
121
  )
121
122
  yield dashboard_request
122
123
  self.register_record(dashboard_request=dashboard_request)
@@ -8,6 +8,7 @@
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
+ # pylint: disable=too-many-lines
11
12
  """
12
13
  Looker source module.
13
14
  Supports:
@@ -46,9 +47,7 @@ from metadata.generated.schema.api.data.createDashboardDataModel import (
46
47
  )
47
48
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
48
49
  from metadata.generated.schema.entity.data.chart import Chart
49
- from metadata.generated.schema.entity.data.dashboard import (
50
- Dashboard as MetadataDashboard,
51
- )
50
+ from metadata.generated.schema.entity.data.dashboard import Dashboard
52
51
  from metadata.generated.schema.entity.data.dashboardDataModel import (
53
52
  DashboardDataModel,
54
53
  DataModelType,
@@ -62,6 +61,9 @@ from metadata.generated.schema.entity.services.dashboardService import (
62
61
  DashboardServiceType,
63
62
  )
64
63
  from metadata.generated.schema.entity.services.databaseService import DatabaseService
64
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
65
+ StackTraceError,
66
+ )
65
67
  from metadata.generated.schema.metadataIngestion.workflow import (
66
68
  Source as WorkflowSource,
67
69
  )
@@ -75,7 +77,7 @@ from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDe
75
77
  from metadata.generated.schema.type.entityLineage import Source as LineageSource
76
78
  from metadata.generated.schema.type.entityReference import EntityReference
77
79
  from metadata.generated.schema.type.usageRequest import UsageRequest
78
- from metadata.ingestion.api.models import Either, StackTraceError
80
+ from metadata.ingestion.api.models import Either
79
81
  from metadata.ingestion.api.steps import InvalidSourceException
80
82
  from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
81
83
  from metadata.ingestion.lineage.parser import LineageParser
@@ -403,7 +405,7 @@ class LookerSource(DashboardServiceSource):
403
405
  project=model.project_name,
404
406
  )
405
407
  yield Either(right=explore_datamodel)
406
- self.register_record_datamodel(datamodel_requst=explore_datamodel)
408
+ self.register_record_datamodel(datamodel_request=explore_datamodel)
407
409
 
408
410
  # build datamodel by our hand since ack_sink=False
409
411
  self.context.dataModel = self._build_data_model(datamodel_name)
@@ -440,7 +442,7 @@ class LookerSource(DashboardServiceSource):
440
442
  left=StackTraceError(
441
443
  name=model.name,
442
444
  error=f"Validation error yielding Data Model [{model.name}]: {err}",
443
- stack_trace=traceback.format_exc(),
445
+ stackTrace=traceback.format_exc(),
444
446
  )
445
447
  )
446
448
  except Exception as err:
@@ -448,7 +450,7 @@ class LookerSource(DashboardServiceSource):
448
450
  left=StackTraceError(
449
451
  name=model.name,
450
452
  error=f"Wild error yielding Data Model [{model.name}]: {err}",
451
- stack_trace=traceback.format_exc(),
453
+ stackTrace=traceback.format_exc(),
452
454
  )
453
455
  )
454
456
 
@@ -504,14 +506,14 @@ class LookerSource(DashboardServiceSource):
504
506
  self._view_data_model = self._build_data_model(
505
507
  build_datamodel_name(explore.model_name, view.name)
506
508
  )
507
- self.register_record_datamodel(datamodel_requst=data_model_request)
509
+ self.register_record_datamodel(datamodel_request=data_model_request)
508
510
  yield from self.add_view_lineage(view, explore)
509
511
  else:
510
512
  yield Either(
511
513
  left=StackTraceError(
512
514
  name=view_name,
513
515
  error=f"Cannot find the view [{view_name}]: empty",
514
- stack_trace=traceback.format_exc(),
516
+ stackTrace=traceback.format_exc(),
515
517
  )
516
518
  )
517
519
 
@@ -579,7 +581,7 @@ class LookerSource(DashboardServiceSource):
579
581
  left=StackTraceError(
580
582
  name=view.name,
581
583
  error=f"Error to yield lineage details for view [{view.name}]: {err}",
582
- stack_trace=traceback.format_exc(),
584
+ stackTrace=traceback.format_exc(),
583
585
  )
584
586
  )
585
587
 
@@ -612,7 +614,7 @@ class LookerSource(DashboardServiceSource):
612
614
  dashboard_id=dashboard.id, fields=",".join(GET_DASHBOARD_FIELDS)
613
615
  )
614
616
 
615
- def get_owner_details(
617
+ def get_owner_ref(
616
618
  self, dashboard_details: LookerDashboard
617
619
  ) -> Optional[EntityReference]:
618
620
  """Get dashboard owner
@@ -630,9 +632,7 @@ class LookerSource(DashboardServiceSource):
630
632
  try:
631
633
  if dashboard_details.user_id is not None:
632
634
  dashboard_owner = self.client.user(dashboard_details.user_id)
633
- user = self.metadata.get_user_by_email(dashboard_owner.email)
634
- if user:
635
- return EntityReference(id=user.id.__root__, type="user")
635
+ return self.metadata.get_reference_by_email(dashboard_owner.email)
636
636
 
637
637
  except Exception as err:
638
638
  logger.debug(traceback.format_exc())
@@ -657,19 +657,19 @@ class LookerSource(DashboardServiceSource):
657
657
  service_name=self.context.dashboard_service,
658
658
  chart_name=chart,
659
659
  )
660
- for chart in self.context.charts
660
+ for chart in self.context.charts or []
661
661
  ],
662
662
  # Dashboards are created from the UI directly. They are not linked to a project
663
663
  # like LookML assets, but rather just organised in folders.
664
- project=self._get_dashboard_project(dashboard_details),
664
+ project=self.get_project_name(dashboard_details),
665
665
  sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/dashboards/{dashboard_details.id}",
666
666
  service=self.context.dashboard_service,
667
+ owner=self.get_owner_ref(dashboard_details=dashboard_details),
667
668
  )
668
669
  yield Either(right=dashboard_request)
669
670
  self.register_record(dashboard_request=dashboard_request)
670
671
 
671
- @staticmethod
672
- def _get_dashboard_project(dashboard_details: LookerDashboard) -> Optional[str]:
672
+ def get_project_name(self, dashboard_details: LookerDashboard) -> Optional[str]:
673
673
  """
674
674
  Get dashboard project if the folder is informed
675
675
  """
@@ -679,7 +679,7 @@ class LookerSource(DashboardServiceSource):
679
679
  logger.debug(
680
680
  f"Cannot get folder name from dashboard [{dashboard_details.title}] - [{exc}]"
681
681
  )
682
- return None
682
+ return None
683
683
 
684
684
  @staticmethod
685
685
  def _clean_table_name(table_name: str) -> str:
@@ -756,12 +756,12 @@ class LookerSource(DashboardServiceSource):
756
756
  if cached_explore:
757
757
  dashboard_fqn = fqn.build(
758
758
  self.metadata,
759
- entity_type=MetadataDashboard,
759
+ entity_type=Dashboard,
760
760
  service_name=self.context.dashboard_service,
761
761
  dashboard_name=self.context.dashboard,
762
762
  )
763
763
  dashboard_entity = self.metadata.get_by_name(
764
- entity=MetadataDashboard, fqn=dashboard_fqn
764
+ entity=Dashboard, fqn=dashboard_fqn
765
765
  )
766
766
  yield Either(
767
767
  right=AddLineageRequest(
@@ -786,7 +786,7 @@ class LookerSource(DashboardServiceSource):
786
786
  left=StackTraceError(
787
787
  name=dashboard_entity.displayName,
788
788
  error=f"Unexpected exception yielding lineage from [{dashboard_entity.displayName}]: {exc}",
789
- stack_trace=traceback.format_exc(),
789
+ stackTrace=traceback.format_exc(),
790
790
  )
791
791
  )
792
792
 
@@ -794,7 +794,7 @@ class LookerSource(DashboardServiceSource):
794
794
  self,
795
795
  source: str,
796
796
  db_service_name: str,
797
- to_entity: Union[MetadataDashboard, DashboardDataModel],
797
+ to_entity: Union[Dashboard, DashboardDataModel],
798
798
  ) -> Optional[Either[AddLineageRequest]]:
799
799
  """
800
800
  Once we have a list of origin data sources, check their components
@@ -832,7 +832,6 @@ class LookerSource(DashboardServiceSource):
832
832
  to_entity.id.__root__
833
833
  not in self._added_lineage[from_entity.id.__root__]
834
834
  ):
835
-
836
835
  self._added_lineage[from_entity.id.__root__].append(
837
836
  to_entity.id.__root__
838
837
  )
@@ -879,7 +878,7 @@ class LookerSource(DashboardServiceSource):
879
878
  left=StackTraceError(
880
879
  name=chart.id,
881
880
  error=f"Error creating chart [{chart}]: {exc}",
882
- stack_trace=traceback.format_exc(),
881
+ stackTrace=traceback.format_exc(),
883
882
  )
884
883
  )
885
884
 
@@ -939,9 +938,22 @@ class LookerSource(DashboardServiceSource):
939
938
  :return: UsageRequest, if not computed
940
939
  """
941
940
 
942
- dashboard: MetadataDashboard = self.context.dashboard
941
+ dashboard_name = self.context.dashboard
943
942
 
944
943
  try:
944
+ dashboard_fqn = fqn.build(
945
+ metadata=self.metadata,
946
+ entity_type=Dashboard,
947
+ service_name=self.context.dashboard_service,
948
+ dashboard_name=dashboard_name,
949
+ )
950
+
951
+ dashboard: Dashboard = self.metadata.get_by_name(
952
+ entity=Dashboard,
953
+ fqn=dashboard_fqn,
954
+ fields=["usageSummary"],
955
+ )
956
+
945
957
  current_views = dashboard_details.view_count
946
958
 
947
959
  if not current_views:
@@ -993,8 +1005,8 @@ class LookerSource(DashboardServiceSource):
993
1005
  except Exception as exc:
994
1006
  yield Either(
995
1007
  left=StackTraceError(
996
- name=f"{dashboard.name} Usage",
997
- error=f"Exception computing dashboard usage for {dashboard.fullyQualifiedName.__root__}: {exc}",
998
- stack_trace=traceback.format_exc(),
1008
+ name=f"{dashboard_name} Usage",
1009
+ error=f"Exception computing dashboard usage for {dashboard_name}: {exc}",
1010
+ stackTrace=traceback.format_exc(),
999
1011
  )
1000
1012
  )
@@ -54,14 +54,16 @@ def _clone_repo(
54
54
  return
55
55
 
56
56
  url = None
57
+ allow_unsafe_protocols = False
57
58
  if isinstance(credential, GitHubCredentials):
58
59
  url = f"https://x-oauth-basic:{credential.token.__root__.get_secret_value()}@github.com/{repo_name}.git"
59
60
  elif isinstance(credential, BitBucketCredentials):
60
- url = f"https://x-token-auth::{credential.token.__root__.get_secret_value()}@bitbucket.or/{repo_name}.git"
61
+ url = f"https://x-token-auth:{credential.token.__root__.get_secret_value()}@bitbucket.org/{repo_name}.git"
62
+ allow_unsafe_protocols = True
61
63
 
62
64
  assert url is not None
63
65
 
64
- Repo.clone_from(url, path)
66
+ Repo.clone_from(url, path, allow_unsafe_protocols=allow_unsafe_protocols)
65
67
 
66
68
  logger.info(f"repo {repo_name} cloned to {path}")
67
69
  except Exception as exc:
@@ -124,6 +124,10 @@ class MetabaseClient:
124
124
  try:
125
125
  resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}")
126
126
  if resp_dashboard:
127
+ # Small hack needed to support Metabase versions older than 0.48
128
+ # https://www.metabase.com/releases/metabase-48#fyi--breaking-changes
129
+ if "ordered_cards" in resp_dashboard:
130
+ resp_dashboard["dashcards"] = resp_dashboard["ordered_cards"]
127
131
  return MetabaseDashboardDetails(**resp_dashboard)
128
132
  except Exception:
129
133
  logger.debug(traceback.format_exc())
@@ -26,11 +26,16 @@ from metadata.generated.schema.entity.services.connections.dashboard.metabaseCon
26
26
  from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
27
27
  OpenMetadataConnection,
28
28
  )
29
+ from metadata.generated.schema.entity.services.databaseService import DatabaseService
30
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
31
+ StackTraceError,
32
+ )
29
33
  from metadata.generated.schema.metadataIngestion.workflow import (
30
34
  Source as WorkflowSource,
31
35
  )
32
- from metadata.ingestion.api.models import Either, StackTraceError
36
+ from metadata.ingestion.api.models import Either
33
37
  from metadata.ingestion.api.steps import InvalidSourceException
38
+ from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
34
39
  from metadata.ingestion.lineage.parser import LineageParser
35
40
  from metadata.ingestion.lineage.sql_lineage import search_table_entities
36
41
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -147,9 +152,10 @@ class MetabaseSource(DashboardServiceSource):
147
152
  service_name=self.context.dashboard_service,
148
153
  chart_name=chart,
149
154
  )
150
- for chart in self.context.charts
155
+ for chart in self.context.charts or []
151
156
  ],
152
157
  service=self.context.dashboard_service,
158
+ owner=self.get_owner_ref(dashboard_details=dashboard_details),
153
159
  )
154
160
  yield Either(right=dashboard_request)
155
161
  self.register_record(dashboard_request=dashboard_request)
@@ -158,7 +164,7 @@ class MetabaseSource(DashboardServiceSource):
158
164
  left=StackTraceError(
159
165
  name=dashboard_details.name,
160
166
  error=f"Error creating dashboard [{dashboard_details.name}]: {exc}",
161
- stack_trace=traceback.format_exc(),
167
+ stackTrace=traceback.format_exc(),
162
168
  )
163
169
  )
164
170
 
@@ -172,7 +178,7 @@ class MetabaseSource(DashboardServiceSource):
172
178
  Returns:
173
179
  Iterable[CreateChartRequest]
174
180
  """
175
- charts = dashboard_details.ordered_cards
181
+ charts = dashboard_details.dashcards
176
182
  for chart in charts:
177
183
  try:
178
184
  chart_details = chart.card
@@ -202,7 +208,7 @@ class MetabaseSource(DashboardServiceSource):
202
208
  left=StackTraceError(
203
209
  name="Chart",
204
210
  error=f"Error creating chart [{chart}]: {exc}",
205
- stack_trace=traceback.format_exc(),
211
+ stackTrace=traceback.format_exc(),
206
212
  )
207
213
  )
208
214
 
@@ -219,7 +225,7 @@ class MetabaseSource(DashboardServiceSource):
219
225
  if not db_service_name:
220
226
  return
221
227
  chart_list, dashboard_name = (
222
- dashboard_details.ordered_cards,
228
+ dashboard_details.dashcards,
223
229
  str(dashboard_details.id),
224
230
  )
225
231
  for chart in chart_list:
@@ -253,10 +259,13 @@ class MetabaseSource(DashboardServiceSource):
253
259
  left=StackTraceError(
254
260
  name="Lineage",
255
261
  error=f"Error adding lineage: {exc}",
256
- stack_trace=traceback.format_exc(),
262
+ stackTrace=traceback.format_exc(),
257
263
  )
258
264
  )
259
265
 
266
+ def _get_database_service(self, db_service_name: str):
267
+ return self.metadata.get_by_name(DatabaseService, db_service_name)
268
+
260
269
  def _yield_lineage_from_query(
261
270
  self, chart_details: MetabaseChart, db_service_name: str, dashboard_name: str
262
271
  ) -> Iterable[Either[AddLineageRequest]]:
@@ -275,7 +284,15 @@ class MetabaseSource(DashboardServiceSource):
275
284
 
276
285
  database_name = database.details.db if database and database.details else None
277
286
 
278
- lineage_parser = LineageParser(query)
287
+ db_service = self._get_database_service(db_service_name)
288
+
289
+ lineage_parser = LineageParser(
290
+ query,
291
+ ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
292
+ if db_service
293
+ else None,
294
+ )
295
+
279
296
  for table in lineage_parser.source_tables:
280
297
  database_schema_name, table = fqn.split(str(table))[-2:]
281
298
  database_schema_name = self.check_database_schema_name(database_schema_name)
@@ -307,8 +324,9 @@ class MetabaseSource(DashboardServiceSource):
307
324
  self, chart_details: MetabaseChart, db_service_name: str, dashboard_name: str
308
325
  ) -> Iterable[Either[AddLineageRequest]]:
309
326
  table = self.client.get_table(chart_details.table_id)
327
+ table_name = table.name or table.display_name
310
328
 
311
- if table is None or table.display_name is None:
329
+ if table is None or table_name is None:
312
330
  return
313
331
 
314
332
  database_name = table.db.details.db if table.db and table.db.details else None
@@ -317,7 +335,7 @@ class MetabaseSource(DashboardServiceSource):
317
335
  database=database_name,
318
336
  service_name=db_service_name,
319
337
  database_schema=table.table_schema,
320
- table=table.display_name,
338
+ table=table_name,
321
339
  )
322
340
 
323
341
  to_fqn = fqn.build(