openmetadata-ingestion 1.6.7.1__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (867) hide show
  1. metadata/automations/execute_runner.py +49 -0
  2. metadata/automations/extended_runner.py +13 -0
  3. metadata/automations/runner.py +34 -35
  4. metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +11 -1
  5. metadata/examples/workflows/cassandra.yaml +35 -0
  6. metadata/examples/workflows/cockroach.yaml +24 -0
  7. metadata/examples/workflows/nifi.yaml +51 -0
  8. metadata/examples/workflows/opensearch.yaml +20 -0
  9. metadata/generated/schema/analytics/__init__.py +1 -1
  10. metadata/generated/schema/analytics/basic.py +1 -1
  11. metadata/generated/schema/analytics/reportData.py +1 -1
  12. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  13. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  14. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  15. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  16. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  17. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  18. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  19. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  20. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  21. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  22. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  23. metadata/generated/schema/api/__init__.py +1 -1
  24. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  25. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  26. metadata/generated/schema/api/analytics/__init__.py +1 -1
  27. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  28. metadata/generated/schema/api/automations/__init__.py +1 -1
  29. metadata/generated/schema/api/automations/createWorkflow.py +14 -4
  30. metadata/generated/schema/api/bulkAssets.py +1 -1
  31. metadata/generated/schema/api/classification/__init__.py +1 -1
  32. metadata/generated/schema/api/classification/createClassification.py +2 -2
  33. metadata/generated/schema/api/classification/createTag.py +2 -2
  34. metadata/generated/schema/api/classification/loadTags.py +1 -1
  35. metadata/generated/schema/api/createBot.py +1 -1
  36. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  37. metadata/generated/schema/api/createType.py +1 -1
  38. metadata/generated/schema/api/data/__init__.py +1 -1
  39. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  40. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  41. metadata/generated/schema/api/data/createChart.py +1 -1
  42. metadata/generated/schema/api/data/createContainer.py +1 -1
  43. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  44. metadata/generated/schema/api/data/createDashboard.py +1 -1
  45. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  46. metadata/generated/schema/api/data/createDatabase.py +1 -1
  47. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  48. metadata/generated/schema/api/data/createGlossary.py +1 -1
  49. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  50. metadata/generated/schema/api/data/createMetric.py +1 -1
  51. metadata/generated/schema/api/data/createMlModel.py +1 -1
  52. metadata/generated/schema/api/data/createPipeline.py +5 -1
  53. metadata/generated/schema/api/data/createQuery.py +1 -1
  54. metadata/generated/schema/api/data/createQueryCostRecord.py +35 -0
  55. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  56. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  57. metadata/generated/schema/api/data/createTable.py +1 -1
  58. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  59. metadata/generated/schema/api/data/createTopic.py +1 -1
  60. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  61. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  62. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  63. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  64. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  65. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  66. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  67. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  68. metadata/generated/schema/api/docStore/__init__.py +1 -1
  69. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  70. metadata/generated/schema/api/domains/__init__.py +1 -1
  71. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  72. metadata/generated/schema/api/domains/createDomain.py +8 -1
  73. metadata/generated/schema/api/feed/__init__.py +1 -1
  74. metadata/generated/schema/api/feed/closeTask.py +1 -1
  75. metadata/generated/schema/api/feed/createPost.py +1 -1
  76. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  77. metadata/generated/schema/api/feed/createThread.py +1 -1
  78. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  79. metadata/generated/schema/api/feed/threadCount.py +25 -1
  80. metadata/generated/schema/api/governance/__init__.py +1 -1
  81. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  82. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  83. metadata/generated/schema/api/lineage/__init__.py +1 -1
  84. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  85. metadata/generated/schema/api/lineage/esLineageData.py +85 -0
  86. metadata/generated/schema/api/lineage/lineageDirection.py +12 -0
  87. metadata/generated/schema/api/lineage/nodeInformation.py +19 -0
  88. metadata/generated/schema/api/lineage/searchLineageRequest.py +45 -0
  89. metadata/generated/schema/api/lineage/searchLineageResult.py +46 -0
  90. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  91. metadata/generated/schema/api/policies/__init__.py +1 -1
  92. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  93. metadata/generated/schema/api/search/__init__.py +3 -0
  94. metadata/generated/schema/api/search/previewSearchRequest.py +47 -0
  95. metadata/generated/schema/api/services/__init__.py +1 -1
  96. metadata/generated/schema/api/services/createApiService.py +9 -2
  97. metadata/generated/schema/api/services/createDashboardService.py +9 -2
  98. metadata/generated/schema/api/services/createDatabaseService.py +9 -2
  99. metadata/generated/schema/api/services/createMessagingService.py +9 -2
  100. metadata/generated/schema/api/services/createMetadataService.py +9 -2
  101. metadata/generated/schema/api/services/createMlModelService.py +9 -2
  102. metadata/generated/schema/api/services/createPipelineService.py +9 -2
  103. metadata/generated/schema/api/services/createSearchService.py +9 -2
  104. metadata/generated/schema/api/services/createStorageService.py +9 -2
  105. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  106. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  107. metadata/generated/schema/api/setOwner.py +1 -1
  108. metadata/generated/schema/api/teams/__init__.py +1 -1
  109. metadata/generated/schema/api/teams/createPersona.py +1 -1
  110. metadata/generated/schema/api/teams/createRole.py +1 -1
  111. metadata/generated/schema/api/teams/createTeam.py +1 -1
  112. metadata/generated/schema/api/teams/createUser.py +1 -1
  113. metadata/generated/schema/api/tests/__init__.py +1 -1
  114. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  115. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  116. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  117. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  118. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  119. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  120. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  121. metadata/generated/schema/api/voteRequest.py +1 -1
  122. metadata/generated/schema/auth/__init__.py +1 -1
  123. metadata/generated/schema/auth/basicAuth.py +1 -1
  124. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  125. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  126. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  127. metadata/generated/schema/auth/emailRequest.py +1 -1
  128. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  129. metadata/generated/schema/auth/generateToken.py +1 -1
  130. metadata/generated/schema/auth/jwtAuth.py +1 -1
  131. metadata/generated/schema/auth/loginRequest.py +1 -1
  132. metadata/generated/schema/auth/logoutRequest.py +1 -1
  133. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  134. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  135. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  136. metadata/generated/schema/auth/refreshToken.py +1 -1
  137. metadata/generated/schema/auth/registrationRequest.py +1 -1
  138. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  139. metadata/generated/schema/auth/revokeToken.py +1 -1
  140. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  141. metadata/generated/schema/auth/ssoAuth.py +1 -1
  142. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  143. metadata/generated/schema/configuration/__init__.py +1 -1
  144. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  145. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  146. metadata/generated/schema/configuration/authConfig.py +1 -1
  147. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  148. metadata/generated/schema/configuration/authorizerConfiguration.py +4 -1
  149. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  150. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  151. metadata/generated/schema/configuration/elasticSearchConfiguration.py +55 -1
  152. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  153. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  154. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  155. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  156. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  157. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  158. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  159. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  160. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  161. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  162. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  163. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  164. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  165. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  166. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  167. metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +21 -0
  168. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  169. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  170. metadata/generated/schema/configuration/searchSettings.py +428 -4
  171. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  172. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  173. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  175. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  176. metadata/generated/schema/configuration/workflowSettings.py +8 -1
  177. metadata/generated/schema/dataInsight/__init__.py +1 -1
  178. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  179. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  180. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  181. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  182. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  183. metadata/generated/schema/dataInsight/custom/lineChart.py +15 -1
  184. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  185. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  186. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  187. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  188. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  189. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  190. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  191. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  192. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  193. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  194. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  195. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  196. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  197. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  198. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  199. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  200. metadata/generated/schema/email/__init__.py +1 -1
  201. metadata/generated/schema/email/emailRequest.py +1 -1
  202. metadata/generated/schema/email/emailTemplate.py +1 -1
  203. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  204. metadata/generated/schema/email/smtpSettings.py +1 -2
  205. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  206. metadata/generated/schema/entity/__init__.py +1 -1
  207. metadata/generated/schema/entity/applications/__init__.py +1 -1
  208. metadata/generated/schema/entity/applications/app.py +14 -1
  209. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  210. metadata/generated/schema/entity/applications/appRunRecord.py +23 -4
  211. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  212. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +9 -3
  213. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  214. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  215. metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
  216. metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +41 -0
  217. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  218. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  219. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  220. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  221. metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +71 -0
  222. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  223. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  224. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  225. metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
  226. metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +33 -0
  227. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  228. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  229. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  230. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  231. metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +45 -0
  232. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  233. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +13 -1
  234. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  235. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  236. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  237. metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +46 -0
  238. metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +46 -0
  239. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +109 -2
  240. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  241. metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +1 -1
  242. metadata/generated/schema/entity/applications/configuration/internal/dayOneExperienceAppConfig.py +48 -0
  243. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  244. metadata/generated/schema/entity/applications/configuration/private/__init__.py +3 -0
  245. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  246. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +3 -22
  247. metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +3 -0
  248. metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +24 -0
  249. metadata/generated/schema/entity/applications/configuration/private/limits.py +33 -0
  250. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  251. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  252. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  253. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  254. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +13 -1
  255. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +13 -1
  256. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  257. metadata/generated/schema/entity/automations/__init__.py +1 -1
  258. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  259. metadata/generated/schema/entity/automations/workflow.py +15 -4
  260. metadata/generated/schema/entity/bot.py +1 -1
  261. metadata/generated/schema/entity/classification/__init__.py +1 -1
  262. metadata/generated/schema/entity/classification/classification.py +1 -1
  263. metadata/generated/schema/entity/classification/tag.py +1 -1
  264. metadata/generated/schema/entity/data/__init__.py +1 -1
  265. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  266. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  267. metadata/generated/schema/entity/data/chart.py +1 -1
  268. metadata/generated/schema/entity/data/container.py +1 -1
  269. metadata/generated/schema/entity/data/dashboard.py +1 -1
  270. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  271. metadata/generated/schema/entity/data/database.py +5 -1
  272. metadata/generated/schema/entity/data/databaseSchema.py +7 -5
  273. metadata/generated/schema/entity/data/glossary.py +1 -1
  274. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  275. metadata/generated/schema/entity/data/metric.py +1 -1
  276. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  277. metadata/generated/schema/entity/data/pipeline.py +14 -1
  278. metadata/generated/schema/entity/data/query.py +1 -1
  279. metadata/generated/schema/entity/data/queryCostRecord.py +47 -0
  280. metadata/generated/schema/entity/data/queryCostSearchResult.py +56 -0
  281. metadata/generated/schema/entity/data/report.py +1 -1
  282. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  283. metadata/generated/schema/entity/data/storedProcedure.py +3 -1
  284. metadata/generated/schema/entity/data/table.py +6 -1
  285. metadata/generated/schema/entity/data/topic.py +1 -1
  286. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  287. metadata/generated/schema/entity/docStore/document.py +1 -1
  288. metadata/generated/schema/entity/domains/__init__.py +1 -1
  289. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  290. metadata/generated/schema/entity/domains/domain.py +8 -1
  291. metadata/generated/schema/entity/events/__init__.py +1 -1
  292. metadata/generated/schema/entity/events/webhook.py +1 -1
  293. metadata/generated/schema/entity/feed/__init__.py +1 -1
  294. metadata/generated/schema/entity/feed/assets.py +1 -1
  295. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  296. metadata/generated/schema/entity/feed/description.py +1 -1
  297. metadata/generated/schema/entity/feed/domain.py +1 -1
  298. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  299. metadata/generated/schema/entity/feed/owner.py +1 -1
  300. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  301. metadata/generated/schema/entity/feed/tag.py +1 -1
  302. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  303. metadata/generated/schema/entity/feed/thread.py +1 -1
  304. metadata/generated/schema/entity/policies/__init__.py +1 -1
  305. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  306. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  307. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  308. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  309. metadata/generated/schema/entity/policies/filters.py +1 -1
  310. metadata/generated/schema/entity/policies/policy.py +1 -1
  311. metadata/generated/schema/entity/services/__init__.py +1 -1
  312. metadata/generated/schema/entity/services/apiService.py +8 -1
  313. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  314. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  315. metadata/generated/schema/entity/services/connections/api/restConnection.py +11 -1
  316. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  317. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  318. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  319. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  320. metadata/generated/schema/entity/services/connections/connectionBasicType.py +7 -1
  321. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  322. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +34 -1
  323. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +34 -1
  324. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +34 -1
  325. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +34 -1
  326. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +34 -1
  327. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +34 -1
  328. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +34 -1
  329. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +34 -1
  330. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +34 -1
  331. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  332. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  333. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  334. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  335. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  336. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +50 -2
  337. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +34 -1
  338. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +34 -1
  339. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +34 -1
  340. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +35 -1
  341. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +34 -1
  342. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +34 -1
  343. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  344. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +26 -1
  345. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +26 -1
  346. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +36 -1
  347. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +26 -1
  348. metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +3 -0
  349. metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +61 -0
  350. metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +106 -0
  351. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +26 -1
  352. metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +136 -0
  353. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +26 -1
  360. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +26 -1
  361. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +37 -2
  362. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +26 -1
  367. metadata/generated/schema/entity/services/connections/database/db2Connection.py +26 -1
  368. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +26 -1
  369. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  372. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +26 -1
  373. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +26 -1
  374. metadata/generated/schema/entity/services/connections/database/druidConnection.py +26 -1
  375. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +26 -1
  376. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +26 -1
  377. metadata/generated/schema/entity/services/connections/database/glueConnection.py +26 -1
  378. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +26 -1
  379. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +26 -1
  380. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +26 -1
  388. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +26 -1
  389. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +26 -1
  390. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +26 -1
  391. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +42 -2
  392. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +32 -2
  393. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +32 -2
  394. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +26 -1
  395. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +37 -2
  396. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +26 -1
  397. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +41 -2
  398. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +26 -1
  399. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +26 -1
  400. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +26 -1
  404. metadata/generated/schema/entity/services/connections/database/sasConnection.py +27 -1
  405. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +26 -1
  406. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +58 -3
  407. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +26 -1
  408. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +26 -1
  409. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +26 -1
  410. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +26 -1
  411. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +37 -2
  412. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +26 -1
  413. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  414. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +10 -1
  415. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +10 -1
  416. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +10 -1
  417. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +10 -1
  418. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +10 -1
  419. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  420. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  421. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +26 -1
  422. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +26 -1
  423. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +26 -1
  424. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +26 -1
  425. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +26 -1
  426. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +26 -2
  427. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  428. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +10 -1
  429. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +10 -1
  430. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +10 -1
  431. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +10 -1
  432. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +10 -1
  433. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  434. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +10 -1
  435. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +10 -1
  436. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -1
  437. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +10 -1
  438. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +10 -1
  439. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +10 -1
  440. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +10 -1
  441. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +11 -1
  442. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +10 -1
  443. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +10 -1
  444. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +10 -1
  445. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +10 -1
  446. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +11 -1
  447. metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
  448. metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
  449. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +10 -1
  450. metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
  451. metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
  452. metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
  453. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +10 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +10 -1
  455. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -1
  456. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +10 -1
  457. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +10 -1
  458. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  459. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +10 -1
  460. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  461. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  462. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  463. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +10 -1
  464. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +37 -29
  465. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  466. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  467. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +10 -1
  468. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +10 -1
  469. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +10 -1
  470. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +10 -1
  471. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  472. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  473. metadata/generated/schema/entity/services/dashboardService.py +8 -1
  474. metadata/generated/schema/entity/services/databaseService.py +14 -1
  475. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  476. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +8 -1
  477. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  478. metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +47 -0
  479. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  480. metadata/generated/schema/entity/services/messagingService.py +8 -1
  481. metadata/generated/schema/entity/services/metadataService.py +8 -1
  482. metadata/generated/schema/entity/services/mlmodelService.py +8 -1
  483. metadata/generated/schema/entity/services/pipelineService.py +8 -1
  484. metadata/generated/schema/entity/services/searchService.py +9 -2
  485. metadata/generated/schema/entity/services/serviceType.py +1 -1
  486. metadata/generated/schema/entity/services/storageService.py +8 -1
  487. metadata/generated/schema/entity/teams/__init__.py +1 -1
  488. metadata/generated/schema/entity/teams/persona.py +5 -1
  489. metadata/generated/schema/entity/teams/role.py +1 -1
  490. metadata/generated/schema/entity/teams/team.py +1 -1
  491. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  492. metadata/generated/schema/entity/teams/user.py +5 -1
  493. metadata/generated/schema/entity/type.py +1 -1
  494. metadata/generated/schema/entity/utils/__init__.py +1 -1
  495. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  496. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  497. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  498. metadata/generated/schema/events/__init__.py +1 -1
  499. metadata/generated/schema/events/alertMetrics.py +1 -1
  500. metadata/generated/schema/events/api/__init__.py +1 -1
  501. metadata/generated/schema/events/api/createEventSubscription.py +9 -1
  502. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  503. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  504. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  505. metadata/generated/schema/events/api/typedEvent.py +1 -1
  506. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  507. metadata/generated/schema/events/eventFilterRule.py +1 -1
  508. metadata/generated/schema/events/eventSubscription.py +9 -2
  509. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  510. metadata/generated/schema/events/failedEvent.py +1 -1
  511. metadata/generated/schema/events/failedEventResponse.py +1 -1
  512. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  513. metadata/generated/schema/events/statusContext.py +1 -1
  514. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  515. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  516. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  517. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  518. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  519. metadata/generated/schema/governance/workflows/elements/edge.py +2 -2
  520. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +4 -1
  521. metadata/generated/schema/governance/workflows/elements/nodeType.py +2 -1
  522. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  523. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  524. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +28 -8
  525. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +86 -0
  526. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +73 -0
  527. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +27 -6
  528. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +28 -7
  529. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  530. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +9 -4
  531. metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +3 -0
  532. metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +30 -0
  533. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  534. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +9 -4
  535. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  536. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +24 -7
  537. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  538. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +8 -4
  539. metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +20 -0
  540. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +18 -5
  541. metadata/generated/schema/governance/workflows/workflowDefinition.py +30 -7
  542. metadata/generated/schema/governance/workflows/workflowInstance.py +11 -8
  543. metadata/generated/schema/governance/workflows/workflowInstanceState.py +4 -8
  544. metadata/generated/schema/jobs/__init__.py +1 -1
  545. metadata/generated/schema/jobs/backgroundJob.py +1 -1
  546. metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
  547. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  548. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  549. metadata/generated/schema/metadataIngestion/application.py +1 -1
  550. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  551. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  552. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  553. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -1
  554. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +4 -4
  555. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +5 -1
  556. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +18 -2
  557. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +9 -1
  558. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  560. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  561. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  562. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  563. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  564. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  565. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  566. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  567. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  568. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  569. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  570. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  571. metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +64 -0
  572. metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +3 -0
  573. metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +24 -0
  574. metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +28 -0
  575. metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +28 -0
  576. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  577. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  578. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  579. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  580. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  581. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  582. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  583. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  584. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  585. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  586. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  587. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  588. metadata/generated/schema/metadataIngestion/workflow.py +3 -1
  589. metadata/generated/schema/monitoring/__init__.py +1 -1
  590. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  591. metadata/generated/schema/search/__init__.py +3 -0
  592. metadata/generated/schema/search/searchRequest.py +106 -0
  593. metadata/generated/schema/security/__init__.py +1 -1
  594. metadata/generated/schema/security/client/__init__.py +1 -1
  595. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  596. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  597. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  598. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  599. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  600. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  601. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  602. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  603. metadata/generated/schema/security/credentials/__init__.py +1 -1
  604. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  605. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  606. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  607. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  608. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  609. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  610. metadata/generated/schema/security/credentials/gcpCredentials.py +42 -22
  611. metadata/generated/schema/security/credentials/gcpExternalAccount.py +2 -2
  612. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  613. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  614. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  615. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  616. metadata/generated/schema/security/sasl/__init__.py +1 -1
  617. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  618. metadata/generated/schema/security/secrets/__init__.py +1 -1
  619. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  620. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  621. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  622. metadata/generated/schema/security/securityConfiguration.py +1 -1
  623. metadata/generated/schema/security/ssl/__init__.py +1 -1
  624. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  625. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  626. metadata/generated/schema/settings/__init__.py +1 -1
  627. metadata/generated/schema/settings/settings.py +4 -1
  628. metadata/generated/schema/system/__init__.py +1 -1
  629. metadata/generated/schema/system/entityError.py +1 -1
  630. metadata/generated/schema/system/eventPublisherJob.py +1 -1
  631. metadata/generated/schema/system/indexingError.py +1 -1
  632. metadata/generated/schema/system/limitsResponse.py +1 -1
  633. metadata/generated/schema/system/ui/__init__.py +1 -1
  634. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  635. metadata/generated/schema/system/ui/navigationItem.py +38 -0
  636. metadata/generated/schema/system/ui/page.py +23 -15
  637. metadata/generated/schema/system/ui/tab.py +36 -0
  638. metadata/generated/schema/system/ui/uiCustomization.py +45 -0
  639. metadata/generated/schema/system/validationResponse.py +1 -1
  640. metadata/generated/schema/tests/__init__.py +1 -1
  641. metadata/generated/schema/tests/assigned.py +1 -1
  642. metadata/generated/schema/tests/basic.py +1 -1
  643. metadata/generated/schema/tests/customMetric.py +1 -1
  644. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  645. metadata/generated/schema/tests/resolved.py +1 -1
  646. metadata/generated/schema/tests/testCase.py +4 -1
  647. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  648. metadata/generated/schema/tests/testDefinition.py +1 -1
  649. metadata/generated/schema/tests/testSuite.py +1 -1
  650. metadata/generated/schema/type/__init__.py +1 -1
  651. metadata/generated/schema/type/apiSchema.py +1 -1
  652. metadata/generated/schema/type/assetCertification.py +1 -1
  653. metadata/generated/schema/type/auditLog.py +1 -1
  654. metadata/generated/schema/type/basic.py +1 -1
  655. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  656. metadata/generated/schema/type/changeEvent.py +1 -1
  657. metadata/generated/schema/type/changeEventType.py +1 -1
  658. metadata/generated/schema/type/changeSummaryMap.py +37 -0
  659. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  660. metadata/generated/schema/type/csvDocumentation.py +1 -1
  661. metadata/generated/schema/type/csvErrorType.py +1 -1
  662. metadata/generated/schema/type/csvFile.py +1 -1
  663. metadata/generated/schema/type/csvImportResult.py +1 -1
  664. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  665. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  666. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  667. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  668. metadata/generated/schema/type/customProperty.py +1 -1
  669. metadata/generated/schema/type/dailyCount.py +1 -1
  670. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  671. metadata/generated/schema/type/entityHierarchy.py +1 -1
  672. metadata/generated/schema/type/entityHistory.py +6 -3
  673. metadata/generated/schema/type/entityLineage.py +27 -1
  674. metadata/generated/schema/type/entityReference.py +1 -1
  675. metadata/generated/schema/type/entityReferenceList.py +1 -1
  676. metadata/generated/schema/type/entityRelationship.py +1 -1
  677. metadata/generated/schema/type/entityUsage.py +1 -1
  678. metadata/generated/schema/type/filterPattern.py +1 -1
  679. metadata/generated/schema/type/function.py +1 -1
  680. metadata/generated/schema/type/include.py +1 -1
  681. metadata/generated/schema/type/jdbcConnection.py +1 -1
  682. metadata/generated/schema/type/lifeCycle.py +1 -1
  683. metadata/generated/schema/type/paging.py +1 -1
  684. metadata/generated/schema/type/profile.py +1 -1
  685. metadata/generated/schema/type/queryParserData.py +4 -1
  686. metadata/generated/schema/type/reaction.py +1 -1
  687. metadata/generated/schema/type/schedule.py +1 -1
  688. metadata/generated/schema/type/schema.py +1 -1
  689. metadata/generated/schema/type/tableQuery.py +4 -1
  690. metadata/generated/schema/type/tableUsageCount.py +18 -1
  691. metadata/generated/schema/type/tagLabel.py +1 -1
  692. metadata/generated/schema/type/usageDetails.py +1 -1
  693. metadata/generated/schema/type/usageRequest.py +1 -1
  694. metadata/generated/schema/type/votes.py +1 -1
  695. metadata/ingestion/api/topology_runner.py +3 -3
  696. metadata/ingestion/bulksink/metadata_usage.py +27 -6
  697. metadata/ingestion/lineage/masker.py +13 -2
  698. metadata/ingestion/models/custom_pydantic.py +44 -6
  699. metadata/ingestion/ometa/client.py +20 -0
  700. metadata/ingestion/ometa/mixins/domain_mixin.py +80 -0
  701. metadata/ingestion/ometa/mixins/es_mixin.py +42 -2
  702. metadata/ingestion/ometa/mixins/lineage_mixin.py +7 -33
  703. metadata/ingestion/ometa/mixins/patch_mixin.py +15 -5
  704. metadata/ingestion/ometa/mixins/pipeline_mixin.py +18 -0
  705. metadata/ingestion/ometa/mixins/query_mixin.py +43 -0
  706. metadata/ingestion/ometa/ometa_api.py +28 -3
  707. metadata/ingestion/ometa/routes.py +7 -0
  708. metadata/ingestion/ometa/utils.py +13 -0
  709. metadata/ingestion/processor/query_parser.py +1 -0
  710. metadata/ingestion/sink/metadata_rest.py +21 -1
  711. metadata/ingestion/source/dashboard/dashboard_service.py +9 -1
  712. metadata/ingestion/source/dashboard/domodashboard/metadata.py +3 -1
  713. metadata/ingestion/source/dashboard/lightdash/metadata.py +1 -1
  714. metadata/ingestion/source/dashboard/looker/metadata.py +8 -2
  715. metadata/ingestion/source/dashboard/metabase/client.py +65 -6
  716. metadata/ingestion/source/dashboard/metabase/metadata.py +88 -37
  717. metadata/ingestion/source/dashboard/metabase/models.py +2 -1
  718. metadata/ingestion/source/dashboard/microstrategy/metadata.py +3 -1
  719. metadata/ingestion/source/dashboard/mode/metadata.py +16 -9
  720. metadata/ingestion/source/dashboard/powerbi/client.py +3 -14
  721. metadata/ingestion/source/dashboard/powerbi/metadata.py +6 -54
  722. metadata/ingestion/source/dashboard/powerbi/models.py +0 -12
  723. metadata/ingestion/source/dashboard/qlikcloud/client.py +25 -2
  724. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +69 -15
  725. metadata/ingestion/source/dashboard/qlikcloud/models.py +44 -1
  726. metadata/ingestion/source/dashboard/qliksense/metadata.py +26 -18
  727. metadata/ingestion/source/dashboard/quicksight/metadata.py +32 -29
  728. metadata/ingestion/source/dashboard/redash/metadata.py +12 -10
  729. metadata/ingestion/source/dashboard/sigma/metadata.py +12 -14
  730. metadata/ingestion/source/dashboard/superset/api_source.py +25 -26
  731. metadata/ingestion/source/dashboard/superset/db_source.py +22 -20
  732. metadata/ingestion/source/dashboard/superset/mixin.py +207 -53
  733. metadata/ingestion/source/dashboard/superset/models.py +3 -0
  734. metadata/ingestion/source/dashboard/superset/queries.py +7 -4
  735. metadata/ingestion/source/dashboard/superset/utils.py +36 -0
  736. metadata/ingestion/source/dashboard/tableau/client.py +56 -1
  737. metadata/ingestion/source/dashboard/tableau/metadata.py +243 -128
  738. metadata/ingestion/source/database/bigquery/helper.py +9 -1
  739. metadata/ingestion/source/database/bigquery/metadata.py +82 -10
  740. metadata/ingestion/source/database/bigquery/queries.py +2 -1
  741. metadata/ingestion/source/database/bigquery/query_parser.py +1 -0
  742. metadata/ingestion/source/database/bigtable/metadata.py +11 -3
  743. metadata/ingestion/source/database/cassandra/__init__.py +0 -0
  744. metadata/ingestion/source/database/cassandra/connection.py +138 -0
  745. metadata/ingestion/source/database/cassandra/helpers.py +100 -0
  746. metadata/ingestion/source/database/cassandra/metadata.py +138 -0
  747. metadata/ingestion/source/database/cassandra/queries.py +45 -0
  748. metadata/ingestion/source/database/cassandra/service_spec.py +10 -0
  749. metadata/ingestion/source/database/cockroach/__init__.py +0 -0
  750. metadata/ingestion/source/database/cockroach/connection.py +71 -0
  751. metadata/ingestion/source/database/cockroach/metadata.py +206 -0
  752. metadata/ingestion/source/database/cockroach/queries.py +90 -0
  753. metadata/ingestion/source/database/cockroach/service_spec.py +4 -0
  754. metadata/ingestion/source/database/common_db_source.py +9 -0
  755. metadata/ingestion/source/database/common_nosql_source.py +78 -29
  756. metadata/ingestion/source/database/common_pg_mappings.py +49 -0
  757. metadata/ingestion/source/database/couchbase/metadata.py +8 -2
  758. metadata/ingestion/source/database/databricks/metadata.py +8 -0
  759. metadata/ingestion/source/database/datalake/metadata.py +15 -9
  760. metadata/ingestion/source/database/db2/service_spec.py +4 -1
  761. metadata/ingestion/source/database/dbt/dbt_config.py +1 -0
  762. metadata/ingestion/source/database/dbt/dbt_utils.py +1 -1
  763. metadata/ingestion/source/database/dbt/metadata.py +5 -1
  764. metadata/ingestion/source/database/doris/metadata.py +7 -2
  765. metadata/ingestion/source/database/druid/service_spec.py +5 -1
  766. metadata/ingestion/source/database/dynamodb/metadata.py +6 -3
  767. metadata/ingestion/source/database/greenplum/metadata.py +6 -37
  768. metadata/ingestion/source/database/greenplum/service_spec.py +5 -1
  769. metadata/ingestion/source/database/hive/metastore_dialects/mysql/dialect.py +35 -13
  770. metadata/ingestion/source/database/hive/metastore_dialects/postgres/dialect.py +38 -14
  771. metadata/ingestion/source/database/hive/service_spec.py +5 -1
  772. metadata/ingestion/source/database/impala/service_spec.py +5 -1
  773. metadata/ingestion/source/database/life_cycle_query_mixin.py +11 -7
  774. metadata/ingestion/source/database/lineage_source.py +75 -2
  775. metadata/ingestion/source/database/mariadb/service_spec.py +9 -1
  776. metadata/ingestion/source/database/mongodb/metadata.py +8 -3
  777. metadata/ingestion/source/database/mssql/metadata.py +68 -2
  778. metadata/ingestion/source/database/mssql/queries.py +43 -0
  779. metadata/ingestion/source/database/mysql/utils.py +5 -3
  780. metadata/ingestion/source/database/oracle/connection.py +24 -2
  781. metadata/ingestion/source/database/oracle/metadata.py +36 -19
  782. metadata/ingestion/source/database/oracle/models.py +5 -4
  783. metadata/ingestion/source/database/oracle/queries.py +22 -1
  784. metadata/ingestion/source/database/pinotdb/service_spec.py +5 -1
  785. metadata/ingestion/source/database/postgres/lineage.py +1 -2
  786. metadata/ingestion/source/database/postgres/metadata.py +32 -56
  787. metadata/ingestion/source/database/postgres/models.py +1 -0
  788. metadata/ingestion/source/database/postgres/queries.py +18 -1
  789. metadata/ingestion/source/database/query_parser_source.py +11 -6
  790. metadata/ingestion/source/database/sample_data.py +11 -0
  791. metadata/ingestion/source/database/sample_usage.py +2 -0
  792. metadata/ingestion/source/database/saphana/lineage.py +1 -1
  793. metadata/ingestion/source/database/singlestore/service_spec.py +6 -1
  794. metadata/ingestion/source/database/snowflake/connection.py +6 -0
  795. metadata/ingestion/source/database/snowflake/constants.py +48 -0
  796. metadata/ingestion/source/database/snowflake/metadata.py +114 -3
  797. metadata/ingestion/source/database/snowflake/models.py +3 -3
  798. metadata/ingestion/source/database/snowflake/queries.py +27 -1
  799. metadata/ingestion/source/database/snowflake/query_parser.py +2 -0
  800. metadata/ingestion/source/database/snowflake/utils.py +110 -26
  801. metadata/ingestion/source/database/sql_column_handler.py +95 -60
  802. metadata/ingestion/source/database/sqlite/service_spec.py +5 -1
  803. metadata/ingestion/source/database/teradata/service_spec.py +5 -1
  804. metadata/ingestion/source/database/trino/lineage.py +95 -0
  805. metadata/ingestion/source/database/unitycatalog/connection.py +23 -0
  806. metadata/ingestion/source/database/unitycatalog/lineage.py +2 -0
  807. metadata/ingestion/source/database/unitycatalog/metadata.py +2 -0
  808. metadata/ingestion/source/database/usage_source.py +2 -0
  809. metadata/ingestion/source/messaging/messaging_service.py +2 -0
  810. metadata/ingestion/source/metadata/alationsink/metadata.py +32 -18
  811. metadata/ingestion/source/metadata/amundsen/metadata.py +7 -4
  812. metadata/ingestion/source/metadata/atlas/metadata.py +2 -1
  813. metadata/ingestion/source/mlmodel/mlmodel_service.py +2 -0
  814. metadata/ingestion/source/pipeline/airflow/metadata.py +25 -0
  815. metadata/ingestion/source/pipeline/airflow/models.py +1 -0
  816. metadata/ingestion/source/pipeline/nifi/client.py +68 -74
  817. metadata/ingestion/source/pipeline/nifi/connection.py +1 -18
  818. metadata/ingestion/source/pipeline/openlineage/metadata.py +1 -0
  819. metadata/ingestion/source/pipeline/pipeline_service.py +121 -2
  820. metadata/ingestion/source/search/opensearch/connection.py +215 -0
  821. metadata/ingestion/source/search/opensearch/metadata.py +270 -0
  822. metadata/ingestion/source/search/opensearch/parser.py +71 -0
  823. metadata/ingestion/source/search/opensearch/service_spec.py +4 -0
  824. metadata/ingestion/source/search/search_service.py +2 -0
  825. metadata/ingestion/source/storage/storage_service.py +2 -0
  826. metadata/ingestion/stage/table_usage.py +53 -1
  827. metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py +3 -4
  828. metadata/profiler/orm/converter/azuresql/converter.py +39 -0
  829. metadata/profiler/orm/converter/converter_registry.py +2 -0
  830. metadata/profiler/orm/functions/length.py +2 -0
  831. metadata/profiler/orm/functions/median.py +19 -0
  832. metadata/profiler/orm/functions/modulo.py +1 -0
  833. metadata/profiler/orm/functions/random_num.py +1 -0
  834. metadata/profiler/orm/functions/sum.py +1 -0
  835. metadata/profiler/orm/registry.py +2 -0
  836. metadata/profiler/processor/core.py +1 -0
  837. metadata/profiler/processor/metric_filter.py +1 -1
  838. metadata/profiler/source/database/base/profiler_source.py +1 -0
  839. metadata/profiler/source/fetcher/fetcher_strategy.py +4 -0
  840. metadata/sampler/config.py +1 -0
  841. metadata/sampler/models.py +3 -0
  842. metadata/sampler/nosql/sampler.py +4 -0
  843. metadata/sampler/pandas/sampler.py +1 -0
  844. metadata/sampler/partition.py +9 -2
  845. metadata/sampler/sampler_interface.py +0 -1
  846. metadata/sampler/sqlalchemy/azuresql/sampler.py +30 -3
  847. metadata/sampler/sqlalchemy/mssql/sampler.py +2 -1
  848. metadata/sampler/sqlalchemy/sampler.py +51 -42
  849. metadata/sampler/sqlalchemy/snowflake/sampler.py +2 -1
  850. metadata/sampler/sqlalchemy/trino/sampler.py +9 -8
  851. metadata/utils/constants.py +1 -0
  852. metadata/utils/credentials.py +25 -4
  853. metadata/utils/db_utils.py +1 -11
  854. metadata/utils/elasticsearch.py +4 -0
  855. metadata/utils/helpers.py +91 -0
  856. metadata/utils/logger.py +9 -0
  857. metadata/utils/ssl_manager.py +35 -0
  858. metadata/utils/tag_utils.py +72 -27
  859. metadata/workflow/classification.py +3 -0
  860. metadata/workflow/profiler.py +13 -5
  861. metadata/workflow/workflow_status_mixin.py +6 -0
  862. {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/METADATA +433 -370
  863. {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/RECORD +867 -798
  864. {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/LICENSE +0 -0
  865. {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/WHEEL +0 -0
  866. {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/entry_points.txt +0 -0
  867. {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/top_level.txt +0 -0
@@ -22,7 +22,6 @@ from metadata.generated.schema.api.data.createDashboardDataModel import (
22
22
  )
23
23
  from metadata.generated.schema.entity.data.chart import Chart
24
24
  from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
25
- from metadata.generated.schema.entity.data.table import Table
26
25
  from metadata.generated.schema.entity.services.databaseService import DatabaseService
27
26
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
28
27
  StackTraceError,
@@ -42,6 +41,7 @@ from metadata.ingestion.source.dashboard.superset.models import (
42
41
  )
43
42
  from metadata.utils import fqn
44
43
  from metadata.utils.filters import filter_by_datamodel
44
+ from metadata.utils.fqn import build_es_fqn_search_string
45
45
  from metadata.utils.helpers import (
46
46
  clean_uri,
47
47
  get_database_name_for_lineage,
@@ -130,10 +130,10 @@ class SupersetAPISource(SupersetSourceMixin):
130
130
  )
131
131
 
132
132
  def _get_datasource_fqn_for_lineage(
133
- self, chart_json: ChartResult, db_service_entity: DatabaseService
133
+ self, chart_json: ChartResult, db_service_name: Optional[str]
134
134
  ):
135
135
  return (
136
- self._get_datasource_fqn(chart_json.datasource_id, db_service_entity)
136
+ self._get_datasource_fqn(chart_json.datasource_id, db_service_name)
137
137
  if chart_json.datasource_id
138
138
  else None
139
139
  )
@@ -173,34 +173,33 @@ class SupersetAPISource(SupersetSourceMixin):
173
173
  )
174
174
 
175
175
  def _get_datasource_fqn(
176
- self, datasource_id: str, db_service_entity: DatabaseService
176
+ self, datasource_id: str, db_service_name: Optional[str]
177
177
  ) -> Optional[str]:
178
178
  try:
179
179
  datasource_json = self.client.fetch_datasource(datasource_id)
180
180
  if datasource_json:
181
- database_json = self.client.fetch_database(
182
- datasource_json.result.database.id
183
- )
184
- default_database_name = (
185
- database_json.result.parameters.database
186
- if database_json.result.parameters
187
- else None
188
- )
189
-
190
- database_name = get_database_name_for_lineage(
191
- db_service_entity, default_database_name
192
- )
193
-
194
- if database_json:
195
- dataset_fqn = fqn.build(
196
- self.metadata,
197
- entity_type=Table,
198
- table_name=datasource_json.result.table_name,
199
- schema_name=datasource_json.result.table_schema,
200
- database_name=database_name,
201
- service_name=db_service_entity.name.root,
181
+ database_name = None
182
+ if db_service_name:
183
+ database_json = self.client.fetch_database(
184
+ datasource_json.result.database.id
202
185
  )
203
- return dataset_fqn
186
+ default_database_name = (
187
+ database_json.result.parameters.database
188
+ if database_json.result.parameters
189
+ else None
190
+ )
191
+ db_service_entity = self.metadata.get_by_name(
192
+ entity=DatabaseService, fqn=db_service_name
193
+ )
194
+ database_name = get_database_name_for_lineage(
195
+ db_service_entity, default_database_name
196
+ )
197
+ return build_es_fqn_search_string(
198
+ database_name=database_name,
199
+ schema_name=datasource_json.result.table_schema,
200
+ service_name=db_service_name or "*",
201
+ table_name=datasource_json.result.table_name,
202
+ )
204
203
  except Exception as err:
205
204
  logger.debug(traceback.format_exc())
206
205
  logger.warning(
@@ -25,7 +25,6 @@ from metadata.generated.schema.api.data.createDashboardDataModel import (
25
25
  )
26
26
  from metadata.generated.schema.entity.data.chart import Chart
27
27
  from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
28
- from metadata.generated.schema.entity.data.table import Table
29
28
  from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
30
29
  MysqlConnection,
31
30
  )
@@ -58,6 +57,7 @@ from metadata.ingestion.source.dashboard.superset.queries import (
58
57
  )
59
58
  from metadata.utils import fqn
60
59
  from metadata.utils.filters import filter_by_datamodel
60
+ from metadata.utils.fqn import build_es_fqn_search_string
61
61
  from metadata.utils.helpers import (
62
62
  clean_uri,
63
63
  get_database_name_for_lineage,
@@ -95,17 +95,15 @@ class SupersetDBSource(SupersetSourceMixin):
95
95
  logger.debug(traceback.format_exc())
96
96
  logger.warning(f"Failed to fetch chart list due to - {err}]")
97
97
 
98
- def get_column_list(self, table_name: str) -> Iterable[FetchChart]:
98
+ def get_column_list(self, table_id: Optional[int]) -> Iterable[FetchChart]:
99
99
  try:
100
- if table_name:
101
- col_list = self.engine.execute(
102
- FETCH_COLUMN, table_name=table_name.lower()
103
- )
100
+ if table_id:
101
+ col_list = self.engine.execute(FETCH_COLUMN, table_id=table_id)
104
102
  return [FetchColumn(**col) for col in col_list]
105
103
  except Exception as err:
106
104
  logger.debug(traceback.format_exc())
107
105
  logger.warning(
108
- f"Failed to fetch column name list for table: [{table_name} due to - {err}]"
106
+ f"Failed to fetch column name list for table: [{table_id} due to - {err}]"
109
107
  )
110
108
  return []
111
109
 
@@ -162,10 +160,10 @@ class SupersetDBSource(SupersetSourceMixin):
162
160
  )
163
161
 
164
162
  def _get_datasource_fqn_for_lineage(
165
- self, chart_json: FetchChart, db_service_entity: DatabaseService
163
+ self, chart_json: FetchChart, db_service_name: Optional[str]
166
164
  ):
167
165
  return (
168
- self._get_datasource_fqn(db_service_entity, chart_json)
166
+ self._get_datasource_fqn(db_service_name, chart_json)
169
167
  if chart_json.table_name
170
168
  else None
171
169
  )
@@ -214,23 +212,27 @@ class SupersetDBSource(SupersetSourceMixin):
214
212
  if sqa_str:
215
213
  sqa_url = make_url(sqa_str)
216
214
  default_db_name = sqa_url.database if sqa_url else None
215
+
217
216
  return get_database_name_for_lineage(db_service_entity, default_db_name)
218
217
 
219
218
  def _get_datasource_fqn(
220
- self, db_service_entity: DatabaseService, chart_json: FetchChart
219
+ self, db_service_name: Optional[str], chart_json: FetchChart
221
220
  ) -> Optional[str]:
222
221
  try:
223
- dataset_fqn = fqn.build(
224
- self.metadata,
225
- entity_type=Table,
226
- table_name=chart_json.table_name,
227
- database_name=self._get_database_name(
222
+ database_name = None
223
+ if db_service_name:
224
+ db_service_entity = self.metadata.get_by_name(
225
+ entity=DatabaseService, fqn=db_service_name
226
+ )
227
+ database_name = self._get_database_name(
228
228
  chart_json.sqlalchemy_uri, db_service_entity
229
- ),
229
+ )
230
+ return build_es_fqn_search_string(
231
+ database_name=database_name,
230
232
  schema_name=chart_json.table_schema,
231
- service_name=db_service_entity.name.root,
233
+ service_name=db_service_name or "*",
234
+ table_name=chart_json.table_name,
232
235
  )
233
- return dataset_fqn
234
236
  except Exception as err:
235
237
  logger.debug(traceback.format_exc())
236
238
  logger.warning(
@@ -255,7 +257,7 @@ class SupersetDBSource(SupersetSourceMixin):
255
257
  self.status.filter(
256
258
  chart_json.table_name, "Data model filtered out."
257
259
  )
258
- col_names = self.get_column_list(chart_json.table_name)
260
+ col_names = self.get_column_list(chart_json.table_id)
259
261
  try:
260
262
  data_model_request = CreateDashboardDataModelRequest(
261
263
  name=EntityName(str(chart_json.datasource_id)),
@@ -285,5 +287,5 @@ class SupersetDBSource(SupersetSourceMixin):
285
287
  Returns:
286
288
  List of columns as str to generate column lineage
287
289
  """
288
- col_list = self.get_column_list(chart_json.table_name)
290
+ col_list = self.get_column_list(chart_json.table_id)
289
291
  return [col.column_name for col in col_list]
@@ -13,7 +13,10 @@ Superset mixin module
13
13
  """
14
14
  import json
15
15
  import traceback
16
- from typing import Iterable, List, Optional, Union
16
+ from typing import Dict, Iterable, List, Optional, Tuple, Union
17
+
18
+ from collate_sqllineage.core.models import Column as LineageColumn
19
+ from collate_sqllineage.core.models import Table as LineageTable
17
20
 
18
21
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
19
22
  from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
@@ -27,16 +30,18 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
27
30
  from metadata.generated.schema.entity.services.dashboardService import (
28
31
  DashboardServiceType,
29
32
  )
30
- from metadata.generated.schema.entity.services.databaseService import DatabaseService
31
33
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
32
34
  StackTraceError,
33
35
  )
34
36
  from metadata.generated.schema.metadataIngestion.workflow import (
35
37
  Source as WorkflowSource,
36
38
  )
39
+ from metadata.generated.schema.type.entityLineage import ColumnLineage
37
40
  from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
38
41
  from metadata.ingestion.api.models import Either
39
42
  from metadata.ingestion.api.steps import InvalidSourceException
43
+ from metadata.ingestion.lineage.parser import LineageParser
44
+ from metadata.ingestion.lineage.sql_lineage import get_column_fqn
40
45
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
41
46
  from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
42
47
  from metadata.ingestion.source.dashboard.superset.models import (
@@ -47,6 +52,9 @@ from metadata.ingestion.source.dashboard.superset.models import (
47
52
  FetchDashboard,
48
53
  SupersetDatasource,
49
54
  )
55
+ from metadata.ingestion.source.dashboard.superset.utils import (
56
+ get_dashboard_data_model_column_fqn,
57
+ )
50
58
  from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
51
59
  from metadata.utils import fqn
52
60
  from metadata.utils.logger import ingestion_logger
@@ -146,63 +154,199 @@ class SupersetSourceMixin(DashboardServiceSource):
146
154
  )
147
155
  return []
148
156
 
157
+ def _is_table_to_table_lineage(self, columns: tuple, table: LineageTable) -> bool:
158
+ from_column: LineageColumn = columns[0]
159
+ to_column: LineageColumn = columns[-1]
160
+
161
+ if not isinstance(from_column.parent, LineageTable):
162
+ return False
163
+
164
+ if not isinstance(to_column.parent, LineageTable):
165
+ return False
166
+
167
+ if from_column.parent.schema.raw_name != table.schema.raw_name:
168
+ return False
169
+
170
+ if from_column.parent.raw_name != table.raw_name:
171
+ return False
172
+
173
+ return True
174
+
175
+ def _append_value_to_dict_list(
176
+ self, input_dict: Dict[str, List[str]], dict_key: str, list_value: str
177
+ ) -> None:
178
+ if input_dict.get(dict_key):
179
+ input_dict[dict_key].append(list_value)
180
+ else:
181
+ input_dict[dict_key] = [list_value]
182
+
183
+ def _get_table_schema(self, table: LineageTable, chart: FetchChart) -> str:
184
+ if table.schema.raw_name == table.schema.unknown:
185
+ return chart.table_schema
186
+
187
+ return table.schema.raw_name
188
+
189
+ def _create_column_lineage_mapping(
190
+ self, parser: LineageParser, table: LineageTable, chart: FetchChart
191
+ ) -> Dict[str, List[str]]:
192
+ result = {}
193
+ table_to_table_lineage = [
194
+ _columns
195
+ for _columns in parser.column_lineage
196
+ if self._is_table_to_table_lineage(_columns, table)
197
+ ]
198
+
199
+ for columns in table_to_table_lineage:
200
+ from_column_name = columns[0].raw_name
201
+ to_column_name = columns[-1].raw_name
202
+
203
+ if from_column_name != "*" and to_column_name != "*":
204
+ self._append_value_to_dict_list(
205
+ result, to_column_name, from_column_name
206
+ )
207
+
208
+ if from_column_name == "*" and to_column_name == "*":
209
+ for col_name in self._get_columns_list_for_lineage(chart):
210
+ self._append_value_to_dict_list(result, col_name, col_name)
211
+
212
+ return result
213
+
214
+ def _parse_lineage_from_dataset_sql(
215
+ self, chart_json: FetchChart
216
+ ) -> List[Tuple[FetchChart, Dict[str, List[str]]]]:
217
+ # Every SQL query in tables is a SQL statement SELECTING data.
218
+ # To get lineage we 'simulate' INSERT INTO query into dummy table.
219
+ result = []
220
+ parser = LineageParser(f"INSERT INTO dummy_table {chart_json.sql}")
221
+
222
+ for table in parser.source_tables:
223
+ table_name = table.raw_name
224
+ table_schema = self._get_table_schema(table, chart_json)
225
+
226
+ column_mapping: Dict[str, List[str]] = self._create_column_lineage_mapping(
227
+ parser, table, chart_json
228
+ )
229
+
230
+ result.append(
231
+ (
232
+ FetchChart(
233
+ table_name=table_name,
234
+ schema=table_schema,
235
+ sqlalchemy_uri=chart_json.sqlalchemy_uri,
236
+ ),
237
+ column_mapping,
238
+ )
239
+ )
240
+
241
+ return result
242
+
243
+ def _enrich_raw_input_tables(
244
+ self,
245
+ from_entities: List[Tuple[FetchChart, Dict[str, List[str]]]],
246
+ to_entity: DashboardDataModel,
247
+ db_service_name: Optional[str],
248
+ ):
249
+ result = []
250
+
251
+ for from_entity in from_entities:
252
+ input_table, _column_lineage = from_entity
253
+ datasource_fqn = self._get_datasource_fqn_for_lineage(
254
+ input_table, db_service_name
255
+ )
256
+ from_entity = self.metadata.search_in_any_service(
257
+ entity_type=Table,
258
+ fqn_search_string=datasource_fqn,
259
+ )
260
+ if not from_entity:
261
+ continue
262
+ column_lineage: List[ColumnLineage] = []
263
+ for to_column, from_columns in _column_lineage.items():
264
+ _from_columns = [
265
+ get_column_fqn(from_entity, from_column)
266
+ for from_column in from_columns
267
+ if get_column_fqn(from_entity, from_column)
268
+ ]
269
+
270
+ _to_column = get_dashboard_data_model_column_fqn(to_entity, to_column)
271
+
272
+ if _from_columns and _to_column:
273
+ column_lineage.append(
274
+ ColumnLineage(
275
+ fromColumns=_from_columns,
276
+ toColumn=_to_column,
277
+ )
278
+ )
279
+
280
+ result.append((from_entity, column_lineage))
281
+
282
+ return result
283
+
284
+ def _get_input_tables(self, chart: FetchChart):
285
+ if getattr(chart, "sql", None):
286
+ result = self._parse_lineage_from_dataset_sql(chart)
287
+ else:
288
+ result = [
289
+ (chart, {c: [c] for c in self._get_columns_list_for_lineage(chart)})
290
+ ]
291
+
292
+ return result
293
+
294
+ def _get_dashboard_data_model_entity(
295
+ self, chart: FetchChart
296
+ ) -> Optional[DashboardDataModel]:
297
+ datamodel_fqn = fqn.build(
298
+ self.metadata,
299
+ entity_type=DashboardDataModel,
300
+ service_name=self.config.serviceName,
301
+ data_model_name=str(chart.datasource_id),
302
+ )
303
+ return self.metadata.get_by_name(
304
+ entity=DashboardDataModel,
305
+ fqn=datamodel_fqn,
306
+ )
307
+
149
308
  def yield_dashboard_lineage_details(
150
309
  self,
151
310
  dashboard_details: Union[FetchDashboard, DashboardResult],
152
- db_service_name: DatabaseService,
311
+ db_service_name: Optional[str] = None,
153
312
  ) -> Iterable[Either[AddLineageRequest]]:
154
313
  """
155
314
  Get lineage between datamodel and table
156
315
  """
157
- db_service_entity = self.metadata.get_by_name(
158
- entity=DatabaseService, fqn=db_service_name
159
- )
160
- if db_service_entity:
161
- for chart_id in self._get_charts_of_dashboard(dashboard_details):
162
- chart_json = self.all_charts.get(chart_id)
163
- if chart_json:
164
- try:
165
- datasource_fqn = self._get_datasource_fqn_for_lineage(
166
- chart_json, db_service_entity
167
- )
168
- if not datasource_fqn:
169
- continue
170
- from_entity = self.metadata.get_by_name(
171
- entity=Table,
172
- fqn=datasource_fqn,
173
- )
174
- datamodel_fqn = fqn.build(
175
- self.metadata,
176
- entity_type=DashboardDataModel,
177
- service_name=self.config.serviceName,
178
- data_model_name=str(chart_json.datasource_id),
179
- )
180
- to_entity = self.metadata.get_by_name(
181
- entity=DashboardDataModel,
182
- fqn=datamodel_fqn,
183
- )
316
+ for chart_json in filter(
317
+ None,
318
+ [
319
+ self.all_charts.get(chart_id)
320
+ for chart_id in self._get_charts_of_dashboard(dashboard_details)
321
+ ],
322
+ ):
323
+ try:
324
+ to_entity = self._get_dashboard_data_model_entity(chart_json)
184
325
 
185
- columns_list = self._get_columns_list_for_lineage(chart_json)
186
- column_lineage = self._get_column_lineage(
187
- from_entity, to_entity, columns_list
188
- )
189
- if from_entity and to_entity:
190
- yield self._get_add_lineage_request(
191
- to_entity=to_entity,
192
- from_entity=from_entity,
193
- column_lineage=column_lineage,
194
- )
195
- except Exception as exc:
196
- yield Either(
197
- left=StackTraceError(
198
- name=db_service_name,
199
- error=(
200
- "Error to yield dashboard lineage details for DB "
201
- f"service name [{db_service_name}]: {exc}"
202
- ),
203
- stackTrace=traceback.format_exc(),
204
- )
326
+ if to_entity:
327
+ _input_tables = self._get_input_tables(chart_json)
328
+ input_tables = self._enrich_raw_input_tables(
329
+ _input_tables, to_entity, db_service_name
330
+ )
331
+ for input_table in input_tables:
332
+ from_entity_table, column_lineage = input_table
333
+
334
+ yield self._get_add_lineage_request(
335
+ to_entity=to_entity,
336
+ from_entity=from_entity_table,
337
+ column_lineage=column_lineage,
205
338
  )
339
+ except Exception as exc:
340
+ yield Either(
341
+ left=StackTraceError(
342
+ name="Dashboard Lineage Details",
343
+ error=(
344
+ "Error to yield dashboard lineage details for DB "
345
+ f"service name [{db_service_name}]: {exc}"
346
+ ),
347
+ stackTrace=traceback.format_exc(),
348
+ )
349
+ )
206
350
 
207
351
  def _get_datamodel(
208
352
  self, datamodel: Union[SupersetDatasource, FetchChart]
@@ -238,6 +382,18 @@ class SupersetSourceMixin(DashboardServiceSource):
238
382
  return DataType(col_parse["arrayDataType"])
239
383
  return None
240
384
 
385
+ def parse_row_data_type(self, col_parse: dict) -> List[Column]:
386
+ """
387
+ Set children to single UNKNOWN column for Trino row columns
388
+ to prevent validation error requiring non empty list of children.
389
+ """
390
+ if col_parse["dataType"] == "ROW" and not col_parse.get("children"):
391
+ return [Column(name="unknown", dataType=DataType.UNKNOWN)]
392
+
393
+ if col_parse.get("children"):
394
+ return col_parse["children"]
395
+ return []
396
+
241
397
  def get_column_info(
242
398
  self, data_source: List[Union[DataSourceResult, FetchColumn]]
243
399
  ) -> Optional[List[Column]]:
@@ -259,9 +415,7 @@ class SupersetSourceMixin(DashboardServiceSource):
259
415
  dataTypeDisplay=field.type,
260
416
  dataType=col_parse["dataType"],
261
417
  arrayDataType=self.parse_array_data_type(col_parse),
262
- children=list(col_parse["children"])
263
- if col_parse.get("children")
264
- else None,
418
+ children=self.parse_row_data_type(col_parse),
265
419
  name=str(field.id),
266
420
  displayName=field.column_name,
267
421
  description=field.description,
@@ -144,17 +144,20 @@ class FetchChart(BaseModel):
144
144
  id: Optional[int] = None
145
145
  slice_name: Optional[str] = None
146
146
  description: Optional[str] = None
147
+ table_id: Optional[int] = None
147
148
  table_name: Optional[str] = None
148
149
  table_schema: Optional[str] = Field(None, alias="schema")
149
150
  database_name: Optional[str] = None
150
151
  sqlalchemy_uri: Optional[str] = None
151
152
  viz_type: Optional[str] = None
152
153
  datasource_id: Optional[int] = None
154
+ sql: Optional[str] = None
153
155
 
154
156
 
155
157
  class FetchColumn(BaseModel):
156
158
  id: Optional[int] = None
157
159
  type: Optional[str] = None
158
160
  column_name: Optional[str] = None
161
+ table_id: Optional[int] = None
159
162
  table_name: Optional[str] = None
160
163
  description: Optional[str] = None
@@ -20,8 +20,10 @@ select
20
20
  s.description,
21
21
  s.datasource_id,
22
22
  s.viz_type,
23
+ t.id AS table_id,
23
24
  t.table_name,
24
25
  t.schema,
26
+ t.sql,
25
27
  db.database_name,
26
28
  db.sqlalchemy_uri
27
29
  from
@@ -91,9 +93,10 @@ LIMIT 1
91
93
  FETCH_COLUMN = """
92
94
  select
93
95
  tc.id,
94
- table_name ,
95
- column_name,
96
- type,
96
+ tc.table_name ,
97
+ tc.column_name,
98
+ tc.table_id,
99
+ tc.type,
97
100
  tc.description
98
101
  from
99
102
  table_columns tc
@@ -102,5 +105,5 @@ inner join
102
105
  on
103
106
  t.id=tc.table_id
104
107
  where
105
- table_name=%(table_name)s
108
+ tc.table_id=%(table_id)s
106
109
  """
@@ -0,0 +1,36 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+ """
12
+ Superset utils module
13
+ """
14
+
15
+ from typing import Optional
16
+
17
+ from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
18
+
19
+
20
+ def get_dashboard_data_model_column_fqn(
21
+ dashboard_data_model_entity: DashboardDataModel, column: str
22
+ ) -> Optional[str]:
23
+ """
24
+ Get fqn of column if exist in dashboard data model entity.
25
+
26
+ This is Superset implementation specific as table name is stored within displayName (table name contains
27
+ numerical id), which is not consistent with implementations of dashboard data model columns of
28
+ other dashboard sources.
29
+ """
30
+ if not dashboard_data_model_entity:
31
+ return None
32
+ for dashboard_data_model_column in dashboard_data_model_entity.columns:
33
+ if column.lower() == dashboard_data_model_column.displayName.lower():
34
+ return dashboard_data_model_column.fullyQualifiedName.root
35
+
36
+ return None
@@ -15,11 +15,13 @@ import math
15
15
  import traceback
16
16
  from typing import Any, Callable, Dict, List, Optional
17
17
 
18
+ import pandas as pd
18
19
  import validators
19
20
  from cached_property import cached_property
20
21
  from packaging import version
21
22
  from tableau_api_lib import TableauServerConnection
22
- from tableau_api_lib.utils import extract_pages
23
+ from tableau_api_lib.utils import extract_pages, flatten_dict_column
24
+ from tableau_api_lib.utils.querying import get_views_dataframe
23
25
 
24
26
  from metadata.ingestion.source.dashboard.tableau import (
25
27
  TABLEAU_GET_VIEWS_PARAM_DICT,
@@ -83,6 +85,7 @@ class TableauClient:
83
85
  self._client.sign_in().json()
84
86
  self.pagination_limit = pagination_limit
85
87
  self.custom_sql_table_queries: Dict[str, List[str]] = {}
88
+ self.usage_metrics: Dict[str, int] = {}
86
89
 
87
90
  @cached_property
88
91
  def server_info(self) -> Callable:
@@ -113,6 +116,58 @@ class TableauClient:
113
116
  "Please check if the user has permissions to access the Owner information"
114
117
  )
115
118
 
119
+ def get_all_workbook_usage_metrics(self) -> None:
120
+ """
121
+ Get the usage metrics for all workbook and store it in self.usage_metrics
122
+ """
123
+ try:
124
+ views_df = get_views_dataframe(self._client, all_fields=False)
125
+ if views_df.empty:
126
+ logger.debug("No views data found to process usage metrics.")
127
+ return
128
+
129
+ if "workbook" not in views_df.columns:
130
+ logger.debug("Expected 'workbook' column not found in views data.")
131
+ return
132
+
133
+ usage_views_df = flatten_dict_column(
134
+ df=views_df, keys=["id"], col_name="workbook"
135
+ )
136
+
137
+ if (
138
+ "workbook_id" not in usage_views_df.columns
139
+ or "usage_totalViewCount" not in usage_views_df.columns
140
+ ):
141
+ logger.debug(
142
+ "Expected columns 'workbook_id' or 'usage_totalViewCount' not found after flattening."
143
+ )
144
+ return
145
+
146
+ usage_views_df = usage_views_df[["workbook_id", "usage_totalViewCount"]]
147
+ usage_views_df = (
148
+ usage_views_df.groupby("workbook_id", as_index=False)[
149
+ "usage_totalViewCount"
150
+ ]
151
+ .apply(lambda x: pd.to_numeric(x, errors="coerce").astype(int).sum())
152
+ .reset_index()
153
+ )
154
+ self.usage_metrics.update(
155
+ usage_views_df.set_index("workbook_id")[
156
+ "usage_totalViewCount"
157
+ ].to_dict()
158
+ )
159
+ except Exception as exc:
160
+ logger.debug(traceback.format_exc())
161
+ logger.warning(f"Error processing workbook usage metrics: {exc}")
162
+
163
+ def get_workbook_view_count_by_id(self, workbook_id: str) -> Optional[int]:
164
+ """
165
+ Get a workbook view count by dashboard_id
166
+ """
167
+ if not self.usage_metrics:
168
+ self.get_all_workbook_usage_metrics()
169
+ return self.usage_metrics.get(workbook_id)
170
+
116
171
  def get_workbooks(self) -> List[TableauDashboard]:
117
172
  return [
118
173
  TableauDashboard(**workbook)