openmetadata-ingestion 1.6.0.0rc1__py3-none-any.whl → 1.6.0.0rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (815) hide show
  1. metadata/cli/classify.py +52 -0
  2. metadata/clients/aws_client.py +2 -2
  3. metadata/cmd.py +9 -0
  4. metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +40 -29
  5. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -25
  6. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -50
  7. metadata/data_quality/interface/test_suite_interface.py +45 -56
  8. metadata/data_quality/processor/test_case_runner.py +9 -9
  9. metadata/data_quality/runner/base_test_suite_source.py +52 -26
  10. metadata/data_quality/validations/base_test_handler.py +14 -9
  11. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
  12. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  13. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  14. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  15. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  16. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  17. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  18. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  19. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
  20. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
  21. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  22. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  23. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  24. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
  25. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  26. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  27. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
  28. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
  29. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +21 -16
  30. metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +7 -1
  31. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +21 -18
  32. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
  33. metadata/examples/workflows/bigquery_classifier.yaml +56 -0
  34. metadata/examples/workflows/bigquery_profiler.yaml +1 -2
  35. metadata/examples/workflows/db2_profiler.yaml +1 -2
  36. metadata/examples/workflows/dbtcloud.yaml +2 -1
  37. metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
  38. metadata/examples/workflows/mongodb.yaml +4 -0
  39. metadata/examples/workflows/mysql_profiler.yaml +0 -1
  40. metadata/examples/workflows/redshift_classifier.yaml +38 -0
  41. metadata/examples/workflows/redshift_profiler.yaml +2 -3
  42. metadata/generated/antlr/FqnLexer.py +15 -15
  43. metadata/generated/schema/analytics/__init__.py +1 -1
  44. metadata/generated/schema/analytics/basic.py +1 -1
  45. metadata/generated/schema/analytics/reportData.py +1 -1
  46. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  47. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  48. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  49. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  50. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  51. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  52. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  53. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  54. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  55. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  56. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  57. metadata/generated/schema/api/__init__.py +1 -1
  58. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  59. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  60. metadata/generated/schema/api/analytics/__init__.py +1 -1
  61. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  62. metadata/generated/schema/api/automations/__init__.py +1 -1
  63. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  64. metadata/generated/schema/api/bulkAssets.py +1 -1
  65. metadata/generated/schema/api/classification/__init__.py +1 -1
  66. metadata/generated/schema/api/classification/createClassification.py +1 -1
  67. metadata/generated/schema/api/classification/createTag.py +1 -1
  68. metadata/generated/schema/api/classification/loadTags.py +1 -1
  69. metadata/generated/schema/api/createBot.py +1 -1
  70. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  71. metadata/generated/schema/api/createType.py +1 -1
  72. metadata/generated/schema/api/data/__init__.py +1 -1
  73. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  74. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  75. metadata/generated/schema/api/data/createChart.py +1 -1
  76. metadata/generated/schema/api/data/createContainer.py +1 -1
  77. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  78. metadata/generated/schema/api/data/createDashboard.py +1 -1
  79. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  80. metadata/generated/schema/api/data/createDatabase.py +1 -1
  81. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  82. metadata/generated/schema/api/data/createGlossary.py +1 -1
  83. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  84. metadata/generated/schema/api/data/createMetric.py +1 -1
  85. metadata/generated/schema/api/data/createMlModel.py +1 -1
  86. metadata/generated/schema/api/data/createPipeline.py +1 -1
  87. metadata/generated/schema/api/data/createQuery.py +1 -1
  88. metadata/generated/schema/api/data/createSearchIndex.py +8 -1
  89. metadata/generated/schema/api/data/createStoredProcedure.py +8 -1
  90. metadata/generated/schema/api/data/createTable.py +1 -1
  91. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  92. metadata/generated/schema/api/data/createTopic.py +1 -1
  93. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  94. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  95. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  96. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  97. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  98. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  99. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  100. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  101. metadata/generated/schema/api/docStore/__init__.py +1 -1
  102. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  103. metadata/generated/schema/api/domains/__init__.py +1 -1
  104. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  105. metadata/generated/schema/api/domains/createDomain.py +1 -1
  106. metadata/generated/schema/api/feed/__init__.py +1 -1
  107. metadata/generated/schema/api/feed/closeTask.py +1 -1
  108. metadata/generated/schema/api/feed/createPost.py +1 -1
  109. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  110. metadata/generated/schema/api/feed/createThread.py +1 -1
  111. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  112. metadata/generated/schema/api/feed/threadCount.py +1 -1
  113. metadata/generated/schema/api/governance/__init__.py +1 -1
  114. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  115. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  116. metadata/generated/schema/api/lineage/__init__.py +1 -1
  117. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  118. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  119. metadata/generated/schema/api/policies/__init__.py +1 -1
  120. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  121. metadata/generated/schema/api/services/__init__.py +1 -1
  122. metadata/generated/schema/api/services/createApiService.py +1 -1
  123. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  124. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  125. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  126. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  127. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  128. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  129. metadata/generated/schema/api/services/createSearchService.py +1 -1
  130. metadata/generated/schema/api/services/createStorageService.py +1 -1
  131. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  132. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  133. metadata/generated/schema/api/setOwner.py +1 -1
  134. metadata/generated/schema/api/teams/__init__.py +1 -1
  135. metadata/generated/schema/api/teams/createPersona.py +1 -1
  136. metadata/generated/schema/api/teams/createRole.py +1 -1
  137. metadata/generated/schema/api/teams/createTeam.py +1 -1
  138. metadata/generated/schema/api/teams/createUser.py +1 -1
  139. metadata/generated/schema/api/tests/__init__.py +1 -1
  140. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  141. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  142. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  143. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  144. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  145. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  146. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  147. metadata/generated/schema/api/voteRequest.py +1 -1
  148. metadata/generated/schema/auth/__init__.py +1 -1
  149. metadata/generated/schema/auth/basicAuth.py +1 -1
  150. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  151. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  152. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  153. metadata/generated/schema/auth/emailRequest.py +1 -1
  154. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  155. metadata/generated/schema/auth/generateToken.py +1 -1
  156. metadata/generated/schema/auth/jwtAuth.py +1 -1
  157. metadata/generated/schema/auth/loginRequest.py +1 -1
  158. metadata/generated/schema/auth/logoutRequest.py +1 -1
  159. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  160. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  161. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  162. metadata/generated/schema/auth/refreshToken.py +1 -1
  163. metadata/generated/schema/auth/registrationRequest.py +1 -1
  164. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  165. metadata/generated/schema/auth/revokeToken.py +1 -1
  166. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  167. metadata/generated/schema/auth/ssoAuth.py +1 -1
  168. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  169. metadata/generated/schema/configuration/__init__.py +1 -1
  170. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  171. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  172. metadata/generated/schema/configuration/authConfig.py +1 -1
  173. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  175. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  176. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  177. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  178. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  179. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  180. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  181. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  182. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  183. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  184. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  185. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  186. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  187. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  188. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  189. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  190. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  191. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  195. metadata/generated/schema/configuration/searchSettings.py +1 -1
  196. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
  197. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  198. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  199. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  200. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  201. metadata/generated/schema/dataInsight/__init__.py +1 -1
  202. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  203. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  204. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  205. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  206. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  207. metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
  208. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  209. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  210. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  211. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  212. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  213. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  214. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  215. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  216. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  217. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  218. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  219. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  220. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  221. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  222. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  223. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  224. metadata/generated/schema/email/__init__.py +1 -1
  225. metadata/generated/schema/email/emailRequest.py +1 -1
  226. metadata/generated/schema/email/emailTemplate.py +1 -1
  227. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  228. metadata/generated/schema/email/smtpSettings.py +1 -1
  229. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  230. metadata/generated/schema/entity/__init__.py +1 -1
  231. metadata/generated/schema/entity/applications/__init__.py +1 -1
  232. metadata/generated/schema/entity/applications/app.py +2 -1
  233. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  234. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  235. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  236. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
  237. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  238. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  240. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  241. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  242. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  243. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  244. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  245. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  246. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  247. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  248. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  249. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  250. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  251. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  252. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  253. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
  254. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  255. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  256. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  257. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  258. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  259. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  260. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  261. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  262. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  263. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  264. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
  265. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
  266. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  267. metadata/generated/schema/entity/automations/__init__.py +1 -1
  268. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  269. metadata/generated/schema/entity/automations/workflow.py +1 -1
  270. metadata/generated/schema/entity/bot.py +1 -1
  271. metadata/generated/schema/entity/classification/__init__.py +1 -1
  272. metadata/generated/schema/entity/classification/classification.py +1 -1
  273. metadata/generated/schema/entity/classification/tag.py +1 -1
  274. metadata/generated/schema/entity/data/__init__.py +1 -1
  275. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  276. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  277. metadata/generated/schema/entity/data/chart.py +1 -1
  278. metadata/generated/schema/entity/data/container.py +1 -1
  279. metadata/generated/schema/entity/data/dashboard.py +1 -1
  280. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  281. metadata/generated/schema/entity/data/database.py +1 -1
  282. metadata/generated/schema/entity/data/databaseSchema.py +1 -1
  283. metadata/generated/schema/entity/data/glossary.py +1 -1
  284. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  285. metadata/generated/schema/entity/data/metric.py +1 -1
  286. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  287. metadata/generated/schema/entity/data/pipeline.py +1 -1
  288. metadata/generated/schema/entity/data/query.py +1 -1
  289. metadata/generated/schema/entity/data/report.py +1 -1
  290. metadata/generated/schema/entity/data/searchIndex.py +13 -1
  291. metadata/generated/schema/entity/data/storedProcedure.py +13 -1
  292. metadata/generated/schema/entity/data/table.py +1 -1
  293. metadata/generated/schema/entity/data/topic.py +1 -1
  294. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  295. metadata/generated/schema/entity/docStore/document.py +1 -1
  296. metadata/generated/schema/entity/domains/__init__.py +1 -1
  297. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  298. metadata/generated/schema/entity/domains/domain.py +1 -1
  299. metadata/generated/schema/entity/events/__init__.py +1 -1
  300. metadata/generated/schema/entity/events/webhook.py +1 -1
  301. metadata/generated/schema/entity/feed/__init__.py +1 -1
  302. metadata/generated/schema/entity/feed/assets.py +1 -1
  303. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  304. metadata/generated/schema/entity/feed/description.py +1 -1
  305. metadata/generated/schema/entity/feed/domain.py +1 -1
  306. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  307. metadata/generated/schema/entity/feed/owner.py +1 -1
  308. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  309. metadata/generated/schema/entity/feed/tag.py +1 -1
  310. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  311. metadata/generated/schema/entity/feed/thread.py +1 -1
  312. metadata/generated/schema/entity/policies/__init__.py +1 -1
  313. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  314. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  315. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  316. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  317. metadata/generated/schema/entity/policies/filters.py +1 -1
  318. metadata/generated/schema/entity/policies/policy.py +1 -1
  319. metadata/generated/schema/entity/services/__init__.py +1 -1
  320. metadata/generated/schema/entity/services/apiService.py +1 -1
  321. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  322. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  323. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  324. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  325. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  326. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  327. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  328. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  329. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  330. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  331. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  332. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  333. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  334. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  335. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
  336. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  337. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  340. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  341. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  342. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  343. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  344. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  345. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  346. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  347. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  352. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  353. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  360. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  361. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  362. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  367. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/db2Connection.py +13 -1
  372. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  389. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  390. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +6 -2
  395. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +6 -3
  397. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  410. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
  411. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  416. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  418. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  422. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  423. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  424. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  425. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  427. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  428. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  429. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  431. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  432. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  435. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  436. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  437. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  438. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  439. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  440. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  441. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  442. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  443. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  444. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
  446. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  449. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  451. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  455. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  456. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  458. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  459. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  460. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  461. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  462. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  463. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  464. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  466. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  467. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  469. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  470. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  471. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  472. metadata/generated/schema/entity/services/dashboardService.py +4 -4
  473. metadata/generated/schema/entity/services/databaseService.py +1 -1
  474. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  475. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
  476. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  477. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  478. metadata/generated/schema/entity/services/messagingService.py +1 -1
  479. metadata/generated/schema/entity/services/metadataService.py +1 -1
  480. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  481. metadata/generated/schema/entity/services/pipelineService.py +1 -1
  482. metadata/generated/schema/entity/services/searchService.py +1 -1
  483. metadata/generated/schema/entity/services/serviceType.py +1 -1
  484. metadata/generated/schema/entity/services/storageService.py +1 -1
  485. metadata/generated/schema/entity/teams/__init__.py +1 -1
  486. metadata/generated/schema/entity/teams/persona.py +1 -1
  487. metadata/generated/schema/entity/teams/role.py +1 -1
  488. metadata/generated/schema/entity/teams/team.py +1 -1
  489. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  490. metadata/generated/schema/entity/teams/user.py +1 -1
  491. metadata/generated/schema/entity/type.py +1 -1
  492. metadata/generated/schema/entity/utils/__init__.py +1 -1
  493. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  494. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  495. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  496. metadata/generated/schema/events/__init__.py +1 -1
  497. metadata/generated/schema/events/alertMetrics.py +1 -1
  498. metadata/generated/schema/events/api/__init__.py +1 -1
  499. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  500. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  501. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  502. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  503. metadata/generated/schema/events/api/typedEvent.py +1 -1
  504. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  505. metadata/generated/schema/events/eventFilterRule.py +1 -1
  506. metadata/generated/schema/events/eventSubscription.py +1 -1
  507. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  508. metadata/generated/schema/events/failedEvent.py +1 -1
  509. metadata/generated/schema/events/failedEventResponse.py +1 -1
  510. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  511. metadata/generated/schema/events/statusContext.py +1 -1
  512. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  513. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  514. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  515. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  516. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  517. metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
  518. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
  519. metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
  520. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  521. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  522. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
  523. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
  524. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
  525. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  526. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
  527. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  528. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
  529. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  530. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
  531. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  532. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
  533. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +2 -2
  534. metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
  535. metadata/generated/schema/governance/workflows/workflowInstance.py +8 -1
  536. metadata/generated/schema/governance/workflows/workflowInstanceState.py +8 -1
  537. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  538. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  539. metadata/generated/schema/metadataIngestion/application.py +1 -1
  540. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  541. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  542. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  543. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +111 -0
  544. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
  545. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -33
  546. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  547. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  548. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  549. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  550. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  551. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  552. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  553. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  554. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
  555. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
  556. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  557. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  558. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  560. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  561. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
  562. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  563. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  564. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  565. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  566. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  567. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  568. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  569. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  570. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  571. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  572. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  573. metadata/generated/schema/metadataIngestion/workflow.py +3 -1
  574. metadata/generated/schema/monitoring/__init__.py +1 -1
  575. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  576. metadata/generated/schema/security/__init__.py +1 -1
  577. metadata/generated/schema/security/client/__init__.py +1 -1
  578. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  579. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  580. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  581. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  582. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  583. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  584. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  585. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  586. metadata/generated/schema/security/credentials/__init__.py +1 -1
  587. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  588. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  589. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  590. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  591. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  592. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  593. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  594. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  595. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  596. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  597. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  598. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  599. metadata/generated/schema/security/sasl/__init__.py +1 -1
  600. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  601. metadata/generated/schema/security/secrets/__init__.py +1 -1
  602. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  603. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  604. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  605. metadata/generated/schema/security/securityConfiguration.py +1 -1
  606. metadata/generated/schema/security/ssl/__init__.py +1 -1
  607. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  608. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  609. metadata/generated/schema/settings/__init__.py +1 -1
  610. metadata/generated/schema/settings/settings.py +1 -1
  611. metadata/generated/schema/system/__init__.py +1 -1
  612. metadata/generated/schema/system/entityError.py +1 -1
  613. metadata/generated/schema/system/eventPublisherJob.py +2 -1
  614. metadata/generated/schema/system/indexingError.py +1 -1
  615. metadata/generated/schema/system/limitsResponse.py +1 -1
  616. metadata/generated/schema/system/ui/__init__.py +1 -1
  617. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  618. metadata/generated/schema/system/ui/page.py +1 -1
  619. metadata/generated/schema/system/validationResponse.py +1 -1
  620. metadata/generated/schema/tests/__init__.py +1 -1
  621. metadata/generated/schema/tests/assigned.py +1 -1
  622. metadata/generated/schema/tests/basic.py +1 -1
  623. metadata/generated/schema/tests/customMetric.py +1 -1
  624. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  625. metadata/generated/schema/tests/resolved.py +1 -1
  626. metadata/generated/schema/tests/testCase.py +1 -1
  627. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  628. metadata/generated/schema/tests/testDefinition.py +1 -1
  629. metadata/generated/schema/tests/testSuite.py +1 -1
  630. metadata/generated/schema/type/__init__.py +1 -1
  631. metadata/generated/schema/type/apiSchema.py +1 -1
  632. metadata/generated/schema/type/assetCertification.py +1 -1
  633. metadata/generated/schema/type/auditLog.py +1 -1
  634. metadata/generated/schema/type/basic.py +1 -1
  635. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  636. metadata/generated/schema/type/changeEvent.py +1 -1
  637. metadata/generated/schema/type/changeEventType.py +1 -1
  638. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  639. metadata/generated/schema/type/csvDocumentation.py +1 -1
  640. metadata/generated/schema/type/csvErrorType.py +1 -1
  641. metadata/generated/schema/type/csvFile.py +1 -1
  642. metadata/generated/schema/type/csvImportResult.py +1 -1
  643. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  644. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  645. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  646. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  647. metadata/generated/schema/type/customProperty.py +1 -1
  648. metadata/generated/schema/type/dailyCount.py +1 -1
  649. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  650. metadata/generated/schema/type/entityHierarchy.py +1 -1
  651. metadata/generated/schema/type/entityHistory.py +1 -1
  652. metadata/generated/schema/type/entityLineage.py +1 -1
  653. metadata/generated/schema/type/entityReference.py +1 -1
  654. metadata/generated/schema/type/entityReferenceList.py +1 -1
  655. metadata/generated/schema/type/entityRelationship.py +2 -1
  656. metadata/generated/schema/type/entityUsage.py +1 -1
  657. metadata/generated/schema/type/filterPattern.py +1 -1
  658. metadata/generated/schema/type/function.py +1 -1
  659. metadata/generated/schema/type/include.py +1 -1
  660. metadata/generated/schema/type/jdbcConnection.py +1 -1
  661. metadata/generated/schema/type/lifeCycle.py +1 -1
  662. metadata/generated/schema/type/paging.py +1 -1
  663. metadata/generated/schema/type/profile.py +1 -1
  664. metadata/generated/schema/type/queryParserData.py +1 -1
  665. metadata/generated/schema/type/reaction.py +1 -1
  666. metadata/generated/schema/type/schedule.py +1 -1
  667. metadata/generated/schema/type/schema.py +1 -1
  668. metadata/generated/schema/type/tableQuery.py +1 -1
  669. metadata/generated/schema/type/tableUsageCount.py +1 -1
  670. metadata/generated/schema/type/tagLabel.py +1 -1
  671. metadata/generated/schema/type/usageDetails.py +1 -1
  672. metadata/generated/schema/type/usageRequest.py +1 -1
  673. metadata/generated/schema/type/votes.py +1 -1
  674. metadata/ingestion/api/models.py +9 -5
  675. metadata/ingestion/lineage/parser.py +6 -4
  676. metadata/ingestion/lineage/sql_lineage.py +258 -29
  677. metadata/ingestion/models/patch_request.py +2 -0
  678. metadata/ingestion/ometa/client.py +13 -8
  679. metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -2
  680. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +5 -3
  681. metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
  682. metadata/ingestion/ometa/mixins/table_mixin.py +1 -1
  683. metadata/ingestion/ometa/models.py +1 -0
  684. metadata/ingestion/ometa/ometa_api.py +6 -3
  685. metadata/ingestion/sink/metadata_rest.py +23 -18
  686. metadata/ingestion/source/dashboard/dashboard_service.py +2 -0
  687. metadata/ingestion/source/dashboard/microstrategy/client.py +268 -0
  688. metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +11 -11
  689. metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +74 -51
  690. metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +9 -3
  691. metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
  692. metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
  693. metadata/ingestion/source/dashboard/superset/db_source.py +1 -1
  694. metadata/ingestion/source/dashboard/tableau/metadata.py +29 -12
  695. metadata/ingestion/source/dashboard/tableau/models.py +10 -0
  696. metadata/ingestion/source/database/athena/metadata.py +8 -0
  697. metadata/ingestion/source/database/azuresql/service_spec.py +2 -0
  698. metadata/ingestion/source/database/bigquery/profiler/profiler.py +1 -1
  699. metadata/ingestion/source/database/bigquery/profiler/system.py +5 -4
  700. metadata/ingestion/source/database/bigquery/service_spec.py +2 -0
  701. metadata/ingestion/source/database/common_nosql_source.py +7 -0
  702. metadata/ingestion/source/database/databricks/metadata.py +1 -3
  703. metadata/ingestion/source/database/databricks/service_spec.py +4 -0
  704. metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
  705. metadata/ingestion/source/database/datalake/clients/base.py +10 -1
  706. metadata/ingestion/source/database/datalake/clients/gcs.py +4 -2
  707. metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
  708. metadata/ingestion/source/database/datalake/service_spec.py +8 -1
  709. metadata/ingestion/source/database/db2/connection.py +14 -1
  710. metadata/ingestion/source/database/dbt/constants.py +12 -0
  711. metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
  712. metadata/ingestion/source/database/dbt/dbt_service.py +25 -1
  713. metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
  714. metadata/ingestion/source/database/dbt/metadata.py +70 -1
  715. metadata/ingestion/source/database/dbt/models.py +2 -0
  716. metadata/ingestion/source/database/dynamodb/service_spec.py +4 -1
  717. metadata/ingestion/source/database/mongodb/service_spec.py +4 -1
  718. metadata/ingestion/source/database/mysql/connection.py +5 -0
  719. metadata/ingestion/source/database/mysql/lineage.py +12 -30
  720. metadata/ingestion/source/database/mysql/queries.py +43 -0
  721. metadata/ingestion/source/database/mysql/query_parser.py +46 -0
  722. metadata/ingestion/source/database/mysql/service_spec.py +7 -1
  723. metadata/{data_quality/builders/pandas_validator_builder.py → ingestion/source/database/mysql/usage.py} +9 -14
  724. metadata/ingestion/source/database/redshift/profiler/system.py +5 -4
  725. metadata/ingestion/source/database/sample_data.py +4 -12
  726. metadata/ingestion/source/database/snowflake/connection.py +8 -3
  727. metadata/ingestion/source/database/snowflake/lineage.py +35 -1
  728. metadata/ingestion/source/database/snowflake/metadata.py +38 -19
  729. metadata/ingestion/source/database/snowflake/models.py +1 -0
  730. metadata/ingestion/source/database/snowflake/profiler/system.py +5 -4
  731. metadata/ingestion/source/database/snowflake/queries.py +22 -1
  732. metadata/ingestion/source/database/snowflake/service_spec.py +6 -0
  733. metadata/ingestion/source/database/stored_procedures_mixin.py +20 -8
  734. metadata/ingestion/source/database/trino/service_spec.py +2 -0
  735. metadata/ingestion/source/database/unitycatalog/service_spec.py +4 -0
  736. metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
  737. metadata/ingestion/source/pipeline/airflow/metadata.py +35 -25
  738. metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
  739. metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
  740. metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
  741. metadata/ingestion/source/pipeline/fivetran/metadata.py +1 -1
  742. metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
  743. metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
  744. metadata/ingestion/source/search/search_service.py +44 -1
  745. metadata/mixins/pandas/pandas_mixin.py +2 -31
  746. metadata/mixins/sqalchemy/sqa_mixin.py +16 -16
  747. metadata/pii/processor.py +10 -9
  748. metadata/profiler/api/models.py +3 -79
  749. metadata/profiler/config.py +39 -0
  750. metadata/profiler/interface/nosql/profiler_interface.py +1 -26
  751. metadata/profiler/interface/pandas/profiler_interface.py +37 -77
  752. metadata/profiler/interface/profiler_interface.py +10 -282
  753. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
  754. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
  755. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
  756. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +2 -2
  757. metadata/profiler/interface/sqlalchemy/profiler_interface.py +46 -109
  758. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  759. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
  760. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +6 -8
  761. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  762. metadata/profiler/orm/converter/base.py +21 -12
  763. metadata/profiler/orm/functions/table_metric_computer.py +9 -8
  764. metadata/profiler/processor/core.py +5 -58
  765. metadata/profiler/processor/handle_partition.py +0 -48
  766. metadata/profiler/processor/runner.py +116 -35
  767. metadata/profiler/processor/sample_data_handler.py +7 -4
  768. metadata/profiler/source/database/base/profiler_source.py +56 -138
  769. metadata/profiler/source/database/bigquery/profiler_source.py +3 -3
  770. metadata/profiler/source/database/databricks/profiler_source.py +2 -3
  771. metadata/profiler/source/fetcher/config.py +44 -0
  772. metadata/profiler/source/fetcher/fetcher_strategy.py +2 -4
  773. metadata/sampler/config.py +237 -0
  774. metadata/sampler/models.py +106 -0
  775. metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
  776. metadata/sampler/pandas/sampler.py +239 -0
  777. metadata/{utils → sampler}/partition.py +66 -51
  778. metadata/sampler/processor.py +154 -0
  779. metadata/sampler/sampler_interface.py +251 -0
  780. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
  781. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
  782. metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
  783. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +93 -98
  784. metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
  785. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
  786. metadata/utils/class_helper.py +7 -0
  787. metadata/utils/constants.py +63 -0
  788. metadata/utils/helpers.py +1 -2
  789. metadata/utils/importer.py +1 -1
  790. metadata/utils/logger.py +9 -0
  791. metadata/utils/profiler_utils.py +42 -0
  792. metadata/utils/service_spec/default.py +6 -0
  793. metadata/utils/service_spec/service_spec.py +47 -0
  794. metadata/utils/sqlalchemy_utils.py +1 -1
  795. metadata/utils/ssl_manager.py +39 -1
  796. metadata/workflow/classification.py +53 -0
  797. metadata/workflow/data_quality.py +5 -1
  798. metadata/workflow/profiler.py +1 -17
  799. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/METADATA +373 -373
  800. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/RECORD +805 -795
  801. metadata/data_quality/builders/sqa_validator_builder.py +0 -25
  802. metadata/data_quality/interface/test_suite_interface_factory.py +0 -158
  803. metadata/data_quality/runner/test_suite_source_factory.py +0 -38
  804. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
  805. metadata/ingestion/source/dashboard/mstr/client.py +0 -209
  806. metadata/ingestion/source/dashboard/mstr/service_spec.py +0 -4
  807. metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
  808. metadata/profiler/processor/sampler/sampler_factory.py +0 -100
  809. metadata/profiler/processor/sampler/sampler_interface.py +0 -74
  810. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -83
  811. /metadata/ingestion/source/dashboard/{mstr → microstrategy}/__init__.py +0 -0
  812. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/LICENSE +0 -0
  813. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/WHEEL +0 -0
  814. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/entry_points.txt +0 -0
  815. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/top_level.txt +0 -0
@@ -8,7 +8,7 @@
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
- """Mstr source module"""
11
+ """MicroStrategy source module"""
12
12
  import traceback
13
13
  from typing import Iterable, List, Optional
14
14
 
@@ -16,8 +16,8 @@ from metadata.generated.schema.api.data.createChart import CreateChartRequest
16
16
  from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
17
17
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
18
18
  from metadata.generated.schema.entity.data.chart import Chart
19
- from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
20
- MstrConnection,
19
+ from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
20
+ MicroStrategyConnection,
21
21
  )
22
22
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
23
23
  StackTraceError,
@@ -34,7 +34,7 @@ from metadata.ingestion.api.models import Either
34
34
  from metadata.ingestion.api.steps import InvalidSourceException
35
35
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
36
36
  from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
37
- from metadata.ingestion.source.dashboard.mstr.models import (
37
+ from metadata.ingestion.source.dashboard.microstrategy.models import (
38
38
  MstrDashboard,
39
39
  MstrDashboardDetails,
40
40
  MstrPage,
@@ -47,9 +47,9 @@ from metadata.utils.logger import ingestion_logger
47
47
  logger = ingestion_logger()
48
48
 
49
49
 
50
- class MstrSource(DashboardServiceSource):
50
+ class MicrostrategySource(DashboardServiceSource):
51
51
  """
52
- MSTR Source Class
52
+ Microstrategy Source Class
53
53
  """
54
54
 
55
55
  @classmethod
@@ -60,10 +60,10 @@ class MstrSource(DashboardServiceSource):
60
60
  pipeline_name: Optional[str] = None,
61
61
  ):
62
62
  config = WorkflowSource.model_validate(config_dict)
63
- connection: MstrConnection = config.serviceConnection.root.config
64
- if not isinstance(connection, MstrConnection):
63
+ connection: MicroStrategyConnection = config.serviceConnection.root.config
64
+ if not isinstance(connection, MicroStrategyConnection):
65
65
  raise InvalidSourceException(
66
- f"Expected MstrConnection, but got {connection}"
66
+ f"Expected MicroStrategyConnection, but got {connection}"
67
67
  )
68
68
  return cls(config, metadata)
69
69
 
@@ -75,14 +75,18 @@ class MstrSource(DashboardServiceSource):
75
75
 
76
76
  if self.client.is_project_name():
77
77
  project = self.client.get_project_by_name()
78
- dashboards.extend(self.client.get_dashboards_list(project.id, project.name))
79
-
80
- if not self.client.is_project_name():
81
- for project in self.client.get_projects_list():
78
+ if project:
82
79
  dashboards.extend(
83
80
  self.client.get_dashboards_list(project.id, project.name)
84
81
  )
85
82
 
83
+ if not self.client.is_project_name():
84
+ for project in self.client.get_projects_list():
85
+ if project:
86
+ dashboards.extend(
87
+ self.client.get_dashboards_list(project.id, project.name)
88
+ )
89
+
86
90
  return dashboards
87
91
 
88
92
  def get_dashboard_name(self, dashboard: MstrDashboard) -> str:
@@ -91,6 +95,18 @@ class MstrSource(DashboardServiceSource):
91
95
  """
92
96
  return dashboard.name
93
97
 
98
+ def get_project_name(self, dashboard_details: MstrDashboard) -> Optional[str]:
99
+ """
100
+ Get dashboard project name
101
+ """
102
+ try:
103
+ return dashboard_details.projectName
104
+ except Exception as exc:
105
+ logger.debug(
106
+ f"Cannot get project name from dashboard [{dashboard_details.name}] - [{exc}]"
107
+ )
108
+ return None
109
+
94
110
  def get_dashboard_details(self, dashboard: MstrDashboard) -> MstrDashboardDetails:
95
111
  """
96
112
  Get Dashboard Details
@@ -106,40 +122,41 @@ class MstrSource(DashboardServiceSource):
106
122
  """
107
123
  Method to Get Dashboard Entity
108
124
  """
109
- try:
110
- dashboard_url = (
111
- f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
112
- f"{dashboard_details.projectId}/{dashboard_details.id}"
113
- )
114
- dashboard_request = CreateDashboardRequest(
115
- name=EntityName(dashboard_details.id),
116
- displayName=dashboard_details.name,
117
- sourceUrl=SourceUrl(dashboard_url),
118
- project=dashboard_details.projectName,
119
- charts=[
120
- FullyQualifiedEntityName(
121
- fqn.build(
122
- self.metadata,
123
- entity_type=Chart,
124
- service_name=self.context.get().dashboard_service,
125
- chart_name=chart,
125
+ if dashboard_details:
126
+ try:
127
+ dashboard_url = (
128
+ f"{clean_uri(str(self.service_connection.hostPort))}/MicroStrategyLibrary/app/"
129
+ f"{dashboard_details.projectId}/{dashboard_details.id}"
130
+ )
131
+ dashboard_request = CreateDashboardRequest(
132
+ name=EntityName(dashboard_details.id),
133
+ displayName=dashboard_details.name,
134
+ sourceUrl=SourceUrl(dashboard_url),
135
+ project=dashboard_details.projectName,
136
+ charts=[
137
+ FullyQualifiedEntityName(
138
+ fqn.build(
139
+ self.metadata,
140
+ entity_type=Chart,
141
+ service_name=self.context.get().dashboard_service,
142
+ chart_name=chart,
143
+ )
126
144
  )
145
+ for chart in self.context.get().charts or []
146
+ ],
147
+ service=self.context.get().dashboard_service,
148
+ owners=self.get_owner_ref(dashboard_details=dashboard_details),
149
+ )
150
+ yield Either(right=dashboard_request)
151
+ self.register_record(dashboard_request=dashboard_request)
152
+ except Exception as exc:
153
+ yield Either(
154
+ left=StackTraceError(
155
+ name=dashboard_details.id,
156
+ error=f"Error yielding dashboard for {dashboard_details}: {exc}",
157
+ stackTrace=traceback.format_exc(),
127
158
  )
128
- for chart in self.context.get().charts or []
129
- ],
130
- service=self.context.get().dashboard_service,
131
- owners=self.get_owner_ref(dashboard_details=dashboard_details),
132
- )
133
- yield Either(right=dashboard_request)
134
- self.register_record(dashboard_request=dashboard_request)
135
- except Exception as exc:
136
- yield Either(
137
- left=StackTraceError(
138
- name=dashboard_details.id,
139
- error=f"Error yielding dashboard for {dashboard_details}: {exc}",
140
- stackTrace=traceback.format_exc(),
141
159
  )
142
- )
143
160
 
144
161
  def yield_dashboard_lineage_details(
145
162
  self, dashboard_details: MstrDashboardDetails, db_service_name: str
@@ -156,14 +173,15 @@ class MstrSource(DashboardServiceSource):
156
173
  Returns:
157
174
  Iterable[CreateChartRequest]
158
175
  """
159
- try:
160
- for chapter in dashboard_details.chapters:
161
- for page in chapter.pages:
162
- yield from self._yield_chart_from_visualization(page)
176
+ if dashboard_details:
177
+ try:
178
+ for chapter in dashboard_details.chapters:
179
+ for page in chapter.pages:
180
+ yield from self._yield_chart_from_visualization(page)
163
181
 
164
- except Exception as exc:
165
- logger.debug(traceback.format_exc())
166
- logger.warning(f"Error creating dashboard: {exc}")
182
+ except Exception as exc:
183
+ logger.debug(traceback.format_exc())
184
+ logger.warning(f"Error creating dashboard: {exc}")
167
185
 
168
186
  def _yield_chart_from_visualization(
169
187
  self, page: MstrPage
@@ -192,3 +210,8 @@ class MstrSource(DashboardServiceSource):
192
210
  stackTrace=traceback.format_exc(),
193
211
  )
194
212
  )
213
+
214
+ def close(self):
215
+ # close the api session
216
+ self.client.close_api_session()
217
+ self.metadata.close()
@@ -9,10 +9,10 @@
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
11
  """
12
- MSTR Models
12
+ MicroStrategy Models
13
13
  """
14
14
  from datetime import datetime
15
- from typing import List, Optional
15
+ from typing import Any, List, Optional
16
16
 
17
17
  from pydantic import BaseModel
18
18
 
@@ -69,7 +69,8 @@ class MstrSearchResult(BaseModel):
69
69
 
70
70
 
71
71
  class MstrSearchResultList(BaseModel):
72
- results: Optional[List[MstrSearchResult]]
72
+ totalItems: Optional[int] = 0
73
+ result: Optional[List[MstrSearchResult]] = None
73
74
 
74
75
 
75
76
  class MstrDashboard(BaseModel):
@@ -142,3 +143,8 @@ class MstrDashboardDetails(BaseModel):
142
143
  projectName: str
143
144
  currentChapter: str
144
145
  chapters: List[MstrChapter]
146
+
147
+
148
+ class AuthHeaderCookie(BaseModel):
149
+ auth_header: dict
150
+ auth_cookies: Any
@@ -0,0 +1,6 @@
1
+ from metadata.ingestion.source.dashboard.microstrategy.metadata import (
2
+ MicrostrategySource,
3
+ )
4
+ from metadata.utils.service_spec.default import DefaultDatabaseSpec
5
+
6
+ ServiceSpec = DefaultDatabaseSpec(metadata_source_class=MicrostrategySource)
@@ -64,11 +64,10 @@ class PowerBiApiClient:
64
64
  client_credential=self.config.clientSecret.get_secret_value(),
65
65
  authority=self.config.authorityURI + self.config.tenantId,
66
66
  )
67
- self.auth_token = self.get_auth_token()
68
67
  client_config = ClientConfig(
69
68
  base_url="https://api.powerbi.com",
70
69
  api_version="v1.0",
71
- auth_token=lambda: self.auth_token,
70
+ auth_token=self.get_auth_token,
72
71
  auth_header="Authorization",
73
72
  allow_redirects=True,
74
73
  retry_codes=[429],
@@ -201,7 +201,7 @@ class SupersetDBSource(SupersetSourceMixin):
201
201
  except Exception as exc:
202
202
  yield Either(
203
203
  left=StackTraceError(
204
- name=chart_json.id,
204
+ name=str(chart_json.id),
205
205
  error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
206
206
  stackTrace=traceback.format_exc(),
207
207
  )
@@ -69,6 +69,7 @@ from metadata.ingestion.source.dashboard.tableau.models import (
69
69
  ChartUrl,
70
70
  DataSource,
71
71
  DatasourceField,
72
+ TableAndQuery,
72
73
  TableauDashboard,
73
74
  TableauTag,
74
75
  UpstreamTable,
@@ -149,7 +150,7 @@ class TableauSource(DashboardServiceSource):
149
150
  return None
150
151
 
151
152
  @staticmethod
152
- def _get_data_models_tags(dataModels: [DataSource]) -> Set[str]:
153
+ def _get_data_models_tags(dataModels: List[DataSource]) -> Set[str]:
153
154
  """
154
155
  Get the tags from the data model in the upstreamDatasources
155
156
  """
@@ -382,9 +383,12 @@ class TableauSource(DashboardServiceSource):
382
383
  column=column.id,
383
384
  )
384
385
  for to_column in to_columns:
385
- column_lineage.append(
386
- ColumnLineage(fromColumns=[from_column], toColumn=to_column)
387
- )
386
+ if from_column and to_column:
387
+ column_lineage.append(
388
+ ColumnLineage(
389
+ fromColumns=[from_column], toColumn=to_column
390
+ )
391
+ )
388
392
  return column_lineage
389
393
  except Exception as exc:
390
394
  logger.debug(f"Error to get column lineage: {exc}")
@@ -451,17 +455,22 @@ class TableauSource(DashboardServiceSource):
451
455
  column.id
452
456
  for field in upstream_data_model.fields
453
457
  for column in field.upstreamColumns
458
+ if column is not None
454
459
  }
455
460
  for table in datamodel.upstreamTables or []:
456
461
  om_tables = self._get_database_tables(db_service_entity, table)
457
- for om_table in om_tables or []:
462
+ for om_table_and_query in om_tables or []:
458
463
  column_lineage = self._get_column_lineage(
459
- table, om_table, upstream_data_model_entity, upstream_col_set
464
+ table,
465
+ om_table_and_query.table,
466
+ upstream_data_model_entity,
467
+ upstream_col_set,
460
468
  )
461
469
  yield self._get_add_lineage_request(
462
470
  to_entity=upstream_data_model_entity,
463
- from_entity=om_table,
471
+ from_entity=om_table_and_query.table,
464
472
  column_lineage=column_lineage,
473
+ sql=om_table_and_query.query,
465
474
  )
466
475
  except Exception as err:
467
476
  yield Either(
@@ -698,7 +707,7 @@ class TableauSource(DashboardServiceSource):
698
707
 
699
708
  def _get_table_entities_from_api(
700
709
  self, db_service_entity: DatabaseService, table: UpstreamTable
701
- ) -> Optional[List[Table]]:
710
+ ) -> Optional[List[TableAndQuery]]:
702
711
  """
703
712
  In case we get the table details from the Graphql APIs we process them
704
713
  """
@@ -734,7 +743,7 @@ class TableauSource(DashboardServiceSource):
734
743
  fqn=table_fqn,
735
744
  )
736
745
  if table_entity:
737
- return [table_entity]
746
+ return [TableAndQuery(table=table_entity)]
738
747
  except Exception as exc:
739
748
  logger.debug(traceback.format_exc())
740
749
  logger.warning(f"Error to get tables for lineage using GraphQL Apis: {exc}")
@@ -742,7 +751,7 @@ class TableauSource(DashboardServiceSource):
742
751
 
743
752
  def _get_table_entities_from_query(
744
753
  self, db_service_entity: DatabaseService, table: UpstreamTable
745
- ) -> Optional[List[Table]]:
754
+ ) -> Optional[List[TableAndQuery]]:
746
755
  """
747
756
  In case we get the table details from the Graphql APIs we process them
748
757
  """
@@ -778,7 +787,12 @@ class TableauSource(DashboardServiceSource):
778
787
  database_schema=schema_name,
779
788
  table=table_name,
780
789
  )
781
- tables_list.extend(from_entities)
790
+ tables_list.extend(
791
+ [
792
+ TableAndQuery(table=table, query=custom_sql_table.query)
793
+ for table in from_entities
794
+ ]
795
+ )
782
796
 
783
797
  except Exception as exc:
784
798
  logger.debug(traceback.format_exc())
@@ -787,7 +801,7 @@ class TableauSource(DashboardServiceSource):
787
801
 
788
802
  def _get_database_tables(
789
803
  self, db_service_entity: DatabaseService, table: UpstreamTable
790
- ) -> Optional[List[Table]]:
804
+ ) -> Optional[List[TableAndQuery]]:
791
805
  """
792
806
  Get the table entities for lineage
793
807
  """
@@ -879,6 +893,9 @@ class TableauSource(DashboardServiceSource):
879
893
  try:
880
894
  return dashboard_details.project.name
881
895
  except Exception as exc:
896
+ logger.info(
897
+ f"Cannot parse project name for dashboard:{dashboard_details.id} from Tableau server"
898
+ )
882
899
  logger.debug(traceback.format_exc())
883
900
  logger.warning(
884
901
  f"Error fetching project name for {dashboard_details.id}: {exc}"
@@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional, Union
18
18
  from pydantic import BaseModel, ConfigDict, Field, validator
19
19
 
20
20
  from metadata.generated.schema.entity.data.chart import ChartType
21
+ from metadata.generated.schema.entity.data.table import Table
21
22
 
22
23
 
23
24
  class TableauBaseModel(BaseModel):
@@ -172,3 +173,12 @@ class TableauDashboard(TableauBaseModel):
172
173
  webpageUrl: Optional[str] = None
173
174
  charts: Optional[List[TableauChart]] = None
174
175
  dataModels: List[DataSource] = []
176
+
177
+
178
+ class TableAndQuery(BaseModel):
179
+ """
180
+ Wrapper class for Table entity and associated Query for lineage
181
+ """
182
+
183
+ table: Table
184
+ query: Optional[str] = None
@@ -157,6 +157,14 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
157
157
  return True, partition_details
158
158
  return False, None
159
159
 
160
+ def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]:
161
+ """
162
+ Method to fetch the location path of the table
163
+ """
164
+ return self.external_location_map.get(
165
+ (self.context.get().database, schema_name, table_name)
166
+ )
167
+
160
168
  def yield_tag(
161
169
  self, schema_name: str
162
170
  ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -1,10 +1,12 @@
1
1
  from metadata.ingestion.source.database.azuresql.lineage import AzuresqlLineageSource
2
2
  from metadata.ingestion.source.database.azuresql.metadata import AzuresqlSource
3
3
  from metadata.ingestion.source.database.azuresql.usage import AzuresqlUsageSource
4
+ from metadata.sampler.sqlalchemy.azuresql.sampler import AzureSQLSampler
4
5
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
5
6
 
6
7
  ServiceSpec = DefaultDatabaseSpec(
7
8
  metadata_source_class=AzuresqlSource,
8
9
  lineage_source_class=AzuresqlLineageSource,
9
10
  usage_source_class=AzuresqlUsageSource,
11
+ sampler_class=AzureSQLSampler,
10
12
  )
@@ -22,7 +22,7 @@ class BigQueryProfiler(BigQueryProfilerInterface):
22
22
  **kwargs,
23
23
  ) -> List[SystemProfile]:
24
24
  return self.system_metrics_computer.get_system_metrics(
25
- table=runner.table,
25
+ table=runner.dataset,
26
26
  usage_location=self.service_connection_config.usageLocation,
27
27
  )
28
28
 
@@ -13,6 +13,7 @@ from metadata.profiler.metrics.system.system import (
13
13
  SQASessionProvider,
14
14
  SystemMetricsComputer,
15
15
  )
16
+ from metadata.profiler.processor.runner import QueryRunner
16
17
  from metadata.utils.logger import profiler_logger
17
18
  from metadata.utils.time_utils import datetime_to_timestamp
18
19
 
@@ -25,11 +26,11 @@ class BigQuerySystemMetricsSource(
25
26
  """BigQuery system metrics source class"""
26
27
 
27
28
  def get_kwargs(self, **kwargs):
28
- table = kwargs.get("table")
29
+ runner: QueryRunner = kwargs.get("runner")
29
30
  return {
30
- "table": table.__table__.name,
31
- "dataset_id": table.__table_args__["schema"],
32
- "project_id": super().get_session().get_bind().url.host,
31
+ "table": runner.table_name,
32
+ "database": runner.session.get_bind().url.database,
33
+ "schema": runner.schema_name,
33
34
  "usage_location": kwargs.get("usage_location"),
34
35
  }
35
36
 
@@ -4,6 +4,7 @@ from metadata.ingestion.source.database.bigquery.profiler.profiler import (
4
4
  BigQueryProfiler,
5
5
  )
6
6
  from metadata.ingestion.source.database.bigquery.usage import BigqueryUsageSource
7
+ from metadata.sampler.sqlalchemy.bigquery.sampler import BigQuerySampler
7
8
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
8
9
 
9
10
  ServiceSpec = DefaultDatabaseSpec(
@@ -11,4 +12,5 @@ ServiceSpec = DefaultDatabaseSpec(
11
12
  lineage_source_class=BigqueryLineageSource,
12
13
  usage_source_class=BigqueryUsageSource,
13
14
  profiler_class=BigQueryProfiler,
15
+ sampler_class=BigQuerySampler,
14
16
  )
@@ -52,6 +52,7 @@ from metadata.utils.constants import DEFAULT_DATABASE
52
52
  from metadata.utils.datalake.datalake_utils import DataFrameColumnParser
53
53
  from metadata.utils.filters import filter_by_schema, filter_by_table
54
54
  from metadata.utils.logger import ingestion_logger
55
+ from metadata.utils.ssl_manager import check_ssl_and_init
55
56
 
56
57
  logger = ingestion_logger()
57
58
 
@@ -73,7 +74,13 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
73
74
  )
74
75
  self.metadata = metadata
75
76
  self.service_connection = self.config.serviceConnection.root.config
77
+ self.ssl_manager = check_ssl_and_init(self.service_connection)
78
+ if self.ssl_manager:
79
+ self.service_connection = self.ssl_manager.setup_ssl(
80
+ self.service_connection
81
+ )
76
82
  self.connection_obj = get_connection(self.service_connection)
83
+
77
84
  self.test_connection()
78
85
 
79
86
  def prepare(self):
@@ -131,7 +131,7 @@ def _get_table_columns(self, connection, table_name, schema, db_name):
131
131
  query = DATABRICKS_GET_TABLE_COMMENTS.format(
132
132
  database_name=db_name, schema_name=schema, table_name=table_name
133
133
  )
134
- cursor = get_table_comment_result(
134
+ rows = get_table_comment_result(
135
135
  self,
136
136
  connection=connection,
137
137
  query=query,
@@ -140,8 +140,6 @@ def _get_table_columns(self, connection, table_name, schema, db_name):
140
140
  schema=schema,
141
141
  )
142
142
 
143
- rows = cursor.fetchall()
144
-
145
143
  except exc.OperationalError as e:
146
144
  # Does the table exist?
147
145
  regex_fmt = r"TExecuteStatementResp.*SemanticException.*Table not found {}"
@@ -1,3 +1,6 @@
1
+ from metadata.data_quality.interface.sqlalchemy.databricks.test_suite_interface import (
2
+ DatabricksTestSuiteInterface,
3
+ )
1
4
  from metadata.ingestion.source.database.databricks.lineage import (
2
5
  DatabricksLineageSource,
3
6
  )
@@ -13,4 +16,5 @@ ServiceSpec = DefaultDatabaseSpec(
13
16
  lineage_source_class=DatabricksLineageSource,
14
17
  usage_source_class=DatabricksUsageSource,
15
18
  profiler_class=DatabricksProfilerInterface,
19
+ test_suite_class=DatabricksTestSuiteInterface,
16
20
  )
@@ -26,9 +26,6 @@ from metadata.utils.constants import DEFAULT_DATABASE
26
26
 
27
27
 
28
28
  class DatalakeAzureBlobClient(DatalakeBaseClient):
29
- def __init__(self, client: BlobServiceClient):
30
- self._client = client
31
-
32
29
  @classmethod
33
30
  def from_config(cls, config: AzureConfig) -> "DatalakeAzureBlobClient":
34
31
  try:
@@ -13,10 +13,19 @@
13
13
  Datalake Base Client
14
14
  """
15
15
  from abc import ABC, abstractmethod
16
- from typing import Callable, Iterable, Optional
16
+ from typing import Any, Callable, Iterable, Optional
17
17
 
18
18
 
19
19
  class DatalakeBaseClient(ABC):
20
+ """Base DL client implementation"""
21
+
22
+ def __init__(self, client: Any, **kwargs):
23
+ self._client = client
24
+
25
+ @property
26
+ def client(self) -> Any:
27
+ return self._client
28
+
20
29
  @classmethod
21
30
  @abstractmethod
22
31
  def from_config(cls, config) -> "DatalakeBaseClient":
@@ -33,9 +33,11 @@ from metadata.utils.credentials import GOOGLE_CREDENTIALS, set_google_credential
33
33
 
34
34
  class DatalakeGcsClient(DatalakeBaseClient):
35
35
  def __init__(
36
- self, client: storage.Client, temp_credentials_file_path_list: List[str]
36
+ self,
37
+ client: storage.Client,
38
+ temp_credentials_file_path_list: List[str],
37
39
  ):
38
- self._client = client
40
+ super().__init__(client=client)
39
41
  self._temp_credentials_file_path_list = temp_credentials_file_path_list
40
42
 
41
43
  @property
@@ -25,9 +25,6 @@ from metadata.utils.s3_utils import list_s3_objects
25
25
 
26
26
 
27
27
  class DatalakeS3Client(DatalakeBaseClient):
28
- def __init__(self, client):
29
- self._client = client
30
-
31
28
  @classmethod
32
29
  def from_config(cls, config: S3Config) -> "DatalakeS3Client":
33
30
  if not config.securityConfig:
@@ -1,9 +1,16 @@
1
+ from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
2
+ PandasTestSuiteInterface,
3
+ )
1
4
  from metadata.ingestion.source.database.datalake.metadata import DatalakeSource
2
5
  from metadata.profiler.interface.pandas.profiler_interface import (
3
6
  PandasProfilerInterface,
4
7
  )
8
+ from metadata.sampler.pandas.sampler import DatalakeSampler
5
9
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
6
10
 
7
11
  ServiceSpec = DefaultDatabaseSpec(
8
- metadata_source_class=DatalakeSource, profiler_class=PandasProfilerInterface
12
+ metadata_source_class=DatalakeSource,
13
+ profiler_class=PandasProfilerInterface,
14
+ test_suite_class=PandasTestSuiteInterface,
15
+ sampler_class=DatalakeSampler,
9
16
  )
@@ -12,6 +12,7 @@
12
12
  """
13
13
  Source connection handler
14
14
  """
15
+ from pathlib import Path
15
16
  from typing import Optional
16
17
 
17
18
  from sqlalchemy.engine import Engine
@@ -32,13 +33,25 @@ from metadata.ingestion.connections.builders import (
32
33
  )
33
34
  from metadata.ingestion.connections.test_connections import test_connection_db_common
34
35
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
35
- from metadata.utils.constants import THREE_MIN
36
+ from metadata.utils.constants import THREE_MIN, UTF_8
36
37
 
37
38
 
38
39
  def get_connection(connection: Db2Connection) -> Engine:
39
40
  """
40
41
  Create connection
41
42
  """
43
+ # prepare license
44
+ # pylint: disable=import-outside-toplevel
45
+ if connection.license and connection.licenseFileName:
46
+ import clidriver
47
+
48
+ with open(
49
+ Path(clidriver.__path__[0], "license", connection.licenseFileName),
50
+ "w",
51
+ encoding=UTF_8,
52
+ ) as file:
53
+ file.write(connection.license)
54
+
42
55
  return create_generic_db_connection(
43
56
  connection=connection,
44
57
  get_connection_url_fn=get_connection_url_common,