openmetadata-ingestion 1.6.0.0rc1__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (791) hide show
  1. metadata/cli/classify.py +52 -0
  2. metadata/cmd.py +9 -0
  3. metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +40 -29
  4. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -25
  5. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -50
  6. metadata/data_quality/interface/test_suite_interface.py +45 -56
  7. metadata/data_quality/processor/test_case_runner.py +9 -9
  8. metadata/data_quality/runner/base_test_suite_source.py +52 -26
  9. metadata/data_quality/validations/base_test_handler.py +10 -5
  10. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
  11. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  12. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  13. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  14. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  15. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  16. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  17. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  18. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
  19. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
  20. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  21. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  22. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  23. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
  24. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  25. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  26. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
  27. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
  28. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +21 -16
  29. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +21 -18
  30. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
  31. metadata/examples/workflows/bigquery_classifier.yaml +56 -0
  32. metadata/examples/workflows/bigquery_profiler.yaml +1 -2
  33. metadata/examples/workflows/db2_profiler.yaml +1 -2
  34. metadata/examples/workflows/dbtcloud.yaml +2 -1
  35. metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
  36. metadata/examples/workflows/mongodb.yaml +4 -0
  37. metadata/examples/workflows/mysql_profiler.yaml +0 -1
  38. metadata/examples/workflows/redshift_classifier.yaml +38 -0
  39. metadata/examples/workflows/redshift_profiler.yaml +2 -3
  40. metadata/generated/antlr/FqnLexer.py +15 -15
  41. metadata/generated/schema/analytics/__init__.py +1 -1
  42. metadata/generated/schema/analytics/basic.py +1 -1
  43. metadata/generated/schema/analytics/reportData.py +1 -1
  44. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  45. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  46. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  47. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  48. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  49. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  50. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  51. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  52. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  53. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  54. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  55. metadata/generated/schema/api/__init__.py +1 -1
  56. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  57. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  58. metadata/generated/schema/api/analytics/__init__.py +1 -1
  59. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  60. metadata/generated/schema/api/automations/__init__.py +1 -1
  61. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  62. metadata/generated/schema/api/bulkAssets.py +1 -1
  63. metadata/generated/schema/api/classification/__init__.py +1 -1
  64. metadata/generated/schema/api/classification/createClassification.py +1 -1
  65. metadata/generated/schema/api/classification/createTag.py +1 -1
  66. metadata/generated/schema/api/classification/loadTags.py +1 -1
  67. metadata/generated/schema/api/createBot.py +1 -1
  68. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  69. metadata/generated/schema/api/createType.py +1 -1
  70. metadata/generated/schema/api/data/__init__.py +1 -1
  71. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  72. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  73. metadata/generated/schema/api/data/createChart.py +1 -1
  74. metadata/generated/schema/api/data/createContainer.py +1 -1
  75. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  76. metadata/generated/schema/api/data/createDashboard.py +1 -1
  77. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  78. metadata/generated/schema/api/data/createDatabase.py +1 -1
  79. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  80. metadata/generated/schema/api/data/createGlossary.py +1 -1
  81. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  82. metadata/generated/schema/api/data/createMetric.py +1 -1
  83. metadata/generated/schema/api/data/createMlModel.py +1 -1
  84. metadata/generated/schema/api/data/createPipeline.py +1 -1
  85. metadata/generated/schema/api/data/createQuery.py +1 -1
  86. metadata/generated/schema/api/data/createSearchIndex.py +8 -1
  87. metadata/generated/schema/api/data/createStoredProcedure.py +8 -1
  88. metadata/generated/schema/api/data/createTable.py +1 -1
  89. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  90. metadata/generated/schema/api/data/createTopic.py +1 -1
  91. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  92. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  93. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  94. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  95. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  96. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  97. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  98. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  99. metadata/generated/schema/api/docStore/__init__.py +1 -1
  100. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  101. metadata/generated/schema/api/domains/__init__.py +1 -1
  102. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  103. metadata/generated/schema/api/domains/createDomain.py +1 -1
  104. metadata/generated/schema/api/feed/__init__.py +1 -1
  105. metadata/generated/schema/api/feed/closeTask.py +1 -1
  106. metadata/generated/schema/api/feed/createPost.py +1 -1
  107. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  108. metadata/generated/schema/api/feed/createThread.py +1 -1
  109. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  110. metadata/generated/schema/api/feed/threadCount.py +1 -1
  111. metadata/generated/schema/api/governance/__init__.py +1 -1
  112. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  113. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  114. metadata/generated/schema/api/lineage/__init__.py +1 -1
  115. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  116. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  117. metadata/generated/schema/api/policies/__init__.py +1 -1
  118. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  119. metadata/generated/schema/api/services/__init__.py +1 -1
  120. metadata/generated/schema/api/services/createApiService.py +1 -1
  121. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  122. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  123. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  124. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  125. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  126. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  127. metadata/generated/schema/api/services/createSearchService.py +1 -1
  128. metadata/generated/schema/api/services/createStorageService.py +1 -1
  129. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  130. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  131. metadata/generated/schema/api/setOwner.py +1 -1
  132. metadata/generated/schema/api/teams/__init__.py +1 -1
  133. metadata/generated/schema/api/teams/createPersona.py +1 -1
  134. metadata/generated/schema/api/teams/createRole.py +1 -1
  135. metadata/generated/schema/api/teams/createTeam.py +1 -1
  136. metadata/generated/schema/api/teams/createUser.py +1 -1
  137. metadata/generated/schema/api/tests/__init__.py +1 -1
  138. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  139. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  140. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  141. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  142. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  143. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  144. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  145. metadata/generated/schema/api/voteRequest.py +1 -1
  146. metadata/generated/schema/auth/__init__.py +1 -1
  147. metadata/generated/schema/auth/basicAuth.py +1 -1
  148. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  149. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  150. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  151. metadata/generated/schema/auth/emailRequest.py +1 -1
  152. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  153. metadata/generated/schema/auth/generateToken.py +1 -1
  154. metadata/generated/schema/auth/jwtAuth.py +1 -1
  155. metadata/generated/schema/auth/loginRequest.py +1 -1
  156. metadata/generated/schema/auth/logoutRequest.py +1 -1
  157. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  158. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  159. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  160. metadata/generated/schema/auth/refreshToken.py +1 -1
  161. metadata/generated/schema/auth/registrationRequest.py +1 -1
  162. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  163. metadata/generated/schema/auth/revokeToken.py +1 -1
  164. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  165. metadata/generated/schema/auth/ssoAuth.py +1 -1
  166. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  167. metadata/generated/schema/configuration/__init__.py +1 -1
  168. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  169. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  170. metadata/generated/schema/configuration/authConfig.py +1 -1
  171. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  172. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  173. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  175. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  176. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  177. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  178. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  179. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  180. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  181. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  182. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  183. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  184. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  185. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  186. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  187. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  188. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  189. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  190. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/searchSettings.py +1 -1
  194. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
  195. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  196. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  197. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  198. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  199. metadata/generated/schema/dataInsight/__init__.py +1 -1
  200. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  201. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  202. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  203. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  204. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  205. metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
  206. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  207. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  208. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  209. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  210. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  211. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  212. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  213. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  214. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  215. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  216. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  217. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  218. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  219. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  220. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  221. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  222. metadata/generated/schema/email/__init__.py +1 -1
  223. metadata/generated/schema/email/emailRequest.py +1 -1
  224. metadata/generated/schema/email/emailTemplate.py +1 -1
  225. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  226. metadata/generated/schema/email/smtpSettings.py +1 -1
  227. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  228. metadata/generated/schema/entity/__init__.py +1 -1
  229. metadata/generated/schema/entity/applications/__init__.py +1 -1
  230. metadata/generated/schema/entity/applications/app.py +2 -1
  231. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  232. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  233. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  234. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
  235. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  236. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  237. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  238. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  240. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  241. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  242. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  243. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  244. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  245. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  246. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  247. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  248. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  249. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  250. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  251. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
  252. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  253. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  254. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  255. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  256. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  257. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  258. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  259. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  260. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  261. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  262. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
  263. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
  264. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  265. metadata/generated/schema/entity/automations/__init__.py +1 -1
  266. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  267. metadata/generated/schema/entity/automations/workflow.py +1 -1
  268. metadata/generated/schema/entity/bot.py +1 -1
  269. metadata/generated/schema/entity/classification/__init__.py +1 -1
  270. metadata/generated/schema/entity/classification/classification.py +1 -1
  271. metadata/generated/schema/entity/classification/tag.py +1 -1
  272. metadata/generated/schema/entity/data/__init__.py +1 -1
  273. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  274. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  275. metadata/generated/schema/entity/data/chart.py +1 -1
  276. metadata/generated/schema/entity/data/container.py +1 -1
  277. metadata/generated/schema/entity/data/dashboard.py +1 -1
  278. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  279. metadata/generated/schema/entity/data/database.py +1 -1
  280. metadata/generated/schema/entity/data/databaseSchema.py +1 -1
  281. metadata/generated/schema/entity/data/glossary.py +1 -1
  282. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  283. metadata/generated/schema/entity/data/metric.py +1 -1
  284. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  285. metadata/generated/schema/entity/data/pipeline.py +1 -1
  286. metadata/generated/schema/entity/data/query.py +1 -1
  287. metadata/generated/schema/entity/data/report.py +1 -1
  288. metadata/generated/schema/entity/data/searchIndex.py +13 -1
  289. metadata/generated/schema/entity/data/storedProcedure.py +13 -1
  290. metadata/generated/schema/entity/data/table.py +1 -1
  291. metadata/generated/schema/entity/data/topic.py +1 -1
  292. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  293. metadata/generated/schema/entity/docStore/document.py +1 -1
  294. metadata/generated/schema/entity/domains/__init__.py +1 -1
  295. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  296. metadata/generated/schema/entity/domains/domain.py +1 -1
  297. metadata/generated/schema/entity/events/__init__.py +1 -1
  298. metadata/generated/schema/entity/events/webhook.py +1 -1
  299. metadata/generated/schema/entity/feed/__init__.py +1 -1
  300. metadata/generated/schema/entity/feed/assets.py +1 -1
  301. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  302. metadata/generated/schema/entity/feed/description.py +1 -1
  303. metadata/generated/schema/entity/feed/domain.py +1 -1
  304. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  305. metadata/generated/schema/entity/feed/owner.py +1 -1
  306. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  307. metadata/generated/schema/entity/feed/tag.py +1 -1
  308. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  309. metadata/generated/schema/entity/feed/thread.py +1 -1
  310. metadata/generated/schema/entity/policies/__init__.py +1 -1
  311. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  312. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  313. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  314. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  315. metadata/generated/schema/entity/policies/filters.py +1 -1
  316. metadata/generated/schema/entity/policies/policy.py +1 -1
  317. metadata/generated/schema/entity/services/__init__.py +1 -1
  318. metadata/generated/schema/entity/services/apiService.py +1 -1
  319. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  320. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  321. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  322. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  323. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  324. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  325. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  326. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  327. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  328. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  329. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  330. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  331. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  332. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  333. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
  334. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  335. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  336. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  337. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  338. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  339. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  340. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  341. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  342. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  343. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  344. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  345. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  346. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
  347. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  350. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  352. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
  353. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  360. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  361. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  367. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  372. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  388. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  389. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +6 -2
  393. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +6 -3
  395. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  410. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  411. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  416. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  422. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  423. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  424. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  425. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  427. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  428. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  429. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  430. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  431. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  432. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  435. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  436. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  437. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  438. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  439. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  440. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  441. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  442. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
  443. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
  444. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  446. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  449. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  451. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
  455. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  456. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  458. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  459. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  460. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  461. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  462. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  463. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  464. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  466. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  467. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  469. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  470. metadata/generated/schema/entity/services/dashboardService.py +4 -4
  471. metadata/generated/schema/entity/services/databaseService.py +1 -1
  472. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  473. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
  474. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  475. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  476. metadata/generated/schema/entity/services/messagingService.py +1 -1
  477. metadata/generated/schema/entity/services/metadataService.py +1 -1
  478. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  479. metadata/generated/schema/entity/services/pipelineService.py +1 -1
  480. metadata/generated/schema/entity/services/searchService.py +1 -1
  481. metadata/generated/schema/entity/services/serviceType.py +1 -1
  482. metadata/generated/schema/entity/services/storageService.py +1 -1
  483. metadata/generated/schema/entity/teams/__init__.py +1 -1
  484. metadata/generated/schema/entity/teams/persona.py +1 -1
  485. metadata/generated/schema/entity/teams/role.py +1 -1
  486. metadata/generated/schema/entity/teams/team.py +1 -1
  487. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  488. metadata/generated/schema/entity/teams/user.py +1 -1
  489. metadata/generated/schema/entity/type.py +1 -1
  490. metadata/generated/schema/entity/utils/__init__.py +1 -1
  491. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  492. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  493. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  494. metadata/generated/schema/events/__init__.py +1 -1
  495. metadata/generated/schema/events/alertMetrics.py +1 -1
  496. metadata/generated/schema/events/api/__init__.py +1 -1
  497. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  498. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  499. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  500. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  501. metadata/generated/schema/events/api/typedEvent.py +1 -1
  502. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  503. metadata/generated/schema/events/eventFilterRule.py +1 -1
  504. metadata/generated/schema/events/eventSubscription.py +1 -1
  505. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  506. metadata/generated/schema/events/failedEvent.py +1 -1
  507. metadata/generated/schema/events/failedEventResponse.py +1 -1
  508. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  509. metadata/generated/schema/events/statusContext.py +1 -1
  510. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  511. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  512. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  513. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  514. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  515. metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
  516. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
  517. metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
  518. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  519. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  520. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
  521. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
  522. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
  523. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  524. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
  525. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  526. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
  527. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  528. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
  529. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  530. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
  531. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +2 -2
  532. metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
  533. metadata/generated/schema/governance/workflows/workflowInstance.py +8 -1
  534. metadata/generated/schema/governance/workflows/workflowInstanceState.py +8 -1
  535. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  536. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  537. metadata/generated/schema/metadataIngestion/application.py +1 -1
  538. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  539. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  540. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  541. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +127 -0
  542. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
  543. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -25
  544. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  545. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  546. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  547. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  548. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  549. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  550. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  551. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  552. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
  553. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
  554. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  555. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  556. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  557. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  558. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
  560. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  561. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  562. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  563. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  564. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  565. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  566. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  567. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  568. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  569. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  570. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  571. metadata/generated/schema/metadataIngestion/workflow.py +3 -1
  572. metadata/generated/schema/monitoring/__init__.py +1 -1
  573. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  574. metadata/generated/schema/security/__init__.py +1 -1
  575. metadata/generated/schema/security/client/__init__.py +1 -1
  576. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  577. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  578. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  579. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  580. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  581. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  582. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  583. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  584. metadata/generated/schema/security/credentials/__init__.py +1 -1
  585. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  586. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  587. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  588. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  589. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  590. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  591. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  592. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  593. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  594. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  595. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  596. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  597. metadata/generated/schema/security/sasl/__init__.py +1 -1
  598. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  599. metadata/generated/schema/security/secrets/__init__.py +1 -1
  600. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  601. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  602. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  603. metadata/generated/schema/security/securityConfiguration.py +1 -1
  604. metadata/generated/schema/security/ssl/__init__.py +1 -1
  605. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  606. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  607. metadata/generated/schema/settings/__init__.py +1 -1
  608. metadata/generated/schema/settings/settings.py +1 -1
  609. metadata/generated/schema/system/__init__.py +1 -1
  610. metadata/generated/schema/system/entityError.py +1 -1
  611. metadata/generated/schema/system/eventPublisherJob.py +2 -1
  612. metadata/generated/schema/system/indexingError.py +1 -1
  613. metadata/generated/schema/system/limitsResponse.py +1 -1
  614. metadata/generated/schema/system/ui/__init__.py +1 -1
  615. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  616. metadata/generated/schema/system/ui/page.py +1 -1
  617. metadata/generated/schema/system/validationResponse.py +1 -1
  618. metadata/generated/schema/tests/__init__.py +1 -1
  619. metadata/generated/schema/tests/assigned.py +1 -1
  620. metadata/generated/schema/tests/basic.py +1 -1
  621. metadata/generated/schema/tests/customMetric.py +1 -1
  622. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  623. metadata/generated/schema/tests/resolved.py +1 -1
  624. metadata/generated/schema/tests/testCase.py +1 -1
  625. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  626. metadata/generated/schema/tests/testDefinition.py +1 -1
  627. metadata/generated/schema/tests/testSuite.py +1 -1
  628. metadata/generated/schema/type/__init__.py +1 -1
  629. metadata/generated/schema/type/apiSchema.py +1 -1
  630. metadata/generated/schema/type/assetCertification.py +1 -1
  631. metadata/generated/schema/type/auditLog.py +1 -1
  632. metadata/generated/schema/type/basic.py +1 -1
  633. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  634. metadata/generated/schema/type/changeEvent.py +1 -1
  635. metadata/generated/schema/type/changeEventType.py +1 -1
  636. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  637. metadata/generated/schema/type/csvDocumentation.py +1 -1
  638. metadata/generated/schema/type/csvErrorType.py +1 -1
  639. metadata/generated/schema/type/csvFile.py +1 -1
  640. metadata/generated/schema/type/csvImportResult.py +1 -1
  641. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  642. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  643. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  644. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  645. metadata/generated/schema/type/customProperty.py +1 -1
  646. metadata/generated/schema/type/dailyCount.py +1 -1
  647. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  648. metadata/generated/schema/type/entityHierarchy.py +1 -1
  649. metadata/generated/schema/type/entityHistory.py +1 -1
  650. metadata/generated/schema/type/entityLineage.py +1 -1
  651. metadata/generated/schema/type/entityReference.py +1 -1
  652. metadata/generated/schema/type/entityReferenceList.py +1 -1
  653. metadata/generated/schema/type/entityRelationship.py +1 -1
  654. metadata/generated/schema/type/entityUsage.py +1 -1
  655. metadata/generated/schema/type/filterPattern.py +1 -1
  656. metadata/generated/schema/type/function.py +1 -1
  657. metadata/generated/schema/type/include.py +1 -1
  658. metadata/generated/schema/type/jdbcConnection.py +1 -1
  659. metadata/generated/schema/type/lifeCycle.py +1 -1
  660. metadata/generated/schema/type/paging.py +1 -1
  661. metadata/generated/schema/type/profile.py +1 -1
  662. metadata/generated/schema/type/queryParserData.py +1 -1
  663. metadata/generated/schema/type/reaction.py +1 -1
  664. metadata/generated/schema/type/schedule.py +1 -1
  665. metadata/generated/schema/type/schema.py +1 -1
  666. metadata/generated/schema/type/tableQuery.py +1 -1
  667. metadata/generated/schema/type/tableUsageCount.py +1 -1
  668. metadata/generated/schema/type/tagLabel.py +1 -1
  669. metadata/generated/schema/type/usageDetails.py +1 -1
  670. metadata/generated/schema/type/usageRequest.py +1 -1
  671. metadata/generated/schema/type/votes.py +1 -1
  672. metadata/ingestion/api/models.py +9 -5
  673. metadata/ingestion/models/patch_request.py +1 -0
  674. metadata/ingestion/ometa/client.py +13 -8
  675. metadata/ingestion/ometa/models.py +1 -0
  676. metadata/ingestion/ometa/ometa_api.py +6 -3
  677. metadata/ingestion/sink/metadata_rest.py +23 -18
  678. metadata/ingestion/source/dashboard/microstrategy/client.py +252 -0
  679. metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +7 -10
  680. metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +74 -51
  681. metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +9 -3
  682. metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
  683. metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
  684. metadata/ingestion/source/dashboard/superset/db_source.py +1 -1
  685. metadata/ingestion/source/database/athena/metadata.py +8 -0
  686. metadata/ingestion/source/database/azuresql/service_spec.py +2 -0
  687. metadata/ingestion/source/database/bigquery/profiler/profiler.py +1 -1
  688. metadata/ingestion/source/database/bigquery/service_spec.py +2 -0
  689. metadata/ingestion/source/database/common_nosql_source.py +7 -0
  690. metadata/ingestion/source/database/databricks/metadata.py +1 -3
  691. metadata/ingestion/source/database/databricks/service_spec.py +4 -0
  692. metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
  693. metadata/ingestion/source/database/datalake/clients/base.py +10 -1
  694. metadata/ingestion/source/database/datalake/clients/gcs.py +4 -2
  695. metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
  696. metadata/ingestion/source/database/datalake/service_spec.py +8 -1
  697. metadata/ingestion/source/database/dbt/constants.py +4 -0
  698. metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
  699. metadata/ingestion/source/database/dbt/dbt_service.py +8 -1
  700. metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
  701. metadata/ingestion/source/database/dbt/metadata.py +70 -1
  702. metadata/ingestion/source/database/dbt/models.py +2 -0
  703. metadata/ingestion/source/database/dynamodb/service_spec.py +4 -1
  704. metadata/ingestion/source/database/mongodb/service_spec.py +4 -1
  705. metadata/ingestion/source/database/mysql/connection.py +5 -0
  706. metadata/ingestion/source/database/mysql/lineage.py +12 -30
  707. metadata/ingestion/source/database/mysql/queries.py +43 -0
  708. metadata/ingestion/source/database/mysql/query_parser.py +46 -0
  709. metadata/ingestion/source/database/mysql/service_spec.py +7 -1
  710. metadata/{data_quality/builders/pandas_validator_builder.py → ingestion/source/database/mysql/usage.py} +9 -14
  711. metadata/ingestion/source/database/sample_data.py +4 -12
  712. metadata/ingestion/source/database/snowflake/connection.py +1 -1
  713. metadata/ingestion/source/database/snowflake/service_spec.py +6 -0
  714. metadata/ingestion/source/database/trino/service_spec.py +2 -0
  715. metadata/ingestion/source/database/unitycatalog/service_spec.py +4 -0
  716. metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
  717. metadata/ingestion/source/pipeline/airflow/metadata.py +35 -25
  718. metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
  719. metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
  720. metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
  721. metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
  722. metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
  723. metadata/ingestion/source/search/search_service.py +44 -1
  724. metadata/mixins/pandas/pandas_mixin.py +2 -31
  725. metadata/mixins/sqalchemy/sqa_mixin.py +16 -16
  726. metadata/pii/processor.py +10 -9
  727. metadata/profiler/api/models.py +3 -79
  728. metadata/profiler/config.py +39 -0
  729. metadata/profiler/interface/nosql/profiler_interface.py +1 -26
  730. metadata/profiler/interface/pandas/profiler_interface.py +37 -77
  731. metadata/profiler/interface/profiler_interface.py +10 -282
  732. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
  733. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
  734. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
  735. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +2 -2
  736. metadata/profiler/interface/sqlalchemy/profiler_interface.py +46 -109
  737. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  738. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
  739. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +6 -8
  740. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  741. metadata/profiler/orm/converter/base.py +21 -12
  742. metadata/profiler/orm/functions/table_metric_computer.py +5 -4
  743. metadata/profiler/processor/core.py +5 -58
  744. metadata/profiler/processor/handle_partition.py +0 -48
  745. metadata/profiler/processor/runner.py +111 -35
  746. metadata/profiler/processor/sample_data_handler.py +7 -4
  747. metadata/profiler/source/database/base/profiler_source.py +57 -138
  748. metadata/profiler/source/database/bigquery/profiler_source.py +3 -3
  749. metadata/profiler/source/database/databricks/profiler_source.py +2 -3
  750. metadata/profiler/source/fetcher/config.py +44 -0
  751. metadata/profiler/source/fetcher/fetcher_strategy.py +2 -4
  752. metadata/sampler/config.py +237 -0
  753. metadata/sampler/models.py +106 -0
  754. metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
  755. metadata/sampler/pandas/sampler.py +239 -0
  756. metadata/{utils → sampler}/partition.py +66 -51
  757. metadata/sampler/processor.py +158 -0
  758. metadata/sampler/sampler_interface.py +251 -0
  759. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
  760. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
  761. metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
  762. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +90 -98
  763. metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
  764. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
  765. metadata/utils/constants.py +63 -0
  766. metadata/utils/helpers.py +1 -2
  767. metadata/utils/logger.py +9 -0
  768. metadata/utils/profiler_utils.py +42 -0
  769. metadata/utils/service_spec/default.py +6 -0
  770. metadata/utils/service_spec/service_spec.py +47 -0
  771. metadata/utils/sqlalchemy_utils.py +1 -1
  772. metadata/utils/ssl_manager.py +39 -1
  773. metadata/workflow/classification.py +53 -0
  774. metadata/workflow/profiler.py +1 -17
  775. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/METADATA +366 -366
  776. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/RECORD +781 -771
  777. metadata/data_quality/builders/sqa_validator_builder.py +0 -25
  778. metadata/data_quality/interface/test_suite_interface_factory.py +0 -158
  779. metadata/data_quality/runner/test_suite_source_factory.py +0 -38
  780. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
  781. metadata/ingestion/source/dashboard/mstr/client.py +0 -209
  782. metadata/ingestion/source/dashboard/mstr/service_spec.py +0 -4
  783. metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
  784. metadata/profiler/processor/sampler/sampler_factory.py +0 -100
  785. metadata/profiler/processor/sampler/sampler_interface.py +0 -74
  786. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -83
  787. /metadata/ingestion/source/dashboard/{mstr → microstrategy}/__init__.py +0 -0
  788. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/LICENSE +0 -0
  789. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/WHEEL +0 -0
  790. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/entry_points.txt +0 -0
  791. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -8,7 +8,7 @@
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
- """Mstr source module"""
11
+ """MicroStrategy source module"""
12
12
  import traceback
13
13
  from typing import Iterable, List, Optional
14
14
 
@@ -16,8 +16,8 @@ from metadata.generated.schema.api.data.createChart import CreateChartRequest
16
16
  from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
17
17
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
18
18
  from metadata.generated.schema.entity.data.chart import Chart
19
- from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
20
- MstrConnection,
19
+ from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
20
+ MicroStrategyConnection,
21
21
  )
22
22
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
23
23
  StackTraceError,
@@ -34,7 +34,7 @@ from metadata.ingestion.api.models import Either
34
34
  from metadata.ingestion.api.steps import InvalidSourceException
35
35
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
36
36
  from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
37
- from metadata.ingestion.source.dashboard.mstr.models import (
37
+ from metadata.ingestion.source.dashboard.microstrategy.models import (
38
38
  MstrDashboard,
39
39
  MstrDashboardDetails,
40
40
  MstrPage,
@@ -47,9 +47,9 @@ from metadata.utils.logger import ingestion_logger
47
47
  logger = ingestion_logger()
48
48
 
49
49
 
50
- class MstrSource(DashboardServiceSource):
50
+ class MicrostrategySource(DashboardServiceSource):
51
51
  """
52
- MSTR Source Class
52
+ Microstrategy Source Class
53
53
  """
54
54
 
55
55
  @classmethod
@@ -60,10 +60,10 @@ class MstrSource(DashboardServiceSource):
60
60
  pipeline_name: Optional[str] = None,
61
61
  ):
62
62
  config = WorkflowSource.model_validate(config_dict)
63
- connection: MstrConnection = config.serviceConnection.root.config
64
- if not isinstance(connection, MstrConnection):
63
+ connection: MicroStrategyConnection = config.serviceConnection.root.config
64
+ if not isinstance(connection, MicroStrategyConnection):
65
65
  raise InvalidSourceException(
66
- f"Expected MstrConnection, but got {connection}"
66
+ f"Expected MicroStrategyConnection, but got {connection}"
67
67
  )
68
68
  return cls(config, metadata)
69
69
 
@@ -75,14 +75,18 @@ class MstrSource(DashboardServiceSource):
75
75
 
76
76
  if self.client.is_project_name():
77
77
  project = self.client.get_project_by_name()
78
- dashboards.extend(self.client.get_dashboards_list(project.id, project.name))
79
-
80
- if not self.client.is_project_name():
81
- for project in self.client.get_projects_list():
78
+ if project:
82
79
  dashboards.extend(
83
80
  self.client.get_dashboards_list(project.id, project.name)
84
81
  )
85
82
 
83
+ if not self.client.is_project_name():
84
+ for project in self.client.get_projects_list():
85
+ if project:
86
+ dashboards.extend(
87
+ self.client.get_dashboards_list(project.id, project.name)
88
+ )
89
+
86
90
  return dashboards
87
91
 
88
92
  def get_dashboard_name(self, dashboard: MstrDashboard) -> str:
@@ -91,6 +95,18 @@ class MstrSource(DashboardServiceSource):
91
95
  """
92
96
  return dashboard.name
93
97
 
98
+ def get_project_name(self, dashboard_details: MstrDashboard) -> Optional[str]:
99
+ """
100
+ Get dashboard project name
101
+ """
102
+ try:
103
+ return dashboard_details.projectName
104
+ except Exception as exc:
105
+ logger.debug(
106
+ f"Cannot get project name from dashboard [{dashboard_details.name}] - [{exc}]"
107
+ )
108
+ return None
109
+
94
110
  def get_dashboard_details(self, dashboard: MstrDashboard) -> MstrDashboardDetails:
95
111
  """
96
112
  Get Dashboard Details
@@ -106,40 +122,41 @@ class MstrSource(DashboardServiceSource):
106
122
  """
107
123
  Method to Get Dashboard Entity
108
124
  """
109
- try:
110
- dashboard_url = (
111
- f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
112
- f"{dashboard_details.projectId}/{dashboard_details.id}"
113
- )
114
- dashboard_request = CreateDashboardRequest(
115
- name=EntityName(dashboard_details.id),
116
- displayName=dashboard_details.name,
117
- sourceUrl=SourceUrl(dashboard_url),
118
- project=dashboard_details.projectName,
119
- charts=[
120
- FullyQualifiedEntityName(
121
- fqn.build(
122
- self.metadata,
123
- entity_type=Chart,
124
- service_name=self.context.get().dashboard_service,
125
- chart_name=chart,
125
+ if dashboard_details:
126
+ try:
127
+ dashboard_url = (
128
+ f"{clean_uri(str(self.service_connection.hostPort))}/MicroStrategyLibrary/app/"
129
+ f"{dashboard_details.projectId}/{dashboard_details.id}"
130
+ )
131
+ dashboard_request = CreateDashboardRequest(
132
+ name=EntityName(dashboard_details.id),
133
+ displayName=dashboard_details.name,
134
+ sourceUrl=SourceUrl(dashboard_url),
135
+ project=dashboard_details.projectName,
136
+ charts=[
137
+ FullyQualifiedEntityName(
138
+ fqn.build(
139
+ self.metadata,
140
+ entity_type=Chart,
141
+ service_name=self.context.get().dashboard_service,
142
+ chart_name=chart,
143
+ )
126
144
  )
145
+ for chart in self.context.get().charts or []
146
+ ],
147
+ service=self.context.get().dashboard_service,
148
+ owners=self.get_owner_ref(dashboard_details=dashboard_details),
149
+ )
150
+ yield Either(right=dashboard_request)
151
+ self.register_record(dashboard_request=dashboard_request)
152
+ except Exception as exc:
153
+ yield Either(
154
+ left=StackTraceError(
155
+ name=dashboard_details.id,
156
+ error=f"Error yielding dashboard for {dashboard_details}: {exc}",
157
+ stackTrace=traceback.format_exc(),
127
158
  )
128
- for chart in self.context.get().charts or []
129
- ],
130
- service=self.context.get().dashboard_service,
131
- owners=self.get_owner_ref(dashboard_details=dashboard_details),
132
- )
133
- yield Either(right=dashboard_request)
134
- self.register_record(dashboard_request=dashboard_request)
135
- except Exception as exc:
136
- yield Either(
137
- left=StackTraceError(
138
- name=dashboard_details.id,
139
- error=f"Error yielding dashboard for {dashboard_details}: {exc}",
140
- stackTrace=traceback.format_exc(),
141
159
  )
142
- )
143
160
 
144
161
  def yield_dashboard_lineage_details(
145
162
  self, dashboard_details: MstrDashboardDetails, db_service_name: str
@@ -156,14 +173,15 @@ class MstrSource(DashboardServiceSource):
156
173
  Returns:
157
174
  Iterable[CreateChartRequest]
158
175
  """
159
- try:
160
- for chapter in dashboard_details.chapters:
161
- for page in chapter.pages:
162
- yield from self._yield_chart_from_visualization(page)
176
+ if dashboard_details:
177
+ try:
178
+ for chapter in dashboard_details.chapters:
179
+ for page in chapter.pages:
180
+ yield from self._yield_chart_from_visualization(page)
163
181
 
164
- except Exception as exc:
165
- logger.debug(traceback.format_exc())
166
- logger.warning(f"Error creating dashboard: {exc}")
182
+ except Exception as exc:
183
+ logger.debug(traceback.format_exc())
184
+ logger.warning(f"Error creating dashboard: {exc}")
167
185
 
168
186
  def _yield_chart_from_visualization(
169
187
  self, page: MstrPage
@@ -192,3 +210,8 @@ class MstrSource(DashboardServiceSource):
192
210
  stackTrace=traceback.format_exc(),
193
211
  )
194
212
  )
213
+
214
+ def close(self):
215
+ # close the api session
216
+ self.client.close_api_session()
217
+ self.metadata.close()
@@ -9,10 +9,10 @@
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
11
  """
12
- MSTR Models
12
+ MicroStrategy Models
13
13
  """
14
14
  from datetime import datetime
15
- from typing import List, Optional
15
+ from typing import Any, List, Optional
16
16
 
17
17
  from pydantic import BaseModel
18
18
 
@@ -69,7 +69,8 @@ class MstrSearchResult(BaseModel):
69
69
 
70
70
 
71
71
  class MstrSearchResultList(BaseModel):
72
- results: Optional[List[MstrSearchResult]]
72
+ totalItems: Optional[int] = 0
73
+ result: Optional[List[MstrSearchResult]] = None
73
74
 
74
75
 
75
76
  class MstrDashboard(BaseModel):
@@ -142,3 +143,8 @@ class MstrDashboardDetails(BaseModel):
142
143
  projectName: str
143
144
  currentChapter: str
144
145
  chapters: List[MstrChapter]
146
+
147
+
148
+ class AuthHeaderCookie(BaseModel):
149
+ auth_header: dict
150
+ auth_cookies: Any
@@ -0,0 +1,6 @@
1
+ from metadata.ingestion.source.dashboard.microstrategy.metadata import (
2
+ MicrostrategySource,
3
+ )
4
+ from metadata.utils.service_spec.default import DefaultDatabaseSpec
5
+
6
+ ServiceSpec = DefaultDatabaseSpec(metadata_source_class=MicrostrategySource)
@@ -64,11 +64,10 @@ class PowerBiApiClient:
64
64
  client_credential=self.config.clientSecret.get_secret_value(),
65
65
  authority=self.config.authorityURI + self.config.tenantId,
66
66
  )
67
- self.auth_token = self.get_auth_token()
68
67
  client_config = ClientConfig(
69
68
  base_url="https://api.powerbi.com",
70
69
  api_version="v1.0",
71
- auth_token=lambda: self.auth_token,
70
+ auth_token=self.get_auth_token,
72
71
  auth_header="Authorization",
73
72
  allow_redirects=True,
74
73
  retry_codes=[429],
@@ -201,7 +201,7 @@ class SupersetDBSource(SupersetSourceMixin):
201
201
  except Exception as exc:
202
202
  yield Either(
203
203
  left=StackTraceError(
204
- name=chart_json.id,
204
+ name=str(chart_json.id),
205
205
  error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
206
206
  stackTrace=traceback.format_exc(),
207
207
  )
@@ -157,6 +157,14 @@ class AthenaSource(ExternalTableLineageMixin, CommonDbSourceService):
157
157
  return True, partition_details
158
158
  return False, None
159
159
 
160
+ def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]:
161
+ """
162
+ Method to fetch the location path of the table
163
+ """
164
+ return self.external_location_map.get(
165
+ (self.context.get().database, schema_name, table_name)
166
+ )
167
+
160
168
  def yield_tag(
161
169
  self, schema_name: str
162
170
  ) -> Iterable[Either[OMetaTagAndClassification]]:
@@ -1,10 +1,12 @@
1
1
  from metadata.ingestion.source.database.azuresql.lineage import AzuresqlLineageSource
2
2
  from metadata.ingestion.source.database.azuresql.metadata import AzuresqlSource
3
3
  from metadata.ingestion.source.database.azuresql.usage import AzuresqlUsageSource
4
+ from metadata.sampler.sqlalchemy.azuresql.sampler import AzureSQLSampler
4
5
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
5
6
 
6
7
  ServiceSpec = DefaultDatabaseSpec(
7
8
  metadata_source_class=AzuresqlSource,
8
9
  lineage_source_class=AzuresqlLineageSource,
9
10
  usage_source_class=AzuresqlUsageSource,
11
+ sampler_class=AzureSQLSampler,
10
12
  )
@@ -22,7 +22,7 @@ class BigQueryProfiler(BigQueryProfilerInterface):
22
22
  **kwargs,
23
23
  ) -> List[SystemProfile]:
24
24
  return self.system_metrics_computer.get_system_metrics(
25
- table=runner.table,
25
+ table=runner.dataset,
26
26
  usage_location=self.service_connection_config.usageLocation,
27
27
  )
28
28
 
@@ -4,6 +4,7 @@ from metadata.ingestion.source.database.bigquery.profiler.profiler import (
4
4
  BigQueryProfiler,
5
5
  )
6
6
  from metadata.ingestion.source.database.bigquery.usage import BigqueryUsageSource
7
+ from metadata.sampler.sqlalchemy.bigquery.sampler import BigQuerySampler
7
8
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
8
9
 
9
10
  ServiceSpec = DefaultDatabaseSpec(
@@ -11,4 +12,5 @@ ServiceSpec = DefaultDatabaseSpec(
11
12
  lineage_source_class=BigqueryLineageSource,
12
13
  usage_source_class=BigqueryUsageSource,
13
14
  profiler_class=BigQueryProfiler,
15
+ sampler_class=BigQuerySampler,
14
16
  )
@@ -52,6 +52,7 @@ from metadata.utils.constants import DEFAULT_DATABASE
52
52
  from metadata.utils.datalake.datalake_utils import DataFrameColumnParser
53
53
  from metadata.utils.filters import filter_by_schema, filter_by_table
54
54
  from metadata.utils.logger import ingestion_logger
55
+ from metadata.utils.ssl_manager import check_ssl_and_init
55
56
 
56
57
  logger = ingestion_logger()
57
58
 
@@ -73,7 +74,13 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
73
74
  )
74
75
  self.metadata = metadata
75
76
  self.service_connection = self.config.serviceConnection.root.config
77
+ self.ssl_manager = check_ssl_and_init(self.service_connection)
78
+ if self.ssl_manager:
79
+ self.service_connection = self.ssl_manager.setup_ssl(
80
+ self.service_connection
81
+ )
76
82
  self.connection_obj = get_connection(self.service_connection)
83
+
77
84
  self.test_connection()
78
85
 
79
86
  def prepare(self):
@@ -131,7 +131,7 @@ def _get_table_columns(self, connection, table_name, schema, db_name):
131
131
  query = DATABRICKS_GET_TABLE_COMMENTS.format(
132
132
  database_name=db_name, schema_name=schema, table_name=table_name
133
133
  )
134
- cursor = get_table_comment_result(
134
+ rows = get_table_comment_result(
135
135
  self,
136
136
  connection=connection,
137
137
  query=query,
@@ -140,8 +140,6 @@ def _get_table_columns(self, connection, table_name, schema, db_name):
140
140
  schema=schema,
141
141
  )
142
142
 
143
- rows = cursor.fetchall()
144
-
145
143
  except exc.OperationalError as e:
146
144
  # Does the table exist?
147
145
  regex_fmt = r"TExecuteStatementResp.*SemanticException.*Table not found {}"
@@ -1,3 +1,6 @@
1
+ from metadata.data_quality.interface.sqlalchemy.databricks.test_suite_interface import (
2
+ DatabricksTestSuiteInterface,
3
+ )
1
4
  from metadata.ingestion.source.database.databricks.lineage import (
2
5
  DatabricksLineageSource,
3
6
  )
@@ -13,4 +16,5 @@ ServiceSpec = DefaultDatabaseSpec(
13
16
  lineage_source_class=DatabricksLineageSource,
14
17
  usage_source_class=DatabricksUsageSource,
15
18
  profiler_class=DatabricksProfilerInterface,
19
+ test_suite_class=DatabricksTestSuiteInterface,
16
20
  )
@@ -26,9 +26,6 @@ from metadata.utils.constants import DEFAULT_DATABASE
26
26
 
27
27
 
28
28
  class DatalakeAzureBlobClient(DatalakeBaseClient):
29
- def __init__(self, client: BlobServiceClient):
30
- self._client = client
31
-
32
29
  @classmethod
33
30
  def from_config(cls, config: AzureConfig) -> "DatalakeAzureBlobClient":
34
31
  try:
@@ -13,10 +13,19 @@
13
13
  Datalake Base Client
14
14
  """
15
15
  from abc import ABC, abstractmethod
16
- from typing import Callable, Iterable, Optional
16
+ from typing import Any, Callable, Iterable, Optional
17
17
 
18
18
 
19
19
  class DatalakeBaseClient(ABC):
20
+ """Base DL client implementation"""
21
+
22
+ def __init__(self, client: Any, **kwargs):
23
+ self._client = client
24
+
25
+ @property
26
+ def client(self) -> Any:
27
+ return self._client
28
+
20
29
  @classmethod
21
30
  @abstractmethod
22
31
  def from_config(cls, config) -> "DatalakeBaseClient":
@@ -33,9 +33,11 @@ from metadata.utils.credentials import GOOGLE_CREDENTIALS, set_google_credential
33
33
 
34
34
  class DatalakeGcsClient(DatalakeBaseClient):
35
35
  def __init__(
36
- self, client: storage.Client, temp_credentials_file_path_list: List[str]
36
+ self,
37
+ client: storage.Client,
38
+ temp_credentials_file_path_list: List[str],
37
39
  ):
38
- self._client = client
40
+ super().__init__(client=client)
39
41
  self._temp_credentials_file_path_list = temp_credentials_file_path_list
40
42
 
41
43
  @property
@@ -25,9 +25,6 @@ from metadata.utils.s3_utils import list_s3_objects
25
25
 
26
26
 
27
27
  class DatalakeS3Client(DatalakeBaseClient):
28
- def __init__(self, client):
29
- self._client = client
30
-
31
28
  @classmethod
32
29
  def from_config(cls, config: S3Config) -> "DatalakeS3Client":
33
30
  if not config.securityConfig:
@@ -1,9 +1,16 @@
1
+ from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
2
+ PandasTestSuiteInterface,
3
+ )
1
4
  from metadata.ingestion.source.database.datalake.metadata import DatalakeSource
2
5
  from metadata.profiler.interface.pandas.profiler_interface import (
3
6
  PandasProfilerInterface,
4
7
  )
8
+ from metadata.sampler.pandas.sampler import DatalakeSampler
5
9
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
6
10
 
7
11
  ServiceSpec = DefaultDatabaseSpec(
8
- metadata_source_class=DatalakeSource, profiler_class=PandasProfilerInterface
12
+ metadata_source_class=DatalakeSource,
13
+ profiler_class=PandasProfilerInterface,
14
+ test_suite_class=PandasTestSuiteInterface,
15
+ sampler_class=DatalakeSampler,
9
16
  )
@@ -82,6 +82,7 @@ NONE_KEYWORDS_LIST = ["none", "null"]
82
82
  DBT_CATALOG_FILE_NAME = "catalog.json"
83
83
  DBT_MANIFEST_FILE_NAME = "manifest.json"
84
84
  DBT_RUN_RESULTS_FILE_NAME = "run_results"
85
+ DBT_SOURCES_FILE_NAME = "sources.json"
85
86
 
86
87
 
87
88
  class SkipResourceTypeEnum(Enum):
@@ -91,6 +92,7 @@ class SkipResourceTypeEnum(Enum):
91
92
 
92
93
  ANALYSIS = "analysis"
93
94
  TEST = "test"
95
+ SOURCE = "source"
94
96
 
95
97
 
96
98
  class CompiledQueriesEnum(Enum):
@@ -127,6 +129,7 @@ class DbtTestFailureEnum(Enum):
127
129
 
128
130
  FAILURE = "failure"
129
131
  FAIL = "fail"
132
+ ERROR = "error"
130
133
 
131
134
 
132
135
  class DbtCommonEnum(Enum):
@@ -137,6 +140,7 @@ class DbtCommonEnum(Enum):
137
140
  OWNER = "owner"
138
141
  NODES = "nodes"
139
142
  SOURCES = "sources"
143
+ SOURCES_FILE = "sources_file"
140
144
  SOURCE = "source"
141
145
  RESOURCETYPE = "resource_type"
142
146
  MANIFEST_NODE = "manifest_node"
@@ -43,6 +43,7 @@ from metadata.ingestion.source.database.dbt.constants import (
43
43
  DBT_CATALOG_FILE_NAME,
44
44
  DBT_MANIFEST_FILE_NAME,
45
45
  DBT_RUN_RESULTS_FILE_NAME,
46
+ DBT_SOURCES_FILE_NAME,
46
47
  )
47
48
  from metadata.ingestion.source.database.dbt.models import DbtFiles
48
49
  from metadata.readers.file.config_source_factory import get_reader
@@ -85,6 +86,7 @@ def _(config: DbtLocalConfig):
85
86
  config.dbtManifestFilePath,
86
87
  config.dbtCatalogFilePath,
87
88
  config.dbtRunResultsFilePath,
89
+ config.dbtSourcesFilePath,
88
90
  ]
89
91
  yield from download_dbt_files(
90
92
  blob_grouped_by_directory=blob_grouped_by_directory,
@@ -123,12 +125,22 @@ def _(config: DbtHttpConfig):
123
125
  dbt_catalog = requests.get( # pylint: disable=missing-timeout
124
126
  config.dbtCatalogHttpPath
125
127
  )
128
+
129
+ dbt_sources = None
130
+ if config.dbtSourcesHttpPath:
131
+ logger.debug(
132
+ f"Requesting [dbtSourcesHttpPath] to: {config.dbtSourcesHttpPath}"
133
+ )
134
+ dbt_sources = requests.get( # pylint: disable=missing-timeout
135
+ config.dbtSourcesHttpPath
136
+ )
126
137
  if not dbt_manifest:
127
138
  raise DBTConfigException("Manifest file not found in file server")
128
139
  yield DbtFiles(
129
140
  dbt_catalog=dbt_catalog.json() if dbt_catalog else None,
130
141
  dbt_manifest=dbt_manifest.json(),
131
142
  dbt_run_results=[dbt_run_results.json()] if dbt_run_results else None,
143
+ dbt_sources=dbt_sources.json() if dbt_sources else None,
132
144
  )
133
145
  except DBTConfigException as exc:
134
146
  raise exc
@@ -164,7 +176,7 @@ def _(config: DbtCloudConfig): # pylint: disable=too-many-locals
164
176
  logger.debug(
165
177
  "Requesting [dbt_catalog], [dbt_manifest] and [dbt_run_results] data"
166
178
  )
167
- params_data = {"order_by": "-finished_at", "limit": "1", "status": "10"}
179
+ params_data = {"order_by": "-finished_at", "limit": "1"}
168
180
  if project_id:
169
181
  params_data["project_id"] = project_id
170
182
 
@@ -243,6 +255,7 @@ def get_blobs_grouped_by_dir(blobs: List[str]) -> Dict[str, List[str]]:
243
255
  return blob_grouped_by_directory
244
256
 
245
257
 
258
+ # pylint: disable=too-many-locals, too-many-branches
246
259
  def download_dbt_files(
247
260
  blob_grouped_by_directory: Dict, config, client, bucket_name: Optional[str]
248
261
  ) -> Iterable[DbtFiles]:
@@ -255,6 +268,7 @@ def download_dbt_files(
255
268
  ) in blob_grouped_by_directory.items():
256
269
  dbt_catalog = None
257
270
  dbt_manifest = None
271
+ dbt_sources = None
258
272
  dbt_run_results = []
259
273
  kwargs = {}
260
274
  if bucket_name:
@@ -285,12 +299,16 @@ def download_dbt_files(
285
299
  logger.warning(
286
300
  f"{DBT_RUN_RESULTS_FILE_NAME} not found in {key}: {exc}"
287
301
  )
302
+ if DBT_SOURCES_FILE_NAME == blob_file_name.lower():
303
+ logger.debug(f"{DBT_SOURCES_FILE_NAME} found in {key}")
304
+ dbt_sources = reader.read(path=blob, **kwargs)
288
305
  if not dbt_manifest:
289
306
  raise DBTConfigException(f"Manifest file not found at: {key}")
290
307
  yield DbtFiles(
291
308
  dbt_catalog=json.loads(dbt_catalog) if dbt_catalog else None,
292
309
  dbt_manifest=json.loads(dbt_manifest),
293
310
  dbt_run_results=dbt_run_results if dbt_run_results else None,
311
+ dbt_sources=json.loads(dbt_sources) if dbt_sources else None,
294
312
  )
295
313
  except DBTConfigException as exc:
296
314
  logger.warning(exc)
@@ -15,7 +15,12 @@ DBT service Topology.
15
15
  from abc import ABC, abstractmethod
16
16
  from typing import Iterable, List
17
17
 
18
- from dbt_artifacts_parser.parser import parse_catalog, parse_manifest, parse_run_results
18
+ from dbt_artifacts_parser.parser import (
19
+ parse_catalog,
20
+ parse_manifest,
21
+ parse_run_results,
22
+ parse_sources,
23
+ )
19
24
  from pydantic import Field
20
25
  from typing_extensions import Annotated
21
26
 
@@ -209,11 +214,13 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
209
214
  self.remove_run_result_non_required_keys(
210
215
  run_results=self.context.get().dbt_file.dbt_run_results
211
216
  )
217
+
212
218
  dbt_objects = DbtObjects(
213
219
  dbt_catalog=parse_catalog(self.context.get().dbt_file.dbt_catalog)
214
220
  if self.context.get().dbt_file.dbt_catalog
215
221
  else None,
216
222
  dbt_manifest=parse_manifest(self.context.get().dbt_file.dbt_manifest),
223
+ dbt_sources=parse_sources(self.context.get().dbt_file.dbt_sources),
217
224
  dbt_run_results=[
218
225
  parse_run_results(run_result_file)
219
226
  for run_result_file in self.context.get().dbt_file.dbt_run_results
@@ -44,6 +44,20 @@ def create_test_case_parameter_definitions(dbt_test):
44
44
  }
45
45
  ]
46
46
  return test_case_param_definition
47
+ if hasattr(dbt_test, "freshness"):
48
+ test_case_param_definition = [
49
+ {
50
+ "name": "warn_after",
51
+ "displayName": "warn_after",
52
+ "required": False,
53
+ },
54
+ {
55
+ "name": "error_after",
56
+ "displayName": "error_after",
57
+ "required": False,
58
+ },
59
+ ]
60
+ return test_case_param_definition
47
61
  except Exception as err: # pylint: disable=broad-except
48
62
  logger.debug(traceback.format_exc())
49
63
  logger.error(
@@ -67,6 +81,21 @@ def create_test_case_parameter_values(dbt_test):
67
81
  {"name": manifest_node.test_metadata.name, "value": dbt_test_values}
68
82
  ]
69
83
  return test_case_param_values
84
+ if hasattr(manifest_node, "freshness"):
85
+ warn_after = manifest_node.freshness.warn_after
86
+ error_after = manifest_node.freshness.error_after
87
+
88
+ test_case_param_values = [
89
+ {
90
+ "name": "error_after",
91
+ "value": f"{error_after.count} {error_after.period.value}",
92
+ },
93
+ {
94
+ "name": "warn_after",
95
+ "value": f"{warn_after.count} {warn_after.period.value}",
96
+ },
97
+ ]
98
+ return test_case_param_values
70
99
  except Exception as err: # pylint: disable=broad-except
71
100
  logger.debug(traceback.format_exc())
72
101
  logger.error(