openmetadata-ingestion 1.5.13.0__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1009) hide show
  1. _openmetadata_testutils/dict.py +23 -0
  2. _openmetadata_testutils/pydantic/test_utils.py +22 -2
  3. _openmetadata_testutils/pytest_openmetadata/plugin.py +86 -0
  4. _openmetadata_testutils/pytest_openmetadata/test_utils.py +99 -0
  5. metadata/cli/app.py +1 -0
  6. metadata/cli/classify.py +52 -0
  7. metadata/cli/dataquality.py +4 -1
  8. metadata/cli/ingest.py +2 -1
  9. metadata/cli/lineage.py +2 -1
  10. metadata/cli/profile.py +4 -1
  11. metadata/cli/usage.py +2 -1
  12. metadata/cmd.py +9 -0
  13. metadata/config/common.py +3 -1
  14. metadata/data_quality/api/models.py +2 -1
  15. metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +45 -31
  16. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -24
  17. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -49
  18. metadata/data_quality/interface/test_suite_interface.py +36 -52
  19. metadata/data_quality/processor/test_case_runner.py +9 -9
  20. metadata/data_quality/runner/base_test_suite_source.py +52 -26
  21. metadata/data_quality/source/test_suite.py +1 -1
  22. metadata/data_quality/validations/base_test_handler.py +13 -8
  23. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +4 -2
  24. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
  25. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  26. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  27. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  28. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  29. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  30. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  31. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  32. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
  33. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
  34. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  35. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  36. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  37. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
  38. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  39. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  40. metadata/data_quality/validations/models.py +6 -1
  41. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
  42. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
  43. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +26 -16
  44. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +192 -17
  45. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
  46. metadata/examples/workflows/bigquery_classifier.yaml +56 -0
  47. metadata/examples/workflows/bigquery_profiler.yaml +1 -2
  48. metadata/examples/workflows/datafactory.yaml +29 -0
  49. metadata/examples/workflows/db2_profiler.yaml +1 -2
  50. metadata/examples/workflows/dbtcloud.yaml +2 -1
  51. metadata/examples/workflows/matillion.yaml +31 -0
  52. metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
  53. metadata/examples/workflows/mysql_profiler.yaml +0 -1
  54. metadata/examples/workflows/redshift_classifier.yaml +38 -0
  55. metadata/examples/workflows/redshift_profiler.yaml +2 -3
  56. metadata/examples/workflows/rest.yaml +20 -0
  57. metadata/examples/workflows/sigma.yaml +25 -0
  58. metadata/examples/workflows/trino_profiler.yaml +20 -0
  59. metadata/generated/antlr/EntityLinkLexer.py +70 -99
  60. metadata/generated/schema/analytics/__init__.py +1 -1
  61. metadata/generated/schema/analytics/basic.py +1 -1
  62. metadata/generated/schema/analytics/reportData.py +1 -1
  63. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  64. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  65. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  66. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  67. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  68. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  69. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  70. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  71. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  72. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  73. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  74. metadata/generated/schema/api/__init__.py +1 -1
  75. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  76. metadata/generated/schema/api/addTagToAssetsRequest.py +43 -0
  77. metadata/generated/schema/api/analytics/__init__.py +1 -1
  78. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  79. metadata/generated/schema/api/automations/__init__.py +1 -1
  80. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  81. metadata/generated/schema/api/bulkAssets.py +1 -1
  82. metadata/generated/schema/api/classification/__init__.py +1 -1
  83. metadata/generated/schema/api/classification/createClassification.py +1 -1
  84. metadata/generated/schema/api/classification/createTag.py +1 -1
  85. metadata/generated/schema/api/classification/loadTags.py +1 -1
  86. metadata/generated/schema/api/createBot.py +1 -1
  87. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  88. metadata/generated/schema/api/createType.py +1 -1
  89. metadata/generated/schema/api/data/__init__.py +1 -1
  90. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  91. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  92. metadata/generated/schema/api/data/createChart.py +1 -1
  93. metadata/generated/schema/api/data/createContainer.py +1 -1
  94. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  95. metadata/generated/schema/api/data/createDashboard.py +1 -1
  96. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  97. metadata/generated/schema/api/data/createDatabase.py +1 -1
  98. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  99. metadata/generated/schema/api/data/createGlossary.py +1 -1
  100. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  101. metadata/generated/schema/api/data/createMetric.py +83 -0
  102. metadata/generated/schema/api/data/createMlModel.py +1 -1
  103. metadata/generated/schema/api/data/createPipeline.py +1 -1
  104. metadata/generated/schema/api/data/createQuery.py +1 -1
  105. metadata/generated/schema/api/data/createSearchIndex.py +8 -1
  106. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  107. metadata/generated/schema/api/data/createTable.py +1 -1
  108. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  109. metadata/generated/schema/api/data/createTopic.py +1 -1
  110. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  111. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  112. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  113. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  114. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  115. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  116. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  117. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  118. metadata/generated/schema/api/docStore/__init__.py +1 -1
  119. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  120. metadata/generated/schema/api/domains/__init__.py +1 -1
  121. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  122. metadata/generated/schema/api/domains/createDomain.py +1 -1
  123. metadata/generated/schema/api/feed/__init__.py +1 -1
  124. metadata/generated/schema/api/feed/closeTask.py +1 -1
  125. metadata/generated/schema/api/feed/createPost.py +1 -1
  126. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  127. metadata/generated/schema/api/feed/createThread.py +1 -1
  128. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  129. metadata/generated/schema/api/feed/threadCount.py +1 -1
  130. metadata/generated/schema/api/governance/__init__.py +3 -0
  131. metadata/generated/schema/api/governance/createWorkflowDefinition.py +83 -0
  132. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +22 -0
  133. metadata/generated/schema/api/lineage/__init__.py +1 -1
  134. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  135. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  136. metadata/generated/schema/api/policies/__init__.py +1 -1
  137. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  138. metadata/generated/schema/api/services/__init__.py +1 -1
  139. metadata/generated/schema/api/services/createApiService.py +2 -2
  140. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  141. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  142. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  143. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  144. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  145. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  146. metadata/generated/schema/api/services/createSearchService.py +1 -1
  147. metadata/generated/schema/api/services/createStorageService.py +1 -1
  148. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  149. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  150. metadata/generated/schema/api/setOwner.py +1 -1
  151. metadata/generated/schema/api/teams/__init__.py +1 -1
  152. metadata/generated/schema/api/teams/createPersona.py +1 -1
  153. metadata/generated/schema/api/teams/createRole.py +1 -1
  154. metadata/generated/schema/api/teams/createTeam.py +1 -1
  155. metadata/generated/schema/api/teams/createUser.py +1 -1
  156. metadata/generated/schema/api/tests/__init__.py +1 -1
  157. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  158. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  159. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  160. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  161. metadata/generated/schema/api/tests/createTestCaseResult.py +71 -0
  162. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  163. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  164. metadata/generated/schema/api/voteRequest.py +1 -1
  165. metadata/generated/schema/auth/__init__.py +1 -1
  166. metadata/generated/schema/auth/basicAuth.py +1 -1
  167. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  168. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  169. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  170. metadata/generated/schema/auth/emailRequest.py +1 -1
  171. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  172. metadata/generated/schema/auth/generateToken.py +1 -1
  173. metadata/generated/schema/auth/jwtAuth.py +1 -1
  174. metadata/generated/schema/auth/loginRequest.py +1 -1
  175. metadata/generated/schema/auth/logoutRequest.py +1 -1
  176. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  177. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  178. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  179. metadata/generated/schema/auth/refreshToken.py +1 -1
  180. metadata/generated/schema/auth/registrationRequest.py +1 -1
  181. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  182. metadata/generated/schema/auth/revokeToken.py +1 -1
  183. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  184. metadata/generated/schema/auth/ssoAuth.py +1 -1
  185. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  186. metadata/generated/schema/configuration/__init__.py +1 -1
  187. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  188. metadata/generated/schema/configuration/assetCertificationSettings.py +22 -0
  189. metadata/generated/schema/configuration/authConfig.py +1 -1
  190. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  195. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  196. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  197. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  198. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  199. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  200. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  201. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  202. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  203. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  204. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  205. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  206. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  207. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  208. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  209. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  210. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  211. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  212. metadata/generated/schema/configuration/searchSettings.py +1 -1
  213. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
  214. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  215. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  216. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  217. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  218. metadata/generated/schema/dataInsight/__init__.py +1 -1
  219. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  220. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  221. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +9 -1
  222. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  223. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  224. metadata/generated/schema/dataInsight/custom/lineChart.py +23 -8
  225. metadata/generated/schema/dataInsight/custom/summaryCard.py +20 -9
  226. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  227. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  228. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  229. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  230. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  231. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  232. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  233. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  234. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  235. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  236. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  237. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  238. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  239. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  240. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  241. metadata/generated/schema/email/__init__.py +1 -1
  242. metadata/generated/schema/email/emailRequest.py +1 -1
  243. metadata/generated/schema/email/emailTemplate.py +1 -1
  244. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  245. metadata/generated/schema/email/smtpSettings.py +1 -1
  246. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  247. metadata/generated/schema/entity/__init__.py +1 -1
  248. metadata/generated/schema/entity/applications/__init__.py +1 -1
  249. metadata/generated/schema/entity/applications/app.py +9 -1
  250. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  251. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  252. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  253. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
  254. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  255. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  256. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  257. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  258. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  259. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  260. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  261. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +54 -2
  262. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  263. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  264. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  265. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  266. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  267. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  268. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  269. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  270. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
  271. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  272. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  273. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  274. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  275. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  276. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  277. metadata/generated/schema/entity/applications/createAppRequest.py +8 -1
  278. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  279. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  280. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  281. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +8 -1
  282. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +8 -1
  283. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  284. metadata/generated/schema/entity/automations/__init__.py +1 -1
  285. metadata/generated/schema/entity/automations/testServiceConnection.py +3 -1
  286. metadata/generated/schema/entity/automations/workflow.py +1 -1
  287. metadata/generated/schema/entity/bot.py +1 -1
  288. metadata/generated/schema/entity/classification/__init__.py +1 -1
  289. metadata/generated/schema/entity/classification/classification.py +2 -2
  290. metadata/generated/schema/entity/classification/tag.py +2 -2
  291. metadata/generated/schema/entity/data/__init__.py +1 -1
  292. metadata/generated/schema/entity/data/apiCollection.py +4 -2
  293. metadata/generated/schema/entity/data/apiEndpoint.py +4 -2
  294. metadata/generated/schema/entity/data/chart.py +4 -3
  295. metadata/generated/schema/entity/data/container.py +4 -2
  296. metadata/generated/schema/entity/data/dashboard.py +4 -2
  297. metadata/generated/schema/entity/data/dashboardDataModel.py +5 -2
  298. metadata/generated/schema/entity/data/database.py +7 -7
  299. metadata/generated/schema/entity/data/databaseSchema.py +5 -2
  300. metadata/generated/schema/entity/data/glossary.py +2 -2
  301. metadata/generated/schema/entity/data/glossaryTerm.py +3 -2
  302. metadata/generated/schema/entity/data/metric.py +196 -0
  303. metadata/generated/schema/entity/data/mlmodel.py +4 -2
  304. metadata/generated/schema/entity/data/pipeline.py +4 -3
  305. metadata/generated/schema/entity/data/query.py +3 -5
  306. metadata/generated/schema/entity/data/report.py +1 -1
  307. metadata/generated/schema/entity/data/searchIndex.py +16 -2
  308. metadata/generated/schema/entity/data/storedProcedure.py +4 -2
  309. metadata/generated/schema/entity/data/table.py +36 -3
  310. metadata/generated/schema/entity/data/topic.py +4 -3
  311. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  312. metadata/generated/schema/entity/docStore/document.py +1 -1
  313. metadata/generated/schema/entity/domains/__init__.py +1 -1
  314. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  315. metadata/generated/schema/entity/domains/domain.py +1 -1
  316. metadata/generated/schema/entity/events/__init__.py +1 -1
  317. metadata/generated/schema/entity/events/webhook.py +19 -2
  318. metadata/generated/schema/entity/feed/__init__.py +1 -1
  319. metadata/generated/schema/entity/feed/assets.py +1 -1
  320. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  321. metadata/generated/schema/entity/feed/description.py +1 -1
  322. metadata/generated/schema/entity/feed/domain.py +1 -1
  323. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  324. metadata/generated/schema/entity/feed/owner.py +1 -1
  325. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  326. metadata/generated/schema/entity/feed/tag.py +1 -1
  327. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  328. metadata/generated/schema/entity/feed/thread.py +1 -1
  329. metadata/generated/schema/entity/policies/__init__.py +1 -1
  330. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  331. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +4 -1
  332. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  333. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  334. metadata/generated/schema/entity/policies/filters.py +1 -1
  335. metadata/generated/schema/entity/policies/policy.py +1 -1
  336. metadata/generated/schema/entity/services/__init__.py +1 -1
  337. metadata/generated/schema/entity/services/apiService.py +7 -7
  338. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  339. metadata/generated/schema/entity/services/connections/{apiService → api}/__init__.py +1 -1
  340. metadata/generated/schema/entity/services/connections/{apiService → api}/restConnection.py +15 -7
  341. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  342. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  343. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  344. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  345. metadata/generated/schema/entity/services/connections/connectionBasicType.py +29 -1
  346. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  347. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  352. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
  353. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  356. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  357. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  358. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  359. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  360. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  361. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  365. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +41 -0
  366. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  367. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +5 -1
  370. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +9 -1
  372. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +11 -0
  380. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/database/db2Connection.py +4 -1
  389. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  391. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  392. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  393. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/database/druidConnection.py +4 -1
  396. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +84 -0
  398. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +4 -1
  400. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +4 -1
  401. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +4 -1
  410. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +4 -1
  411. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +3 -2
  412. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -1
  413. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +11 -7
  414. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +5 -1
  415. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +4 -1
  416. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +5 -1
  417. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +9 -1
  419. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  422. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  423. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  424. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +5 -1
  425. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +4 -1
  427. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +9 -1
  428. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +4 -1
  429. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +4 -1
  431. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +7 -2
  432. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  435. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  436. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  437. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  438. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  439. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  440. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  441. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  442. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  443. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  444. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  446. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  449. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  451. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  455. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  456. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  458. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  459. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  460. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  461. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +69 -0
  462. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
  463. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  464. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  466. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  467. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +65 -0
  469. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  470. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  471. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  472. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  473. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +43 -0
  474. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  475. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  476. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  477. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  478. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  479. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  480. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  481. metadata/generated/schema/entity/services/connections/serviceConnection.py +3 -3
  482. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  483. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  484. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  485. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  486. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  487. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  488. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  489. metadata/generated/schema/entity/services/dashboardService.py +8 -5
  490. metadata/generated/schema/entity/services/databaseService.py +5 -2
  491. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  492. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
  493. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  494. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  495. metadata/generated/schema/entity/services/messagingService.py +2 -2
  496. metadata/generated/schema/entity/services/metadataService.py +2 -2
  497. metadata/generated/schema/entity/services/mlmodelService.py +2 -2
  498. metadata/generated/schema/entity/services/pipelineService.py +11 -2
  499. metadata/generated/schema/entity/services/searchService.py +2 -2
  500. metadata/generated/schema/entity/services/serviceType.py +2 -2
  501. metadata/generated/schema/entity/services/storageService.py +2 -2
  502. metadata/generated/schema/entity/teams/__init__.py +1 -1
  503. metadata/generated/schema/entity/teams/persona.py +1 -1
  504. metadata/generated/schema/entity/teams/role.py +1 -1
  505. metadata/generated/schema/entity/teams/team.py +1 -1
  506. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  507. metadata/generated/schema/entity/teams/user.py +1 -1
  508. metadata/generated/schema/entity/type.py +1 -1
  509. metadata/generated/schema/entity/utils/__init__.py +1 -1
  510. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  511. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  512. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  513. metadata/generated/schema/events/__init__.py +1 -1
  514. metadata/generated/schema/events/alertMetrics.py +1 -1
  515. metadata/generated/schema/events/api/__init__.py +1 -1
  516. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  517. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +69 -0
  518. metadata/generated/schema/events/api/eventsRecord.py +37 -0
  519. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -4
  520. metadata/generated/schema/events/api/typedEvent.py +46 -0
  521. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  522. metadata/generated/schema/events/eventFilterRule.py +1 -1
  523. metadata/generated/schema/events/eventSubscription.py +15 -46
  524. metadata/generated/schema/events/eventSubscriptionOffset.py +7 -2
  525. metadata/generated/schema/events/failedEvent.py +1 -1
  526. metadata/generated/schema/events/failedEventResponse.py +38 -0
  527. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  528. metadata/generated/schema/events/statusContext.py +43 -0
  529. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  530. metadata/generated/schema/events/subscriptionStatus.py +78 -0
  531. metadata/generated/schema/events/testDestinationStatus.py +88 -0
  532. metadata/generated/schema/governance/workflows/__init__.py +3 -0
  533. metadata/generated/schema/governance/workflows/elements/__init__.py +5 -0
  534. metadata/generated/schema/governance/workflows/elements/edge.py +29 -0
  535. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +16 -0
  536. metadata/generated/schema/governance/workflows/elements/nodeType.py +14 -0
  537. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +3 -0
  538. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +5 -0
  539. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +44 -0
  540. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +49 -0
  541. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +42 -0
  542. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +3 -0
  543. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +30 -0
  544. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +3 -0
  545. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +30 -0
  546. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +3 -0
  547. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +57 -0
  548. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +3 -0
  549. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +43 -0
  550. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +44 -0
  551. metadata/generated/schema/{entity/data/metrics.py → governance/workflows/workflowDefinition.py} +36 -49
  552. metadata/generated/schema/governance/workflows/workflowInstance.py +50 -0
  553. metadata/generated/schema/governance/workflows/workflowInstanceState.py +73 -0
  554. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  555. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +53 -0
  556. metadata/generated/schema/metadataIngestion/application.py +1 -1
  557. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  558. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  560. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +127 -0
  561. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -9
  562. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +28 -17
  563. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +42 -1
  564. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  565. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  566. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  567. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  568. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  569. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  570. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  571. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
  572. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
  573. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  574. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  575. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  576. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  577. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  578. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
  579. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  580. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  581. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  582. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  583. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  584. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  585. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  586. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  587. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  588. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  589. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +4 -1
  590. metadata/generated/schema/metadataIngestion/workflow.py +15 -1
  591. metadata/generated/schema/monitoring/__init__.py +1 -1
  592. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  593. metadata/generated/schema/security/__init__.py +1 -1
  594. metadata/generated/schema/security/client/__init__.py +1 -1
  595. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  596. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  597. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  598. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  599. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  600. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  601. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  602. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  603. metadata/generated/schema/security/credentials/__init__.py +1 -1
  604. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  605. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  606. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  607. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  608. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  609. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  610. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  611. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  612. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  613. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  614. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  615. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  616. metadata/generated/schema/security/sasl/__init__.py +1 -1
  617. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  618. metadata/generated/schema/security/secrets/__init__.py +1 -1
  619. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  620. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  621. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  622. metadata/generated/schema/security/securityConfiguration.py +1 -1
  623. metadata/generated/schema/security/ssl/__init__.py +1 -1
  624. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  625. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  626. metadata/generated/schema/settings/__init__.py +1 -1
  627. metadata/generated/schema/settings/settings.py +7 -1
  628. metadata/generated/schema/system/__init__.py +1 -1
  629. metadata/generated/schema/system/entityError.py +1 -1
  630. metadata/generated/schema/system/eventPublisherJob.py +2 -1
  631. metadata/generated/schema/system/indexingError.py +1 -1
  632. metadata/generated/schema/system/limitsResponse.py +1 -1
  633. metadata/generated/schema/system/ui/__init__.py +1 -1
  634. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  635. metadata/generated/schema/system/ui/page.py +1 -1
  636. metadata/generated/schema/system/validationResponse.py +1 -1
  637. metadata/generated/schema/tests/__init__.py +1 -1
  638. metadata/generated/schema/tests/assigned.py +1 -1
  639. metadata/generated/schema/tests/basic.py +19 -4
  640. metadata/generated/schema/tests/customMetric.py +1 -1
  641. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  642. metadata/generated/schema/tests/resolved.py +1 -1
  643. metadata/generated/schema/tests/testCase.py +6 -2
  644. metadata/generated/schema/tests/testCaseResolutionStatus.py +17 -2
  645. metadata/generated/schema/tests/testDefinition.py +1 -1
  646. metadata/generated/schema/tests/testSuite.py +9 -2
  647. metadata/generated/schema/type/__init__.py +1 -1
  648. metadata/generated/schema/type/apiSchema.py +1 -1
  649. metadata/generated/schema/type/assetCertification.py +26 -0
  650. metadata/generated/schema/type/auditLog.py +1 -1
  651. metadata/generated/schema/type/basic.py +1 -1
  652. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  653. metadata/generated/schema/type/changeEvent.py +1 -1
  654. metadata/generated/schema/type/changeEventType.py +1 -1
  655. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  656. metadata/generated/schema/type/csvDocumentation.py +1 -1
  657. metadata/generated/schema/type/csvErrorType.py +1 -1
  658. metadata/generated/schema/type/csvFile.py +1 -1
  659. metadata/generated/schema/type/csvImportResult.py +1 -1
  660. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  661. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  662. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  663. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  664. metadata/generated/schema/type/customProperty.py +8 -1
  665. metadata/generated/schema/type/dailyCount.py +1 -1
  666. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  667. metadata/generated/schema/type/entityHierarchy.py +1 -1
  668. metadata/generated/schema/type/entityHistory.py +1 -1
  669. metadata/generated/schema/type/entityLineage.py +1 -1
  670. metadata/generated/schema/type/entityReference.py +1 -1
  671. metadata/generated/schema/type/entityReferenceList.py +1 -1
  672. metadata/generated/schema/type/entityRelationship.py +1 -1
  673. metadata/generated/schema/type/entityUsage.py +1 -1
  674. metadata/generated/schema/type/filterPattern.py +1 -1
  675. metadata/generated/schema/type/function.py +1 -1
  676. metadata/generated/schema/type/include.py +1 -1
  677. metadata/generated/schema/type/jdbcConnection.py +1 -1
  678. metadata/generated/schema/type/lifeCycle.py +1 -1
  679. metadata/generated/schema/type/paging.py +1 -1
  680. metadata/generated/schema/type/profile.py +1 -1
  681. metadata/generated/schema/type/queryParserData.py +1 -1
  682. metadata/generated/schema/type/reaction.py +1 -1
  683. metadata/generated/schema/type/schedule.py +1 -1
  684. metadata/generated/schema/type/schema.py +1 -1
  685. metadata/generated/schema/type/tableQuery.py +1 -1
  686. metadata/generated/schema/type/tableUsageCount.py +1 -1
  687. metadata/generated/schema/type/tagLabel.py +1 -1
  688. metadata/generated/schema/type/usageDetails.py +1 -1
  689. metadata/generated/schema/type/usageRequest.py +1 -1
  690. metadata/generated/schema/type/votes.py +1 -1
  691. metadata/ingestion/api/models.py +9 -5
  692. metadata/ingestion/api/parser.py +14 -0
  693. metadata/ingestion/lineage/parser.py +4 -6
  694. metadata/ingestion/lineage/sql_lineage.py +29 -185
  695. metadata/ingestion/models/custom_basemodel_validation.py +80 -0
  696. metadata/ingestion/models/custom_pydantic.py +33 -1
  697. metadata/ingestion/models/custom_types.py +2 -0
  698. metadata/ingestion/models/entity_interface.py +71 -0
  699. metadata/ingestion/models/patch_request.py +0 -1
  700. metadata/ingestion/ometa/mixins/es_mixin.py +66 -8
  701. metadata/ingestion/ometa/models.py +1 -0
  702. metadata/ingestion/ometa/ometa_api.py +6 -3
  703. metadata/ingestion/ometa/routes.py +4 -2
  704. metadata/ingestion/processor/query_parser.py +2 -2
  705. metadata/ingestion/sink/metadata_rest.py +23 -18
  706. metadata/ingestion/source/api/api_service.py +212 -0
  707. metadata/ingestion/source/api/rest/connection.py +93 -0
  708. metadata/ingestion/source/api/rest/metadata.py +278 -0
  709. metadata/ingestion/source/api/rest/models.py +41 -0
  710. metadata/ingestion/source/api/rest/service_spec.py +4 -0
  711. metadata/ingestion/source/dashboard/domodashboard/service_spec.py +6 -0
  712. metadata/ingestion/source/dashboard/lightdash/service_spec.py +4 -0
  713. metadata/ingestion/source/dashboard/looker/metadata.py +33 -4
  714. metadata/ingestion/source/dashboard/looker/service_spec.py +4 -0
  715. metadata/ingestion/source/dashboard/metabase/service_spec.py +4 -0
  716. metadata/ingestion/source/dashboard/microstrategy/__init__.py +0 -0
  717. metadata/ingestion/source/dashboard/{mstr → microstrategy}/client.py +27 -32
  718. metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +7 -10
  719. metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +18 -14
  720. metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +1 -1
  721. metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
  722. metadata/ingestion/source/dashboard/mode/service_spec.py +4 -0
  723. metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
  724. metadata/ingestion/source/dashboard/powerbi/service_spec.py +4 -0
  725. metadata/ingestion/source/dashboard/qlikcloud/service_spec.py +4 -0
  726. metadata/ingestion/source/dashboard/qliksense/service_spec.py +4 -0
  727. metadata/ingestion/source/dashboard/quicksight/service_spec.py +4 -0
  728. metadata/ingestion/source/dashboard/redash/service_spec.py +4 -0
  729. metadata/ingestion/source/dashboard/sigma/__init__.py +0 -0
  730. metadata/ingestion/source/dashboard/sigma/client.py +186 -0
  731. metadata/ingestion/source/dashboard/sigma/connection.py +67 -0
  732. metadata/ingestion/source/dashboard/sigma/metadata.py +331 -0
  733. metadata/ingestion/source/dashboard/sigma/models.py +97 -0
  734. metadata/ingestion/source/dashboard/sigma/service_spec.py +4 -0
  735. metadata/ingestion/source/dashboard/superset/service_spec.py +4 -0
  736. metadata/ingestion/source/dashboard/tableau/service_spec.py +4 -0
  737. metadata/ingestion/source/database/athena/service_spec.py +10 -0
  738. metadata/ingestion/source/database/azuresql/service_spec.py +12 -0
  739. metadata/ingestion/source/database/bigquery/connection.py +28 -2
  740. metadata/ingestion/source/database/bigquery/lineage.py +29 -2
  741. metadata/ingestion/source/database/bigquery/metadata.py +1 -25
  742. metadata/ingestion/source/database/bigquery/profiler/__init__.py +0 -0
  743. metadata/ingestion/source/database/bigquery/profiler/profiler.py +30 -0
  744. metadata/ingestion/source/database/bigquery/profiler/system.py +165 -0
  745. metadata/ingestion/source/database/bigquery/queries.py +69 -1
  746. metadata/ingestion/source/database/bigquery/service_spec.py +16 -0
  747. metadata/ingestion/source/database/bigtable/service_spec.py +4 -0
  748. metadata/ingestion/source/database/clickhouse/service_spec.py +12 -0
  749. metadata/ingestion/source/database/common_db_source.py +150 -147
  750. metadata/ingestion/source/database/common_nosql_source.py +0 -14
  751. metadata/ingestion/source/database/couchbase/metadata.py +62 -2
  752. metadata/{data_quality/builders/sqa_validator_builder.py → ingestion/source/database/couchbase/models.py} +15 -9
  753. metadata/ingestion/source/database/couchbase/queries.py +6 -1
  754. metadata/ingestion/source/database/couchbase/service_spec.py +4 -0
  755. metadata/ingestion/source/database/database_service.py +15 -22
  756. metadata/ingestion/source/database/databricks/metadata.py +12 -1
  757. metadata/ingestion/source/database/databricks/service_spec.py +20 -0
  758. metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
  759. metadata/ingestion/source/database/datalake/clients/base.py +10 -1
  760. metadata/ingestion/source/database/datalake/clients/gcs.py +10 -6
  761. metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
  762. metadata/ingestion/source/database/datalake/metadata.py +2 -13
  763. metadata/ingestion/source/database/datalake/service_spec.py +16 -0
  764. metadata/ingestion/source/database/db2/lineage.py +46 -0
  765. metadata/ingestion/source/database/db2/service_spec.py +9 -0
  766. metadata/ingestion/source/database/dbt/constants.py +4 -0
  767. metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
  768. metadata/ingestion/source/database/dbt/dbt_service.py +8 -1
  769. metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
  770. metadata/ingestion/source/database/dbt/metadata.py +73 -7
  771. metadata/ingestion/source/database/dbt/models.py +2 -0
  772. metadata/ingestion/source/database/dbt/service_spec.py +4 -0
  773. metadata/ingestion/source/database/deltalake/metadata.py +1 -12
  774. metadata/ingestion/source/database/deltalake/service_spec.py +4 -0
  775. metadata/ingestion/source/database/domodatabase/metadata.py +1 -12
  776. metadata/ingestion/source/database/domodatabase/service_spec.py +4 -0
  777. metadata/ingestion/source/database/doris/metadata.py +1 -1
  778. metadata/ingestion/source/database/doris/service_spec.py +4 -0
  779. metadata/ingestion/source/database/druid/lineage.py +46 -0
  780. metadata/ingestion/source/database/druid/service_spec.py +4 -0
  781. metadata/ingestion/source/database/dynamodb/service_spec.py +10 -0
  782. metadata/ingestion/source/database/exasol/__init__.py +0 -0
  783. metadata/ingestion/source/database/exasol/connection.py +87 -0
  784. metadata/ingestion/source/database/exasol/metadata.py +27 -0
  785. metadata/ingestion/source/database/exasol/service_spec.py +4 -0
  786. metadata/ingestion/source/database/glue/metadata.py +49 -19
  787. metadata/ingestion/source/database/glue/models.py +8 -0
  788. metadata/ingestion/source/database/glue/service_spec.py +4 -0
  789. metadata/ingestion/source/database/greenplum/lineage.py +46 -0
  790. metadata/ingestion/source/database/greenplum/service_spec.py +4 -0
  791. metadata/ingestion/source/database/hive/lineage.py +46 -0
  792. metadata/ingestion/source/database/hive/service_spec.py +4 -0
  793. metadata/ingestion/source/database/iceberg/metadata.py +1 -16
  794. metadata/ingestion/source/database/iceberg/service_spec.py +4 -0
  795. metadata/ingestion/source/database/impala/lineage.py +46 -0
  796. metadata/ingestion/source/database/impala/service_spec.py +4 -0
  797. metadata/ingestion/source/database/life_cycle_query_mixin.py +3 -5
  798. metadata/ingestion/source/database/lineage_source.py +124 -32
  799. metadata/ingestion/source/database/mariadb/lineage.py +46 -0
  800. metadata/ingestion/source/database/mariadb/service_spec.py +4 -0
  801. metadata/ingestion/source/database/mongodb/service_spec.py +10 -0
  802. metadata/ingestion/source/database/mssql/lineage.py +51 -2
  803. metadata/ingestion/source/database/mssql/metadata.py +2 -44
  804. metadata/ingestion/source/database/mssql/service_spec.py +10 -0
  805. metadata/ingestion/source/database/mysql/lineage.py +0 -2
  806. metadata/ingestion/source/database/mysql/queries.py +0 -2
  807. metadata/ingestion/source/database/mysql/service_spec.py +10 -0
  808. metadata/ingestion/source/database/mysql/usage.py +0 -2
  809. metadata/ingestion/source/database/oracle/lineage.py +28 -1
  810. metadata/ingestion/source/database/oracle/metadata.py +2 -23
  811. metadata/ingestion/source/database/oracle/service_spec.py +10 -0
  812. metadata/ingestion/source/database/pinotdb/lineage.py +46 -0
  813. metadata/ingestion/source/database/pinotdb/service_spec.py +4 -0
  814. metadata/ingestion/source/database/postgres/metadata.py +4 -13
  815. metadata/ingestion/source/database/postgres/service_spec.py +10 -0
  816. metadata/ingestion/source/database/postgres/utils.py +26 -0
  817. metadata/ingestion/source/database/presto/service_spec.py +4 -0
  818. metadata/ingestion/source/database/query/service_spec.py +9 -0
  819. metadata/ingestion/source/database/query_parser_source.py +1 -0
  820. metadata/ingestion/source/database/redshift/connection.py +9 -2
  821. metadata/ingestion/source/database/redshift/lineage.py +27 -3
  822. metadata/ingestion/source/database/redshift/metadata.py +2 -27
  823. metadata/ingestion/source/database/redshift/profiler/__init__.py +0 -0
  824. metadata/ingestion/source/database/redshift/profiler/profiler.py +14 -0
  825. metadata/ingestion/source/database/redshift/profiler/system.py +148 -0
  826. metadata/ingestion/source/database/redshift/queries.py +99 -8
  827. metadata/ingestion/source/database/redshift/service_spec.py +14 -0
  828. metadata/ingestion/source/database/redshift/utils.py +3 -3
  829. metadata/ingestion/source/database/salesforce/metadata.py +1 -12
  830. metadata/ingestion/source/database/salesforce/service_spec.py +4 -0
  831. metadata/ingestion/source/database/sample_data.py +129 -34
  832. metadata/ingestion/source/database/saperp/service_spec.py +4 -0
  833. metadata/ingestion/source/database/saphana/service_spec.py +7 -0
  834. metadata/ingestion/source/database/sas/metadata.py +1 -10
  835. metadata/ingestion/source/database/sas/service_spec.py +4 -0
  836. metadata/ingestion/source/database/singlestore/lineage.py +46 -0
  837. metadata/ingestion/source/database/singlestore/service_spec.py +9 -0
  838. metadata/ingestion/source/database/snowflake/connection.py +6 -1
  839. metadata/ingestion/source/database/snowflake/lineage.py +31 -2
  840. metadata/ingestion/source/database/snowflake/metadata.py +20 -62
  841. metadata/ingestion/source/database/snowflake/models.py +50 -2
  842. metadata/ingestion/source/database/snowflake/profiler/__init__.py +0 -0
  843. metadata/ingestion/source/database/snowflake/profiler/profiler.py +26 -0
  844. metadata/ingestion/source/database/snowflake/profiler/system.py +436 -0
  845. metadata/ingestion/source/database/snowflake/queries.py +28 -22
  846. metadata/ingestion/source/database/snowflake/service_spec.py +20 -0
  847. metadata/ingestion/source/database/sql_column_handler.py +26 -0
  848. metadata/ingestion/source/database/sqlite/lineage.py +46 -0
  849. metadata/ingestion/source/database/sqlite/service_spec.py +4 -0
  850. metadata/ingestion/source/database/stored_procedures_mixin.py +78 -52
  851. metadata/ingestion/source/database/teradata/lineage.py +46 -0
  852. metadata/ingestion/source/database/teradata/service_spec.py +4 -0
  853. metadata/ingestion/source/database/trino/connection.py +60 -23
  854. metadata/ingestion/source/database/trino/profiler/__init__.py +0 -0
  855. metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py +167 -0
  856. metadata/ingestion/source/database/trino/service_spec.py +16 -0
  857. metadata/ingestion/source/database/unitycatalog/metadata.py +3 -24
  858. metadata/ingestion/source/database/unitycatalog/service_spec.py +22 -0
  859. metadata/ingestion/source/database/usage_source.py +4 -1
  860. metadata/ingestion/source/database/vertica/service_spec.py +10 -0
  861. metadata/ingestion/source/messaging/kafka/service_spec.py +4 -0
  862. metadata/ingestion/source/messaging/kinesis/service_spec.py +4 -0
  863. metadata/ingestion/source/messaging/redpanda/service_spec.py +4 -0
  864. metadata/ingestion/source/metadata/alationsink/service_spec.py +4 -0
  865. metadata/ingestion/source/metadata/amundsen/service_spec.py +4 -0
  866. metadata/ingestion/source/metadata/atlas/service_spec.py +4 -0
  867. metadata/ingestion/source/mlmodel/mlflow/service_spec.py +4 -0
  868. metadata/ingestion/source/mlmodel/sagemaker/service_spec.py +4 -0
  869. metadata/ingestion/source/pipeline/airbyte/service_spec.py +4 -0
  870. metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
  871. metadata/ingestion/source/pipeline/airflow/metadata.py +71 -27
  872. metadata/ingestion/source/pipeline/airflow/service_spec.py +4 -0
  873. metadata/ingestion/source/pipeline/dagster/metadata.py +3 -1
  874. metadata/ingestion/source/pipeline/dagster/service_spec.py +4 -0
  875. metadata/ingestion/source/pipeline/databrickspipeline/service_spec.py +6 -0
  876. metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
  877. metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
  878. metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
  879. metadata/ingestion/source/pipeline/dbtcloud/service_spec.py +4 -0
  880. metadata/ingestion/source/pipeline/domopipeline/service_spec.py +4 -0
  881. metadata/ingestion/source/pipeline/fivetran/service_spec.py +4 -0
  882. metadata/ingestion/source/pipeline/flink/service_spec.py +4 -0
  883. metadata/ingestion/source/pipeline/gluepipeline/service_spec.py +4 -0
  884. metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
  885. metadata/ingestion/source/pipeline/kafkaconnect/service_spec.py +4 -0
  886. metadata/ingestion/source/pipeline/nifi/service_spec.py +4 -0
  887. metadata/ingestion/source/pipeline/openlineage/service_spec.py +4 -0
  888. metadata/ingestion/source/pipeline/pipeline_service.py +3 -1
  889. metadata/ingestion/source/pipeline/spline/service_spec.py +4 -0
  890. metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
  891. metadata/ingestion/source/search/elasticsearch/service_spec.py +4 -0
  892. metadata/ingestion/source/search/search_service.py +44 -1
  893. metadata/ingestion/source/storage/gcs/metadata.py +2 -2
  894. metadata/ingestion/source/storage/gcs/service_spec.py +4 -0
  895. metadata/ingestion/source/storage/s3/service_spec.py +4 -0
  896. metadata/mixins/pandas/pandas_mixin.py +2 -31
  897. metadata/mixins/sqalchemy/sqa_mixin.py +18 -17
  898. metadata/parsers/avro_parser.py +51 -16
  899. metadata/parsers/json_schema_parser.py +45 -16
  900. metadata/pii/processor.py +10 -9
  901. metadata/pii/scanners/ner_scanner.py +2 -1
  902. metadata/profiler/api/models.py +4 -75
  903. metadata/profiler/config.py +39 -0
  904. metadata/profiler/interface/nosql/profiler_interface.py +1 -26
  905. metadata/profiler/interface/pandas/profiler_interface.py +37 -77
  906. metadata/profiler/interface/profiler_interface.py +25 -279
  907. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
  908. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
  909. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
  910. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +7 -5
  911. metadata/profiler/interface/sqlalchemy/profiler_interface.py +73 -125
  912. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +5 -5
  913. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
  914. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +145 -0
  915. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +9 -4
  916. metadata/profiler/metrics/system/system.py +69 -374
  917. metadata/profiler/orm/converter/base.py +23 -13
  918. metadata/profiler/orm/converter/bigquery/converter.py +1 -1
  919. metadata/profiler/orm/functions/count.py +9 -0
  920. metadata/profiler/orm/functions/md5.py +39 -0
  921. metadata/{data_quality/builders/pandas_validator_builder.py → profiler/orm/functions/substr.py} +13 -10
  922. metadata/profiler/orm/functions/table_metric_computer.py +6 -5
  923. metadata/profiler/orm/functions/unique_count.py +6 -0
  924. metadata/profiler/orm/registry.py +27 -2
  925. metadata/profiler/processor/core.py +13 -62
  926. metadata/profiler/processor/handle_partition.py +0 -48
  927. metadata/profiler/processor/processor.py +1 -1
  928. metadata/profiler/processor/runner.py +111 -35
  929. metadata/profiler/processor/sample_data_handler.py +7 -4
  930. metadata/profiler/source/database/base/profiler_source.py +215 -0
  931. metadata/profiler/source/{bigquery → database/bigquery}/profiler_source.py +4 -4
  932. metadata/profiler/source/{databricks → database/databricks}/profiler_source.py +6 -7
  933. metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/first_quartile.py +1 -1
  934. metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/median.py +1 -1
  935. metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/third_quartile.py +1 -1
  936. metadata/profiler/source/{single_store → database/single_store}/metrics/window/first_quartile.py +3 -1
  937. metadata/profiler/source/{single_store → database/single_store}/metrics/window/median.py +3 -1
  938. metadata/profiler/source/{single_store → database/single_store}/metrics/window/third_quartile.py +3 -1
  939. metadata/profiler/source/fetcher/config.py +44 -0
  940. metadata/profiler/source/fetcher/entity_fetcher.py +62 -0
  941. metadata/profiler/source/fetcher/fetcher_strategy.py +299 -0
  942. metadata/profiler/source/fetcher/profiler_source_factory.py +89 -0
  943. metadata/profiler/source/metadata.py +16 -201
  944. metadata/profiler/source/metadata_ext.py +23 -18
  945. metadata/profiler/source/model.py +29 -0
  946. metadata/sampler/config.py +237 -0
  947. metadata/sampler/models.py +106 -0
  948. metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
  949. metadata/sampler/pandas/sampler.py +239 -0
  950. metadata/{utils → sampler}/partition.py +66 -51
  951. metadata/sampler/processor.py +158 -0
  952. metadata/sampler/sampler_interface.py +251 -0
  953. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
  954. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
  955. metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
  956. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +90 -98
  957. metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
  958. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
  959. metadata/utils/class_helper.py +5 -0
  960. metadata/utils/collaborative_super.py +28 -0
  961. metadata/utils/constants.py +65 -0
  962. metadata/utils/constraints.py +65 -0
  963. metadata/utils/dict.py +8 -0
  964. metadata/utils/dispatch.py +1 -53
  965. metadata/utils/entity_link.py +8 -2
  966. metadata/utils/entity_utils.py +93 -0
  967. metadata/utils/filters.py +15 -0
  968. metadata/utils/fqn.py +15 -0
  969. metadata/utils/helpers.py +3 -2
  970. metadata/utils/importer.py +12 -24
  971. metadata/utils/logger.py +31 -1
  972. metadata/utils/lru_cache.py +76 -15
  973. metadata/utils/profiler_utils.py +43 -9
  974. metadata/utils/service_spec/__init__.py +5 -0
  975. metadata/utils/service_spec/default.py +21 -0
  976. metadata/utils/service_spec/service_spec.py +146 -0
  977. metadata/utils/ssl_manager.py +33 -0
  978. metadata/utils/time_utils.py +33 -5
  979. metadata/workflow/application.py +7 -36
  980. metadata/workflow/base.py +47 -17
  981. metadata/workflow/classification.py +53 -0
  982. metadata/workflow/ingestion.py +6 -38
  983. metadata/workflow/profiler.py +1 -17
  984. metadata/workflow/workflow_output_handler.py +11 -8
  985. metadata/workflow/workflow_status_mixin.py +0 -2
  986. openmetadata_ingestion-1.6.0.0rc2.dist-info/LICENSE +114 -0
  987. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/METADATA +473 -548
  988. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/RECORD +995 -809
  989. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/entry_points.txt +3 -0
  990. metadata/data_quality/interface/test_suite_interface_factory.py +0 -128
  991. metadata/data_quality/runner/test_suite_source_factory.py +0 -38
  992. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
  993. metadata/profiler/interface/profiler_interface_factory.py +0 -107
  994. metadata/profiler/metrics/system/queries/bigquery.py +0 -54
  995. metadata/profiler/metrics/system/queries/redshift.py +0 -103
  996. metadata/profiler/metrics/system/queries/snowflake.py +0 -191
  997. metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
  998. metadata/profiler/processor/sampler/sampler_factory.py +0 -100
  999. metadata/profiler/processor/sampler/sampler_interface.py +0 -74
  1000. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -60
  1001. metadata/profiler/source/base/profiler_source.py +0 -299
  1002. metadata/profiler/source/profiler_source_factory.py +0 -54
  1003. openmetadata_ingestion-1.5.13.0.dist-info/LICENSE +0 -201
  1004. {metadata/ingestion/source/dashboard/mstr → _openmetadata_testutils/pytest_openmetadata}/__init__.py +0 -0
  1005. /metadata/profiler/source/{bigquery → database/bigquery}/type_mapper.py +0 -0
  1006. /metadata/profiler/source/{mariadb → database/mariadb}/functions/median.py +0 -0
  1007. /metadata/profiler/source/{single_store → database/single_store}/functions/median.py +0 -0
  1008. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/WHEEL +0 -0
  1009. {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,278 @@
1
+ # Copyright 2024 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+ """REST source module"""
12
+
13
+ import traceback
14
+ from typing import Iterable, List, Optional
15
+
16
+ from pydantic import AnyUrl
17
+
18
+ from metadata.generated.schema.api.data.createAPICollection import (
19
+ CreateAPICollectionRequest,
20
+ )
21
+ from metadata.generated.schema.api.data.createAPIEndpoint import (
22
+ CreateAPIEndpointRequest,
23
+ )
24
+ from metadata.generated.schema.entity.data.apiCollection import APICollection
25
+ from metadata.generated.schema.entity.data.apiEndpoint import ApiRequestMethod
26
+ from metadata.generated.schema.entity.services.connections.api.restConnection import (
27
+ RestConnection,
28
+ )
29
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
30
+ StackTraceError,
31
+ )
32
+ from metadata.generated.schema.metadataIngestion.workflow import (
33
+ Source as WorkflowSource,
34
+ )
35
+ from metadata.generated.schema.type.apiSchema import APISchema
36
+ from metadata.generated.schema.type.basic import FullyQualifiedEntityName
37
+ from metadata.generated.schema.type.schema import DataTypeTopic, FieldModel
38
+ from metadata.ingestion.api.models import Either
39
+ from metadata.ingestion.api.steps import InvalidSourceException
40
+ from metadata.ingestion.ometa.ometa_api import OpenMetadata
41
+ from metadata.ingestion.source.api.api_service import ApiServiceSource
42
+ from metadata.ingestion.source.api.rest.models import RESTCollection, RESTEndpoint
43
+ from metadata.utils import fqn
44
+ from metadata.utils.logger import ingestion_logger
45
+
46
+ logger = ingestion_logger()
47
+
48
+
49
+ class RestSource(ApiServiceSource):
50
+ """
51
+ Source implementation to ingest REST data.
52
+
53
+ We will iterate on the registered collections, endpoints
54
+ and prepare an iterator of
55
+ """
56
+
57
+ def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
58
+ super().__init__(config, metadata)
59
+
60
+ @classmethod
61
+ def create(
62
+ cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
63
+ ):
64
+ config: WorkflowSource = WorkflowSource.model_validate(config_dict)
65
+ connection: RestConnection = config.serviceConnection.root.config
66
+ if not isinstance(connection, RestConnection):
67
+ raise InvalidSourceException(
68
+ f"Expected RestConnection, but got {connection}"
69
+ )
70
+ return cls(config, metadata)
71
+
72
+ def get_api_collections(self, *args, **kwargs) -> Iterable[RESTCollection]:
73
+ """
74
+ Method to list all collections to process.
75
+ Here is where filtering happens
76
+ """
77
+ try:
78
+ self.json_response = self.connection.json()
79
+ if self.json_response.get("tags", []):
80
+ # Works only if list of tags are present in schema so we can fetch collection names
81
+ for collection in self.json_response.get("tags", []):
82
+ if not collection.get("name"):
83
+ continue
84
+ yield RESTCollection(**collection)
85
+ else:
86
+ # in other case collect tags from paths because we have to yield collection/tags first
87
+ collections_set = set()
88
+ for path, methods in self.json_response.get("paths", {}).items():
89
+ for method_type, info in methods.items():
90
+ collections_set.update({tag for tag in info.get("tags", [])})
91
+ for collection_name in collections_set:
92
+ data = {"name": collection_name}
93
+ yield RESTCollection(**data)
94
+ except Exception as err:
95
+ logger.error(f"Error while fetching collections from schema URL :{err}")
96
+
97
+ def yield_api_collection(
98
+ self, collection: RESTCollection
99
+ ) -> Iterable[Either[CreateAPICollectionRequest]]:
100
+ """Method to return api collection Entities"""
101
+ try:
102
+ collection.url = self._generate_collection_url(collection.name.root)
103
+ collection_request = CreateAPICollectionRequest(
104
+ name=collection.name,
105
+ displayName=collection.display_name,
106
+ description=collection.description,
107
+ service=FullyQualifiedEntityName(self.context.get().api_service),
108
+ endpointURL=collection.url,
109
+ )
110
+ yield Either(right=collection_request)
111
+ self.register_record(collection_request=collection_request)
112
+ except Exception as exc:
113
+ yield Either(
114
+ left=StackTraceError(
115
+ name=collection.name.root,
116
+ error=f"Error creating api collection request: {exc}",
117
+ stackTrace=traceback.format_exc(),
118
+ )
119
+ )
120
+
121
+ def yield_api_endpoint(
122
+ self, collection: RESTCollection
123
+ ) -> Iterable[Either[CreateAPIEndpointRequest]]:
124
+ """Method to return api endpoint Entities"""
125
+ filtered_endpoints = self._filter_collection_endpoints(collection) or {}
126
+ for path, methods in filtered_endpoints.items():
127
+ for method_type, info in methods.items():
128
+ try:
129
+ endpoint = self._prepare_endpoint_data(path, method_type, info)
130
+ if not endpoint:
131
+ continue
132
+ yield Either(
133
+ right=CreateAPIEndpointRequest(
134
+ name=endpoint.name,
135
+ displayName=endpoint.display_name,
136
+ description=endpoint.description,
137
+ endpointURL=endpoint.url,
138
+ requestMethod=self._get_api_request_method(method_type),
139
+ requestSchema=self._get_request_schema(info),
140
+ responseSchema=self._get_response_schema(info),
141
+ apiCollection=FullyQualifiedEntityName(
142
+ fqn.build(
143
+ self.metadata,
144
+ entity_type=APICollection,
145
+ service_name=self.context.get().api_service,
146
+ api_collection_name=collection.name.root,
147
+ )
148
+ ),
149
+ )
150
+ )
151
+ except Exception as exc: # pylint: disable=broad-except
152
+ yield Either(
153
+ left=StackTraceError(
154
+ name=endpoint.name,
155
+ error=f"Error creating API Endpoint request [{info.get('operationId')}]: {exc}",
156
+ stackTrace=traceback.format_exc(),
157
+ )
158
+ )
159
+
160
+ def _filter_collection_endpoints(
161
+ self, collection: RESTCollection
162
+ ) -> Optional[dict]:
163
+ """filter endpoints related to specific collection"""
164
+ try:
165
+ filtered_paths = {}
166
+ for path, methods in self.json_response.get("paths", {}).items():
167
+ for method_type, info in methods.items():
168
+ if collection.name.root in info.get("tags", []):
169
+ # path & methods are part of collection
170
+ filtered_paths.update({path: methods})
171
+ break
172
+ return filtered_paths
173
+ except Exception as err:
174
+ logger.info(
175
+ f"Error while filtering endpoints for collection {collection.name.root}"
176
+ )
177
+ return None
178
+
179
+ def _prepare_endpoint_data(self, path, method_type, info) -> Optional[RESTEndpoint]:
180
+ try:
181
+ endpoint = RESTEndpoint(**info)
182
+ endpoint.url = self._generate_endpoint_url(endpoint.name)
183
+ if not endpoint.name:
184
+ endpoint.name = f"{path} - {method_type}"
185
+ return endpoint
186
+ except Exception as err:
187
+ logger.info(f"Error while parsing endpoint data: {err}")
188
+ return None
189
+
190
+ def _generate_collection_url(self, collection_name: str) -> Optional[AnyUrl]:
191
+ """generate collection url"""
192
+ try:
193
+ if collection_name:
194
+ return AnyUrl(
195
+ f"{self.config.serviceConnection.root.config.openAPISchemaURL}#tag/{collection_name}"
196
+ )
197
+ except Exception as err:
198
+ logger.info(f"Error while generating collection url: {err}")
199
+ return None
200
+
201
+ def _generate_endpoint_url(self, endpoint_name: str) -> AnyUrl:
202
+ """generate endpoint url"""
203
+ base_url = self.config.serviceConnection.root.config.openAPISchemaURL
204
+ if endpoint_name:
205
+ return AnyUrl(f"{base_url}#operation/{endpoint_name}")
206
+ else:
207
+ return AnyUrl(base_url)
208
+
209
+ def _get_api_request_method(self, method_type: str) -> Optional[str]:
210
+ """fetch endpoint request method"""
211
+ try:
212
+ return ApiRequestMethod[method_type.upper()]
213
+ except KeyError as err:
214
+ logger.info(f"Keyerror while fetching request method: {err}")
215
+ return None
216
+
217
+ def _get_request_schema(self, info: dict) -> Optional[APISchema]:
218
+ """fetch request schema"""
219
+ try:
220
+ schema_ref = (
221
+ info.get("requestBody", {})
222
+ .get("content", {})
223
+ .get("application/json", {})
224
+ .get("schema", {})
225
+ .get("$ref")
226
+ )
227
+ if not schema_ref:
228
+ logger.debug("No request schema found for the endpoint")
229
+ return None
230
+ return self._process_schema(schema_ref)
231
+ except Exception as err:
232
+ logger.info(f"Error while parsing request schema: {err}")
233
+ return None
234
+
235
+ def _get_response_schema(self, info: dict) -> Optional[APISchema]:
236
+ """fetch response schema"""
237
+ try:
238
+ schema_ref = (
239
+ info.get("responses", {})
240
+ .get("200", {})
241
+ .get("content", {})
242
+ .get("application/json", {})
243
+ .get("schema", {})
244
+ .get("$ref", {})
245
+ )
246
+ if not schema_ref:
247
+ logger.debug("No response schema found for the endpoint")
248
+ return None
249
+ return self._process_schema(schema_ref)
250
+ except Exception as err:
251
+ logger.info(f"Error while parsing response schema: {err}")
252
+ return None
253
+
254
+ def _process_schema(self, schema_ref: str) -> Optional[List[APISchema]]:
255
+ """process schema"""
256
+ try:
257
+ schema_ref = schema_ref.split("/")[-1]
258
+ schema_fields = (
259
+ self.json_response.get("components").get("schemas").get(schema_ref)
260
+ )
261
+
262
+ fetched_fields = []
263
+ for key, val in schema_fields.get("properties", {}).items():
264
+ dtype = val.get("type")
265
+ if not dtype:
266
+ continue
267
+ fetched_fields.append(
268
+ FieldModel(
269
+ name=key,
270
+ dataType=DataTypeTopic[dtype.upper()]
271
+ if dtype.upper() in DataTypeTopic.__members__
272
+ else DataTypeTopic.UNKNOWN,
273
+ )
274
+ )
275
+ return APISchema(schemaFields=fetched_fields)
276
+ except Exception as err:
277
+ logger.info(f"Error while processing request schema: {err}")
278
+ return None
@@ -0,0 +1,41 @@
1
+ # Copyright 2024 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+ """
12
+ OpenAPI REST API Models
13
+ """
14
+ from typing import Optional
15
+
16
+ from pydantic import AnyUrl, BaseModel, Field
17
+
18
+ from metadata.generated.schema.entity.data.apiEndpoint import ApiRequestMethod
19
+ from metadata.generated.schema.type import basic
20
+ from metadata.generated.schema.type.apiSchema import APISchema
21
+
22
+
23
+ class RESTCollection(BaseModel):
24
+ """REST colleciton model"""
25
+
26
+ name: basic.EntityName
27
+ display_name: Optional[str] = None
28
+ description: Optional[basic.Markdown] = None
29
+ url: Optional[AnyUrl] = None
30
+
31
+
32
+ class RESTEndpoint(BaseModel):
33
+ """REST endpoint model"""
34
+
35
+ name: Optional[str] = Field(None, alias="operationId")
36
+ display_name: Optional[str] = None
37
+ description: Optional[basic.Markdown] = None
38
+ url: Optional[AnyUrl] = None
39
+ request_method: Optional[ApiRequestMethod] = None
40
+ request_schema: Optional[APISchema] = None
41
+ response_schema: Optional[APISchema] = None
@@ -0,0 +1,4 @@
1
+ from metadata.ingestion.source.api.rest.metadata import RestSource
2
+ from metadata.utils.service_spec import BaseSpec
3
+
4
+ ServiceSpec = BaseSpec(metadata_source_class=RestSource)
@@ -0,0 +1,6 @@
1
+ from metadata.ingestion.source.dashboard.domodashboard.metadata import (
2
+ DomodashboardSource,
3
+ )
4
+ from metadata.utils.service_spec import BaseSpec
5
+
6
+ ServiceSpec = BaseSpec(metadata_source_class=DomodashboardSource)
@@ -0,0 +1,4 @@
1
+ from metadata.ingestion.source.dashboard.lightdash.metadata import LightdashSource
2
+ from metadata.utils.service_spec import BaseSpec
3
+
4
+ ServiceSpec = BaseSpec(metadata_source_class=LightdashSource)
@@ -28,6 +28,7 @@ from typing import Dict, Iterable, List, Optional, Sequence, Set, Type, Union, c
28
28
 
29
29
  import giturlparse
30
30
  import lkml
31
+ from liquid import Template
31
32
  from looker_sdk.sdk.api40.methods import Looker40SDK
32
33
  from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard
33
34
  from looker_sdk.sdk.api40.models import (
@@ -450,11 +451,11 @@ class LookerSource(DashboardServiceSource):
450
451
  view.name, "Data model (View) filtered out."
451
452
  )
452
453
  continue
453
-
454
+ view_name = view.from_ if view.from_ else view.name
454
455
  yield from self._process_view(
455
- view_name=ViewName(view.name), explore=model
456
+ view_name=ViewName(view_name), explore=model
456
457
  )
457
- if len(model.joins) == 0 and model.sql_table_name:
458
+ if model.view_name:
458
459
  yield from self._process_view(
459
460
  view_name=ViewName(model.view_name), explore=model
460
461
  )
@@ -570,7 +571,8 @@ class LookerSource(DashboardServiceSource):
570
571
  db_service_names = self.get_db_service_names()
571
572
 
572
573
  if view.sql_table_name:
573
- source_table_name = self._clean_table_name(view.sql_table_name)
574
+ sql_table_name = self._render_table_name(view.sql_table_name)
575
+ source_table_name = self._clean_table_name(sql_table_name)
574
576
 
575
577
  # View to the source is only there if we are informing the dbServiceNames
576
578
  for db_service_name in db_service_names or []:
@@ -726,6 +728,33 @@ class LookerSource(DashboardServiceSource):
726
728
 
727
729
  return table_name.lower().split(" as ")[0].strip()
728
730
 
731
+ @staticmethod
732
+ def _render_table_name(table_name: str) -> str:
733
+ """
734
+ sql_table_names might contain Liquid templates
735
+ when defining an explore. e.g,:
736
+ sql_table_name:
737
+ {% if openmetadata %}
738
+ event
739
+ {% elsif event.created_week._in_query %}
740
+ event_by_week
741
+ {% else %}
742
+ event
743
+ {% endif %} ;;
744
+ we should render the template and give the option
745
+ to render a specific value during metadata ingestion
746
+ using the "openmetadata" context argument
747
+ :param table_name: table name with possible templating
748
+ :return: rendered table name
749
+ """
750
+ try:
751
+ context = {"openmetadata": True}
752
+ template = Template(table_name)
753
+ sql_table_name = template.render(context)
754
+ except Exception:
755
+ sql_table_name = table_name
756
+ return sql_table_name
757
+
729
758
  @staticmethod
730
759
  def get_dashboard_sources(dashboard_details: LookerDashboard) -> Set[str]:
731
760
  """
@@ -0,0 +1,4 @@
1
+ from metadata.ingestion.source.dashboard.looker.metadata import LookerSource
2
+ from metadata.utils.service_spec import BaseSpec
3
+
4
+ ServiceSpec = BaseSpec(metadata_source_class=LookerSource)
@@ -0,0 +1,4 @@
1
+ from metadata.ingestion.source.dashboard.metabase.metadata import MetabaseSource
2
+ from metadata.utils.service_spec import BaseSpec
3
+
4
+ ServiceSpec = BaseSpec(metadata_source_class=MetabaseSource)
@@ -9,19 +9,19 @@
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
11
  """
12
- REST Auth & Client for Mstr
12
+ REST Auth & Client for MicroStrategy
13
13
  """
14
14
  import traceback
15
15
  from typing import List, Optional
16
16
 
17
17
  import requests
18
18
 
19
- from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
20
- MstrConnection,
19
+ from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
20
+ MicroStrategyConnection,
21
21
  )
22
22
  from metadata.ingestion.connections.test_connections import SourceConnectionException
23
23
  from metadata.ingestion.ometa.client import REST, ClientConfig
24
- from metadata.ingestion.source.dashboard.mstr.models import (
24
+ from metadata.ingestion.source.dashboard.microstrategy.models import (
25
25
  AuthHeaderCookie,
26
26
  MstrDashboard,
27
27
  MstrDashboardDetails,
@@ -37,30 +37,29 @@ from metadata.utils.logger import ingestion_logger
37
37
  logger = ingestion_logger()
38
38
 
39
39
  API_VERSION = "MicroStrategyLibrary/api"
40
- LOGIN_MODE_GUEST = 1
41
40
  APPLICATION_TYPE = 35
42
41
 
43
42
 
44
- class MSTRClient:
43
+ class MicroStrategyClient:
45
44
  """
46
45
  Client Handling API communication with Metabase
47
46
  """
48
47
 
49
48
  def _get_base_url(self, path=None):
50
49
  if not path:
51
- return f"{clean_uri(self.config.hostPort)}/{API_VERSION}"
52
- return f"{clean_uri(self.config.hostPort)}/{API_VERSION}/{path}"
50
+ return f"{clean_uri(str(self.config.hostPort))}/{API_VERSION}"
51
+ return f"{clean_uri(str(self.config.hostPort))}/{API_VERSION}/{path}"
53
52
 
54
53
  def __init__(
55
54
  self,
56
- config: MstrConnection,
55
+ config: MicroStrategyConnection,
57
56
  ):
58
57
  self.config = config
59
58
 
60
59
  self.auth_params: AuthHeaderCookie = self._get_auth_header_and_cookies()
61
60
 
62
61
  client_config = ClientConfig(
63
- base_url=clean_uri(config.hostPort),
62
+ base_url=clean_uri(str(self.config.hostPort)),
64
63
  api_version=API_VERSION,
65
64
  extra_headers=self.auth_params.auth_header,
66
65
  allow_redirects=True,
@@ -77,31 +76,27 @@ class MSTRClient:
77
76
  To know about the data params below please visit
78
77
  https://demo.microstrategy.com/MicroStrategyLibrary/api-docs/index.html#/Authentication/postLogin
79
78
  """
80
- data = {
81
- "username": self.config.username,
82
- "password": self.config.password.get_secret_value(),
83
- "loginMode": LOGIN_MODE_GUEST,
84
- "applicationType": APPLICATION_TYPE,
85
- }
86
- response = requests.post(
87
- url=self._get_base_url("auth/login"), data=data, timeout=60
88
- )
89
- if not response.status_code == 204:
90
- logger.error(
91
- f"Auth Failed with login mode: {LOGIN_MODE_GUEST}, Error: {response.text}"
92
- )
93
- logger.error("Trying with login mode: 8")
94
- data["loginMode"] = 8
79
+ try:
80
+ data = {
81
+ "username": self.config.username,
82
+ "password": self.config.password.get_secret_value(),
83
+ "loginMode": self.config.loginMode,
84
+ "applicationType": APPLICATION_TYPE,
85
+ }
95
86
  response = requests.post(
96
87
  url=self._get_base_url("auth/login"), data=data, timeout=60
97
88
  )
98
- if not response.status_code == 204:
99
- raise SourceConnectionException(
100
- f"Auth Failed with login mode: 8 and {LOGIN_MODE_GUEST}, Error: {response.text}"
101
- )
102
- return AuthHeaderCookie(
103
- auth_header=response.headers, auth_cookies=response.cookies
104
- )
89
+ if not response:
90
+ raise SourceConnectionException()
91
+ return AuthHeaderCookie(
92
+ auth_header=response.headers, auth_cookies=response.cookies
93
+ )
94
+ except Exception as exc:
95
+ logger.debug(traceback.format_exc())
96
+ logger.error(
97
+ f"Failed to fetch the auth header and cookies due to [{exc}], please validate credentials"
98
+ )
99
+ return None
105
100
 
106
101
  def _set_api_session(self) -> bool:
107
102
  """
@@ -17,31 +17,29 @@ from typing import Optional
17
17
  from metadata.generated.schema.entity.automations.workflow import (
18
18
  Workflow as AutomationWorkflow,
19
19
  )
20
- from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
21
- MstrConnection,
20
+ from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
21
+ MicroStrategyConnection,
22
22
  )
23
23
  from metadata.generated.schema.entity.services.connections.testConnectionResult import (
24
24
  TestConnectionResult,
25
25
  )
26
26
  from metadata.ingestion.connections.test_connections import test_connection_steps
27
27
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
28
- from metadata.ingestion.source.dashboard.mstr.client import MSTRClient
29
- from metadata.utils.constants import THREE_MIN
28
+ from metadata.ingestion.source.dashboard.microstrategy.client import MicroStrategyClient
30
29
 
31
30
 
32
- def get_connection(connection: MstrConnection) -> MSTRClient:
31
+ def get_connection(connection: MicroStrategyConnection) -> MicroStrategyClient:
33
32
  """
34
33
  Create connection
35
34
  """
36
- return MSTRClient(connection)
35
+ return MicroStrategyClient(connection)
37
36
 
38
37
 
39
38
  def test_connection(
40
39
  metadata: OpenMetadata,
41
- client: MSTRClient,
42
- service_connection: MstrConnection,
40
+ client: MicroStrategyClient,
41
+ service_connection: MicroStrategyConnection,
43
42
  automation_workflow: Optional[AutomationWorkflow] = None,
44
- timeout_seconds: Optional[int] = THREE_MIN,
45
43
  ) -> TestConnectionResult:
46
44
  """
47
45
  Test connection. This can be executed either as part
@@ -55,5 +53,4 @@ def test_connection(
55
53
  test_fn=test_fn,
56
54
  service_type=service_connection.type.value,
57
55
  automation_workflow=automation_workflow,
58
- timeout_seconds=timeout_seconds,
59
56
  )
@@ -8,7 +8,7 @@
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
- """Mstr source module"""
11
+ """MicroStrategy source module"""
12
12
  import traceback
13
13
  from typing import Iterable, List, Optional
14
14
 
@@ -16,8 +16,8 @@ from metadata.generated.schema.api.data.createChart import CreateChartRequest
16
16
  from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
17
17
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
18
18
  from metadata.generated.schema.entity.data.chart import Chart
19
- from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
20
- MstrConnection,
19
+ from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
20
+ MicroStrategyConnection,
21
21
  )
22
22
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
23
23
  StackTraceError,
@@ -34,7 +34,7 @@ from metadata.ingestion.api.models import Either
34
34
  from metadata.ingestion.api.steps import InvalidSourceException
35
35
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
36
36
  from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
37
- from metadata.ingestion.source.dashboard.mstr.models import (
37
+ from metadata.ingestion.source.dashboard.microstrategy.models import (
38
38
  MstrDashboard,
39
39
  MstrDashboardDetails,
40
40
  MstrPage,
@@ -47,9 +47,9 @@ from metadata.utils.logger import ingestion_logger
47
47
  logger = ingestion_logger()
48
48
 
49
49
 
50
- class MstrSource(DashboardServiceSource):
50
+ class MicrostrategySource(DashboardServiceSource):
51
51
  """
52
- MSTR Source Class
52
+ Microstrategy Source Class
53
53
  """
54
54
 
55
55
  @classmethod
@@ -60,10 +60,10 @@ class MstrSource(DashboardServiceSource):
60
60
  pipeline_name: Optional[str] = None,
61
61
  ):
62
62
  config = WorkflowSource.model_validate(config_dict)
63
- connection: MstrConnection = config.serviceConnection.root.config
64
- if not isinstance(connection, MstrConnection):
63
+ connection: MicroStrategyConnection = config.serviceConnection.root.config
64
+ if not isinstance(connection, MicroStrategyConnection):
65
65
  raise InvalidSourceException(
66
- f"Expected MstrConnection, but got {connection}"
66
+ f"Expected MicroStrategyConnection, but got {connection}"
67
67
  )
68
68
  return cls(config, metadata)
69
69
 
@@ -75,14 +75,18 @@ class MstrSource(DashboardServiceSource):
75
75
 
76
76
  if self.client.is_project_name():
77
77
  project = self.client.get_project_by_name()
78
- dashboards.extend(self.client.get_dashboards_list(project.id, project.name))
79
-
80
- if not self.client.is_project_name():
81
- for project in self.client.get_projects_list():
78
+ if project:
82
79
  dashboards.extend(
83
80
  self.client.get_dashboards_list(project.id, project.name)
84
81
  )
85
82
 
83
+ if not self.client.is_project_name():
84
+ for project in self.client.get_projects_list():
85
+ if project:
86
+ dashboards.extend(
87
+ self.client.get_dashboards_list(project.id, project.name)
88
+ )
89
+
86
90
  return dashboards
87
91
 
88
92
  def get_dashboard_name(self, dashboard: MstrDashboard) -> str:
@@ -121,7 +125,7 @@ class MstrSource(DashboardServiceSource):
121
125
  if dashboard_details:
122
126
  try:
123
127
  dashboard_url = (
124
- f"{clean_uri(self.service_connection.hostPort)}/MicroStrategyLibrary/app/"
128
+ f"{clean_uri(str(self.service_connection.hostPort))}/MicroStrategyLibrary/app/"
125
129
  f"{dashboard_details.projectId}/{dashboard_details.id}"
126
130
  )
127
131
  dashboard_request = CreateDashboardRequest(
@@ -9,7 +9,7 @@
9
9
  # See the License for the specific language governing permissions and
10
10
  # limitations under the License.
11
11
  """
12
- MSTR Models
12
+ MicroStrategy Models
13
13
  """
14
14
  from datetime import datetime
15
15
  from typing import Any, List, Optional