openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1074) hide show
  1. _openmetadata_testutils/data/dvdrental.zip +0 -0
  2. _openmetadata_testutils/helpers/assumption.py +118 -0
  3. _openmetadata_testutils/helpers/docker.py +38 -0
  4. _openmetadata_testutils/helpers/markers.py +5 -0
  5. _openmetadata_testutils/kafka/load_csv_data.py +133 -0
  6. _openmetadata_testutils/kafka/schema_registry_container.py +31 -0
  7. _openmetadata_testutils/ometa.py +23 -0
  8. _openmetadata_testutils/postgres/conftest.py +75 -0
  9. _openmetadata_testutils/pydantic/test_utils.py +79 -0
  10. airflow_provider_openmetadata/lineage/callback.py +3 -3
  11. airflow_provider_openmetadata/lineage/config/loader.py +1 -1
  12. airflow_provider_openmetadata/lineage/runner.py +5 -5
  13. airflow_provider_openmetadata/lineage/status.py +2 -2
  14. metadata/automations/runner.py +1 -1
  15. metadata/cli/app.py +1 -3
  16. metadata/cli/dataquality.py +8 -8
  17. metadata/cli/ingest.py +8 -8
  18. metadata/cli/lineage.py +10 -5
  19. metadata/cli/profile.py +8 -8
  20. metadata/cli/usage.py +6 -8
  21. metadata/clients/aws_client.py +4 -4
  22. metadata/clients/azure_client.py +1 -1
  23. metadata/clients/domo_client.py +12 -18
  24. metadata/cmd.py +16 -20
  25. metadata/config/common.py +3 -4
  26. metadata/data_quality/api/models.py +2 -2
  27. metadata/data_quality/builders/i_validator_builder.py +100 -0
  28. metadata/{data_insight/producer/producer_interface.py → data_quality/builders/pandas_validator_builder.py} +12 -12
  29. metadata/data_quality/{validations/validator.py → builders/sqa_validator_builder.py} +7 -8
  30. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +9 -42
  31. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +7 -40
  32. metadata/data_quality/interface/test_suite_interface.py +84 -2
  33. metadata/data_quality/processor/test_case_runner.py +76 -40
  34. metadata/data_quality/runner/base_test_suite_source.py +1 -1
  35. metadata/data_quality/runner/core.py +2 -2
  36. metadata/data_quality/source/test_suite.py +17 -15
  37. metadata/data_quality/validations/base_test_handler.py +25 -5
  38. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +2 -0
  39. metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +2 -0
  40. metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +2 -0
  41. metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +2 -0
  42. metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +2 -0
  43. metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +2 -0
  44. metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +2 -0
  45. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +24 -14
  46. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +13 -1
  47. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +1 -1
  48. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +1 -1
  49. metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py +1 -1
  50. metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py +1 -1
  51. metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py +1 -1
  52. metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py +1 -1
  53. metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py +1 -1
  54. metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py +1 -1
  55. metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py +1 -1
  56. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +1 -1
  57. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +1 -1
  58. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +1 -1
  59. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +1 -1
  60. metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py +1 -1
  61. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +1 -1
  62. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +1 -1
  63. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +8 -7
  64. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  65. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  66. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  67. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  68. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  69. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  70. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  71. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +8 -3
  72. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  73. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  74. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  75. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +1 -1
  76. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  77. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  78. metadata/data_quality/validations/models.py +21 -0
  79. metadata/data_quality/validations/runtime_param_setter/param_setter.py +42 -0
  80. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +54 -0
  81. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +190 -0
  82. metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +4 -13
  83. metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +2 -2
  84. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +434 -0
  85. metadata/examples/workflows/alationsink.yaml +34 -0
  86. metadata/examples/workflows/bigquery_profiler.yaml +1 -1
  87. metadata/examples/workflows/dbtcloud.yaml +26 -0
  88. metadata/examples/workflows/deltalake.yaml +6 -5
  89. metadata/examples/workflows/flink.yaml +33 -0
  90. metadata/examples/workflows/saperp.yaml +24 -0
  91. metadata/examples/workflows/synapse.yaml +24 -0
  92. metadata/generated/antlr/EntityLinkLexer.py +380 -352
  93. metadata/generated/schema/analytics/__init__.py +1 -1
  94. metadata/generated/schema/analytics/basic.py +19 -16
  95. metadata/generated/schema/analytics/reportData.py +26 -18
  96. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  97. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +64 -43
  98. metadata/generated/schema/analytics/reportDataType/entityReportData.py +35 -24
  99. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +18 -11
  100. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +24 -16
  101. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +23 -12
  102. metadata/generated/schema/analytics/webAnalyticEvent.py +75 -42
  103. metadata/generated/schema/analytics/webAnalyticEventData.py +17 -12
  104. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  105. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +24 -16
  106. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +31 -21
  107. metadata/generated/schema/api/__init__.py +1 -1
  108. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +28 -18
  109. metadata/generated/schema/api/analytics/__init__.py +1 -1
  110. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +33 -20
  111. metadata/generated/schema/api/automations/__init__.py +1 -1
  112. metadata/generated/schema/api/automations/createWorkflow.py +46 -27
  113. metadata/generated/schema/api/bulkAssets.py +14 -8
  114. metadata/generated/schema/api/classification/__init__.py +1 -1
  115. metadata/generated/schema/api/classification/createClassification.py +28 -15
  116. metadata/generated/schema/api/classification/createTag.py +43 -24
  117. metadata/generated/schema/api/classification/loadTags.py +11 -6
  118. metadata/generated/schema/api/createBot.py +30 -14
  119. metadata/generated/schema/api/createEventPublisherJob.py +42 -26
  120. metadata/generated/schema/api/createType.py +33 -21
  121. metadata/generated/schema/api/data/__init__.py +1 -1
  122. metadata/generated/schema/api/data/createAPICollection.py +97 -0
  123. metadata/generated/schema/api/data/createAPIEndpoint.py +107 -0
  124. metadata/generated/schema/api/data/createChart.py +71 -39
  125. metadata/generated/schema/api/data/createContainer.py +105 -68
  126. metadata/generated/schema/api/data/createCustomProperty.py +22 -14
  127. metadata/generated/schema/api/data/createDashboard.py +95 -58
  128. metadata/generated/schema/api/data/createDashboardDataModel.py +88 -51
  129. metadata/generated/schema/api/data/createDatabase.py +85 -53
  130. metadata/generated/schema/api/data/createDatabaseSchema.py +81 -50
  131. metadata/generated/schema/api/data/createGlossary.py +51 -34
  132. metadata/generated/schema/api/data/createGlossaryTerm.py +75 -48
  133. metadata/generated/schema/api/data/createMlModel.py +105 -68
  134. metadata/generated/schema/api/data/createPipeline.py +95 -58
  135. metadata/generated/schema/api/data/createQuery.py +85 -54
  136. metadata/generated/schema/api/data/createSearchIndex.py +76 -49
  137. metadata/generated/schema/api/data/createStoredProcedure.py +70 -46
  138. metadata/generated/schema/api/data/createTable.py +87 -58
  139. metadata/generated/schema/api/data/createTableProfile.py +16 -11
  140. metadata/generated/schema/api/data/createTopic.py +120 -75
  141. metadata/generated/schema/api/data/loadGlossary.py +12 -8
  142. metadata/generated/schema/api/data/restoreEntity.py +11 -7
  143. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  144. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +41 -26
  145. metadata/generated/schema/api/dataInsight/custom/__init__.py +3 -0
  146. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +53 -0
  147. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  148. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +49 -21
  149. metadata/generated/schema/api/docStore/__init__.py +1 -1
  150. metadata/generated/schema/api/docStore/createDocument.py +24 -12
  151. metadata/generated/schema/api/domains/__init__.py +1 -1
  152. metadata/generated/schema/api/domains/createDataProduct.py +50 -30
  153. metadata/generated/schema/api/domains/createDomain.py +38 -25
  154. metadata/generated/schema/api/feed/__init__.py +1 -1
  155. metadata/generated/schema/api/feed/closeTask.py +17 -10
  156. metadata/generated/schema/api/feed/createPost.py +16 -11
  157. metadata/generated/schema/api/feed/createSuggestion.py +56 -16
  158. metadata/generated/schema/api/feed/createThread.py +55 -34
  159. metadata/generated/schema/api/feed/resolveTask.py +21 -14
  160. metadata/generated/schema/api/feed/threadCount.py +29 -19
  161. metadata/generated/schema/api/lineage/__init__.py +1 -1
  162. metadata/generated/schema/api/lineage/addLineage.py +11 -6
  163. metadata/generated/schema/api/openMetadataServerVersion.py +18 -11
  164. metadata/generated/schema/api/policies/__init__.py +1 -1
  165. metadata/generated/schema/api/policies/createPolicy.py +38 -18
  166. metadata/generated/schema/api/services/__init__.py +1 -1
  167. metadata/generated/schema/api/services/createApiService.py +60 -0
  168. metadata/generated/schema/api/services/createDashboardService.py +42 -28
  169. metadata/generated/schema/api/services/createDatabaseService.py +42 -28
  170. metadata/generated/schema/api/services/createMessagingService.py +45 -29
  171. metadata/generated/schema/api/services/createMetadataService.py +34 -20
  172. metadata/generated/schema/api/services/createMlModelService.py +42 -28
  173. metadata/generated/schema/api/services/createPipelineService.py +50 -34
  174. metadata/generated/schema/api/services/createSearchService.py +45 -29
  175. metadata/generated/schema/api/services/createStorageService.py +45 -29
  176. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  177. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +45 -24
  178. metadata/generated/schema/api/setOwner.py +17 -8
  179. metadata/generated/schema/api/teams/__init__.py +1 -1
  180. metadata/generated/schema/api/teams/createPersona.py +32 -15
  181. metadata/generated/schema/api/teams/createRole.py +31 -16
  182. metadata/generated/schema/api/teams/createTeam.py +72 -46
  183. metadata/generated/schema/api/teams/createUser.py +75 -47
  184. metadata/generated/schema/api/tests/__init__.py +1 -1
  185. metadata/generated/schema/api/tests/createCustomMetric.py +38 -26
  186. metadata/generated/schema/api/tests/createLogicalTestCases.py +14 -10
  187. metadata/generated/schema/api/tests/createTestCase.py +46 -26
  188. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +25 -17
  189. metadata/generated/schema/api/tests/createTestDefinition.py +29 -15
  190. metadata/generated/schema/api/tests/createTestSuite.py +50 -27
  191. metadata/generated/schema/api/voteRequest.py +12 -6
  192. metadata/generated/schema/auth/__init__.py +1 -1
  193. metadata/generated/schema/auth/basicAuth.py +9 -6
  194. metadata/generated/schema/auth/basicLoginRequest.py +10 -7
  195. metadata/generated/schema/auth/changePasswordRequest.py +18 -13
  196. metadata/generated/schema/auth/createPersonalToken.py +9 -6
  197. metadata/generated/schema/auth/emailRequest.py +9 -6
  198. metadata/generated/schema/auth/emailVerificationToken.py +17 -13
  199. metadata/generated/schema/auth/generateToken.py +7 -5
  200. metadata/generated/schema/auth/jwtAuth.py +15 -9
  201. metadata/generated/schema/auth/loginRequest.py +10 -7
  202. metadata/generated/schema/auth/logoutRequest.py +14 -9
  203. metadata/generated/schema/auth/passwordResetRequest.py +16 -12
  204. metadata/generated/schema/auth/passwordResetToken.py +27 -18
  205. metadata/generated/schema/auth/personalAccessToken.py +21 -16
  206. metadata/generated/schema/auth/refreshToken.py +21 -16
  207. metadata/generated/schema/auth/registrationRequest.py +13 -10
  208. metadata/generated/schema/auth/revokePersonalToken.py +11 -7
  209. metadata/generated/schema/auth/revokeToken.py +7 -5
  210. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  211. metadata/generated/schema/auth/ssoAuth.py +24 -17
  212. metadata/generated/schema/auth/tokenRefreshRequest.py +11 -6
  213. metadata/generated/schema/configuration/__init__.py +1 -1
  214. metadata/generated/schema/configuration/appsPrivateConfiguration.py +23 -17
  215. metadata/generated/schema/configuration/authConfig.py +31 -22
  216. metadata/generated/schema/configuration/authenticationConfiguration.py +46 -33
  217. metadata/generated/schema/configuration/authorizerConfiguration.py +43 -37
  218. metadata/generated/schema/configuration/changeEventConfiguration.py +7 -5
  219. metadata/generated/schema/configuration/dataQualityConfiguration.py +10 -7
  220. metadata/generated/schema/configuration/elasticSearchConfiguration.py +52 -34
  221. metadata/generated/schema/configuration/eventHandlerConfiguration.py +10 -7
  222. metadata/generated/schema/configuration/fernetConfiguration.py +9 -6
  223. metadata/generated/schema/configuration/jwtTokenConfiguration.py +15 -12
  224. metadata/generated/schema/configuration/kafkaEventConfiguration.py +50 -36
  225. metadata/generated/schema/configuration/ldapConfiguration.py +72 -57
  226. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  227. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +23 -17
  228. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +13 -8
  229. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +10 -7
  230. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +11 -7
  231. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +23 -16
  232. metadata/generated/schema/configuration/limitsConfiguration.py +30 -0
  233. metadata/generated/schema/configuration/loginConfiguration.py +21 -14
  234. metadata/generated/schema/configuration/logoConfiguration.py +16 -13
  235. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +80 -49
  236. metadata/generated/schema/configuration/profilerConfiguration.py +18 -11
  237. metadata/generated/schema/configuration/slackAppConfiguration.py +27 -11
  238. metadata/generated/schema/configuration/taskNotificationConfiguration.py +11 -6
  239. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +21 -14
  240. metadata/generated/schema/configuration/themeConfiguration.py +42 -23
  241. metadata/generated/schema/configuration/uiThemePreference.py +19 -12
  242. metadata/generated/schema/dataInsight/__init__.py +1 -1
  243. metadata/generated/schema/dataInsight/custom/__init__.py +3 -0
  244. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +111 -0
  245. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +21 -0
  246. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +25 -0
  247. metadata/generated/schema/dataInsight/custom/formulaHolder.py +24 -0
  248. metadata/generated/schema/dataInsight/custom/lineChart.py +76 -0
  249. metadata/generated/schema/dataInsight/custom/summaryCard.py +49 -0
  250. metadata/generated/schema/dataInsight/dataInsightChart.py +79 -48
  251. metadata/generated/schema/dataInsight/dataInsightChartResult.py +40 -31
  252. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  253. metadata/generated/schema/dataInsight/kpi/basic.py +43 -32
  254. metadata/generated/schema/dataInsight/kpi/kpi.py +81 -52
  255. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  256. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +19 -12
  257. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +19 -12
  258. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +32 -19
  259. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +32 -18
  260. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +16 -8
  261. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +28 -19
  262. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +18 -13
  263. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +17 -9
  264. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +35 -17
  265. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +31 -17
  266. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +33 -17
  267. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +31 -17
  268. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +26 -14
  269. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +27 -14
  270. metadata/generated/schema/dataInsight/type/unusedAssets.py +17 -9
  271. metadata/generated/schema/email/__init__.py +1 -1
  272. metadata/generated/schema/email/emailRequest.py +26 -18
  273. metadata/generated/schema/email/emailTemplate.py +25 -0
  274. metadata/generated/schema/email/emailTemplatePlaceholder.py +20 -0
  275. metadata/generated/schema/email/smtpSettings.py +34 -18
  276. metadata/generated/schema/email/templateValidationReponse.py +29 -0
  277. metadata/generated/schema/entity/__init__.py +1 -1
  278. metadata/generated/schema/entity/applications/__init__.py +1 -1
  279. metadata/generated/schema/entity/applications/app.py +164 -107
  280. metadata/generated/schema/entity/applications/appRunRecord.py +42 -30
  281. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  282. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +24 -7
  283. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  284. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  285. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +28 -18
  286. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +22 -13
  287. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +23 -14
  288. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +28 -18
  289. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +20 -13
  290. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +19 -12
  291. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +11 -7
  292. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +19 -12
  293. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +11 -7
  294. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +11 -7
  295. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +20 -13
  296. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +11 -7
  297. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +68 -39
  298. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +47 -19
  299. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  300. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +65 -6
  301. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +17 -11
  302. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +34 -24
  303. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  304. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +19 -16
  305. metadata/generated/schema/entity/applications/createAppRequest.py +35 -23
  306. metadata/generated/schema/entity/applications/jobStatus.py +10 -6
  307. metadata/generated/schema/entity/applications/liveExecutionContext.py +15 -6
  308. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  309. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +144 -89
  310. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +95 -64
  311. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +8 -3
  312. metadata/generated/schema/entity/automations/__init__.py +1 -1
  313. metadata/generated/schema/entity/automations/testServiceConnection.py +47 -29
  314. metadata/generated/schema/entity/automations/workflow.py +82 -48
  315. metadata/generated/schema/entity/bot.py +64 -37
  316. metadata/generated/schema/entity/classification/__init__.py +1 -1
  317. metadata/generated/schema/entity/classification/classification.py +90 -50
  318. metadata/generated/schema/entity/classification/tag.py +104 -53
  319. metadata/generated/schema/entity/data/__init__.py +1 -1
  320. metadata/generated/schema/entity/data/apiCollection.py +145 -0
  321. metadata/generated/schema/entity/data/apiEndpoint.py +173 -0
  322. metadata/generated/schema/entity/data/chart.py +120 -72
  323. metadata/generated/schema/entity/data/container.py +178 -116
  324. metadata/generated/schema/entity/data/dashboard.py +141 -89
  325. metadata/generated/schema/entity/data/dashboardDataModel.py +129 -77
  326. metadata/generated/schema/entity/data/database.py +167 -104
  327. metadata/generated/schema/entity/data/databaseSchema.py +157 -98
  328. metadata/generated/schema/entity/data/glossary.py +118 -64
  329. metadata/generated/schema/entity/data/glossaryTerm.py +173 -105
  330. metadata/generated/schema/entity/data/metrics.py +93 -55
  331. metadata/generated/schema/entity/data/mlmodel.py +213 -148
  332. metadata/generated/schema/entity/data/pipeline.py +227 -150
  333. metadata/generated/schema/entity/data/query.py +126 -80
  334. metadata/generated/schema/entity/data/report.py +90 -49
  335. metadata/generated/schema/entity/data/searchIndex.py +171 -114
  336. metadata/generated/schema/entity/data/storedProcedure.py +140 -86
  337. metadata/generated/schema/entity/data/table.py +636 -436
  338. metadata/generated/schema/entity/data/topic.py +175 -111
  339. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  340. metadata/generated/schema/entity/docStore/document.py +52 -31
  341. metadata/generated/schema/entity/domains/__init__.py +1 -1
  342. metadata/generated/schema/entity/domains/dataProduct.py +72 -43
  343. metadata/generated/schema/entity/domains/domain.py +69 -45
  344. metadata/generated/schema/entity/events/__init__.py +1 -1
  345. metadata/generated/schema/entity/events/webhook.py +46 -25
  346. metadata/generated/schema/entity/feed/__init__.py +1 -1
  347. metadata/generated/schema/entity/feed/assets.py +7 -5
  348. metadata/generated/schema/entity/feed/customProperty.py +14 -7
  349. metadata/generated/schema/entity/feed/description.py +21 -13
  350. metadata/generated/schema/entity/feed/domain.py +15 -10
  351. metadata/generated/schema/entity/feed/entityInfo.py +14 -8
  352. metadata/generated/schema/entity/feed/owner.py +16 -11
  353. metadata/generated/schema/entity/feed/suggestion.py +112 -35
  354. metadata/generated/schema/entity/feed/tag.py +15 -10
  355. metadata/generated/schema/entity/feed/testCaseResult.py +19 -13
  356. metadata/generated/schema/entity/feed/thread.py +226 -148
  357. metadata/generated/schema/entity/policies/__init__.py +1 -1
  358. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  359. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +25 -12
  360. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +42 -28
  361. metadata/generated/schema/entity/policies/accessControl/rule.py +36 -23
  362. metadata/generated/schema/entity/policies/filters.py +14 -12
  363. metadata/generated/schema/entity/policies/policy.py +107 -67
  364. metadata/generated/schema/entity/services/__init__.py +1 -1
  365. metadata/generated/schema/entity/services/apiService.py +114 -0
  366. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  367. metadata/generated/schema/entity/services/connections/apiService/__init__.py +5 -0
  368. metadata/generated/schema/entity/services/connections/apiService/restConnection.py +38 -0
  369. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -3
  370. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +21 -13
  371. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +30 -19
  372. metadata/generated/schema/entity/services/connections/common/sslConfig.py +14 -8
  373. metadata/generated/schema/entity/services/connections/connectionBasicType.py +71 -55
  374. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  375. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +23 -15
  376. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +44 -30
  377. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +50 -36
  378. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +51 -37
  379. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +31 -24
  380. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +35 -27
  381. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +35 -27
  382. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +97 -63
  383. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  384. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +25 -18
  385. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +23 -14
  386. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +25 -18
  387. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +25 -18
  388. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +27 -19
  389. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +53 -36
  390. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +38 -23
  391. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +30 -21
  392. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +30 -22
  393. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +46 -35
  394. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  395. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +70 -52
  396. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +131 -89
  397. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +83 -56
  398. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +31 -22
  399. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +109 -72
  400. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +11 -7
  402. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +10 -8
  403. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +11 -7
  404. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +10 -8
  405. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +46 -32
  406. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +23 -15
  407. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +87 -60
  408. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +11 -7
  410. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +11 -7
  411. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +11 -7
  412. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +59 -43
  413. metadata/generated/schema/entity/services/connections/database/db2Connection.py +64 -48
  414. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +41 -85
  415. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +3 -0
  416. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +109 -0
  417. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +42 -0
  418. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +52 -33
  419. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +73 -52
  420. metadata/generated/schema/entity/services/connections/database/druidConnection.py +66 -47
  421. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +38 -27
  422. metadata/generated/schema/entity/services/connections/database/glueConnection.py +34 -24
  423. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +85 -56
  424. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +108 -75
  425. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  426. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +11 -6
  427. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +7 -5
  428. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +14 -9
  429. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +36 -25
  430. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +7 -5
  431. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +67 -46
  432. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +23 -15
  433. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +102 -67
  434. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +75 -52
  435. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +56 -38
  436. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +89 -60
  437. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +88 -60
  438. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +105 -74
  439. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +78 -54
  440. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +97 -65
  441. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +90 -61
  442. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +78 -55
  443. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +75 -47
  444. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +84 -0
  445. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  446. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +15 -9
  447. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +33 -24
  448. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +53 -38
  449. metadata/generated/schema/entity/services/connections/database/sasConnection.py +45 -31
  450. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +82 -54
  451. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +121 -80
  452. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +82 -56
  453. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +168 -0
  454. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +92 -62
  455. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +108 -72
  456. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +89 -60
  457. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +71 -50
  458. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  459. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +23 -15
  460. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +101 -61
  461. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +18 -13
  462. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +15 -10
  463. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +85 -54
  464. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  465. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  466. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +117 -74
  467. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +72 -0
  468. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +43 -28
  469. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +40 -27
  470. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +50 -40
  471. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +151 -92
  472. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  473. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +23 -15
  474. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +29 -20
  475. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +20 -13
  476. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +15 -10
  477. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  478. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +25 -18
  479. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +34 -26
  480. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -7
  481. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +23 -15
  482. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +29 -20
  483. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +37 -27
  484. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +43 -0
  485. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +44 -28
  486. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +37 -27
  487. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +35 -0
  488. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +18 -13
  489. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +62 -40
  490. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +71 -45
  491. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +71 -41
  492. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -7
  493. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +28 -18
  494. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  495. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +23 -15
  496. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  497. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +22 -15
  498. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +14 -11
  499. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +48 -31
  500. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +54 -35
  501. metadata/generated/schema/entity/services/connections/serviceConnection.py +38 -22
  502. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  503. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +27 -19
  504. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +23 -15
  505. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +34 -20
  506. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +32 -22
  507. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +96 -57
  508. metadata/generated/schema/entity/services/connections/testConnectionResult.py +40 -26
  509. metadata/generated/schema/entity/services/dashboardService.py +93 -62
  510. metadata/generated/schema/entity/services/databaseService.py +101 -63
  511. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  512. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +198 -122
  513. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +24 -13
  514. metadata/generated/schema/entity/services/ingestionPipelines/status.py +47 -37
  515. metadata/generated/schema/entity/services/messagingService.py +103 -69
  516. metadata/generated/schema/entity/services/metadataService.py +96 -56
  517. metadata/generated/schema/entity/services/mlmodelService.py +96 -63
  518. metadata/generated/schema/entity/services/pipelineService.py +102 -63
  519. metadata/generated/schema/entity/services/searchService.py +92 -62
  520. metadata/generated/schema/entity/services/serviceType.py +2 -1
  521. metadata/generated/schema/entity/services/storageService.py +93 -62
  522. metadata/generated/schema/entity/teams/__init__.py +1 -1
  523. metadata/generated/schema/entity/teams/persona.py +59 -35
  524. metadata/generated/schema/entity/teams/role.py +82 -51
  525. metadata/generated/schema/entity/teams/team.py +128 -82
  526. metadata/generated/schema/entity/teams/teamHierarchy.py +52 -35
  527. metadata/generated/schema/entity/teams/user.py +122 -81
  528. metadata/generated/schema/entity/type.py +94 -48
  529. metadata/generated/schema/entity/utils/__init__.py +1 -1
  530. metadata/generated/schema/entity/utils/entitiesCount.py +25 -18
  531. metadata/generated/schema/entity/utils/servicesCount.py +25 -22
  532. metadata/generated/schema/entity/utils/supersetApiConnection.py +19 -14
  533. metadata/generated/schema/events/__init__.py +1 -1
  534. metadata/generated/schema/events/alertMetrics.py +22 -14
  535. metadata/generated/schema/events/api/__init__.py +1 -1
  536. metadata/generated/schema/events/api/createEventSubscription.py +63 -35
  537. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +27 -0
  538. metadata/generated/schema/events/emailAlertConfig.py +20 -12
  539. metadata/generated/schema/events/eventFilterRule.py +40 -24
  540. metadata/generated/schema/events/eventSubscription.py +220 -142
  541. metadata/generated/schema/events/eventSubscriptionOffset.py +12 -8
  542. metadata/generated/schema/events/failedEvent.py +25 -13
  543. metadata/generated/schema/events/filterResourceDescriptor.py +24 -14
  544. metadata/generated/schema/events/subscriptionResourceDescriptor.py +20 -11
  545. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  546. metadata/generated/schema/metadataIngestion/application.py +39 -25
  547. metadata/generated/schema/metadataIngestion/applicationPipeline.py +23 -16
  548. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +125 -67
  549. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +9 -6
  550. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +182 -106
  551. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +117 -73
  552. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +75 -47
  553. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +51 -32
  554. metadata/generated/schema/metadataIngestion/dbtPipeline.py +75 -51
  555. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  556. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +37 -25
  557. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +23 -14
  558. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +36 -27
  559. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +37 -25
  560. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +31 -20
  561. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +31 -20
  562. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +37 -25
  563. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +45 -22
  564. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +65 -43
  565. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +35 -17
  566. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +99 -50
  567. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +51 -27
  568. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  569. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +60 -33
  570. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +59 -38
  571. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +22 -14
  572. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +15 -10
  573. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +15 -10
  574. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +14 -9
  575. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +14 -9
  576. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +14 -10
  577. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +57 -28
  578. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +32 -17
  579. metadata/generated/schema/metadataIngestion/workflow.py +114 -62
  580. metadata/generated/schema/monitoring/__init__.py +1 -1
  581. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  582. metadata/generated/schema/security/__init__.py +1 -1
  583. metadata/generated/schema/security/client/__init__.py +1 -1
  584. metadata/generated/schema/security/client/auth0SSOClientConfig.py +10 -9
  585. metadata/generated/schema/security/client/azureSSOClientConfig.py +12 -11
  586. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +11 -10
  587. metadata/generated/schema/security/client/googleSSOClientConfig.py +17 -12
  588. metadata/generated/schema/security/client/oidcClientConfig.py +45 -30
  589. metadata/generated/schema/security/client/oktaSSOClientConfig.py +14 -11
  590. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +9 -8
  591. metadata/generated/schema/security/client/samlSSOClientConfig.py +104 -67
  592. metadata/generated/schema/security/credentials/__init__.py +1 -1
  593. metadata/generated/schema/security/credentials/accessTokenAuth.py +17 -13
  594. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +10 -8
  595. metadata/generated/schema/security/credentials/awsCredentials.py +57 -38
  596. metadata/generated/schema/security/credentials/azureCredentials.py +47 -29
  597. metadata/generated/schema/security/credentials/basicAuth.py +13 -11
  598. metadata/generated/schema/security/credentials/bitbucketCredentials.py +22 -14
  599. metadata/generated/schema/security/credentials/gcpCredentials.py +66 -33
  600. metadata/generated/schema/security/credentials/gcpExternalAccount.py +55 -26
  601. metadata/generated/schema/security/credentials/gcpValues.py +70 -49
  602. metadata/generated/schema/security/credentials/gitCredentials.py +32 -25
  603. metadata/generated/schema/security/credentials/githubCredentials.py +13 -7
  604. metadata/generated/schema/security/credentials/gitlabCredentials.py +34 -0
  605. metadata/generated/schema/security/sasl/__init__.py +3 -0
  606. metadata/generated/schema/security/sasl/saslClientConfig.py +40 -0
  607. metadata/generated/schema/security/secrets/__init__.py +1 -1
  608. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  609. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +39 -24
  610. metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -1
  611. metadata/generated/schema/security/securityConfiguration.py +15 -9
  612. metadata/generated/schema/security/ssl/__init__.py +1 -1
  613. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +30 -20
  614. metadata/generated/schema/security/ssl/verifySSLConfig.py +16 -18
  615. metadata/generated/schema/settings/__init__.py +1 -1
  616. metadata/generated/schema/settings/settings.py +10 -5
  617. metadata/generated/schema/system/__init__.py +1 -1
  618. metadata/generated/schema/system/entityError.py +7 -5
  619. metadata/generated/schema/system/eventPublisherJob.py +58 -37
  620. metadata/generated/schema/system/indexingError.py +7 -5
  621. metadata/generated/schema/system/limitsResponse.py +20 -0
  622. metadata/generated/schema/system/ui/__init__.py +1 -1
  623. metadata/generated/schema/system/ui/knowledgePanel.py +12 -8
  624. metadata/generated/schema/system/ui/page.py +24 -18
  625. metadata/generated/schema/system/validationResponse.py +40 -28
  626. metadata/generated/schema/tests/__init__.py +1 -1
  627. metadata/generated/schema/tests/assigned.py +11 -7
  628. metadata/generated/schema/tests/basic.py +98 -55
  629. metadata/generated/schema/tests/customMetric.py +41 -29
  630. metadata/generated/schema/tests/dataQualityReport.py +40 -0
  631. metadata/generated/schema/tests/resolved.py +17 -13
  632. metadata/generated/schema/tests/testCase.py +137 -76
  633. metadata/generated/schema/tests/testCaseResolutionStatus.py +49 -33
  634. metadata/generated/schema/tests/testDefinition.py +129 -67
  635. metadata/generated/schema/tests/testSuite.py +137 -75
  636. metadata/generated/schema/type/__init__.py +1 -1
  637. metadata/generated/schema/type/apiSchema.py +27 -0
  638. metadata/generated/schema/type/auditLog.py +29 -20
  639. metadata/generated/schema/type/basic.py +177 -110
  640. metadata/generated/schema/type/bulkOperationResult.py +45 -30
  641. metadata/generated/schema/type/changeEvent.py +72 -40
  642. metadata/generated/schema/type/changeEventType.py +1 -1
  643. metadata/generated/schema/type/collectionDescriptor.py +24 -15
  644. metadata/generated/schema/type/csvDocumentation.py +11 -8
  645. metadata/generated/schema/type/csvErrorType.py +1 -1
  646. metadata/generated/schema/type/csvFile.py +25 -17
  647. metadata/generated/schema/type/csvImportResult.py +34 -21
  648. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  649. metadata/generated/schema/type/customProperties/complexTypes.py +110 -64
  650. metadata/generated/schema/type/customProperties/enumConfig.py +7 -5
  651. metadata/generated/schema/type/customProperty.py +34 -25
  652. metadata/generated/schema/type/dailyCount.py +10 -7
  653. metadata/generated/schema/type/databaseConnectionConfig.py +69 -42
  654. metadata/generated/schema/type/entityHierarchy.py +35 -26
  655. metadata/generated/schema/type/entityHistory.py +71 -45
  656. metadata/generated/schema/type/entityLineage.py +89 -57
  657. metadata/generated/schema/type/entityReference.py +51 -30
  658. metadata/generated/schema/type/entityReferenceList.py +11 -8
  659. metadata/generated/schema/type/entityRelationship.py +68 -39
  660. metadata/generated/schema/type/entityUsage.py +15 -10
  661. metadata/generated/schema/type/filterPattern.py +21 -12
  662. metadata/generated/schema/type/function.py +30 -21
  663. metadata/generated/schema/type/include.py +1 -1
  664. metadata/generated/schema/type/jdbcConnection.py +24 -19
  665. metadata/generated/schema/type/lifeCycle.py +47 -26
  666. metadata/generated/schema/type/paging.py +32 -21
  667. metadata/generated/schema/type/profile.py +13 -11
  668. metadata/generated/schema/type/queryParserData.py +47 -32
  669. metadata/generated/schema/type/reaction.py +13 -8
  670. metadata/generated/schema/type/schedule.py +18 -11
  671. metadata/generated/schema/type/schema.py +64 -44
  672. metadata/generated/schema/type/tableQuery.py +53 -40
  673. metadata/generated/schema/type/tableUsageCount.py +46 -33
  674. metadata/generated/schema/type/tagLabel.py +36 -24
  675. metadata/generated/schema/type/usageDetails.py +40 -24
  676. metadata/generated/schema/type/usageRequest.py +10 -7
  677. metadata/generated/schema/type/votes.py +21 -12
  678. metadata/great_expectations/action.py +14 -12
  679. metadata/great_expectations/utils/ometa_config_handler.py +1 -1
  680. metadata/ingestion/api/common.py +3 -4
  681. metadata/ingestion/api/delete.py +1 -1
  682. metadata/ingestion/api/models.py +6 -8
  683. metadata/ingestion/api/parser.py +13 -17
  684. metadata/ingestion/api/status.py +35 -13
  685. metadata/ingestion/api/topology_runner.py +2 -1
  686. metadata/ingestion/bulksink/metadata_usage.py +17 -14
  687. metadata/ingestion/connections/builders.py +12 -12
  688. metadata/ingestion/connections/secrets.py +8 -6
  689. metadata/ingestion/connections/test_connections.py +7 -2
  690. metadata/ingestion/lineage/models.py +5 -4
  691. metadata/ingestion/lineage/sql_lineage.py +43 -13
  692. metadata/ingestion/models/custom_properties.py +9 -9
  693. metadata/ingestion/models/custom_pydantic.py +70 -44
  694. metadata/ingestion/models/lf_tags_model.py +3 -3
  695. metadata/ingestion/models/ometa_classification.py +1 -1
  696. metadata/ingestion/models/ometa_lineage.py +24 -0
  697. metadata/ingestion/models/patch_request.py +96 -56
  698. metadata/ingestion/models/table_metadata.py +2 -2
  699. metadata/ingestion/models/topology.py +18 -12
  700. metadata/ingestion/ometa/client.py +34 -8
  701. metadata/ingestion/ometa/client_utils.py +6 -7
  702. metadata/ingestion/ometa/mixins/custom_property_mixin.py +2 -4
  703. metadata/ingestion/ometa/mixins/dashboard_mixin.py +2 -2
  704. metadata/ingestion/ometa/mixins/data_insight_mixin.py +10 -6
  705. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +5 -3
  706. metadata/ingestion/ometa/mixins/lineage_mixin.py +36 -15
  707. metadata/ingestion/ometa/mixins/mlmodel_mixin.py +2 -2
  708. metadata/ingestion/ometa/mixins/patch_mixin.py +32 -30
  709. metadata/ingestion/ometa/mixins/pipeline_mixin.py +3 -3
  710. metadata/ingestion/ometa/mixins/query_mixin.py +4 -4
  711. metadata/ingestion/ometa/mixins/role_policy_mixin.py +13 -13
  712. metadata/ingestion/ometa/mixins/search_index_mixin.py +5 -5
  713. metadata/ingestion/ometa/mixins/server_mixin.py +4 -5
  714. metadata/ingestion/ometa/mixins/service_mixin.py +2 -2
  715. metadata/ingestion/ometa/mixins/suggestions_mixin.py +2 -2
  716. metadata/ingestion/ometa/mixins/table_mixin.py +20 -20
  717. metadata/ingestion/ometa/mixins/tests_mixin.py +21 -18
  718. metadata/ingestion/ometa/mixins/topic_mixin.py +2 -2
  719. metadata/ingestion/ometa/mixins/user_mixin.py +44 -23
  720. metadata/ingestion/ometa/models.py +1 -1
  721. metadata/ingestion/ometa/ometa_api.py +6 -7
  722. metadata/ingestion/ometa/routes.py +18 -0
  723. metadata/ingestion/ometa/ttl_cache.py +47 -0
  724. metadata/ingestion/ometa/utils.py +3 -3
  725. metadata/ingestion/processor/query_parser.py +3 -3
  726. metadata/ingestion/sink/file.py +2 -2
  727. metadata/ingestion/sink/metadata_rest.py +59 -29
  728. metadata/ingestion/source/dashboard/dashboard_service.py +45 -45
  729. metadata/ingestion/source/dashboard/domodashboard/metadata.py +31 -19
  730. metadata/ingestion/source/dashboard/lightdash/client.py +1 -1
  731. metadata/ingestion/source/dashboard/lightdash/metadata.py +26 -14
  732. metadata/ingestion/source/dashboard/lightdash/models.py +11 -11
  733. metadata/ingestion/source/dashboard/looker/bulk_parser.py +3 -3
  734. metadata/ingestion/source/dashboard/looker/connection.py +1 -1
  735. metadata/ingestion/source/dashboard/looker/metadata.py +67 -42
  736. metadata/ingestion/source/dashboard/looker/parser.py +3 -3
  737. metadata/ingestion/source/dashboard/looker/utils.py +8 -5
  738. metadata/ingestion/source/dashboard/metabase/client.py +19 -2
  739. metadata/ingestion/source/dashboard/metabase/metadata.py +45 -14
  740. metadata/ingestion/source/dashboard/metabase/models.py +46 -28
  741. metadata/ingestion/source/dashboard/mode/client.py +1 -1
  742. metadata/ingestion/source/dashboard/mode/metadata.py +24 -14
  743. metadata/ingestion/source/dashboard/mstr/client.py +1 -1
  744. metadata/ingestion/source/dashboard/mstr/metadata.py +18 -11
  745. metadata/ingestion/source/dashboard/mstr/models.py +4 -4
  746. metadata/ingestion/source/dashboard/powerbi/client.py +2 -3
  747. metadata/ingestion/source/dashboard/powerbi/metadata.py +52 -31
  748. metadata/ingestion/source/dashboard/powerbi/models.py +25 -22
  749. metadata/ingestion/source/dashboard/qlikcloud/client.py +2 -2
  750. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +28 -16
  751. metadata/ingestion/source/dashboard/qlikcloud/models.py +4 -4
  752. metadata/ingestion/source/dashboard/qliksense/client.py +3 -3
  753. metadata/ingestion/source/dashboard/qliksense/metadata.py +35 -19
  754. metadata/ingestion/source/dashboard/qliksense/models.py +8 -8
  755. metadata/ingestion/source/dashboard/quicksight/metadata.py +26 -18
  756. metadata/ingestion/source/dashboard/quicksight/models.py +16 -16
  757. metadata/ingestion/source/dashboard/redash/client.py +1 -1
  758. metadata/ingestion/source/dashboard/redash/metadata.py +31 -19
  759. metadata/ingestion/source/dashboard/superset/api_source.py +46 -18
  760. metadata/ingestion/source/dashboard/superset/client.py +2 -2
  761. metadata/ingestion/source/dashboard/superset/db_source.py +44 -18
  762. metadata/ingestion/source/dashboard/superset/metadata.py +2 -2
  763. metadata/ingestion/source/dashboard/superset/mixin.py +12 -13
  764. metadata/ingestion/source/dashboard/superset/models.py +68 -68
  765. metadata/ingestion/source/dashboard/tableau/connection.py +2 -2
  766. metadata/ingestion/source/dashboard/tableau/metadata.py +398 -94
  767. metadata/ingestion/source/dashboard/tableau/models.py +33 -35
  768. metadata/ingestion/source/dashboard/tableau/queries.py +14 -0
  769. metadata/ingestion/source/database/athena/connection.py +34 -5
  770. metadata/ingestion/source/database/athena/metadata.py +43 -168
  771. metadata/ingestion/source/database/athena/models.py +10 -10
  772. metadata/ingestion/source/database/athena/query_parser.py +2 -2
  773. metadata/ingestion/source/database/athena/utils.py +191 -0
  774. metadata/ingestion/source/database/azuresql/metadata.py +4 -4
  775. metadata/ingestion/source/database/azuresql/query_parser.py +2 -2
  776. metadata/ingestion/source/database/bigquery/connection.py +17 -5
  777. metadata/ingestion/source/database/bigquery/helper.py +1 -1
  778. metadata/ingestion/source/database/bigquery/metadata.py +91 -52
  779. metadata/ingestion/source/database/bigquery/query_parser.py +3 -3
  780. metadata/ingestion/source/database/bigtable/connection.py +2 -2
  781. metadata/ingestion/source/database/bigtable/metadata.py +2 -2
  782. metadata/ingestion/source/database/clickhouse/connection.py +3 -3
  783. metadata/ingestion/source/database/clickhouse/metadata.py +11 -8
  784. metadata/ingestion/source/database/clickhouse/query_parser.py +4 -4
  785. metadata/ingestion/source/database/column_type_parser.py +19 -11
  786. metadata/ingestion/source/database/common_db_source.py +106 -48
  787. metadata/ingestion/source/database/common_nosql_source.py +22 -17
  788. metadata/ingestion/source/database/couchbase/connection.py +1 -1
  789. metadata/ingestion/source/database/couchbase/metadata.py +3 -3
  790. metadata/ingestion/source/database/database_service.py +30 -17
  791. metadata/ingestion/source/database/databricks/connection.py +1 -1
  792. metadata/ingestion/source/database/databricks/lineage.py +4 -3
  793. metadata/ingestion/source/database/databricks/metadata.py +93 -24
  794. metadata/ingestion/source/database/databricks/queries.py +5 -1
  795. metadata/ingestion/source/database/databricks/query_parser.py +2 -2
  796. metadata/ingestion/source/database/databricks/usage.py +4 -3
  797. metadata/ingestion/source/database/datalake/clients/azure_blob.py +72 -0
  798. metadata/ingestion/source/database/datalake/clients/base.py +47 -0
  799. metadata/ingestion/source/database/datalake/clients/gcs.py +136 -0
  800. metadata/ingestion/source/database/datalake/clients/s3.py +79 -0
  801. metadata/ingestion/source/database/datalake/connection.py +12 -70
  802. metadata/ingestion/source/database/datalake/metadata.py +75 -219
  803. metadata/ingestion/source/database/db2/metadata.py +13 -13
  804. metadata/ingestion/source/database/dbt/dbt_config.py +3 -2
  805. metadata/ingestion/source/database/dbt/dbt_service.py +15 -6
  806. metadata/ingestion/source/database/dbt/metadata.py +128 -83
  807. metadata/ingestion/source/database/dbt/models.py +9 -9
  808. metadata/ingestion/source/database/deltalake/clients/__init__.py +0 -0
  809. metadata/ingestion/source/database/deltalake/clients/base.py +82 -0
  810. metadata/ingestion/source/database/deltalake/clients/pyspark.py +304 -0
  811. metadata/ingestion/source/database/deltalake/clients/s3.py +200 -0
  812. metadata/ingestion/source/database/deltalake/connection.py +56 -84
  813. metadata/ingestion/source/database/deltalake/metadata.py +67 -193
  814. metadata/ingestion/source/database/domodatabase/metadata.py +83 -38
  815. metadata/ingestion/source/database/domodatabase/models.py +6 -6
  816. metadata/ingestion/source/database/doris/metadata.py +3 -3
  817. metadata/ingestion/source/database/druid/metadata.py +2 -2
  818. metadata/ingestion/source/database/dynamodb/metadata.py +5 -5
  819. metadata/ingestion/source/database/dynamodb/models.py +2 -2
  820. metadata/ingestion/source/database/extended_sample_data.py +41 -41
  821. metadata/ingestion/source/database/external_table_lineage_mixin.py +58 -3
  822. metadata/ingestion/source/database/glue/metadata.py +23 -18
  823. metadata/ingestion/source/database/glue/models.py +6 -6
  824. metadata/ingestion/source/database/greenplum/metadata.py +4 -4
  825. metadata/ingestion/source/database/hive/connection.py +2 -2
  826. metadata/ingestion/source/database/hive/metadata.py +2 -2
  827. metadata/ingestion/source/database/iceberg/catalog/hive.py +1 -1
  828. metadata/ingestion/source/database/iceberg/catalog/rest.py +1 -1
  829. metadata/ingestion/source/database/iceberg/fs/__init__.py +1 -1
  830. metadata/ingestion/source/database/iceberg/fs/s3.py +1 -1
  831. metadata/ingestion/source/database/iceberg/helper.py +4 -4
  832. metadata/ingestion/source/database/iceberg/metadata.py +28 -22
  833. metadata/ingestion/source/database/iceberg/models.py +6 -6
  834. metadata/ingestion/source/database/impala/connection.py +3 -3
  835. metadata/ingestion/source/database/impala/metadata.py +2 -2
  836. metadata/ingestion/source/database/incremental_metadata_extraction.py +1 -1
  837. metadata/ingestion/source/database/life_cycle_query_mixin.py +12 -8
  838. metadata/ingestion/source/database/lineage_source.py +25 -13
  839. metadata/ingestion/source/database/mariadb/metadata.py +2 -2
  840. metadata/ingestion/source/database/mongodb/connection.py +1 -1
  841. metadata/ingestion/source/database/mongodb/metadata.py +2 -2
  842. metadata/ingestion/source/database/mssql/constants.py +23 -0
  843. metadata/ingestion/source/database/mssql/lineage.py +25 -0
  844. metadata/ingestion/source/database/mssql/metadata.py +30 -10
  845. metadata/ingestion/source/database/mssql/queries.py +2 -0
  846. metadata/ingestion/source/database/mssql/query_parser.py +2 -2
  847. metadata/ingestion/source/database/mssql/usage.py +42 -0
  848. metadata/ingestion/source/database/mssql/utils.py +16 -1
  849. metadata/ingestion/source/database/mysql/metadata.py +2 -2
  850. metadata/ingestion/source/database/oracle/metadata.py +16 -13
  851. metadata/ingestion/source/database/oracle/models.py +1 -1
  852. metadata/ingestion/source/database/oracle/query_parser.py +2 -2
  853. metadata/ingestion/source/database/pinotdb/metadata.py +2 -2
  854. metadata/ingestion/source/database/postgres/lineage.py +2 -1
  855. metadata/ingestion/source/database/postgres/metadata.py +28 -6
  856. metadata/ingestion/source/database/postgres/pgspider/lineage.py +2 -2
  857. metadata/ingestion/source/database/postgres/queries.py +7 -0
  858. metadata/ingestion/source/database/postgres/query_parser.py +4 -4
  859. metadata/ingestion/source/database/postgres/usage.py +15 -2
  860. metadata/ingestion/source/database/presto/connection.py +2 -2
  861. metadata/ingestion/source/database/presto/metadata.py +2 -2
  862. metadata/ingestion/source/database/query/lineage.py +1 -1
  863. metadata/ingestion/source/database/query/usage.py +5 -3
  864. metadata/ingestion/source/database/query_parser_source.py +1 -1
  865. metadata/ingestion/source/database/redshift/metadata.py +7 -7
  866. metadata/ingestion/source/database/redshift/models.py +1 -1
  867. metadata/ingestion/source/database/redshift/queries.py +26 -2
  868. metadata/ingestion/source/database/redshift/query_parser.py +2 -2
  869. metadata/ingestion/source/database/salesforce/connection.py +1 -3
  870. metadata/ingestion/source/database/salesforce/metadata.py +22 -17
  871. metadata/ingestion/source/database/sample_data.py +198 -98
  872. metadata/ingestion/source/database/sample_usage.py +5 -7
  873. metadata/ingestion/source/database/saperp/__init__.py +0 -0
  874. metadata/ingestion/source/database/saperp/client.py +170 -0
  875. metadata/ingestion/source/database/saperp/connection.py +50 -0
  876. metadata/ingestion/source/database/saperp/constants.py +27 -0
  877. metadata/ingestion/source/database/saperp/metadata.py +322 -0
  878. metadata/ingestion/source/database/saperp/models.py +100 -0
  879. metadata/ingestion/source/database/saphana/metadata.py +2 -2
  880. metadata/ingestion/source/database/sas/client.py +2 -2
  881. metadata/ingestion/source/database/sas/metadata.py +35 -44
  882. metadata/ingestion/source/database/singlestore/metadata.py +2 -2
  883. metadata/ingestion/source/database/snowflake/connection.py +3 -3
  884. metadata/ingestion/source/database/snowflake/metadata.py +39 -12
  885. metadata/ingestion/source/database/snowflake/models.py +7 -7
  886. metadata/ingestion/source/database/snowflake/query_parser.py +2 -2
  887. metadata/ingestion/source/database/sql_column_handler.py +4 -3
  888. metadata/ingestion/source/database/sqlite/metadata.py +2 -2
  889. metadata/ingestion/source/database/stored_procedures_mixin.py +10 -11
  890. metadata/ingestion/source/database/teradata/metadata.py +4 -4
  891. metadata/ingestion/source/database/trino/connection.py +5 -7
  892. metadata/ingestion/source/database/trino/metadata.py +2 -2
  893. metadata/ingestion/source/database/trino/query_parser.py +2 -2
  894. metadata/ingestion/source/database/unitycatalog/client.py +21 -0
  895. metadata/ingestion/source/database/unitycatalog/lineage.py +7 -7
  896. metadata/ingestion/source/database/unitycatalog/metadata.py +82 -42
  897. metadata/ingestion/source/database/unitycatalog/models.py +16 -16
  898. metadata/ingestion/source/database/unitycatalog/query_parser.py +2 -2
  899. metadata/ingestion/source/database/usage_source.py +51 -34
  900. metadata/ingestion/source/database/vertica/metadata.py +3 -3
  901. metadata/ingestion/source/database/vertica/query_parser.py +4 -4
  902. metadata/ingestion/source/messaging/common_broker_source.py +18 -4
  903. metadata/ingestion/source/messaging/kafka/connection.py +1 -1
  904. metadata/ingestion/source/messaging/kafka/metadata.py +7 -9
  905. metadata/ingestion/source/messaging/kinesis/metadata.py +10 -5
  906. metadata/ingestion/source/messaging/kinesis/models.py +3 -5
  907. metadata/ingestion/source/messaging/messaging_service.py +12 -7
  908. metadata/ingestion/source/messaging/redpanda/metadata.py +2 -2
  909. metadata/ingestion/source/metadata/alationsink/__init__.py +0 -0
  910. metadata/ingestion/source/metadata/alationsink/client.py +213 -0
  911. metadata/ingestion/source/metadata/alationsink/connection.py +53 -0
  912. metadata/ingestion/source/metadata/alationsink/constants.py +60 -0
  913. metadata/ingestion/source/metadata/alationsink/metadata.py +397 -0
  914. metadata/ingestion/source/metadata/alationsink/models.py +148 -0
  915. metadata/ingestion/source/metadata/amundsen/connection.py +1 -1
  916. metadata/ingestion/source/metadata/amundsen/metadata.py +14 -10
  917. metadata/ingestion/source/metadata/atlas/client.py +1 -1
  918. metadata/ingestion/source/metadata/atlas/metadata.py +11 -12
  919. metadata/ingestion/source/mlmodel/mlflow/metadata.py +12 -6
  920. metadata/ingestion/source/mlmodel/mlmodel_service.py +13 -6
  921. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +34 -21
  922. metadata/ingestion/source/pipeline/airbyte/client.py +1 -1
  923. metadata/ingestion/source/pipeline/airbyte/metadata.py +15 -11
  924. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +3 -4
  925. metadata/ingestion/source/pipeline/airflow/metadata.py +28 -17
  926. metadata/ingestion/source/pipeline/airflow/models.py +21 -24
  927. metadata/ingestion/source/pipeline/dagster/client.py +3 -3
  928. metadata/ingestion/source/pipeline/dagster/metadata.py +31 -17
  929. metadata/ingestion/source/pipeline/dagster/models.py +14 -14
  930. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +120 -126
  931. metadata/ingestion/source/pipeline/databrickspipeline/models.py +72 -0
  932. metadata/ingestion/source/pipeline/dbtcloud/__init__.py +0 -0
  933. metadata/ingestion/source/pipeline/dbtcloud/client.py +143 -0
  934. metadata/ingestion/source/pipeline/dbtcloud/connection.py +60 -0
  935. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +345 -0
  936. metadata/ingestion/source/pipeline/dbtcloud/models.py +72 -0
  937. metadata/ingestion/source/pipeline/dbtcloud/queries.py +45 -0
  938. metadata/ingestion/source/pipeline/domopipeline/metadata.py +21 -12
  939. metadata/ingestion/source/pipeline/fivetran/client.py +20 -6
  940. metadata/ingestion/source/pipeline/fivetran/metadata.py +124 -50
  941. metadata/ingestion/source/pipeline/fivetran/models.py +20 -0
  942. metadata/ingestion/source/pipeline/flink/client.py +53 -0
  943. metadata/ingestion/source/pipeline/flink/connection.py +52 -0
  944. metadata/ingestion/source/pipeline/flink/metadata.py +188 -0
  945. metadata/ingestion/source/pipeline/flink/models.py +44 -0
  946. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +23 -11
  947. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +7 -8
  948. metadata/ingestion/source/pipeline/nifi/metadata.py +17 -8
  949. metadata/ingestion/source/pipeline/openlineage/connection.py +12 -3
  950. metadata/ingestion/source/pipeline/openlineage/metadata.py +3 -27
  951. metadata/ingestion/source/pipeline/pipeline_service.py +52 -8
  952. metadata/ingestion/source/pipeline/spline/client.py +1 -1
  953. metadata/ingestion/source/pipeline/spline/metadata.py +3 -3
  954. metadata/ingestion/source/pipeline/spline/models.py +12 -12
  955. metadata/ingestion/source/search/elasticsearch/connection.py +2 -2
  956. metadata/ingestion/source/search/elasticsearch/metadata.py +5 -4
  957. metadata/ingestion/source/search/search_service.py +13 -6
  958. metadata/ingestion/source/storage/gcs/client.py +58 -0
  959. metadata/ingestion/source/storage/gcs/connection.py +159 -0
  960. metadata/ingestion/source/storage/gcs/metadata.py +465 -0
  961. metadata/ingestion/source/storage/gcs/models.py +86 -0
  962. metadata/ingestion/source/storage/s3/metadata.py +268 -18
  963. metadata/ingestion/source/storage/s3/models.py +30 -11
  964. metadata/ingestion/source/storage/storage_service.py +43 -7
  965. metadata/ingestion/stage/table_usage.py +3 -3
  966. metadata/mixins/pandas/pandas_mixin.py +2 -2
  967. metadata/mixins/sqalchemy/sqa_mixin.py +1 -1
  968. metadata/parsers/avro_parser.py +12 -10
  969. metadata/parsers/json_schema_parser.py +4 -4
  970. metadata/parsers/protobuf_parser.py +4 -5
  971. metadata/pii/processor.py +4 -6
  972. metadata/profiler/adaptors/dynamodb.py +2 -2
  973. metadata/profiler/adaptors/mongodb.py +5 -17
  974. metadata/profiler/api/models.py +9 -9
  975. metadata/profiler/interface/nosql/profiler_interface.py +5 -8
  976. metadata/profiler/interface/pandas/profiler_interface.py +8 -10
  977. metadata/profiler/interface/profiler_interface.py +3 -3
  978. metadata/profiler/interface/profiler_interface_factory.py +46 -51
  979. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +28 -17
  980. metadata/profiler/interface/sqlalchemy/profiler_interface.py +3 -5
  981. metadata/profiler/metrics/composed/null_ratio.py +5 -4
  982. metadata/profiler/metrics/static/mean.py +9 -0
  983. metadata/profiler/metrics/system/queries/snowflake.py +1 -1
  984. metadata/profiler/metrics/window/percentille_mixin.py +1 -1
  985. metadata/profiler/orm/converter/base.py +21 -13
  986. metadata/profiler/orm/converter/common.py +3 -4
  987. metadata/profiler/orm/functions/datetime.py +1 -1
  988. metadata/profiler/orm/functions/sum.py +8 -1
  989. metadata/profiler/orm/functions/table_metric_computer.py +4 -5
  990. metadata/profiler/orm/registry.py +5 -0
  991. metadata/profiler/orm/types/custom_hex_byte_string.py +5 -3
  992. metadata/profiler/orm/types/undetermined_type.py +38 -0
  993. metadata/profiler/processor/core.py +19 -13
  994. metadata/profiler/processor/default.py +4 -1
  995. metadata/profiler/processor/processor.py +4 -4
  996. metadata/profiler/processor/sample_data_handler.py +5 -5
  997. metadata/profiler/processor/sampler/nosql/sampler.py +9 -4
  998. metadata/profiler/processor/sampler/sampler_factory.py +36 -6
  999. metadata/profiler/processor/sampler/sqlalchemy/azuresql/sampler.py +40 -0
  1000. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +33 -62
  1001. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +60 -0
  1002. metadata/profiler/source/base/profiler_source.py +12 -14
  1003. metadata/profiler/source/bigquery/profiler_source.py +2 -2
  1004. metadata/profiler/source/bigquery/type_mapper.py +2 -2
  1005. metadata/profiler/source/metadata.py +14 -16
  1006. metadata/profiler/source/metadata_ext.py +3 -3
  1007. metadata/readers/dataframe/avro.py +1 -3
  1008. metadata/readers/dataframe/models.py +29 -12
  1009. metadata/readers/dataframe/parquet.py +3 -3
  1010. metadata/readers/dataframe/reader_factory.py +6 -0
  1011. metadata/readers/file/api_reader.py +5 -2
  1012. metadata/readers/file/bitbucket.py +4 -4
  1013. metadata/readers/file/credentials.py +5 -5
  1014. metadata/readers/file/github.py +6 -6
  1015. metadata/readers/file/gitlab.py +172 -0
  1016. metadata/utils/class_helper.py +3 -5
  1017. metadata/utils/credentials.py +2 -2
  1018. metadata/utils/datalake/datalake_utils.py +4 -2
  1019. metadata/utils/entity_link.py +8 -0
  1020. metadata/utils/execution_time_tracker.py +2 -2
  1021. metadata/utils/filters.py +25 -17
  1022. metadata/utils/fqn.py +59 -14
  1023. metadata/utils/helpers.py +15 -12
  1024. metadata/utils/importer.py +3 -2
  1025. metadata/utils/life_cycle_utils.py +3 -3
  1026. metadata/utils/logger.py +9 -24
  1027. metadata/utils/lru_cache.py +3 -0
  1028. metadata/utils/metadata_service_helper.py +3 -1
  1029. metadata/utils/secrets/gcp_secrets_manager.py +128 -0
  1030. metadata/utils/secrets/secrets_manager_factory.py +3 -0
  1031. metadata/utils/source_hash.py +1 -1
  1032. metadata/utils/ssl_manager.py +20 -26
  1033. metadata/utils/ssl_registry.py +5 -1
  1034. metadata/utils/storage_metadata_config.py +6 -5
  1035. metadata/utils/tag_utils.py +21 -13
  1036. metadata/utils/time_utils.py +45 -5
  1037. metadata/workflow/application.py +4 -4
  1038. metadata/workflow/application_output_handler.py +6 -19
  1039. metadata/workflow/base.py +42 -5
  1040. metadata/workflow/data_quality.py +5 -5
  1041. metadata/workflow/ingestion.py +4 -6
  1042. metadata/workflow/metadata.py +4 -4
  1043. metadata/workflow/output_handler.py +35 -214
  1044. metadata/workflow/profiler.py +5 -5
  1045. metadata/workflow/usage.py +8 -6
  1046. metadata/workflow/workflow_init_error_handler.py +154 -0
  1047. metadata/workflow/workflow_output_handler.py +189 -98
  1048. metadata/workflow/workflow_status_mixin.py +20 -11
  1049. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/METADATA +387 -312
  1050. openmetadata_ingestion-1.5.0.0rc2.dist-info/RECORD +1598 -0
  1051. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/top_level.txt +1 -0
  1052. metadata/cli/insight.py +0 -51
  1053. metadata/data_insight/processor/kpi/kpi_runner.py +0 -156
  1054. metadata/data_insight/processor/kpi/run_result_registry.py +0 -162
  1055. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +0 -253
  1056. metadata/data_insight/processor/reports/data_processor.py +0 -72
  1057. metadata/data_insight/processor/reports/entity_report_data_processor.py +0 -262
  1058. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +0 -384
  1059. metadata/data_insight/producer/cost_analysis_producer.py +0 -126
  1060. metadata/data_insight/producer/entity_producer.py +0 -63
  1061. metadata/data_insight/producer/producer_factory.py +0 -53
  1062. metadata/data_insight/producer/web_analytics_producer.py +0 -84
  1063. metadata/data_insight/source/metadata.py +0 -175
  1064. metadata/workflow/data_insight.py +0 -78
  1065. openmetadata_ingestion-1.4.8.0.dist-info/RECORD +0 -1509
  1066. {metadata/cli → _openmetadata_testutils}/__init__.py +0 -0
  1067. {metadata/data_insight/processor → _openmetadata_testutils/helpers}/__init__.py +0 -0
  1068. {metadata/data_insight/processor/kpi → _openmetadata_testutils/kafka}/__init__.py +0 -0
  1069. {metadata/data_insight/processor/reports → _openmetadata_testutils/postgres}/__init__.py +0 -0
  1070. {metadata/data_insight/producer → _openmetadata_testutils/pydantic}/__init__.py +0 -0
  1071. /metadata/{data_insight/source → ingestion/source/database/datalake/clients}/__init__.py +0 -0
  1072. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/LICENSE +0 -0
  1073. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/WHEEL +0 -0
  1074. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/entry_points.txt +0 -0
@@ -11,9 +11,9 @@
11
11
  """
12
12
  Python API REST wrapper and helpers
13
13
  """
14
- import datetime
15
14
  import time
16
15
  import traceback
16
+ from datetime import datetime, timezone
17
17
  from typing import Callable, Dict, List, Optional, Union
18
18
 
19
19
  import requests
@@ -21,6 +21,7 @@ from requests.exceptions import HTTPError
21
21
 
22
22
  from metadata.config.common import ConfigModel
23
23
  from metadata.ingestion.ometa.credentials import URL, get_api_version
24
+ from metadata.ingestion.ometa.ttl_cache import TTLCache
24
25
  from metadata.utils.execution_time_tracker import calculate_execution_time
25
26
  from metadata.utils.logger import ometa_logger
26
27
 
@@ -33,6 +34,12 @@ class RetryException(Exception):
33
34
  """
34
35
 
35
36
 
37
+ class LimitsException(Exception):
38
+ """
39
+ API Client Feature Limit exception
40
+ """
41
+
42
+
36
43
  class APIError(Exception):
37
44
  """
38
45
  Represent API related error.
@@ -97,7 +104,8 @@ class ClientConfig(ConfigModel):
97
104
  api_version: Optional[str] = "v1"
98
105
  retry: Optional[int] = 3
99
106
  retry_wait: Optional[int] = 30
100
- retry_codes: List[int] = [429, 504]
107
+ limit_codes: List[int] = [429]
108
+ retry_codes: List[int] = [504]
101
109
  auth_token: Optional[Callable] = None
102
110
  access_token: Optional[str] = None
103
111
  expires_in: Optional[int] = None
@@ -107,8 +115,10 @@ class ClientConfig(ConfigModel):
107
115
  allow_redirects: Optional[bool] = False
108
116
  auth_token_mode: Optional[str] = "Bearer"
109
117
  verify: Optional[Union[bool, str]] = None
118
+ ttl_cache: int = 60
110
119
 
111
120
 
121
+ # pylint: disable=too-many-instance-attributes
112
122
  class REST:
113
123
  """
114
124
  REST client wrapper to manage requests with
@@ -124,20 +134,27 @@ class REST:
124
134
  self._retry = self.config.retry
125
135
  self._retry_wait = self.config.retry_wait
126
136
  self._retry_codes = self.config.retry_codes
137
+ self._limit_codes = self.config.limit_codes
127
138
  self._auth_token = self.config.auth_token
128
139
  self._auth_token_mode = self.config.auth_token_mode
129
140
  self._verify = self.config.verify
130
141
 
131
- def _request(
142
+ self._limits_reached = TTLCache(config.ttl_cache)
143
+
144
+ def _request( # pylint: disable=too-many-arguments
132
145
  self,
133
146
  method,
134
147
  path,
135
148
  data=None,
149
+ json=None,
136
150
  base_url: URL = None,
137
151
  api_version: str = None,
138
152
  headers: dict = None,
139
153
  ):
140
154
  # pylint: disable=too-many-locals
155
+ if path in self._limits_reached:
156
+ raise LimitsException(f"Skipping request - limits reached for {path}")
157
+
141
158
  if not headers:
142
159
  headers = {"Content-type": "application/json"}
143
160
  base_url = base_url or self._base_url
@@ -145,16 +162,16 @@ class REST:
145
162
  url: URL = URL(base_url + "/" + version + path)
146
163
  if (
147
164
  self.config.expires_in
148
- and datetime.datetime.utcnow().timestamp() >= self.config.expires_in
165
+ and datetime.now(timezone.utc).timestamp() >= self.config.expires_in
149
166
  or not self.config.access_token
150
167
  ):
151
168
  self.config.access_token, expiry = self._auth_token()
152
169
  if not self.config.access_token == "no_token":
153
- if isinstance(expiry, datetime.datetime):
170
+ if isinstance(expiry, datetime):
154
171
  self.config.expires_in = expiry.timestamp() - 120
155
172
  else:
156
173
  self.config.expires_in = (
157
- datetime.datetime.utcnow().timestamp() + expiry - 120
174
+ datetime.now(timezone.utc).timestamp() + expiry - 120
158
175
  )
159
176
 
160
177
  headers[self.config.auth_header] = (
@@ -185,12 +202,18 @@ class REST:
185
202
 
186
203
  method_key = "params" if method.upper() == "GET" else "data"
187
204
  opts[method_key] = data
205
+ if json:
206
+ opts["json"] = json
188
207
 
189
208
  total_retries = self._retry if self._retry > 0 else 0
190
209
  retry = total_retries
191
210
  while retry >= 0:
192
211
  try:
193
212
  return self._one_request(method, url, opts, retry)
213
+ except LimitsException as exc:
214
+ logger.error(f"Feature limit exceeded for {url}")
215
+ self._limits_reached.add(path)
216
+ raise exc
194
217
  except RetryException:
195
218
  retry_wait = self._retry_wait * (total_retries - retry + 1)
196
219
  logger.warning(
@@ -214,6 +237,7 @@ class REST:
214
237
  Returns the body json in the 200 status.
215
238
  """
216
239
  retry_codes = self._retry_codes
240
+ limit_codes = self._limit_codes
217
241
  try:
218
242
  resp = self._session.request(method, url, **opts)
219
243
  resp.raise_for_status()
@@ -231,6 +255,8 @@ class REST:
231
255
  # retry if we hit Rate Limit
232
256
  if resp.status_code in retry_codes and retry > 0:
233
257
  raise RetryException() from http_error
258
+ if resp.status_code in limit_codes:
259
+ raise LimitsException() from http_error
234
260
  if "code" in resp.text:
235
261
  error = resp.json()
236
262
  if "code" in error:
@@ -270,7 +296,7 @@ class REST:
270
296
  return self._request("GET", path, data)
271
297
 
272
298
  @calculate_execution_time(context="POST")
273
- def post(self, path, data=None):
299
+ def post(self, path, data=None, json=None):
274
300
  """
275
301
  POST method
276
302
 
@@ -281,7 +307,7 @@ class REST:
281
307
  Returns:
282
308
  Response
283
309
  """
284
- return self._request("POST", path, data)
310
+ return self._request("POST", path, data, json)
285
311
 
286
312
  @calculate_execution_time(context="PUT")
287
313
  def put(self, path, data=None):
@@ -18,8 +18,8 @@ from metadata.generated.schema.entity.data.chart import Chart
18
18
  from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
19
19
  OpenMetadataConnection,
20
20
  )
21
- from metadata.generated.schema.type.entityReference import EntityReference
22
- from metadata.ingestion.ometa.ometa_api import OpenMetadata
21
+ from metadata.generated.schema.type.basic import FullyQualifiedEntityName
22
+ from metadata.ingestion.ometa.ometa_api import C, OpenMetadata, T
23
23
  from metadata.utils import fqn
24
24
  from metadata.utils.logger import ometa_logger
25
25
 
@@ -28,7 +28,7 @@ logger = ometa_logger()
28
28
 
29
29
  def create_ometa_client(
30
30
  metadata_config: OpenMetadataConnection,
31
- ) -> OpenMetadata:
31
+ ) -> OpenMetadata[T, C]: # pyright: ignore[reportInvalidTypeVarUse]
32
32
  """Create an OpenMetadata client
33
33
 
34
34
  Args:
@@ -38,7 +38,7 @@ def create_ometa_client(
38
38
  OpenMetadata: an OM client
39
39
  """
40
40
  try:
41
- metadata = OpenMetadata(metadata_config)
41
+ metadata = OpenMetadata[T, C](metadata_config)
42
42
  metadata.health_check()
43
43
  return metadata
44
44
  except Exception as exc:
@@ -49,7 +49,7 @@ def create_ometa_client(
49
49
 
50
50
  def get_chart_entities_from_id(
51
51
  chart_ids: List[str], metadata: OpenMetadata, service_name: str
52
- ) -> List[EntityReference]:
52
+ ) -> List[FullyQualifiedEntityName]:
53
53
  """
54
54
  Method to get the chart entity using get_by_name api
55
55
  """
@@ -63,6 +63,5 @@ def get_chart_entities_from_id(
63
63
  ),
64
64
  )
65
65
  if chart:
66
- entity = EntityReference(id=chart.id, type="chart")
67
- entities.append(entity)
66
+ entities.append(chart.fullyQualifiedName)
68
67
  return entities
@@ -61,7 +61,7 @@ class OMetaCustomPropertyMixin:
61
61
 
62
62
  resp = self.client.put(
63
63
  f"/metadata/types/{entity_schema.get('id')}",
64
- data=ometa_custom_property.createCustomPropertyRequest.json(),
64
+ data=ometa_custom_property.createCustomPropertyRequest.model_dump_json(),
65
65
  )
66
66
  return resp
67
67
 
@@ -79,9 +79,7 @@ class OMetaCustomPropertyMixin:
79
79
  Get the PropertyType for custom properties
80
80
  """
81
81
  custom_property_type = self.get_custom_property_type(data_type=data_type)
82
- return PropertyType(
83
- __root__=EntityReference(id=custom_property_type.id, type="type")
84
- )
82
+ return PropertyType(EntityReference(id=custom_property_type.id, type="type"))
85
83
 
86
84
  def get_entity_custom_properties(self, entity_type: Type[T]) -> Optional[List]:
87
85
  """
@@ -41,7 +41,7 @@ class OMetaDashboardMixin:
41
41
  :param dashboard_usage_request: Usage data to add
42
42
  """
43
43
  resp = self.client.put(
44
- f"/usage/dashboard/{dashboard.id.__root__}",
45
- data=dashboard_usage_request.json(),
44
+ f"/usage/dashboard/{dashboard.id.root}",
45
+ data=dashboard_usage_request.model_dump_json(),
46
46
  )
47
47
  logger.debug("Published dashboard usage %s", resp)
@@ -44,7 +44,9 @@ class DataInsightMixin:
44
44
  record (ReportData): report data
45
45
  """
46
46
 
47
- resp = self.client.post("/analytics/dataInsights/data", record.json())
47
+ resp = self.client.post(
48
+ "/analytics/dataInsights/data", record.model_dump_json()
49
+ )
48
50
 
49
51
  return resp
50
52
 
@@ -56,7 +58,7 @@ class DataInsightMixin:
56
58
  record (ReportData): report data
57
59
  """
58
60
 
59
- resp = self.client.put(f"/kpi/{fqn}/kpiResult", record.json())
61
+ resp = self.client.put(f"/kpi/{fqn}/kpiResult", record.model_dump_json())
60
62
 
61
63
  return resp
62
64
 
@@ -66,7 +68,9 @@ class DataInsightMixin:
66
68
  ) -> List[WebAnalyticEventData]:
67
69
  """Get web analytic event"""
68
70
 
69
- resp = self.client.put("/analytics/web/events/collect", event_data.json())
71
+ resp = self.client.put(
72
+ "/analytics/web/events/collect", event_data.model_dump_json()
73
+ )
70
74
 
71
75
  return resp
72
76
 
@@ -127,7 +131,7 @@ class DataInsightMixin:
127
131
  request_params,
128
132
  )
129
133
 
130
- return DataInsightChartResult.parse_obj(resp)
134
+ return DataInsightChartResult.model_validate(resp)
131
135
 
132
136
  def get_kpi_result(self, fqn: str, start_ts, end_ts) -> list[KpiResult]:
133
137
  """Given FQN return KPI results
@@ -146,9 +150,9 @@ class DataInsightMixin:
146
150
  return [KpiResult(**data) for data in resp["data"]]
147
151
 
148
152
  def create_kpi(self, create: CreateKpiRequest) -> Kpi:
149
- resp = self.client.post("/kpi", create.json())
153
+ resp = self.client.post("/kpi", create.model_dump_json())
150
154
 
151
- return Kpi.parse_obj(resp)
155
+ return Kpi.model_validate(resp)
152
156
 
153
157
  def get_web_analytic_events(
154
158
  self, event_type: WebAnalyticEventType, start_ts: int, end_ts: int
@@ -47,10 +47,10 @@ class OMetaIngestionPipelineMixin:
47
47
  """
48
48
  resp = self.client.put(
49
49
  f"{self.get_suffix(IngestionPipeline)}/{ingestion_pipeline_fqn}/pipelineStatus",
50
- data=pipeline_status.json(),
50
+ data=pipeline_status.model_dump_json(),
51
51
  )
52
52
  logger.debug(
53
- f"Created Pipeline Status for pipeline {ingestion_pipeline_fqn}: {resp}"
53
+ f"Created Pipeline Status for pipeline {ingestion_pipeline_fqn}: {pipeline_status}"
54
54
  )
55
55
  return resp
56
56
 
@@ -104,7 +104,9 @@ class OMetaIngestionPipelineMixin:
104
104
  )
105
105
 
106
106
  if resp:
107
- return [PipelineStatus.parse_obj(status) for status in resp.get("data")]
107
+ return [
108
+ PipelineStatus.model_validate(status) for status in resp.get("data")
109
+ ]
108
110
  return None
109
111
 
110
112
  def get_ingestion_pipeline_by_name(
@@ -13,6 +13,7 @@ Mixin class containing Lineage specific methods
13
13
 
14
14
  To be used by OpenMetadata class
15
15
  """
16
+ import functools
16
17
  import traceback
17
18
  from copy import deepcopy
18
19
  from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union
@@ -82,11 +83,11 @@ class OMetaLineageMixin(Generic[T]):
82
83
  def _update_cache(self, request: AddLineageRequest, response: Dict[str, Any]):
83
84
  try:
84
85
  for res in response.get("downstreamEdges", []):
85
- if str(request.edge.toEntity.id.__root__) == res.get("toEntity"):
86
+ if str(request.edge.toEntity.id.root) == res.get("toEntity"):
86
87
  search_cache.put(
87
88
  (
88
- request.edge.fromEntity.id.__root__,
89
- request.edge.toEntity.id.__root__,
89
+ request.edge.fromEntity.id.root,
90
+ request.edge.toEntity.id.root,
90
91
  ),
91
92
  {"edge": res.get("lineageDetails")},
92
93
  )
@@ -97,8 +98,8 @@ class OMetaLineageMixin(Generic[T]):
97
98
  # discard the cache if failed to update
98
99
  search_cache.put(
99
100
  (
100
- request.edge.fromEntity.id.__root__,
101
- request.edge.toEntity.id.__root__,
101
+ request.edge.fromEntity.id.root,
102
+ request.edge.toEntity.id.root,
102
103
  ),
103
104
  None,
104
105
  )
@@ -113,8 +114,8 @@ class OMetaLineageMixin(Generic[T]):
113
114
  try:
114
115
  patch_op_success = False
115
116
  if check_patch and data.edge.lineageDetails:
116
- from_id = data.edge.fromEntity.id.__root__
117
- to_id = data.edge.toEntity.id.__root__
117
+ from_id = data.edge.fromEntity.id.root
118
+ to_id = data.edge.toEntity.id.root
118
119
  edge = self.get_lineage_edge(from_id, to_id)
119
120
  if edge:
120
121
  original: AddLineageRequest = deepcopy(data)
@@ -151,20 +152,22 @@ class OMetaLineageMixin(Generic[T]):
151
152
  patch_op_success = True
152
153
 
153
154
  if patch_op_success is False:
154
- self.client.put(self.get_suffix(AddLineageRequest), data=data.json())
155
+ self.client.put(
156
+ self.get_suffix(AddLineageRequest), data=data.model_dump_json()
157
+ )
155
158
 
156
159
  except APIError as err:
157
160
  logger.debug(traceback.format_exc())
158
161
  logger.error(
159
162
  "Error %s trying to PUT lineage for %s: %s",
160
163
  err.status_code,
161
- data.json(),
164
+ data.model_dump_json(),
162
165
  str(err),
163
166
  )
164
167
  raise err
165
168
 
166
169
  from_entity_lineage = self.get_lineage_by_id(
167
- data.edge.fromEntity.type, str(data.edge.fromEntity.id.__root__)
170
+ data.edge.fromEntity.type, str(data.edge.fromEntity.id.root)
168
171
  )
169
172
 
170
173
  self._update_cache(data, from_entity_lineage)
@@ -229,8 +232,8 @@ class OMetaLineageMixin(Generic[T]):
229
232
  if patch:
230
233
  self.client.patch(
231
234
  f"{self.get_suffix(AddLineageRequest)}/{original.edge.fromEntity.type}/"
232
- f"{original.edge.fromEntity.id.__root__}/{original.edge.toEntity.type}"
233
- f"/{original.edge.toEntity.id.__root__}",
235
+ f"{original.edge.fromEntity.id.root}/{original.edge.toEntity.type}"
236
+ f"/{original.edge.toEntity.id.root}",
234
237
  data=str(patch),
235
238
  )
236
239
  return True
@@ -319,13 +322,31 @@ class OMetaLineageMixin(Generic[T]):
319
322
  """
320
323
  try:
321
324
  self.client.delete(
322
- f"{self.get_suffix(AddLineageRequest)}/{edge.fromEntity.type}/{edge.fromEntity.id.__root__}/"
323
- f"{edge.toEntity.type}/{edge.toEntity.id.__root__}"
325
+ f"{self.get_suffix(AddLineageRequest)}/{edge.fromEntity.type}/{edge.fromEntity.id.root}/"
326
+ f"{edge.toEntity.type}/{edge.toEntity.id.root}"
324
327
  )
325
328
  except APIError as err:
326
329
  logger.debug(traceback.format_exc())
327
330
  logger.error(f"Error {err.status_code} trying to DELETE linage for {edge}")
328
331
 
332
+ @functools.lru_cache(maxsize=LRU_CACHE_SIZE)
333
+ def delete_lineage_by_source(
334
+ self, entity_type: str, entity_id: str, source: str
335
+ ) -> None:
336
+ """
337
+ Remove the given Edge
338
+ """
339
+ try:
340
+ self.client.delete(
341
+ f"{self.get_suffix(AddLineageRequest)}/{entity_type}/{entity_id}/"
342
+ f"type/{source}"
343
+ )
344
+ except APIError as err:
345
+ logger.debug(traceback.format_exc())
346
+ logger.error(
347
+ f"Error {err.status_code} trying to DELETE linage for {entity_id} of type {source}"
348
+ )
349
+
329
350
  def add_lineage_by_query(
330
351
  self,
331
352
  database_service: DatabaseService,
@@ -348,7 +369,7 @@ class OMetaLineageMixin(Generic[T]):
348
369
  connection_type = database_service.serviceType.value
349
370
  add_lineage_request = get_lineage_by_query(
350
371
  metadata=self,
351
- service_name=database_service.name.__root__,
372
+ service_name=database_service.name.root,
352
373
  dialect=ConnectionTypeDialectMapper.dialect_of(connection_type),
353
374
  query=sql,
354
375
  database_name=database_name,
@@ -90,7 +90,7 @@ class OMetaMlModelMixin(OMetaLineageMixin):
90
90
  )
91
91
  )
92
92
 
93
- mlmodel_lineage = self.get_lineage_by_id(MlModel, str(model.id.__root__))
93
+ mlmodel_lineage = self.get_lineage_by_id(MlModel, str(model.id.root))
94
94
 
95
95
  return mlmodel_lineage
96
96
 
@@ -151,7 +151,7 @@ class OMetaMlModelMixin(OMetaLineageMixin):
151
151
  mlHyperParameters=[
152
152
  MlHyperParameter(
153
153
  name=key,
154
- value=value,
154
+ value=str(value),
155
155
  )
156
156
  for key, value in model.get_params().items()
157
157
  ],
@@ -30,8 +30,9 @@ from metadata.generated.schema.entity.services.connections.testConnectionResult
30
30
  TestConnectionResult,
31
31
  )
32
32
  from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
33
- from metadata.generated.schema.type.basic import EntityLink
33
+ from metadata.generated.schema.type.basic import EntityLink, Markdown
34
34
  from metadata.generated.schema.type.entityReference import EntityReference
35
+ from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
35
36
  from metadata.generated.schema.type.lifeCycle import LifeCycle
36
37
  from metadata.generated.schema.type.tagLabel import TagLabel
37
38
  from metadata.ingestion.api.models import Entity
@@ -64,10 +65,7 @@ def update_column_tags(
64
65
  Inplace update for the incoming column list
65
66
  """
66
67
  for col in columns:
67
- if (
68
- str(col.fullyQualifiedName.__root__).lower()
69
- == column_tag.column_fqn.lower()
70
- ):
68
+ if str(col.fullyQualifiedName.root).lower() == column_tag.column_fqn.lower():
71
69
  if operation == PatchOperation.REMOVE:
72
70
  for tag in col.tags:
73
71
  if tag.tagFQN == column_tag.tag_label.tagFQN:
@@ -92,14 +90,14 @@ def update_column_description(
92
90
  for col in columns:
93
91
  # For dbt the column names in OM and dbt are not always in the same case.
94
92
  # We'll match the column names in case insensitive way
95
- desc_column = col_dict.get(col.fullyQualifiedName.__root__.lower())
93
+ desc_column = col_dict.get(col.fullyQualifiedName.root.lower())
96
94
  if desc_column:
97
95
  if col.description and not force:
98
96
  # If the description is already present and force is not passed,
99
97
  # description will not be overridden
100
98
  continue
101
99
 
102
- col.description = desc_column.__root__
100
+ col.description = desc_column # Keep the Markdown type
103
101
 
104
102
  if col.children:
105
103
  update_column_description(col.children, column_descriptions, force)
@@ -114,7 +112,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
114
112
 
115
113
  client: REST
116
114
 
117
- def patch(
115
+ def patch( # pylint: disable=too-many-arguments
118
116
  self,
119
117
  entity: Type[T],
120
118
  source: T,
@@ -122,6 +120,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
122
120
  allowed_fields: Optional[Dict] = None,
123
121
  restrict_update_fields: Optional[List] = None,
124
122
  array_entity_fields: Optional[List] = None,
123
+ override_metadata: Optional[bool] = False,
125
124
  ) -> Optional[T]:
126
125
  """
127
126
  Given an Entity type and Source entity and Destination entity,
@@ -144,6 +143,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
144
143
  allowed_fields=allowed_fields,
145
144
  restrict_update_fields=restrict_update_fields,
146
145
  array_entity_fields=array_entity_fields,
146
+ override_metadata=override_metadata,
147
147
  )
148
148
 
149
149
  if not patch:
@@ -200,8 +200,8 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
200
200
  return None
201
201
 
202
202
  # https://docs.pydantic.dev/latest/usage/exporting_models/#modelcopy
203
- destination = source.copy(deep=True)
204
- destination.description = description
203
+ destination = source.model_copy(deep=True)
204
+ destination.description = Markdown(description)
205
205
 
206
206
  return self.patch(entity=entity, source=source, destination=destination)
207
207
 
@@ -229,7 +229,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
229
229
 
230
230
  table.tableConstraints = instance.tableConstraints
231
231
 
232
- destination = table.copy(deep=True)
232
+ destination = table.model_copy(deep=True)
233
233
  destination.tableConstraints = constraints
234
234
 
235
235
  return self.patch(entity=Table, source=table, destination=destination)
@@ -254,9 +254,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
254
254
  if not source:
255
255
  return None
256
256
 
257
- destination = source.copy(deep=True)
257
+ destination = source.model_copy(deep=True)
258
258
 
259
- destination.entityLink = EntityLink(__root__=entity_link)
259
+ destination.entityLink = EntityLink(entity_link)
260
260
  if test_case_parameter_values:
261
261
  destination.parameterValues = test_case_parameter_values
262
262
  if compute_passed_failed_row_count != source.computePassedFailedRowCount:
@@ -292,13 +292,13 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
292
292
 
293
293
  # Initialize empty tag list or the last updated tags
294
294
  source.tags = instance.tags or []
295
- destination = source.copy(deep=True)
295
+ destination = source.model_copy(deep=True)
296
296
 
297
- tag_fqns = {label.tagFQN.__root__ for label in tag_labels}
297
+ tag_fqns = {label.tagFQN.root for label in tag_labels}
298
298
 
299
299
  if operation == PatchOperation.REMOVE:
300
300
  for tag in destination.tags:
301
- if tag.tagFQN.__root__ in tag_fqns:
301
+ if tag.tagFQN.root in tag_fqns:
302
302
  destination.tags.remove(tag)
303
303
  else:
304
304
  destination.tags.extend(tag_labels)
@@ -324,7 +324,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
324
324
  self,
325
325
  entity: Type[T],
326
326
  source: T,
327
- owner: EntityReference = None,
327
+ owners: EntityReferenceList = None,
328
328
  force: bool = False,
329
329
  ) -> Optional[T]:
330
330
  """
@@ -341,20 +341,20 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
341
341
  Updated Entity
342
342
  """
343
343
  instance: Optional[T] = self._fetch_entity_if_exists(
344
- entity=entity, entity_id=source.id, fields=["owner"]
344
+ entity=entity, entity_id=source.id, fields=["owners"]
345
345
  )
346
346
 
347
347
  if not instance:
348
348
  return None
349
349
 
350
350
  # Don't change existing data without force
351
- if instance.owner and not force:
351
+ if instance.owners and instance.owners.root and not force:
352
352
  # If a owner is already present and force is not passed,
353
353
  # owner will not be overridden
354
354
  return None
355
355
 
356
356
  destination = deepcopy(instance)
357
- destination.owner = owner
357
+ destination.owners = owners
358
358
 
359
359
  return self.patch(entity=entity, source=instance, destination=destination)
360
360
 
@@ -386,7 +386,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
386
386
  # Make sure we run the patch against the last updated data from the API
387
387
  table.columns = instance.columns
388
388
 
389
- destination = table.copy(deep=True)
389
+ destination = table.model_copy(deep=True)
390
390
  for column_tag in column_tags or []:
391
391
  update_column_tags(destination.columns, column_tag, operation)
392
392
 
@@ -394,7 +394,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
394
394
  if patched_entity is None:
395
395
  logger.debug(
396
396
  f"Empty PATCH result. Either everything is up to date or the "
397
- f"column names are not in [{table.fullyQualifiedName.__root__}]"
397
+ f"column names are not in [{table.fullyQualifiedName.root}]"
398
398
  )
399
399
 
400
400
  return patched_entity
@@ -440,7 +440,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
440
440
  return self.patch_column_descriptions(
441
441
  table=table,
442
442
  column_descriptions=[
443
- ColumnDescription(column_fqn=column_fqn, description=description)
443
+ ColumnDescription(
444
+ column_fqn=column_fqn, description=Markdown(description)
445
+ )
444
446
  ],
445
447
  force=force,
446
448
  )
@@ -471,14 +473,14 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
471
473
  # Make sure we run the patch against the last updated data from the API
472
474
  table.columns = instance.columns
473
475
 
474
- destination = table.copy(deep=True)
476
+ destination = table.model_copy(deep=True)
475
477
  update_column_description(destination.columns, column_descriptions, force)
476
478
 
477
479
  patched_entity = self.patch(entity=Table, source=table, destination=destination)
478
480
  if patched_entity is None:
479
481
  logger.debug(
480
482
  f"Empty PATCH result. Either everything is up to date or "
481
- f"columns are not matching for [{table.fullyQualifiedName.__root__}]"
483
+ f"columns are not matching for [{table.fullyQualifiedName.root}]"
482
484
  )
483
485
 
484
486
  return patched_entity
@@ -494,7 +496,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
494
496
  """
495
497
  result_data: Dict = {
496
498
  PatchField.PATH: PatchPath.RESPONSE,
497
- PatchField.VALUE: test_connection_result.dict(),
499
+ PatchField.VALUE: test_connection_result.model_dump(),
498
500
  PatchField.OPERATION: PatchOperation.ADD,
499
501
  }
500
502
 
@@ -530,7 +532,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
530
532
  :param life_cycle_data: Life Cycle data to add
531
533
  """
532
534
  try:
533
- destination = entity.copy(deep=True)
535
+ destination = entity.model_copy(deep=True)
534
536
  destination.lifeCycle = life_cycle
535
537
  return self.patch(
536
538
  entity=type(entity), source=entity, destination=destination
@@ -538,14 +540,14 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
538
540
  except Exception as exc:
539
541
  logger.debug(traceback.format_exc())
540
542
  logger.warning(
541
- f"Error trying to Patch life cycle data for {entity.fullyQualifiedName.__root__}: {exc}"
543
+ f"Error trying to Patch life cycle data for {entity.fullyQualifiedName.root}: {exc}"
542
544
  )
543
545
  return None
544
546
 
545
547
  def patch_domain(self, entity: Entity, domain: Domain) -> Optional[Entity]:
546
548
  """Patch domain data for an Entity"""
547
549
  try:
548
- destination: Entity = entity.copy(deep=True)
550
+ destination: Entity = entity.model_copy(deep=True)
549
551
  destination.domain = EntityReference(id=domain.id, type="domain")
550
552
  return self.patch(
551
553
  entity=type(entity), source=entity, destination=destination
@@ -553,6 +555,6 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
553
555
  except Exception as exc:
554
556
  logger.debug(traceback.format_exc())
555
557
  logger.warning(
556
- f"Error trying to Patch Domain for {entity.fullyQualifiedName.__root__}: {exc}"
558
+ f"Error trying to Patch Domain for {entity.fullyQualifiedName.root}: {exc}"
557
559
  )
558
560
  return None