openmetadata-ingestion 1.4.6.0rc1__py3-none-any.whl → 1.5.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1064) hide show
  1. _openmetadata_testutils/data/dvdrental.zip +0 -0
  2. _openmetadata_testutils/helpers/assumption.py +118 -0
  3. _openmetadata_testutils/helpers/docker.py +38 -0
  4. _openmetadata_testutils/helpers/markers.py +5 -0
  5. _openmetadata_testutils/ometa.py +23 -0
  6. _openmetadata_testutils/postgres/conftest.py +70 -0
  7. _openmetadata_testutils/pydantic/test_utils.py +78 -0
  8. airflow_provider_openmetadata/lineage/callback.py +3 -3
  9. airflow_provider_openmetadata/lineage/config/loader.py +1 -1
  10. airflow_provider_openmetadata/lineage/runner.py +5 -5
  11. airflow_provider_openmetadata/lineage/status.py +2 -2
  12. metadata/automations/runner.py +1 -1
  13. metadata/cli/app.py +1 -2
  14. metadata/cli/dataquality.py +8 -7
  15. metadata/cli/ingest.py +8 -7
  16. metadata/cli/lineage.py +10 -5
  17. metadata/cli/profile.py +8 -7
  18. metadata/cli/usage.py +6 -7
  19. metadata/clients/aws_client.py +4 -4
  20. metadata/clients/azure_client.py +1 -1
  21. metadata/clients/domo_client.py +11 -17
  22. metadata/cmd.py +16 -20
  23. metadata/config/common.py +3 -4
  24. metadata/data_quality/api/models.py +2 -2
  25. metadata/data_quality/builders/i_validator_builder.py +100 -0
  26. metadata/{data_insight/producer/producer_interface.py → data_quality/builders/pandas_validator_builder.py} +12 -12
  27. metadata/data_quality/{validations/validator.py → builders/sqa_validator_builder.py} +7 -8
  28. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +9 -42
  29. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +7 -40
  30. metadata/data_quality/interface/test_suite_interface.py +84 -2
  31. metadata/data_quality/processor/test_case_runner.py +76 -40
  32. metadata/data_quality/runner/base_test_suite_source.py +1 -1
  33. metadata/data_quality/runner/core.py +2 -2
  34. metadata/data_quality/source/test_suite.py +17 -15
  35. metadata/data_quality/validations/base_test_handler.py +25 -5
  36. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +2 -0
  37. metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +2 -0
  38. metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +2 -0
  39. metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +2 -0
  40. metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +2 -0
  41. metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +2 -0
  42. metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +2 -0
  43. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +24 -14
  44. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +13 -1
  45. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +1 -1
  46. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +1 -1
  47. metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py +1 -1
  48. metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py +1 -1
  49. metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py +1 -1
  50. metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py +1 -1
  51. metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py +1 -1
  52. metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py +1 -1
  53. metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py +1 -1
  54. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +1 -1
  55. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +1 -1
  56. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +1 -1
  57. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +1 -1
  58. metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py +1 -1
  59. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +1 -1
  60. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +1 -1
  61. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
  62. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  63. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  64. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  65. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  66. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  67. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  68. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  69. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
  70. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  71. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  72. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  73. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +1 -1
  74. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  75. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  76. metadata/data_quality/validations/models.py +21 -0
  77. metadata/data_quality/validations/runtime_param_setter/param_setter.py +42 -0
  78. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +54 -0
  79. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +190 -0
  80. metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +4 -13
  81. metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +2 -2
  82. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +402 -0
  83. metadata/examples/workflows/alationsink.yaml +34 -0
  84. metadata/examples/workflows/bigquery_profiler.yaml +1 -1
  85. metadata/examples/workflows/dbtcloud.yaml +26 -0
  86. metadata/examples/workflows/deltalake.yaml +6 -5
  87. metadata/examples/workflows/flink.yaml +33 -0
  88. metadata/examples/workflows/saperp.yaml +24 -0
  89. metadata/examples/workflows/synapse.yaml +24 -0
  90. metadata/generated/antlr/EntityLinkLexer.py +279 -249
  91. metadata/generated/schema/analytics/__init__.py +1 -1
  92. metadata/generated/schema/analytics/basic.py +19 -16
  93. metadata/generated/schema/analytics/reportData.py +26 -18
  94. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  95. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +64 -43
  96. metadata/generated/schema/analytics/reportDataType/entityReportData.py +35 -24
  97. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +18 -11
  98. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +24 -16
  99. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +23 -12
  100. metadata/generated/schema/analytics/webAnalyticEvent.py +75 -42
  101. metadata/generated/schema/analytics/webAnalyticEventData.py +17 -12
  102. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  103. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +24 -16
  104. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +31 -21
  105. metadata/generated/schema/api/__init__.py +1 -1
  106. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +28 -18
  107. metadata/generated/schema/api/analytics/__init__.py +1 -1
  108. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +33 -20
  109. metadata/generated/schema/api/automations/__init__.py +1 -1
  110. metadata/generated/schema/api/automations/createWorkflow.py +46 -27
  111. metadata/generated/schema/api/bulkAssets.py +14 -8
  112. metadata/generated/schema/api/classification/__init__.py +1 -1
  113. metadata/generated/schema/api/classification/createClassification.py +28 -15
  114. metadata/generated/schema/api/classification/createTag.py +43 -24
  115. metadata/generated/schema/api/classification/loadTags.py +11 -6
  116. metadata/generated/schema/api/createBot.py +30 -14
  117. metadata/generated/schema/api/createEventPublisherJob.py +42 -26
  118. metadata/generated/schema/api/createType.py +33 -21
  119. metadata/generated/schema/api/data/__init__.py +1 -1
  120. metadata/generated/schema/api/data/createAPICollection.py +97 -0
  121. metadata/generated/schema/api/data/createAPIEndpoint.py +107 -0
  122. metadata/generated/schema/api/data/createChart.py +71 -39
  123. metadata/generated/schema/api/data/createContainer.py +105 -68
  124. metadata/generated/schema/api/data/createCustomProperty.py +22 -14
  125. metadata/generated/schema/api/data/createDashboard.py +95 -58
  126. metadata/generated/schema/api/data/createDashboardDataModel.py +88 -51
  127. metadata/generated/schema/api/data/createDatabase.py +85 -53
  128. metadata/generated/schema/api/data/createDatabaseSchema.py +81 -50
  129. metadata/generated/schema/api/data/createGlossary.py +51 -34
  130. metadata/generated/schema/api/data/createGlossaryTerm.py +75 -48
  131. metadata/generated/schema/api/data/createMlModel.py +105 -68
  132. metadata/generated/schema/api/data/createPipeline.py +95 -58
  133. metadata/generated/schema/api/data/createQuery.py +85 -54
  134. metadata/generated/schema/api/data/createSearchIndex.py +76 -49
  135. metadata/generated/schema/api/data/createStoredProcedure.py +70 -46
  136. metadata/generated/schema/api/data/createTable.py +87 -58
  137. metadata/generated/schema/api/data/createTableProfile.py +16 -11
  138. metadata/generated/schema/api/data/createTopic.py +120 -75
  139. metadata/generated/schema/api/data/loadGlossary.py +12 -8
  140. metadata/generated/schema/api/data/restoreEntity.py +11 -7
  141. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  142. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +41 -26
  143. metadata/generated/schema/api/dataInsight/custom/__init__.py +3 -0
  144. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +53 -0
  145. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  146. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +49 -21
  147. metadata/generated/schema/api/docStore/__init__.py +1 -1
  148. metadata/generated/schema/api/docStore/createDocument.py +24 -12
  149. metadata/generated/schema/api/domains/__init__.py +1 -1
  150. metadata/generated/schema/api/domains/createDataProduct.py +50 -30
  151. metadata/generated/schema/api/domains/createDomain.py +38 -25
  152. metadata/generated/schema/api/feed/__init__.py +1 -1
  153. metadata/generated/schema/api/feed/closeTask.py +17 -10
  154. metadata/generated/schema/api/feed/createPost.py +16 -11
  155. metadata/generated/schema/api/feed/createSuggestion.py +56 -16
  156. metadata/generated/schema/api/feed/createThread.py +55 -34
  157. metadata/generated/schema/api/feed/resolveTask.py +21 -14
  158. metadata/generated/schema/api/feed/threadCount.py +29 -19
  159. metadata/generated/schema/api/lineage/__init__.py +1 -1
  160. metadata/generated/schema/api/lineage/addLineage.py +11 -6
  161. metadata/generated/schema/api/openMetadataServerVersion.py +18 -11
  162. metadata/generated/schema/api/policies/__init__.py +1 -1
  163. metadata/generated/schema/api/policies/createPolicy.py +38 -18
  164. metadata/generated/schema/api/services/__init__.py +1 -1
  165. metadata/generated/schema/api/services/createApiService.py +60 -0
  166. metadata/generated/schema/api/services/createDashboardService.py +42 -28
  167. metadata/generated/schema/api/services/createDatabaseService.py +42 -28
  168. metadata/generated/schema/api/services/createMessagingService.py +45 -29
  169. metadata/generated/schema/api/services/createMetadataService.py +34 -20
  170. metadata/generated/schema/api/services/createMlModelService.py +42 -28
  171. metadata/generated/schema/api/services/createPipelineService.py +50 -34
  172. metadata/generated/schema/api/services/createSearchService.py +45 -29
  173. metadata/generated/schema/api/services/createStorageService.py +45 -29
  174. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  175. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +45 -24
  176. metadata/generated/schema/api/setOwner.py +17 -8
  177. metadata/generated/schema/api/teams/__init__.py +1 -1
  178. metadata/generated/schema/api/teams/createPersona.py +32 -15
  179. metadata/generated/schema/api/teams/createRole.py +31 -16
  180. metadata/generated/schema/api/teams/createTeam.py +72 -46
  181. metadata/generated/schema/api/teams/createUser.py +75 -47
  182. metadata/generated/schema/api/tests/__init__.py +1 -1
  183. metadata/generated/schema/api/tests/createCustomMetric.py +38 -26
  184. metadata/generated/schema/api/tests/createLogicalTestCases.py +14 -10
  185. metadata/generated/schema/api/tests/createTestCase.py +46 -26
  186. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +25 -17
  187. metadata/generated/schema/api/tests/createTestDefinition.py +29 -15
  188. metadata/generated/schema/api/tests/createTestSuite.py +50 -27
  189. metadata/generated/schema/api/voteRequest.py +12 -6
  190. metadata/generated/schema/auth/__init__.py +1 -1
  191. metadata/generated/schema/auth/basicAuth.py +9 -6
  192. metadata/generated/schema/auth/basicLoginRequest.py +10 -7
  193. metadata/generated/schema/auth/changePasswordRequest.py +18 -13
  194. metadata/generated/schema/auth/createPersonalToken.py +9 -6
  195. metadata/generated/schema/auth/emailRequest.py +9 -6
  196. metadata/generated/schema/auth/emailVerificationToken.py +17 -13
  197. metadata/generated/schema/auth/generateToken.py +7 -5
  198. metadata/generated/schema/auth/jwtAuth.py +15 -9
  199. metadata/generated/schema/auth/loginRequest.py +10 -7
  200. metadata/generated/schema/auth/logoutRequest.py +14 -9
  201. metadata/generated/schema/auth/passwordResetRequest.py +16 -12
  202. metadata/generated/schema/auth/passwordResetToken.py +27 -18
  203. metadata/generated/schema/auth/personalAccessToken.py +21 -16
  204. metadata/generated/schema/auth/refreshToken.py +21 -16
  205. metadata/generated/schema/auth/registrationRequest.py +13 -10
  206. metadata/generated/schema/auth/revokePersonalToken.py +11 -7
  207. metadata/generated/schema/auth/revokeToken.py +7 -5
  208. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  209. metadata/generated/schema/auth/ssoAuth.py +24 -17
  210. metadata/generated/schema/auth/tokenRefreshRequest.py +11 -6
  211. metadata/generated/schema/configuration/__init__.py +1 -1
  212. metadata/generated/schema/configuration/appsPrivateConfiguration.py +23 -17
  213. metadata/generated/schema/configuration/authConfig.py +31 -22
  214. metadata/generated/schema/configuration/authenticationConfiguration.py +46 -33
  215. metadata/generated/schema/configuration/authorizerConfiguration.py +43 -37
  216. metadata/generated/schema/configuration/changeEventConfiguration.py +7 -5
  217. metadata/generated/schema/configuration/dataQualityConfiguration.py +10 -7
  218. metadata/generated/schema/configuration/elasticSearchConfiguration.py +52 -34
  219. metadata/generated/schema/configuration/eventHandlerConfiguration.py +10 -7
  220. metadata/generated/schema/configuration/fernetConfiguration.py +9 -6
  221. metadata/generated/schema/configuration/jwtTokenConfiguration.py +15 -12
  222. metadata/generated/schema/configuration/kafkaEventConfiguration.py +50 -36
  223. metadata/generated/schema/configuration/ldapConfiguration.py +72 -57
  224. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  225. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +23 -17
  226. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +13 -8
  227. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +10 -7
  228. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +11 -7
  229. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +23 -16
  230. metadata/generated/schema/configuration/limitsConfiguration.py +30 -0
  231. metadata/generated/schema/configuration/loginConfiguration.py +21 -14
  232. metadata/generated/schema/configuration/logoConfiguration.py +16 -13
  233. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +80 -49
  234. metadata/generated/schema/configuration/profilerConfiguration.py +18 -11
  235. metadata/generated/schema/configuration/slackAppConfiguration.py +15 -10
  236. metadata/generated/schema/configuration/taskNotificationConfiguration.py +11 -6
  237. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +21 -14
  238. metadata/generated/schema/configuration/themeConfiguration.py +42 -23
  239. metadata/generated/schema/configuration/uiThemePreference.py +19 -12
  240. metadata/generated/schema/dataInsight/__init__.py +1 -1
  241. metadata/generated/schema/dataInsight/custom/__init__.py +3 -0
  242. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +110 -0
  243. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +21 -0
  244. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +25 -0
  245. metadata/generated/schema/dataInsight/custom/formulaHolder.py +24 -0
  246. metadata/generated/schema/dataInsight/custom/lineChart.py +55 -0
  247. metadata/generated/schema/dataInsight/custom/summaryCard.py +42 -0
  248. metadata/generated/schema/dataInsight/dataInsightChart.py +79 -48
  249. metadata/generated/schema/dataInsight/dataInsightChartResult.py +40 -31
  250. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  251. metadata/generated/schema/dataInsight/kpi/basic.py +43 -32
  252. metadata/generated/schema/dataInsight/kpi/kpi.py +81 -52
  253. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  254. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +19 -12
  255. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +19 -12
  256. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +32 -19
  257. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +32 -18
  258. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +16 -8
  259. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +28 -19
  260. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +18 -13
  261. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +17 -9
  262. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +35 -17
  263. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +31 -17
  264. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +33 -17
  265. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +31 -17
  266. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +26 -14
  267. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +27 -14
  268. metadata/generated/schema/dataInsight/type/unusedAssets.py +17 -9
  269. metadata/generated/schema/email/__init__.py +1 -1
  270. metadata/generated/schema/email/emailRequest.py +26 -18
  271. metadata/generated/schema/email/emailTemplate.py +25 -0
  272. metadata/generated/schema/email/emailTemplatePlaceholder.py +20 -0
  273. metadata/generated/schema/email/smtpSettings.py +34 -18
  274. metadata/generated/schema/email/templateValidationReponse.py +29 -0
  275. metadata/generated/schema/entity/__init__.py +1 -1
  276. metadata/generated/schema/entity/applications/__init__.py +1 -1
  277. metadata/generated/schema/entity/applications/app.py +164 -107
  278. metadata/generated/schema/entity/applications/appRunRecord.py +42 -30
  279. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  280. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +24 -7
  281. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  282. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  283. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +28 -18
  284. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +22 -13
  285. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +23 -14
  286. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +28 -18
  287. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +20 -13
  288. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +19 -12
  289. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +11 -7
  290. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +19 -12
  291. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +11 -7
  292. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +11 -7
  293. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +20 -13
  294. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +11 -7
  295. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +68 -39
  296. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +39 -19
  297. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  298. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +57 -6
  299. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +17 -11
  300. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +34 -24
  301. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  302. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +19 -16
  303. metadata/generated/schema/entity/applications/createAppRequest.py +35 -23
  304. metadata/generated/schema/entity/applications/jobStatus.py +10 -6
  305. metadata/generated/schema/entity/applications/liveExecutionContext.py +15 -6
  306. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  307. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +144 -89
  308. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +95 -64
  309. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +8 -3
  310. metadata/generated/schema/entity/automations/__init__.py +1 -1
  311. metadata/generated/schema/entity/automations/testServiceConnection.py +47 -29
  312. metadata/generated/schema/entity/automations/workflow.py +82 -48
  313. metadata/generated/schema/entity/bot.py +64 -37
  314. metadata/generated/schema/entity/classification/__init__.py +1 -1
  315. metadata/generated/schema/entity/classification/classification.py +90 -50
  316. metadata/generated/schema/entity/classification/tag.py +104 -53
  317. metadata/generated/schema/entity/data/__init__.py +1 -1
  318. metadata/generated/schema/entity/data/apiCollection.py +145 -0
  319. metadata/generated/schema/entity/data/apiEndpoint.py +173 -0
  320. metadata/generated/schema/entity/data/chart.py +120 -72
  321. metadata/generated/schema/entity/data/container.py +178 -116
  322. metadata/generated/schema/entity/data/dashboard.py +141 -89
  323. metadata/generated/schema/entity/data/dashboardDataModel.py +129 -77
  324. metadata/generated/schema/entity/data/database.py +167 -104
  325. metadata/generated/schema/entity/data/databaseSchema.py +157 -98
  326. metadata/generated/schema/entity/data/glossary.py +118 -64
  327. metadata/generated/schema/entity/data/glossaryTerm.py +173 -105
  328. metadata/generated/schema/entity/data/metrics.py +93 -55
  329. metadata/generated/schema/entity/data/mlmodel.py +213 -148
  330. metadata/generated/schema/entity/data/pipeline.py +227 -150
  331. metadata/generated/schema/entity/data/query.py +126 -80
  332. metadata/generated/schema/entity/data/report.py +90 -49
  333. metadata/generated/schema/entity/data/searchIndex.py +171 -114
  334. metadata/generated/schema/entity/data/storedProcedure.py +140 -86
  335. metadata/generated/schema/entity/data/table.py +636 -436
  336. metadata/generated/schema/entity/data/topic.py +175 -111
  337. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  338. metadata/generated/schema/entity/docStore/document.py +52 -31
  339. metadata/generated/schema/entity/domains/__init__.py +1 -1
  340. metadata/generated/schema/entity/domains/dataProduct.py +72 -43
  341. metadata/generated/schema/entity/domains/domain.py +69 -45
  342. metadata/generated/schema/entity/events/__init__.py +1 -1
  343. metadata/generated/schema/entity/events/webhook.py +46 -25
  344. metadata/generated/schema/entity/feed/__init__.py +1 -1
  345. metadata/generated/schema/entity/feed/assets.py +7 -5
  346. metadata/generated/schema/entity/feed/customProperty.py +14 -7
  347. metadata/generated/schema/entity/feed/description.py +21 -13
  348. metadata/generated/schema/entity/feed/domain.py +15 -10
  349. metadata/generated/schema/entity/feed/entityInfo.py +14 -8
  350. metadata/generated/schema/entity/feed/owner.py +16 -11
  351. metadata/generated/schema/entity/feed/suggestion.py +112 -35
  352. metadata/generated/schema/entity/feed/tag.py +15 -10
  353. metadata/generated/schema/entity/feed/testCaseResult.py +19 -13
  354. metadata/generated/schema/entity/feed/thread.py +226 -148
  355. metadata/generated/schema/entity/policies/__init__.py +1 -1
  356. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  357. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +25 -12
  358. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +42 -28
  359. metadata/generated/schema/entity/policies/accessControl/rule.py +36 -23
  360. metadata/generated/schema/entity/policies/filters.py +14 -12
  361. metadata/generated/schema/entity/policies/policy.py +107 -67
  362. metadata/generated/schema/entity/services/__init__.py +1 -1
  363. metadata/generated/schema/entity/services/apiService.py +114 -0
  364. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  365. metadata/generated/schema/entity/services/connections/apiService/__init__.py +5 -0
  366. metadata/generated/schema/entity/services/connections/apiService/restConnection.py +38 -0
  367. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -3
  368. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +21 -13
  369. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +30 -19
  370. metadata/generated/schema/entity/services/connections/common/sslConfig.py +14 -8
  371. metadata/generated/schema/entity/services/connections/connectionBasicType.py +71 -55
  372. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  373. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +23 -15
  374. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +44 -30
  375. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +50 -36
  376. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +45 -36
  377. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +31 -24
  378. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +35 -27
  379. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +35 -27
  380. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +97 -63
  381. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  382. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +25 -18
  383. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +23 -14
  384. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +25 -18
  385. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +25 -18
  386. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +27 -19
  387. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +53 -36
  388. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +38 -23
  389. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +30 -21
  390. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +30 -22
  391. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +46 -35
  392. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  393. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +70 -52
  394. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +131 -89
  395. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +83 -56
  396. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +31 -22
  397. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +109 -72
  398. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +11 -7
  400. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +10 -8
  401. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +11 -7
  402. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +10 -8
  403. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +46 -32
  404. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +23 -15
  405. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +87 -60
  406. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +11 -7
  408. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +11 -7
  409. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +11 -7
  410. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +59 -43
  411. metadata/generated/schema/entity/services/connections/database/db2Connection.py +64 -48
  412. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +41 -85
  413. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +3 -0
  414. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +109 -0
  415. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +42 -0
  416. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +52 -33
  417. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +73 -52
  418. metadata/generated/schema/entity/services/connections/database/druidConnection.py +66 -47
  419. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +38 -27
  420. metadata/generated/schema/entity/services/connections/database/glueConnection.py +34 -24
  421. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +85 -56
  422. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +108 -75
  423. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +11 -6
  425. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +7 -5
  426. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +14 -9
  427. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +36 -25
  428. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +7 -5
  429. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +67 -46
  430. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +23 -15
  431. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +102 -67
  432. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +75 -52
  433. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +56 -38
  434. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +89 -60
  435. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +88 -60
  436. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +105 -74
  437. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +78 -54
  438. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +97 -65
  439. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +90 -61
  440. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +78 -55
  441. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +75 -47
  442. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +84 -0
  443. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  444. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +15 -9
  445. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +33 -24
  446. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +53 -38
  447. metadata/generated/schema/entity/services/connections/database/sasConnection.py +45 -31
  448. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +82 -54
  449. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +121 -80
  450. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +82 -56
  451. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +168 -0
  452. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +91 -62
  453. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +108 -72
  454. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +89 -60
  455. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +71 -50
  456. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  457. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +23 -15
  458. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +101 -61
  459. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +18 -13
  460. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +15 -10
  461. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +85 -54
  462. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  463. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  464. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +117 -74
  465. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +72 -0
  466. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +43 -28
  467. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +40 -27
  468. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +50 -40
  469. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +151 -92
  470. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  471. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +23 -15
  472. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +29 -20
  473. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +20 -13
  474. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +15 -10
  475. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  476. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +25 -18
  477. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +34 -26
  478. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -7
  479. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +23 -15
  480. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +29 -20
  481. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +37 -27
  482. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +43 -0
  483. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +44 -28
  484. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +37 -27
  485. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +35 -0
  486. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +18 -13
  487. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +62 -40
  488. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +71 -45
  489. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +71 -41
  490. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -7
  491. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +28 -18
  492. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  493. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +23 -15
  494. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  495. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +22 -15
  496. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +14 -11
  497. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +48 -31
  498. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +54 -35
  499. metadata/generated/schema/entity/services/connections/serviceConnection.py +38 -22
  500. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  501. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +27 -19
  502. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +23 -15
  503. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +34 -20
  504. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +32 -22
  505. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +96 -57
  506. metadata/generated/schema/entity/services/connections/testConnectionResult.py +40 -26
  507. metadata/generated/schema/entity/services/dashboardService.py +93 -62
  508. metadata/generated/schema/entity/services/databaseService.py +101 -63
  509. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  510. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +198 -122
  511. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +24 -13
  512. metadata/generated/schema/entity/services/ingestionPipelines/status.py +47 -37
  513. metadata/generated/schema/entity/services/messagingService.py +103 -69
  514. metadata/generated/schema/entity/services/metadataService.py +96 -56
  515. metadata/generated/schema/entity/services/mlmodelService.py +96 -63
  516. metadata/generated/schema/entity/services/pipelineService.py +102 -63
  517. metadata/generated/schema/entity/services/searchService.py +92 -62
  518. metadata/generated/schema/entity/services/serviceType.py +2 -1
  519. metadata/generated/schema/entity/services/storageService.py +93 -62
  520. metadata/generated/schema/entity/teams/__init__.py +1 -1
  521. metadata/generated/schema/entity/teams/persona.py +59 -35
  522. metadata/generated/schema/entity/teams/role.py +82 -51
  523. metadata/generated/schema/entity/teams/team.py +128 -82
  524. metadata/generated/schema/entity/teams/teamHierarchy.py +52 -35
  525. metadata/generated/schema/entity/teams/user.py +122 -81
  526. metadata/generated/schema/entity/type.py +94 -48
  527. metadata/generated/schema/entity/utils/__init__.py +1 -1
  528. metadata/generated/schema/entity/utils/entitiesCount.py +25 -18
  529. metadata/generated/schema/entity/utils/servicesCount.py +25 -22
  530. metadata/generated/schema/entity/utils/supersetApiConnection.py +19 -14
  531. metadata/generated/schema/events/__init__.py +1 -1
  532. metadata/generated/schema/events/alertMetrics.py +22 -14
  533. metadata/generated/schema/events/api/__init__.py +1 -1
  534. metadata/generated/schema/events/api/createEventSubscription.py +63 -35
  535. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +27 -0
  536. metadata/generated/schema/events/emailAlertConfig.py +20 -12
  537. metadata/generated/schema/events/eventFilterRule.py +40 -24
  538. metadata/generated/schema/events/eventSubscription.py +220 -142
  539. metadata/generated/schema/events/eventSubscriptionOffset.py +12 -8
  540. metadata/generated/schema/events/failedEvent.py +25 -13
  541. metadata/generated/schema/events/filterResourceDescriptor.py +24 -14
  542. metadata/generated/schema/events/subscriptionResourceDescriptor.py +20 -11
  543. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  544. metadata/generated/schema/metadataIngestion/application.py +39 -25
  545. metadata/generated/schema/metadataIngestion/applicationPipeline.py +23 -16
  546. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +125 -67
  547. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +9 -6
  548. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +182 -106
  549. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +117 -73
  550. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +75 -47
  551. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +51 -32
  552. metadata/generated/schema/metadataIngestion/dbtPipeline.py +75 -51
  553. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  554. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +37 -25
  555. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +23 -14
  556. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +36 -27
  557. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +37 -25
  558. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +31 -20
  559. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +31 -20
  560. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +37 -25
  561. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +45 -22
  562. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +65 -43
  563. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +35 -17
  564. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +99 -50
  565. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +51 -27
  566. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  567. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +60 -33
  568. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +59 -38
  569. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +22 -14
  570. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +15 -10
  571. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +15 -10
  572. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +14 -9
  573. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +14 -9
  574. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +14 -10
  575. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +57 -28
  576. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +32 -17
  577. metadata/generated/schema/metadataIngestion/workflow.py +114 -62
  578. metadata/generated/schema/monitoring/__init__.py +1 -1
  579. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  580. metadata/generated/schema/security/__init__.py +1 -1
  581. metadata/generated/schema/security/client/__init__.py +1 -1
  582. metadata/generated/schema/security/client/auth0SSOClientConfig.py +10 -9
  583. metadata/generated/schema/security/client/azureSSOClientConfig.py +12 -11
  584. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +11 -10
  585. metadata/generated/schema/security/client/googleSSOClientConfig.py +17 -12
  586. metadata/generated/schema/security/client/oidcClientConfig.py +45 -30
  587. metadata/generated/schema/security/client/oktaSSOClientConfig.py +14 -11
  588. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +9 -8
  589. metadata/generated/schema/security/client/samlSSOClientConfig.py +102 -67
  590. metadata/generated/schema/security/credentials/__init__.py +1 -1
  591. metadata/generated/schema/security/credentials/accessTokenAuth.py +17 -13
  592. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +10 -8
  593. metadata/generated/schema/security/credentials/awsCredentials.py +57 -38
  594. metadata/generated/schema/security/credentials/azureCredentials.py +47 -29
  595. metadata/generated/schema/security/credentials/basicAuth.py +13 -11
  596. metadata/generated/schema/security/credentials/bitbucketCredentials.py +22 -14
  597. metadata/generated/schema/security/credentials/gcpCredentials.py +66 -33
  598. metadata/generated/schema/security/credentials/gcpExternalAccount.py +55 -26
  599. metadata/generated/schema/security/credentials/gcpValues.py +70 -49
  600. metadata/generated/schema/security/credentials/gitCredentials.py +32 -25
  601. metadata/generated/schema/security/credentials/githubCredentials.py +13 -7
  602. metadata/generated/schema/security/sasl/__init__.py +3 -0
  603. metadata/generated/schema/security/sasl/saslClientConfig.py +40 -0
  604. metadata/generated/schema/security/secrets/__init__.py +1 -1
  605. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  606. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +39 -24
  607. metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -1
  608. metadata/generated/schema/security/securityConfiguration.py +15 -9
  609. metadata/generated/schema/security/ssl/__init__.py +1 -1
  610. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +30 -20
  611. metadata/generated/schema/security/ssl/verifySSLConfig.py +16 -18
  612. metadata/generated/schema/settings/__init__.py +1 -1
  613. metadata/generated/schema/settings/settings.py +9 -5
  614. metadata/generated/schema/system/__init__.py +1 -1
  615. metadata/generated/schema/system/entityError.py +7 -5
  616. metadata/generated/schema/system/eventPublisherJob.py +54 -37
  617. metadata/generated/schema/system/indexingError.py +7 -5
  618. metadata/generated/schema/system/limitsResponse.py +20 -0
  619. metadata/generated/schema/system/ui/__init__.py +1 -1
  620. metadata/generated/schema/system/ui/knowledgePanel.py +12 -8
  621. metadata/generated/schema/system/ui/page.py +24 -18
  622. metadata/generated/schema/system/validationResponse.py +40 -28
  623. metadata/generated/schema/tests/__init__.py +1 -1
  624. metadata/generated/schema/tests/assigned.py +11 -7
  625. metadata/generated/schema/tests/basic.py +98 -55
  626. metadata/generated/schema/tests/customMetric.py +41 -29
  627. metadata/generated/schema/tests/dataQualityReport.py +40 -0
  628. metadata/generated/schema/tests/resolved.py +17 -13
  629. metadata/generated/schema/tests/testCase.py +137 -76
  630. metadata/generated/schema/tests/testCaseResolutionStatus.py +49 -33
  631. metadata/generated/schema/tests/testDefinition.py +129 -67
  632. metadata/generated/schema/tests/testSuite.py +137 -75
  633. metadata/generated/schema/type/__init__.py +1 -1
  634. metadata/generated/schema/type/apiSchema.py +27 -0
  635. metadata/generated/schema/type/auditLog.py +29 -20
  636. metadata/generated/schema/type/basic.py +177 -110
  637. metadata/generated/schema/type/bulkOperationResult.py +45 -30
  638. metadata/generated/schema/type/changeEvent.py +72 -40
  639. metadata/generated/schema/type/changeEventType.py +1 -1
  640. metadata/generated/schema/type/collectionDescriptor.py +24 -15
  641. metadata/generated/schema/type/csvDocumentation.py +11 -8
  642. metadata/generated/schema/type/csvErrorType.py +1 -1
  643. metadata/generated/schema/type/csvFile.py +25 -17
  644. metadata/generated/schema/type/csvImportResult.py +34 -21
  645. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  646. metadata/generated/schema/type/customProperties/complexTypes.py +110 -64
  647. metadata/generated/schema/type/customProperties/enumConfig.py +7 -5
  648. metadata/generated/schema/type/customProperty.py +34 -25
  649. metadata/generated/schema/type/dailyCount.py +10 -7
  650. metadata/generated/schema/type/databaseConnectionConfig.py +69 -42
  651. metadata/generated/schema/type/entityHierarchy.py +35 -26
  652. metadata/generated/schema/type/entityHistory.py +71 -45
  653. metadata/generated/schema/type/entityLineage.py +89 -57
  654. metadata/generated/schema/type/entityReference.py +51 -30
  655. metadata/generated/schema/type/entityReferenceList.py +11 -8
  656. metadata/generated/schema/type/entityRelationship.py +68 -39
  657. metadata/generated/schema/type/entityUsage.py +15 -10
  658. metadata/generated/schema/type/filterPattern.py +21 -12
  659. metadata/generated/schema/type/function.py +30 -21
  660. metadata/generated/schema/type/include.py +1 -1
  661. metadata/generated/schema/type/jdbcConnection.py +24 -19
  662. metadata/generated/schema/type/lifeCycle.py +47 -26
  663. metadata/generated/schema/type/paging.py +32 -21
  664. metadata/generated/schema/type/profile.py +13 -11
  665. metadata/generated/schema/type/queryParserData.py +47 -32
  666. metadata/generated/schema/type/reaction.py +13 -8
  667. metadata/generated/schema/type/schedule.py +18 -11
  668. metadata/generated/schema/type/schema.py +64 -44
  669. metadata/generated/schema/type/tableQuery.py +53 -40
  670. metadata/generated/schema/type/tableUsageCount.py +46 -33
  671. metadata/generated/schema/type/tagLabel.py +36 -24
  672. metadata/generated/schema/type/usageDetails.py +40 -24
  673. metadata/generated/schema/type/usageRequest.py +10 -7
  674. metadata/generated/schema/type/votes.py +21 -12
  675. metadata/great_expectations/action.py +14 -12
  676. metadata/great_expectations/utils/ometa_config_handler.py +1 -1
  677. metadata/ingestion/api/common.py +3 -4
  678. metadata/ingestion/api/delete.py +1 -1
  679. metadata/ingestion/api/models.py +6 -8
  680. metadata/ingestion/api/parser.py +13 -17
  681. metadata/ingestion/api/status.py +35 -13
  682. metadata/ingestion/api/topology_runner.py +2 -1
  683. metadata/ingestion/bulksink/metadata_usage.py +17 -14
  684. metadata/ingestion/connections/builders.py +12 -12
  685. metadata/ingestion/connections/secrets.py +8 -6
  686. metadata/ingestion/connections/test_connections.py +10 -3
  687. metadata/ingestion/lineage/models.py +5 -4
  688. metadata/ingestion/lineage/sql_lineage.py +43 -13
  689. metadata/ingestion/models/custom_properties.py +9 -9
  690. metadata/ingestion/models/custom_pydantic.py +70 -44
  691. metadata/ingestion/models/lf_tags_model.py +3 -3
  692. metadata/ingestion/models/ometa_classification.py +1 -1
  693. metadata/ingestion/models/ometa_lineage.py +24 -0
  694. metadata/ingestion/models/patch_request.py +96 -56
  695. metadata/ingestion/models/table_metadata.py +2 -2
  696. metadata/ingestion/models/topology.py +18 -12
  697. metadata/ingestion/ometa/client.py +34 -8
  698. metadata/ingestion/ometa/client_utils.py +6 -7
  699. metadata/ingestion/ometa/mixins/custom_property_mixin.py +2 -4
  700. metadata/ingestion/ometa/mixins/dashboard_mixin.py +2 -2
  701. metadata/ingestion/ometa/mixins/data_insight_mixin.py +10 -6
  702. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +4 -2
  703. metadata/ingestion/ometa/mixins/lineage_mixin.py +52 -24
  704. metadata/ingestion/ometa/mixins/mlmodel_mixin.py +2 -2
  705. metadata/ingestion/ometa/mixins/patch_mixin.py +32 -30
  706. metadata/ingestion/ometa/mixins/pipeline_mixin.py +3 -3
  707. metadata/ingestion/ometa/mixins/query_mixin.py +4 -4
  708. metadata/ingestion/ometa/mixins/role_policy_mixin.py +13 -13
  709. metadata/ingestion/ometa/mixins/search_index_mixin.py +5 -5
  710. metadata/ingestion/ometa/mixins/server_mixin.py +4 -5
  711. metadata/ingestion/ometa/mixins/service_mixin.py +2 -2
  712. metadata/ingestion/ometa/mixins/suggestions_mixin.py +2 -2
  713. metadata/ingestion/ometa/mixins/table_mixin.py +20 -20
  714. metadata/ingestion/ometa/mixins/tests_mixin.py +21 -18
  715. metadata/ingestion/ometa/mixins/topic_mixin.py +2 -2
  716. metadata/ingestion/ometa/mixins/user_mixin.py +44 -23
  717. metadata/ingestion/ometa/models.py +1 -1
  718. metadata/ingestion/ometa/ometa_api.py +6 -7
  719. metadata/ingestion/ometa/routes.py +18 -0
  720. metadata/ingestion/ometa/ttl_cache.py +47 -0
  721. metadata/ingestion/ometa/utils.py +3 -3
  722. metadata/ingestion/processor/query_parser.py +3 -3
  723. metadata/ingestion/sink/file.py +2 -2
  724. metadata/ingestion/sink/metadata_rest.py +59 -29
  725. metadata/ingestion/source/dashboard/dashboard_service.py +45 -45
  726. metadata/ingestion/source/dashboard/domodashboard/metadata.py +29 -17
  727. metadata/ingestion/source/dashboard/lightdash/client.py +1 -1
  728. metadata/ingestion/source/dashboard/lightdash/metadata.py +26 -14
  729. metadata/ingestion/source/dashboard/lightdash/models.py +11 -11
  730. metadata/ingestion/source/dashboard/looker/bulk_parser.py +3 -3
  731. metadata/ingestion/source/dashboard/looker/connection.py +1 -1
  732. metadata/ingestion/source/dashboard/looker/metadata.py +59 -39
  733. metadata/ingestion/source/dashboard/looker/parser.py +3 -3
  734. metadata/ingestion/source/dashboard/looker/utils.py +2 -2
  735. metadata/ingestion/source/dashboard/metabase/client.py +18 -2
  736. metadata/ingestion/source/dashboard/metabase/metadata.py +45 -14
  737. metadata/ingestion/source/dashboard/metabase/models.py +46 -28
  738. metadata/ingestion/source/dashboard/mode/client.py +1 -1
  739. metadata/ingestion/source/dashboard/mode/metadata.py +24 -14
  740. metadata/ingestion/source/dashboard/mstr/client.py +1 -1
  741. metadata/ingestion/source/dashboard/mstr/metadata.py +18 -11
  742. metadata/ingestion/source/dashboard/mstr/models.py +4 -4
  743. metadata/ingestion/source/dashboard/powerbi/client.py +3 -3
  744. metadata/ingestion/source/dashboard/powerbi/metadata.py +50 -30
  745. metadata/ingestion/source/dashboard/powerbi/models.py +25 -22
  746. metadata/ingestion/source/dashboard/qlikcloud/client.py +1 -1
  747. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +28 -16
  748. metadata/ingestion/source/dashboard/qlikcloud/models.py +4 -4
  749. metadata/ingestion/source/dashboard/qliksense/client.py +3 -3
  750. metadata/ingestion/source/dashboard/qliksense/metadata.py +35 -19
  751. metadata/ingestion/source/dashboard/qliksense/models.py +8 -8
  752. metadata/ingestion/source/dashboard/quicksight/metadata.py +26 -18
  753. metadata/ingestion/source/dashboard/quicksight/models.py +16 -16
  754. metadata/ingestion/source/dashboard/redash/client.py +1 -1
  755. metadata/ingestion/source/dashboard/redash/metadata.py +31 -19
  756. metadata/ingestion/source/dashboard/superset/api_source.py +46 -18
  757. metadata/ingestion/source/dashboard/superset/client.py +2 -2
  758. metadata/ingestion/source/dashboard/superset/db_source.py +44 -18
  759. metadata/ingestion/source/dashboard/superset/metadata.py +2 -2
  760. metadata/ingestion/source/dashboard/superset/mixin.py +7 -13
  761. metadata/ingestion/source/dashboard/superset/models.py +68 -68
  762. metadata/ingestion/source/dashboard/tableau/connection.py +2 -2
  763. metadata/ingestion/source/dashboard/tableau/metadata.py +398 -94
  764. metadata/ingestion/source/dashboard/tableau/models.py +33 -35
  765. metadata/ingestion/source/dashboard/tableau/queries.py +14 -0
  766. metadata/ingestion/source/database/athena/connection.py +34 -5
  767. metadata/ingestion/source/database/athena/metadata.py +43 -168
  768. metadata/ingestion/source/database/athena/models.py +10 -10
  769. metadata/ingestion/source/database/athena/query_parser.py +2 -2
  770. metadata/ingestion/source/database/athena/utils.py +191 -0
  771. metadata/ingestion/source/database/azuresql/metadata.py +4 -4
  772. metadata/ingestion/source/database/azuresql/query_parser.py +2 -2
  773. metadata/ingestion/source/database/bigquery/connection.py +17 -5
  774. metadata/ingestion/source/database/bigquery/helper.py +1 -1
  775. metadata/ingestion/source/database/bigquery/metadata.py +86 -51
  776. metadata/ingestion/source/database/bigquery/query_parser.py +3 -3
  777. metadata/ingestion/source/database/bigtable/connection.py +2 -2
  778. metadata/ingestion/source/database/bigtable/metadata.py +2 -2
  779. metadata/ingestion/source/database/clickhouse/connection.py +3 -3
  780. metadata/ingestion/source/database/clickhouse/metadata.py +11 -8
  781. metadata/ingestion/source/database/clickhouse/query_parser.py +4 -4
  782. metadata/ingestion/source/database/column_type_parser.py +17 -10
  783. metadata/ingestion/source/database/common_db_source.py +106 -48
  784. metadata/ingestion/source/database/common_nosql_source.py +22 -17
  785. metadata/ingestion/source/database/couchbase/connection.py +1 -1
  786. metadata/ingestion/source/database/couchbase/metadata.py +3 -3
  787. metadata/ingestion/source/database/database_service.py +29 -16
  788. metadata/ingestion/source/database/databricks/connection.py +1 -1
  789. metadata/ingestion/source/database/databricks/lineage.py +2 -1
  790. metadata/ingestion/source/database/databricks/metadata.py +26 -7
  791. metadata/ingestion/source/database/databricks/queries.py +2 -0
  792. metadata/ingestion/source/database/databricks/query_parser.py +2 -2
  793. metadata/ingestion/source/database/databricks/usage.py +2 -1
  794. metadata/ingestion/source/database/datalake/clients/azure_blob.py +72 -0
  795. metadata/ingestion/source/database/datalake/clients/base.py +47 -0
  796. metadata/ingestion/source/database/datalake/clients/gcs.py +132 -0
  797. metadata/ingestion/source/database/datalake/clients/s3.py +79 -0
  798. metadata/ingestion/source/database/datalake/connection.py +12 -70
  799. metadata/ingestion/source/database/datalake/metadata.py +75 -219
  800. metadata/ingestion/source/database/db2/metadata.py +5 -12
  801. metadata/ingestion/source/database/dbt/dbt_config.py +3 -2
  802. metadata/ingestion/source/database/dbt/dbt_service.py +15 -6
  803. metadata/ingestion/source/database/dbt/metadata.py +129 -88
  804. metadata/ingestion/source/database/dbt/models.py +9 -9
  805. metadata/ingestion/source/database/deltalake/clients/base.py +82 -0
  806. metadata/ingestion/source/database/deltalake/clients/pyspark.py +304 -0
  807. metadata/ingestion/source/database/deltalake/clients/s3.py +200 -0
  808. metadata/ingestion/source/database/deltalake/connection.py +56 -84
  809. metadata/ingestion/source/database/deltalake/metadata.py +67 -193
  810. metadata/ingestion/source/database/domodatabase/metadata.py +83 -38
  811. metadata/ingestion/source/database/domodatabase/models.py +6 -6
  812. metadata/ingestion/source/database/doris/metadata.py +3 -3
  813. metadata/ingestion/source/database/druid/metadata.py +2 -2
  814. metadata/ingestion/source/database/dynamodb/metadata.py +5 -5
  815. metadata/ingestion/source/database/dynamodb/models.py +2 -2
  816. metadata/ingestion/source/database/extended_sample_data.py +41 -41
  817. metadata/ingestion/source/database/external_table_lineage_mixin.py +58 -3
  818. metadata/ingestion/source/database/glue/metadata.py +23 -18
  819. metadata/ingestion/source/database/glue/models.py +6 -6
  820. metadata/ingestion/source/database/greenplum/metadata.py +4 -4
  821. metadata/ingestion/source/database/hive/connection.py +2 -2
  822. metadata/ingestion/source/database/hive/metadata.py +2 -2
  823. metadata/ingestion/source/database/iceberg/catalog/hive.py +1 -1
  824. metadata/ingestion/source/database/iceberg/catalog/rest.py +1 -1
  825. metadata/ingestion/source/database/iceberg/fs/__init__.py +1 -1
  826. metadata/ingestion/source/database/iceberg/fs/s3.py +1 -1
  827. metadata/ingestion/source/database/iceberg/helper.py +4 -4
  828. metadata/ingestion/source/database/iceberg/metadata.py +28 -22
  829. metadata/ingestion/source/database/iceberg/models.py +6 -6
  830. metadata/ingestion/source/database/impala/connection.py +3 -3
  831. metadata/ingestion/source/database/impala/metadata.py +2 -2
  832. metadata/ingestion/source/database/incremental_metadata_extraction.py +1 -1
  833. metadata/ingestion/source/database/life_cycle_query_mixin.py +12 -8
  834. metadata/ingestion/source/database/lineage_source.py +25 -13
  835. metadata/ingestion/source/database/mariadb/metadata.py +2 -2
  836. metadata/ingestion/source/database/mongodb/connection.py +1 -1
  837. metadata/ingestion/source/database/mongodb/metadata.py +2 -2
  838. metadata/ingestion/source/database/mssql/constants.py +23 -0
  839. metadata/ingestion/source/database/mssql/lineage.py +29 -0
  840. metadata/ingestion/source/database/mssql/metadata.py +30 -10
  841. metadata/ingestion/source/database/mssql/queries.py +2 -0
  842. metadata/ingestion/source/database/mssql/query_parser.py +2 -2
  843. metadata/ingestion/source/database/mssql/utils.py +16 -1
  844. metadata/ingestion/source/database/mysql/metadata.py +2 -2
  845. metadata/ingestion/source/database/oracle/metadata.py +16 -13
  846. metadata/ingestion/source/database/oracle/models.py +1 -1
  847. metadata/ingestion/source/database/oracle/query_parser.py +2 -2
  848. metadata/ingestion/source/database/pinotdb/metadata.py +2 -2
  849. metadata/ingestion/source/database/postgres/lineage.py +2 -1
  850. metadata/ingestion/source/database/postgres/metadata.py +28 -6
  851. metadata/ingestion/source/database/postgres/pgspider/lineage.py +2 -2
  852. metadata/ingestion/source/database/postgres/queries.py +7 -0
  853. metadata/ingestion/source/database/postgres/query_parser.py +4 -4
  854. metadata/ingestion/source/database/postgres/usage.py +9 -2
  855. metadata/ingestion/source/database/presto/connection.py +2 -2
  856. metadata/ingestion/source/database/presto/metadata.py +2 -2
  857. metadata/ingestion/source/database/query/lineage.py +1 -1
  858. metadata/ingestion/source/database/query/usage.py +5 -3
  859. metadata/ingestion/source/database/query_parser_source.py +1 -1
  860. metadata/ingestion/source/database/redshift/metadata.py +7 -7
  861. metadata/ingestion/source/database/redshift/models.py +1 -1
  862. metadata/ingestion/source/database/redshift/queries.py +26 -2
  863. metadata/ingestion/source/database/redshift/query_parser.py +2 -2
  864. metadata/ingestion/source/database/salesforce/connection.py +1 -3
  865. metadata/ingestion/source/database/salesforce/metadata.py +22 -17
  866. metadata/ingestion/source/database/sample_data.py +198 -98
  867. metadata/ingestion/source/database/sample_usage.py +5 -7
  868. metadata/ingestion/source/database/saperp/__init__.py +0 -0
  869. metadata/ingestion/source/database/saperp/client.py +170 -0
  870. metadata/ingestion/source/database/saperp/connection.py +50 -0
  871. metadata/ingestion/source/database/saperp/constants.py +27 -0
  872. metadata/ingestion/source/database/saperp/metadata.py +322 -0
  873. metadata/ingestion/source/database/saperp/models.py +100 -0
  874. metadata/ingestion/source/database/saphana/metadata.py +2 -2
  875. metadata/ingestion/source/database/sas/client.py +2 -2
  876. metadata/ingestion/source/database/sas/metadata.py +35 -44
  877. metadata/ingestion/source/database/singlestore/metadata.py +2 -2
  878. metadata/ingestion/source/database/snowflake/connection.py +3 -3
  879. metadata/ingestion/source/database/snowflake/metadata.py +33 -12
  880. metadata/ingestion/source/database/snowflake/models.py +7 -7
  881. metadata/ingestion/source/database/snowflake/query_parser.py +2 -2
  882. metadata/ingestion/source/database/sql_column_handler.py +4 -3
  883. metadata/ingestion/source/database/sqlite/metadata.py +2 -2
  884. metadata/ingestion/source/database/stored_procedures_mixin.py +10 -11
  885. metadata/ingestion/source/database/teradata/metadata.py +4 -4
  886. metadata/ingestion/source/database/trino/connection.py +5 -7
  887. metadata/ingestion/source/database/trino/metadata.py +2 -2
  888. metadata/ingestion/source/database/trino/query_parser.py +2 -2
  889. metadata/ingestion/source/database/unitycatalog/lineage.py +7 -7
  890. metadata/ingestion/source/database/unitycatalog/metadata.py +25 -21
  891. metadata/ingestion/source/database/unitycatalog/models.py +16 -16
  892. metadata/ingestion/source/database/unitycatalog/query_parser.py +2 -2
  893. metadata/ingestion/source/database/usage_source.py +51 -34
  894. metadata/ingestion/source/database/vertica/metadata.py +3 -3
  895. metadata/ingestion/source/database/vertica/query_parser.py +4 -4
  896. metadata/ingestion/source/messaging/common_broker_source.py +18 -4
  897. metadata/ingestion/source/messaging/kafka/connection.py +1 -1
  898. metadata/ingestion/source/messaging/kafka/metadata.py +7 -10
  899. metadata/ingestion/source/messaging/kinesis/metadata.py +10 -5
  900. metadata/ingestion/source/messaging/kinesis/models.py +3 -5
  901. metadata/ingestion/source/messaging/messaging_service.py +12 -7
  902. metadata/ingestion/source/messaging/redpanda/metadata.py +2 -2
  903. metadata/ingestion/source/metadata/alationsink/__init__.py +0 -0
  904. metadata/ingestion/source/metadata/alationsink/client.py +213 -0
  905. metadata/ingestion/source/metadata/alationsink/connection.py +53 -0
  906. metadata/ingestion/source/metadata/alationsink/constants.py +60 -0
  907. metadata/ingestion/source/metadata/alationsink/metadata.py +397 -0
  908. metadata/ingestion/source/metadata/alationsink/models.py +148 -0
  909. metadata/ingestion/source/metadata/amundsen/connection.py +1 -1
  910. metadata/ingestion/source/metadata/amundsen/metadata.py +14 -10
  911. metadata/ingestion/source/metadata/atlas/client.py +1 -1
  912. metadata/ingestion/source/metadata/atlas/metadata.py +11 -12
  913. metadata/ingestion/source/mlmodel/mlflow/metadata.py +12 -6
  914. metadata/ingestion/source/mlmodel/mlmodel_service.py +13 -6
  915. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +34 -21
  916. metadata/ingestion/source/pipeline/airbyte/client.py +1 -1
  917. metadata/ingestion/source/pipeline/airbyte/metadata.py +15 -11
  918. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +3 -4
  919. metadata/ingestion/source/pipeline/airflow/metadata.py +28 -17
  920. metadata/ingestion/source/pipeline/airflow/models.py +21 -24
  921. metadata/ingestion/source/pipeline/dagster/client.py +3 -3
  922. metadata/ingestion/source/pipeline/dagster/metadata.py +19 -8
  923. metadata/ingestion/source/pipeline/dagster/models.py +14 -14
  924. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +120 -126
  925. metadata/ingestion/source/pipeline/databrickspipeline/models.py +72 -0
  926. metadata/ingestion/source/pipeline/dbtcloud/__init__.py +0 -0
  927. metadata/ingestion/source/pipeline/dbtcloud/client.py +118 -0
  928. metadata/ingestion/source/pipeline/dbtcloud/connection.py +60 -0
  929. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +322 -0
  930. metadata/ingestion/source/pipeline/dbtcloud/models.py +72 -0
  931. metadata/ingestion/source/pipeline/dbtcloud/queries.py +34 -0
  932. metadata/ingestion/source/pipeline/domopipeline/metadata.py +20 -11
  933. metadata/ingestion/source/pipeline/fivetran/client.py +19 -6
  934. metadata/ingestion/source/pipeline/fivetran/metadata.py +124 -50
  935. metadata/ingestion/source/pipeline/fivetran/models.py +20 -0
  936. metadata/ingestion/source/pipeline/flink/client.py +53 -0
  937. metadata/ingestion/source/pipeline/flink/connection.py +52 -0
  938. metadata/ingestion/source/pipeline/flink/metadata.py +188 -0
  939. metadata/ingestion/source/pipeline/flink/models.py +44 -0
  940. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +23 -11
  941. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +7 -9
  942. metadata/ingestion/source/pipeline/nifi/metadata.py +17 -8
  943. metadata/ingestion/source/pipeline/openlineage/connection.py +12 -3
  944. metadata/ingestion/source/pipeline/openlineage/metadata.py +3 -27
  945. metadata/ingestion/source/pipeline/pipeline_service.py +52 -8
  946. metadata/ingestion/source/pipeline/spline/client.py +1 -1
  947. metadata/ingestion/source/pipeline/spline/metadata.py +3 -3
  948. metadata/ingestion/source/pipeline/spline/models.py +12 -12
  949. metadata/ingestion/source/search/elasticsearch/connection.py +2 -2
  950. metadata/ingestion/source/search/elasticsearch/metadata.py +5 -4
  951. metadata/ingestion/source/search/search_service.py +13 -6
  952. metadata/ingestion/source/storage/gcs/client.py +63 -0
  953. metadata/ingestion/source/storage/gcs/connection.py +159 -0
  954. metadata/ingestion/source/storage/gcs/metadata.py +461 -0
  955. metadata/ingestion/source/storage/gcs/models.py +86 -0
  956. metadata/ingestion/source/storage/s3/metadata.py +268 -18
  957. metadata/ingestion/source/storage/s3/models.py +30 -11
  958. metadata/ingestion/source/storage/storage_service.py +43 -7
  959. metadata/ingestion/stage/table_usage.py +3 -3
  960. metadata/mixins/pandas/pandas_mixin.py +2 -2
  961. metadata/mixins/sqalchemy/sqa_mixin.py +1 -1
  962. metadata/parsers/avro_parser.py +12 -10
  963. metadata/parsers/json_schema_parser.py +4 -4
  964. metadata/parsers/protobuf_parser.py +4 -5
  965. metadata/pii/processor.py +4 -6
  966. metadata/profiler/adaptors/dynamodb.py +2 -2
  967. metadata/profiler/adaptors/mongodb.py +5 -17
  968. metadata/profiler/api/models.py +9 -9
  969. metadata/profiler/interface/nosql/profiler_interface.py +5 -8
  970. metadata/profiler/interface/pandas/profiler_interface.py +8 -10
  971. metadata/profiler/interface/profiler_interface.py +3 -3
  972. metadata/profiler/interface/profiler_interface_factory.py +46 -51
  973. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +28 -17
  974. metadata/profiler/interface/sqlalchemy/profiler_interface.py +3 -5
  975. metadata/profiler/metrics/static/mean.py +9 -0
  976. metadata/profiler/metrics/system/queries/snowflake.py +1 -1
  977. metadata/profiler/metrics/window/percentille_mixin.py +1 -1
  978. metadata/profiler/orm/converter/base.py +21 -13
  979. metadata/profiler/orm/converter/common.py +3 -4
  980. metadata/profiler/orm/functions/datetime.py +1 -1
  981. metadata/profiler/orm/functions/sum.py +7 -0
  982. metadata/profiler/orm/functions/table_metric_computer.py +4 -5
  983. metadata/profiler/orm/registry.py +5 -0
  984. metadata/profiler/orm/types/custom_hex_byte_string.py +5 -3
  985. metadata/profiler/orm/types/undetermined_type.py +38 -0
  986. metadata/profiler/processor/core.py +19 -13
  987. metadata/profiler/processor/processor.py +4 -4
  988. metadata/profiler/processor/sample_data_handler.py +5 -5
  989. metadata/profiler/processor/sampler/nosql/sampler.py +9 -4
  990. metadata/profiler/processor/sampler/sampler_factory.py +27 -6
  991. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +33 -62
  992. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +60 -0
  993. metadata/profiler/source/base/profiler_source.py +11 -14
  994. metadata/profiler/source/bigquery/profiler_source.py +2 -2
  995. metadata/profiler/source/bigquery/type_mapper.py +2 -2
  996. metadata/profiler/source/metadata.py +14 -16
  997. metadata/profiler/source/metadata_ext.py +3 -3
  998. metadata/readers/dataframe/avro.py +1 -3
  999. metadata/readers/dataframe/models.py +29 -12
  1000. metadata/readers/dataframe/parquet.py +3 -3
  1001. metadata/readers/dataframe/reader_factory.py +6 -0
  1002. metadata/readers/file/api_reader.py +1 -1
  1003. metadata/readers/file/bitbucket.py +4 -4
  1004. metadata/readers/file/credentials.py +5 -5
  1005. metadata/readers/file/github.py +6 -6
  1006. metadata/utils/class_helper.py +3 -5
  1007. metadata/utils/credentials.py +2 -2
  1008. metadata/utils/datalake/datalake_utils.py +4 -2
  1009. metadata/utils/entity_link.py +8 -0
  1010. metadata/utils/execution_time_tracker.py +2 -2
  1011. metadata/utils/filters.py +25 -17
  1012. metadata/utils/fqn.py +59 -14
  1013. metadata/utils/helpers.py +12 -12
  1014. metadata/utils/importer.py +3 -2
  1015. metadata/utils/life_cycle_utils.py +3 -3
  1016. metadata/utils/logger.py +9 -24
  1017. metadata/utils/lru_cache.py +3 -0
  1018. metadata/utils/metadata_service_helper.py +3 -1
  1019. metadata/utils/secrets/gcp_secrets_manager.py +128 -0
  1020. metadata/utils/secrets/secrets_manager_factory.py +3 -0
  1021. metadata/utils/source_hash.py +1 -1
  1022. metadata/utils/ssl_manager.py +20 -26
  1023. metadata/utils/ssl_registry.py +5 -1
  1024. metadata/utils/storage_metadata_config.py +6 -5
  1025. metadata/utils/tag_utils.py +21 -13
  1026. metadata/utils/time_utils.py +45 -5
  1027. metadata/workflow/application.py +4 -4
  1028. metadata/workflow/application_output_handler.py +6 -19
  1029. metadata/workflow/base.py +42 -5
  1030. metadata/workflow/data_quality.py +5 -5
  1031. metadata/workflow/ingestion.py +4 -6
  1032. metadata/workflow/metadata.py +4 -4
  1033. metadata/workflow/output_handler.py +35 -214
  1034. metadata/workflow/profiler.py +5 -5
  1035. metadata/workflow/usage.py +8 -6
  1036. metadata/workflow/workflow_init_error_handler.py +154 -0
  1037. metadata/workflow/workflow_output_handler.py +189 -98
  1038. metadata/workflow/workflow_status_mixin.py +20 -11
  1039. {openmetadata_ingestion-1.4.6.0rc1.dist-info → openmetadata_ingestion-1.5.0.0rc1.dist-info}/METADATA +372 -301
  1040. openmetadata_ingestion-1.5.0.0rc1.dist-info/RECORD +1592 -0
  1041. {openmetadata_ingestion-1.4.6.0rc1.dist-info → openmetadata_ingestion-1.5.0.0rc1.dist-info}/WHEEL +1 -1
  1042. {openmetadata_ingestion-1.4.6.0rc1.dist-info → openmetadata_ingestion-1.5.0.0rc1.dist-info}/top_level.txt +1 -0
  1043. metadata/cli/insight.py +0 -51
  1044. metadata/data_insight/processor/kpi/kpi_runner.py +0 -156
  1045. metadata/data_insight/processor/kpi/run_result_registry.py +0 -162
  1046. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +0 -253
  1047. metadata/data_insight/processor/reports/data_processor.py +0 -72
  1048. metadata/data_insight/processor/reports/entity_report_data_processor.py +0 -262
  1049. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +0 -384
  1050. metadata/data_insight/producer/cost_analysis_producer.py +0 -126
  1051. metadata/data_insight/producer/entity_producer.py +0 -63
  1052. metadata/data_insight/producer/producer_factory.py +0 -53
  1053. metadata/data_insight/producer/web_analytics_producer.py +0 -84
  1054. metadata/data_insight/source/metadata.py +0 -175
  1055. metadata/workflow/data_insight.py +0 -78
  1056. openmetadata_ingestion-1.4.6.0rc1.dist-info/RECORD +0 -1509
  1057. {metadata/cli → _openmetadata_testutils}/__init__.py +0 -0
  1058. {metadata/data_insight/processor → _openmetadata_testutils/helpers}/__init__.py +0 -0
  1059. {metadata/data_insight/processor/kpi → _openmetadata_testutils/postgres}/__init__.py +0 -0
  1060. {metadata/data_insight/processor/reports → _openmetadata_testutils/pydantic}/__init__.py +0 -0
  1061. /metadata/{data_insight/producer → ingestion/source/database/datalake/clients}/__init__.py +0 -0
  1062. /metadata/{data_insight/source → ingestion/source/database/deltalake/clients}/__init__.py +0 -0
  1063. {openmetadata_ingestion-1.4.6.0rc1.dist-info → openmetadata_ingestion-1.5.0.0rc1.dist-info}/LICENSE +0 -0
  1064. {openmetadata_ingestion-1.4.6.0rc1.dist-info → openmetadata_ingestion-1.5.0.0rc1.dist-info}/entry_points.txt +0 -0
@@ -30,8 +30,9 @@ from metadata.generated.schema.entity.services.connections.testConnectionResult
30
30
  TestConnectionResult,
31
31
  )
32
32
  from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
33
- from metadata.generated.schema.type.basic import EntityLink
33
+ from metadata.generated.schema.type.basic import EntityLink, Markdown
34
34
  from metadata.generated.schema.type.entityReference import EntityReference
35
+ from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
35
36
  from metadata.generated.schema.type.lifeCycle import LifeCycle
36
37
  from metadata.generated.schema.type.tagLabel import TagLabel
37
38
  from metadata.ingestion.api.models import Entity
@@ -64,10 +65,7 @@ def update_column_tags(
64
65
  Inplace update for the incoming column list
65
66
  """
66
67
  for col in columns:
67
- if (
68
- str(col.fullyQualifiedName.__root__).lower()
69
- == column_tag.column_fqn.lower()
70
- ):
68
+ if str(col.fullyQualifiedName.root).lower() == column_tag.column_fqn.lower():
71
69
  if operation == PatchOperation.REMOVE:
72
70
  for tag in col.tags:
73
71
  if tag.tagFQN == column_tag.tag_label.tagFQN:
@@ -92,14 +90,14 @@ def update_column_description(
92
90
  for col in columns:
93
91
  # For dbt the column names in OM and dbt are not always in the same case.
94
92
  # We'll match the column names in case insensitive way
95
- desc_column = col_dict.get(col.fullyQualifiedName.__root__.lower())
93
+ desc_column = col_dict.get(col.fullyQualifiedName.root.lower())
96
94
  if desc_column:
97
95
  if col.description and not force:
98
96
  # If the description is already present and force is not passed,
99
97
  # description will not be overridden
100
98
  continue
101
99
 
102
- col.description = desc_column.__root__
100
+ col.description = desc_column # Keep the Markdown type
103
101
 
104
102
  if col.children:
105
103
  update_column_description(col.children, column_descriptions, force)
@@ -114,7 +112,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
114
112
 
115
113
  client: REST
116
114
 
117
- def patch(
115
+ def patch( # pylint: disable=too-many-arguments
118
116
  self,
119
117
  entity: Type[T],
120
118
  source: T,
@@ -122,6 +120,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
122
120
  allowed_fields: Optional[Dict] = None,
123
121
  restrict_update_fields: Optional[List] = None,
124
122
  array_entity_fields: Optional[List] = None,
123
+ override_metadata: Optional[bool] = False,
125
124
  ) -> Optional[T]:
126
125
  """
127
126
  Given an Entity type and Source entity and Destination entity,
@@ -144,6 +143,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
144
143
  allowed_fields=allowed_fields,
145
144
  restrict_update_fields=restrict_update_fields,
146
145
  array_entity_fields=array_entity_fields,
146
+ override_metadata=override_metadata,
147
147
  )
148
148
 
149
149
  if not patch:
@@ -200,8 +200,8 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
200
200
  return None
201
201
 
202
202
  # https://docs.pydantic.dev/latest/usage/exporting_models/#modelcopy
203
- destination = source.copy(deep=True)
204
- destination.description = description
203
+ destination = source.model_copy(deep=True)
204
+ destination.description = Markdown(description)
205
205
 
206
206
  return self.patch(entity=entity, source=source, destination=destination)
207
207
 
@@ -229,7 +229,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
229
229
 
230
230
  table.tableConstraints = instance.tableConstraints
231
231
 
232
- destination = table.copy(deep=True)
232
+ destination = table.model_copy(deep=True)
233
233
  destination.tableConstraints = constraints
234
234
 
235
235
  return self.patch(entity=Table, source=table, destination=destination)
@@ -254,9 +254,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
254
254
  if not source:
255
255
  return None
256
256
 
257
- destination = source.copy(deep=True)
257
+ destination = source.model_copy(deep=True)
258
258
 
259
- destination.entityLink = EntityLink(__root__=entity_link)
259
+ destination.entityLink = EntityLink(entity_link)
260
260
  if test_case_parameter_values:
261
261
  destination.parameterValues = test_case_parameter_values
262
262
  if compute_passed_failed_row_count != source.computePassedFailedRowCount:
@@ -292,13 +292,13 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
292
292
 
293
293
  # Initialize empty tag list or the last updated tags
294
294
  source.tags = instance.tags or []
295
- destination = source.copy(deep=True)
295
+ destination = source.model_copy(deep=True)
296
296
 
297
- tag_fqns = {label.tagFQN.__root__ for label in tag_labels}
297
+ tag_fqns = {label.tagFQN.root for label in tag_labels}
298
298
 
299
299
  if operation == PatchOperation.REMOVE:
300
300
  for tag in destination.tags:
301
- if tag.tagFQN.__root__ in tag_fqns:
301
+ if tag.tagFQN.root in tag_fqns:
302
302
  destination.tags.remove(tag)
303
303
  else:
304
304
  destination.tags.extend(tag_labels)
@@ -324,7 +324,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
324
324
  self,
325
325
  entity: Type[T],
326
326
  source: T,
327
- owner: EntityReference = None,
327
+ owners: EntityReferenceList = None,
328
328
  force: bool = False,
329
329
  ) -> Optional[T]:
330
330
  """
@@ -341,20 +341,20 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
341
341
  Updated Entity
342
342
  """
343
343
  instance: Optional[T] = self._fetch_entity_if_exists(
344
- entity=entity, entity_id=source.id, fields=["owner"]
344
+ entity=entity, entity_id=source.id, fields=["owners"]
345
345
  )
346
346
 
347
347
  if not instance:
348
348
  return None
349
349
 
350
350
  # Don't change existing data without force
351
- if instance.owner and not force:
351
+ if instance.owners and instance.owners.root and not force:
352
352
  # If a owner is already present and force is not passed,
353
353
  # owner will not be overridden
354
354
  return None
355
355
 
356
356
  destination = deepcopy(instance)
357
- destination.owner = owner
357
+ destination.owners = owners
358
358
 
359
359
  return self.patch(entity=entity, source=instance, destination=destination)
360
360
 
@@ -386,7 +386,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
386
386
  # Make sure we run the patch against the last updated data from the API
387
387
  table.columns = instance.columns
388
388
 
389
- destination = table.copy(deep=True)
389
+ destination = table.model_copy(deep=True)
390
390
  for column_tag in column_tags or []:
391
391
  update_column_tags(destination.columns, column_tag, operation)
392
392
 
@@ -394,7 +394,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
394
394
  if patched_entity is None:
395
395
  logger.debug(
396
396
  f"Empty PATCH result. Either everything is up to date or the "
397
- f"column names are not in [{table.fullyQualifiedName.__root__}]"
397
+ f"column names are not in [{table.fullyQualifiedName.root}]"
398
398
  )
399
399
 
400
400
  return patched_entity
@@ -440,7 +440,9 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
440
440
  return self.patch_column_descriptions(
441
441
  table=table,
442
442
  column_descriptions=[
443
- ColumnDescription(column_fqn=column_fqn, description=description)
443
+ ColumnDescription(
444
+ column_fqn=column_fqn, description=Markdown(description)
445
+ )
444
446
  ],
445
447
  force=force,
446
448
  )
@@ -471,14 +473,14 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
471
473
  # Make sure we run the patch against the last updated data from the API
472
474
  table.columns = instance.columns
473
475
 
474
- destination = table.copy(deep=True)
476
+ destination = table.model_copy(deep=True)
475
477
  update_column_description(destination.columns, column_descriptions, force)
476
478
 
477
479
  patched_entity = self.patch(entity=Table, source=table, destination=destination)
478
480
  if patched_entity is None:
479
481
  logger.debug(
480
482
  f"Empty PATCH result. Either everything is up to date or "
481
- f"columns are not matching for [{table.fullyQualifiedName.__root__}]"
483
+ f"columns are not matching for [{table.fullyQualifiedName.root}]"
482
484
  )
483
485
 
484
486
  return patched_entity
@@ -494,7 +496,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
494
496
  """
495
497
  result_data: Dict = {
496
498
  PatchField.PATH: PatchPath.RESPONSE,
497
- PatchField.VALUE: test_connection_result.dict(),
499
+ PatchField.VALUE: test_connection_result.model_dump(),
498
500
  PatchField.OPERATION: PatchOperation.ADD,
499
501
  }
500
502
 
@@ -530,7 +532,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
530
532
  :param life_cycle_data: Life Cycle data to add
531
533
  """
532
534
  try:
533
- destination = entity.copy(deep=True)
535
+ destination = entity.model_copy(deep=True)
534
536
  destination.lifeCycle = life_cycle
535
537
  return self.patch(
536
538
  entity=type(entity), source=entity, destination=destination
@@ -538,14 +540,14 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
538
540
  except Exception as exc:
539
541
  logger.debug(traceback.format_exc())
540
542
  logger.warning(
541
- f"Error trying to Patch life cycle data for {entity.fullyQualifiedName.__root__}: {exc}"
543
+ f"Error trying to Patch life cycle data for {entity.fullyQualifiedName.root}: {exc}"
542
544
  )
543
545
  return None
544
546
 
545
547
  def patch_domain(self, entity: Entity, domain: Domain) -> Optional[Entity]:
546
548
  """Patch domain data for an Entity"""
547
549
  try:
548
- destination: Entity = entity.copy(deep=True)
550
+ destination: Entity = entity.model_copy(deep=True)
549
551
  destination.domain = EntityReference(id=domain.id, type="domain")
550
552
  return self.patch(
551
553
  entity=type(entity), source=entity, destination=destination
@@ -553,6 +555,6 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
553
555
  except Exception as exc:
554
556
  logger.debug(traceback.format_exc())
555
557
  logger.warning(
556
- f"Error trying to Patch Domain for {entity.fullyQualifiedName.__root__}: {exc}"
558
+ f"Error trying to Patch Domain for {entity.fullyQualifiedName.root}: {exc}"
557
559
  )
558
560
  return None
@@ -43,7 +43,7 @@ class OMetaPipelineMixin:
43
43
  """
44
44
  resp = self.client.put(
45
45
  f"{self.get_suffix(Pipeline)}/{fqn}/status",
46
- data=status.json(),
46
+ data=status.model_dump_json(),
47
47
  )
48
48
 
49
49
  return Pipeline(**resp)
@@ -86,7 +86,7 @@ class OMetaPipelineMixin:
86
86
  startDate=pipeline.startDate,
87
87
  service=pipeline.service.fullyQualifiedName,
88
88
  tasks=all_tasks,
89
- owner=pipeline.owner,
89
+ owners=pipeline.owners,
90
90
  tags=pipeline.tags,
91
91
  )
92
92
 
@@ -113,7 +113,7 @@ class OMetaPipelineMixin:
113
113
  startDate=pipeline.startDate,
114
114
  service=pipeline.service.fullyQualifiedName,
115
115
  tasks=[task for task in pipeline.tasks if task.name in task_ids],
116
- owner=pipeline.owner,
116
+ owners=pipeline.owners,
117
117
  tags=pipeline.tags,
118
118
  )
119
119
 
@@ -41,10 +41,10 @@ class OMetaQueryMixin:
41
41
  return str(result.hexdigest())
42
42
 
43
43
  def _get_or_create_query(self, query: CreateQueryRequest) -> Optional[Query]:
44
- query_hash = self._get_query_hash(query=query.query.__root__)
44
+ query_hash = self._get_query_hash(query=query.query.root)
45
45
  query_entity = self.get_by_name(entity=Query, fqn=query_hash)
46
46
  if query_entity is None:
47
- resp = self.client.put(self.get_suffix(Query), data=query.json())
47
+ resp = self.client.put(self.get_suffix(Query), data=query.model_dump_json())
48
48
  if resp and resp.get("id"):
49
49
  query_entity = Query(**resp)
50
50
  return query_entity
@@ -63,9 +63,9 @@ class OMetaQueryMixin:
63
63
  query = self._get_or_create_query(create_query)
64
64
  if query:
65
65
  # Add Query Usage
66
- table_ref = EntityReference(id=entity.id.__root__, type="table")
66
+ table_ref = EntityReference(id=entity.id.root, type="table")
67
67
  # convert object to json array string
68
- table_ref_json = "[" + table_ref.json() + "]"
68
+ table_ref_json = "[" + table_ref.model_dump_json() + "]"
69
69
  self.client.put(
70
70
  f"{self.get_suffix(Query)}/{model_str(query.id)}/usage",
71
71
  data=table_ref_json,
@@ -125,7 +125,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
125
125
  if previous is None
126
126
  else PatchOperation.REPLACE,
127
127
  PatchField.PATH: path.format(rule_index=rule_index),
128
- PatchField.VALUE: str(current.__root__),
128
+ PatchField.VALUE: str(current.root),
129
129
  }
130
130
  ]
131
131
  return data
@@ -158,10 +158,10 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
158
158
  if not instance:
159
159
  return None
160
160
 
161
- policy_index: int = len(instance.policies.__root__) - 1
161
+ policy_index: int = len(instance.policies.root) - 1
162
162
  data: List
163
163
  if operation is PatchOperation.REMOVE:
164
- if len(instance.policies.__root__) == 1:
164
+ if len(instance.policies.root) == 1:
165
165
  logger.error(
166
166
  f"The Role with id [{model_str(entity_id)}] has only one (1)"
167
167
  f" policy. Unable to remove."
@@ -177,7 +177,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
177
177
 
178
178
  index: int = 0
179
179
  is_policy_found: bool = False
180
- for policy in instance.policies.__root__:
180
+ for policy in instance.policies.root:
181
181
  if model_str(policy.id) == model_str(policy_id):
182
182
  is_policy_found = True
183
183
  continue
@@ -187,7 +187,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
187
187
  PatchField.PATH: PatchPath.POLICIES_DESCRIPTION.format(
188
188
  index=index
189
189
  ),
190
- PatchField.VALUE: model_str(policy.description.__root__),
190
+ PatchField.VALUE: model_str(policy.description.root),
191
191
  }
192
192
  )
193
193
  data.append(
@@ -294,7 +294,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
294
294
  if not instance:
295
295
  return None
296
296
 
297
- rule_index: int = len(instance.rules.__root__) - 1
297
+ rule_index: int = len(instance.rules.root) - 1
298
298
  data: List[Dict]
299
299
  if operation == PatchOperation.ADD:
300
300
  data = [
@@ -303,7 +303,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
303
303
  PatchField.PATH: PatchPath.RULES.format(rule_index=rule_index + 1),
304
304
  PatchField.VALUE: {
305
305
  PatchValue.NAME: rule.name,
306
- PatchValue.CONDITION: rule.condition.__root__,
306
+ PatchValue.CONDITION: rule.condition.root,
307
307
  PatchValue.EFFECT: rule.effect.name,
308
308
  PatchValue.OPERATIONS: [
309
309
  operation.name for operation in rule.operations
@@ -314,12 +314,12 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
314
314
  ]
315
315
  if rule.description is not None:
316
316
  data[0][PatchField.VALUE][PatchValue.DESCRIPTION] = str(
317
- rule.description.__root__
317
+ rule.description.root
318
318
  )
319
319
 
320
320
  if rule.fullyQualifiedName is not None:
321
321
  data[0][PatchField.VALUE][PatchValue.FQN] = str(
322
- rule.fullyQualifiedName.__root__
322
+ rule.fullyQualifiedName.root
323
323
  )
324
324
 
325
325
  else:
@@ -334,8 +334,8 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
334
334
  }
335
335
  ]
336
336
 
337
- for rule_index in range(len(instance.rules.__root__) - 1, -1, -1):
338
- current_rule: Rule = instance.rules.__root__[rule_index]
337
+ for rule_index in range(len(instance.rules.root) - 1, -1, -1):
338
+ current_rule: Rule = instance.rules.root[rule_index]
339
339
  if current_rule.name == rule.name:
340
340
  break
341
341
 
@@ -345,7 +345,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
345
345
  )
346
346
  return None
347
347
 
348
- previous_rule: Rule = instance.rules.__root__[rule_index - 1]
348
+ previous_rule: Rule = instance.rules.root[rule_index - 1]
349
349
  # Condition
350
350
  data.append(
351
351
  {
@@ -353,7 +353,7 @@ class OMetaRolePolicyMixin(OMetaPatchMixinBase):
353
353
  PatchField.PATH: PatchPath.RULES_CONDITION.format(
354
354
  rule_index=rule_index - 1
355
355
  ),
356
- PatchField.VALUE: current_rule.condition.__root__,
356
+ PatchField.VALUE: current_rule.condition.root,
357
357
  }
358
358
  )
359
359
  # Description - Optional
@@ -47,13 +47,13 @@ class OMetaSearchIndexMixin:
47
47
  resp = None
48
48
  try:
49
49
  resp = self.client.put(
50
- f"{self.get_suffix(SearchIndex)}/{search_index.id.__root__}/sampleData",
51
- data=sample_data.json(),
50
+ f"{self.get_suffix(SearchIndex)}/{search_index.id.root}/sampleData",
51
+ data=sample_data.model_dump_json(),
52
52
  )
53
53
  except Exception as exc:
54
54
  logger.debug(traceback.format_exc())
55
55
  logger.warning(
56
- f"Error trying to PUT sample data for {search_index.fullyQualifiedName.__root__}: {exc}"
56
+ f"Error trying to PUT sample data for {search_index.fullyQualifiedName.root}: {exc}"
57
57
  )
58
58
 
59
59
  if resp:
@@ -63,13 +63,13 @@ class OMetaSearchIndexMixin:
63
63
  logger.debug(traceback.format_exc())
64
64
  logger.warning(
65
65
  "Unicode Error parsing the sample data response "
66
- f"from {search_index.fullyQualifiedName.__root__}: {err}"
66
+ f"from {search_index.fullyQualifiedName.root}: {err}"
67
67
  )
68
68
  except Exception as exc:
69
69
  logger.debug(traceback.format_exc())
70
70
  logger.warning(
71
71
  "Error trying to parse sample data results"
72
- f"from {search_index.fullyQualifiedName.__root__}: {exc}"
72
+ f"from {search_index.fullyQualifiedName.root}: {exc}"
73
73
  )
74
74
 
75
75
  return None
@@ -21,7 +21,6 @@ from metadata.__version__ import (
21
21
  match_versions,
22
22
  )
23
23
  from metadata.generated.schema.settings.settings import Settings, SettingType
24
- from metadata.ingestion.models.encoders import show_secrets_encoder
25
24
  from metadata.ingestion.ometa.client import REST
26
25
  from metadata.ingestion.ometa.routes import ROUTES
27
26
  from metadata.utils.logger import ometa_logger
@@ -91,9 +90,9 @@ class OMetaServerMixin:
91
90
  Returns:
92
91
  Settings
93
92
  """
94
- data = settings.json(encoder=show_secrets_encoder)
93
+ data = settings.model_dump_json()
95
94
  response = self.client.put(ROUTES.get(Settings.__name__), data)
96
- return Settings.parse_obj(response)
95
+ return Settings.model_validate(response)
97
96
 
98
97
  def get_settings_by_name(self, setting_type: SettingType) -> Optional[Settings]:
99
98
  """Get setting by name
@@ -106,7 +105,7 @@ class OMetaServerMixin:
106
105
  )
107
106
  if not response:
108
107
  return None
109
- return Settings.parse_obj(response)
108
+ return Settings.model_validate(response)
110
109
 
111
110
  def get_profiler_config_settings(self) -> Optional[Settings]:
112
111
  """Get profiler config setting
@@ -117,4 +116,4 @@ class OMetaServerMixin:
117
116
  response = self.client.get("/system/settings/profilerConfiguration")
118
117
  if not response:
119
118
  return None
120
- return Settings.parse_obj(response)
119
+ return Settings.model_validate(response)
@@ -55,8 +55,8 @@ class OMetaServiceMixin:
55
55
  create_entity_class = self.get_create_entity_type(entity=entity)
56
56
  return create_entity_class(
57
57
  name=config.serviceName,
58
- serviceType=config.serviceConnection.__root__.config.type.value,
59
- connection=config.serviceConnection.__root__
58
+ serviceType=config.serviceConnection.root.config.type.value,
59
+ connection=config.serviceConnection.root
60
60
  if self.config.storeServiceConnection
61
61
  else None,
62
62
  )
@@ -34,8 +34,8 @@ class OMetaSuggestionsMixin:
34
34
  Update an existing Suggestion with new fields
35
35
  """
36
36
  resp = self.client.put(
37
- f"{self.get_suffix(Suggestion)}/{str(suggestion.id.__root__)}",
38
- data=suggestion.json(),
37
+ f"{self.get_suffix(Suggestion)}/{str(suggestion.root.id.root)}",
38
+ data=suggestion.model_dump_json(),
39
39
  )
40
40
 
41
41
  return Suggestion(**resp)
@@ -69,13 +69,13 @@ class OMetaTableMixin:
69
69
  resp = None
70
70
  try:
71
71
  resp = self.client.put(
72
- f"{self.get_suffix(Table)}/{table.id.__root__}/sampleData",
73
- data=sample_data.json(),
72
+ f"{self.get_suffix(Table)}/{table.id.root}/sampleData",
73
+ data=sample_data.model_dump_json(),
74
74
  )
75
75
  except Exception as exc:
76
76
  logger.debug(traceback.format_exc())
77
77
  logger.warning(
78
- f"Error trying to PUT sample data for {table.fullyQualifiedName.__root__}: {exc}"
78
+ f"Error trying to PUT sample data for {table.fullyQualifiedName.root}: {exc}"
79
79
  )
80
80
 
81
81
  if resp:
@@ -84,12 +84,12 @@ class OMetaTableMixin:
84
84
  except UnicodeError as err:
85
85
  logger.debug(traceback.format_exc())
86
86
  logger.warning(
87
- f"Unicode Error parsing the sample data response from {table.fullyQualifiedName.__root__}: {err}"
87
+ f"Unicode Error parsing the sample data response from {table.fullyQualifiedName.root}: {err}"
88
88
  )
89
89
  except Exception as exc:
90
90
  logger.debug(traceback.format_exc())
91
91
  logger.warning(
92
- f"Error trying to parse sample data results from {table.fullyQualifiedName.__root__}: {exc}"
92
+ f"Error trying to parse sample data results from {table.fullyQualifiedName.root}: {exc}"
93
93
  )
94
94
 
95
95
  return None
@@ -103,12 +103,12 @@ class OMetaTableMixin:
103
103
  resp = None
104
104
  try:
105
105
  resp = self.client.get(
106
- f"{self.get_suffix(Table)}/{table.id.__root__}/sampleData",
106
+ f"{self.get_suffix(Table)}/{table.id.root}/sampleData",
107
107
  )
108
108
  except Exception as exc:
109
109
  logger.debug(traceback.format_exc())
110
110
  logger.warning(
111
- f"Error trying to GET sample data for {table.fullyQualifiedName.__root__}: {exc}"
111
+ f"Error trying to GET sample data for {table.fullyQualifiedName.root}: {exc}"
112
112
  )
113
113
 
114
114
  if resp:
@@ -117,12 +117,12 @@ class OMetaTableMixin:
117
117
  except UnicodeError as err:
118
118
  logger.debug(traceback.format_exc())
119
119
  logger.warning(
120
- f"Unicode Error parsing the sample data response from {table.fullyQualifiedName.__root__}: {err}"
120
+ f"Unicode Error parsing the sample data response from {table.fullyQualifiedName.root}: {err}"
121
121
  )
122
122
  except Exception as exc:
123
123
  logger.debug(traceback.format_exc())
124
124
  logger.warning(
125
- f"Error trying to parse sample data results from {table.fullyQualifiedName.__root__}: {exc}"
125
+ f"Error trying to parse sample data results from {table.fullyQualifiedName.root}: {exc}"
126
126
  )
127
127
 
128
128
  return None
@@ -137,8 +137,8 @@ class OMetaTableMixin:
137
137
  :param table_profile: Profile data to add
138
138
  """
139
139
  resp = self.client.put(
140
- f"{self.get_suffix(Table)}/{table.id.__root__}/tableProfile",
141
- data=profile_request.json(),
140
+ f"{self.get_suffix(Table)}/{table.id.root}/tableProfile",
141
+ data=profile_request.model_dump_json(),
142
142
  )
143
143
  return Table(**resp)
144
144
 
@@ -150,8 +150,8 @@ class OMetaTableMixin:
150
150
  :param data_model: Model to add
151
151
  """
152
152
  resp = self.client.put(
153
- f"{self.get_suffix(Table)}/{table.id.__root__}/dataModel",
154
- data=data_model.json(),
153
+ f"{self.get_suffix(Table)}/{table.id.root}/dataModel",
154
+ data=data_model.model_dump_json(),
155
155
  )
156
156
  return Table(**resp)
157
157
 
@@ -165,7 +165,7 @@ class OMetaTableMixin:
165
165
  :param table_usage_request: Usage data to add
166
166
  """
167
167
  resp = self.client.post(
168
- f"/usage/table/{table.id.__root__}", data=table_usage_request.json()
168
+ f"/usage/table/{table.id.root}", data=table_usage_request.model_dump_json()
169
169
  )
170
170
  logger.debug("published table usage %s", resp)
171
171
 
@@ -179,10 +179,10 @@ class OMetaTableMixin:
179
179
  :param table_join_request: Join data to add
180
180
  """
181
181
 
182
- logger.info("table join request %s", table_join_request.json())
182
+ logger.info("table join request %s", table_join_request.model_dump_json())
183
183
  resp = self.client.put(
184
- f"{self.get_suffix(Table)}/{table.id.__root__}/joins",
185
- data=table_join_request.json(),
184
+ f"{self.get_suffix(Table)}/{table.id.root}/joins",
185
+ data=table_join_request.model_dump_json(),
186
186
  )
187
187
  logger.debug("published frequently joined with %s", resp)
188
188
 
@@ -203,7 +203,7 @@ class OMetaTableMixin:
203
203
  """
204
204
  resp = self.client.put(
205
205
  f"{self.get_suffix(Table)}/{model_str(table_id)}/tableProfilerConfig",
206
- data=table_profiler_config.json(),
206
+ data=table_profiler_config.model_dump_json(),
207
207
  )
208
208
  return Table(**resp)
209
209
 
@@ -290,7 +290,7 @@ class OMetaTableMixin:
290
290
  Returns:
291
291
  Optional[Table]: OM table object
292
292
  """
293
- return self._get(Table, f"{quote(model_str(fqn))}/tableProfile/latest")
293
+ return self._get(Table, f"{quote(model_str(fqn), safe='')}/tableProfile/latest")
294
294
 
295
295
  def create_or_update_custom_metric(
296
296
  self, custom_metric: CreateCustomMetricRequest, table_id: str
@@ -303,6 +303,6 @@ class OMetaTableMixin:
303
303
  """
304
304
  resp = self.client.put(
305
305
  f"{self.get_suffix(Table)}/{table_id}/customMetric",
306
- data=custom_metric.json(),
306
+ data=custom_metric.model_dump_json(),
307
307
  )
308
308
  return Table(**resp)