openmetadata-ingestion 1.4.8.0__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1074) hide show
  1. _openmetadata_testutils/data/dvdrental.zip +0 -0
  2. _openmetadata_testutils/helpers/assumption.py +118 -0
  3. _openmetadata_testutils/helpers/docker.py +38 -0
  4. _openmetadata_testutils/helpers/markers.py +5 -0
  5. _openmetadata_testutils/kafka/load_csv_data.py +133 -0
  6. _openmetadata_testutils/kafka/schema_registry_container.py +31 -0
  7. _openmetadata_testutils/ometa.py +23 -0
  8. _openmetadata_testutils/postgres/conftest.py +75 -0
  9. _openmetadata_testutils/pydantic/test_utils.py +79 -0
  10. airflow_provider_openmetadata/lineage/callback.py +3 -3
  11. airflow_provider_openmetadata/lineage/config/loader.py +1 -1
  12. airflow_provider_openmetadata/lineage/runner.py +5 -5
  13. airflow_provider_openmetadata/lineage/status.py +2 -2
  14. metadata/automations/runner.py +1 -1
  15. metadata/cli/app.py +1 -3
  16. metadata/cli/dataquality.py +8 -8
  17. metadata/cli/ingest.py +8 -8
  18. metadata/cli/lineage.py +10 -5
  19. metadata/cli/profile.py +8 -8
  20. metadata/cli/usage.py +6 -8
  21. metadata/clients/aws_client.py +4 -4
  22. metadata/clients/azure_client.py +1 -1
  23. metadata/clients/domo_client.py +12 -18
  24. metadata/cmd.py +16 -20
  25. metadata/config/common.py +3 -4
  26. metadata/data_quality/api/models.py +2 -2
  27. metadata/data_quality/builders/i_validator_builder.py +100 -0
  28. metadata/{data_insight/producer/producer_interface.py → data_quality/builders/pandas_validator_builder.py} +12 -12
  29. metadata/data_quality/{validations/validator.py → builders/sqa_validator_builder.py} +7 -8
  30. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +9 -42
  31. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +7 -40
  32. metadata/data_quality/interface/test_suite_interface.py +84 -2
  33. metadata/data_quality/processor/test_case_runner.py +76 -40
  34. metadata/data_quality/runner/base_test_suite_source.py +1 -1
  35. metadata/data_quality/runner/core.py +2 -2
  36. metadata/data_quality/source/test_suite.py +17 -15
  37. metadata/data_quality/validations/base_test_handler.py +25 -5
  38. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +2 -0
  39. metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +2 -0
  40. metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +2 -0
  41. metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +2 -0
  42. metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +2 -0
  43. metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +2 -0
  44. metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +2 -0
  45. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +24 -14
  46. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +13 -1
  47. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +1 -1
  48. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +1 -1
  49. metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py +1 -1
  50. metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py +1 -1
  51. metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py +1 -1
  52. metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py +1 -1
  53. metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py +1 -1
  54. metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py +1 -1
  55. metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py +1 -1
  56. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +1 -1
  57. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +1 -1
  58. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +1 -1
  59. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +1 -1
  60. metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py +1 -1
  61. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +1 -1
  62. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +1 -1
  63. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +8 -7
  64. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  65. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  66. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  67. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  68. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  69. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  70. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  71. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +8 -3
  72. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  73. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  74. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  75. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +1 -1
  76. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  77. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  78. metadata/data_quality/validations/models.py +21 -0
  79. metadata/data_quality/validations/runtime_param_setter/param_setter.py +42 -0
  80. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +54 -0
  81. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +190 -0
  82. metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +4 -13
  83. metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +2 -2
  84. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +434 -0
  85. metadata/examples/workflows/alationsink.yaml +34 -0
  86. metadata/examples/workflows/bigquery_profiler.yaml +1 -1
  87. metadata/examples/workflows/dbtcloud.yaml +26 -0
  88. metadata/examples/workflows/deltalake.yaml +6 -5
  89. metadata/examples/workflows/flink.yaml +33 -0
  90. metadata/examples/workflows/saperp.yaml +24 -0
  91. metadata/examples/workflows/synapse.yaml +24 -0
  92. metadata/generated/antlr/EntityLinkLexer.py +380 -352
  93. metadata/generated/schema/analytics/__init__.py +1 -1
  94. metadata/generated/schema/analytics/basic.py +19 -16
  95. metadata/generated/schema/analytics/reportData.py +26 -18
  96. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  97. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +64 -43
  98. metadata/generated/schema/analytics/reportDataType/entityReportData.py +35 -24
  99. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +18 -11
  100. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +24 -16
  101. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +23 -12
  102. metadata/generated/schema/analytics/webAnalyticEvent.py +75 -42
  103. metadata/generated/schema/analytics/webAnalyticEventData.py +17 -12
  104. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  105. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +24 -16
  106. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +31 -21
  107. metadata/generated/schema/api/__init__.py +1 -1
  108. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +28 -18
  109. metadata/generated/schema/api/analytics/__init__.py +1 -1
  110. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +33 -20
  111. metadata/generated/schema/api/automations/__init__.py +1 -1
  112. metadata/generated/schema/api/automations/createWorkflow.py +46 -27
  113. metadata/generated/schema/api/bulkAssets.py +14 -8
  114. metadata/generated/schema/api/classification/__init__.py +1 -1
  115. metadata/generated/schema/api/classification/createClassification.py +28 -15
  116. metadata/generated/schema/api/classification/createTag.py +43 -24
  117. metadata/generated/schema/api/classification/loadTags.py +11 -6
  118. metadata/generated/schema/api/createBot.py +30 -14
  119. metadata/generated/schema/api/createEventPublisherJob.py +42 -26
  120. metadata/generated/schema/api/createType.py +33 -21
  121. metadata/generated/schema/api/data/__init__.py +1 -1
  122. metadata/generated/schema/api/data/createAPICollection.py +97 -0
  123. metadata/generated/schema/api/data/createAPIEndpoint.py +107 -0
  124. metadata/generated/schema/api/data/createChart.py +71 -39
  125. metadata/generated/schema/api/data/createContainer.py +105 -68
  126. metadata/generated/schema/api/data/createCustomProperty.py +22 -14
  127. metadata/generated/schema/api/data/createDashboard.py +95 -58
  128. metadata/generated/schema/api/data/createDashboardDataModel.py +88 -51
  129. metadata/generated/schema/api/data/createDatabase.py +85 -53
  130. metadata/generated/schema/api/data/createDatabaseSchema.py +81 -50
  131. metadata/generated/schema/api/data/createGlossary.py +51 -34
  132. metadata/generated/schema/api/data/createGlossaryTerm.py +75 -48
  133. metadata/generated/schema/api/data/createMlModel.py +105 -68
  134. metadata/generated/schema/api/data/createPipeline.py +95 -58
  135. metadata/generated/schema/api/data/createQuery.py +85 -54
  136. metadata/generated/schema/api/data/createSearchIndex.py +76 -49
  137. metadata/generated/schema/api/data/createStoredProcedure.py +70 -46
  138. metadata/generated/schema/api/data/createTable.py +87 -58
  139. metadata/generated/schema/api/data/createTableProfile.py +16 -11
  140. metadata/generated/schema/api/data/createTopic.py +120 -75
  141. metadata/generated/schema/api/data/loadGlossary.py +12 -8
  142. metadata/generated/schema/api/data/restoreEntity.py +11 -7
  143. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  144. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +41 -26
  145. metadata/generated/schema/api/dataInsight/custom/__init__.py +3 -0
  146. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +53 -0
  147. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  148. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +49 -21
  149. metadata/generated/schema/api/docStore/__init__.py +1 -1
  150. metadata/generated/schema/api/docStore/createDocument.py +24 -12
  151. metadata/generated/schema/api/domains/__init__.py +1 -1
  152. metadata/generated/schema/api/domains/createDataProduct.py +50 -30
  153. metadata/generated/schema/api/domains/createDomain.py +38 -25
  154. metadata/generated/schema/api/feed/__init__.py +1 -1
  155. metadata/generated/schema/api/feed/closeTask.py +17 -10
  156. metadata/generated/schema/api/feed/createPost.py +16 -11
  157. metadata/generated/schema/api/feed/createSuggestion.py +56 -16
  158. metadata/generated/schema/api/feed/createThread.py +55 -34
  159. metadata/generated/schema/api/feed/resolveTask.py +21 -14
  160. metadata/generated/schema/api/feed/threadCount.py +29 -19
  161. metadata/generated/schema/api/lineage/__init__.py +1 -1
  162. metadata/generated/schema/api/lineage/addLineage.py +11 -6
  163. metadata/generated/schema/api/openMetadataServerVersion.py +18 -11
  164. metadata/generated/schema/api/policies/__init__.py +1 -1
  165. metadata/generated/schema/api/policies/createPolicy.py +38 -18
  166. metadata/generated/schema/api/services/__init__.py +1 -1
  167. metadata/generated/schema/api/services/createApiService.py +60 -0
  168. metadata/generated/schema/api/services/createDashboardService.py +42 -28
  169. metadata/generated/schema/api/services/createDatabaseService.py +42 -28
  170. metadata/generated/schema/api/services/createMessagingService.py +45 -29
  171. metadata/generated/schema/api/services/createMetadataService.py +34 -20
  172. metadata/generated/schema/api/services/createMlModelService.py +42 -28
  173. metadata/generated/schema/api/services/createPipelineService.py +50 -34
  174. metadata/generated/schema/api/services/createSearchService.py +45 -29
  175. metadata/generated/schema/api/services/createStorageService.py +45 -29
  176. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  177. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +45 -24
  178. metadata/generated/schema/api/setOwner.py +17 -8
  179. metadata/generated/schema/api/teams/__init__.py +1 -1
  180. metadata/generated/schema/api/teams/createPersona.py +32 -15
  181. metadata/generated/schema/api/teams/createRole.py +31 -16
  182. metadata/generated/schema/api/teams/createTeam.py +72 -46
  183. metadata/generated/schema/api/teams/createUser.py +75 -47
  184. metadata/generated/schema/api/tests/__init__.py +1 -1
  185. metadata/generated/schema/api/tests/createCustomMetric.py +38 -26
  186. metadata/generated/schema/api/tests/createLogicalTestCases.py +14 -10
  187. metadata/generated/schema/api/tests/createTestCase.py +46 -26
  188. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +25 -17
  189. metadata/generated/schema/api/tests/createTestDefinition.py +29 -15
  190. metadata/generated/schema/api/tests/createTestSuite.py +50 -27
  191. metadata/generated/schema/api/voteRequest.py +12 -6
  192. metadata/generated/schema/auth/__init__.py +1 -1
  193. metadata/generated/schema/auth/basicAuth.py +9 -6
  194. metadata/generated/schema/auth/basicLoginRequest.py +10 -7
  195. metadata/generated/schema/auth/changePasswordRequest.py +18 -13
  196. metadata/generated/schema/auth/createPersonalToken.py +9 -6
  197. metadata/generated/schema/auth/emailRequest.py +9 -6
  198. metadata/generated/schema/auth/emailVerificationToken.py +17 -13
  199. metadata/generated/schema/auth/generateToken.py +7 -5
  200. metadata/generated/schema/auth/jwtAuth.py +15 -9
  201. metadata/generated/schema/auth/loginRequest.py +10 -7
  202. metadata/generated/schema/auth/logoutRequest.py +14 -9
  203. metadata/generated/schema/auth/passwordResetRequest.py +16 -12
  204. metadata/generated/schema/auth/passwordResetToken.py +27 -18
  205. metadata/generated/schema/auth/personalAccessToken.py +21 -16
  206. metadata/generated/schema/auth/refreshToken.py +21 -16
  207. metadata/generated/schema/auth/registrationRequest.py +13 -10
  208. metadata/generated/schema/auth/revokePersonalToken.py +11 -7
  209. metadata/generated/schema/auth/revokeToken.py +7 -5
  210. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  211. metadata/generated/schema/auth/ssoAuth.py +24 -17
  212. metadata/generated/schema/auth/tokenRefreshRequest.py +11 -6
  213. metadata/generated/schema/configuration/__init__.py +1 -1
  214. metadata/generated/schema/configuration/appsPrivateConfiguration.py +23 -17
  215. metadata/generated/schema/configuration/authConfig.py +31 -22
  216. metadata/generated/schema/configuration/authenticationConfiguration.py +46 -33
  217. metadata/generated/schema/configuration/authorizerConfiguration.py +43 -37
  218. metadata/generated/schema/configuration/changeEventConfiguration.py +7 -5
  219. metadata/generated/schema/configuration/dataQualityConfiguration.py +10 -7
  220. metadata/generated/schema/configuration/elasticSearchConfiguration.py +52 -34
  221. metadata/generated/schema/configuration/eventHandlerConfiguration.py +10 -7
  222. metadata/generated/schema/configuration/fernetConfiguration.py +9 -6
  223. metadata/generated/schema/configuration/jwtTokenConfiguration.py +15 -12
  224. metadata/generated/schema/configuration/kafkaEventConfiguration.py +50 -36
  225. metadata/generated/schema/configuration/ldapConfiguration.py +72 -57
  226. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  227. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +23 -17
  228. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +13 -8
  229. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +10 -7
  230. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +11 -7
  231. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +23 -16
  232. metadata/generated/schema/configuration/limitsConfiguration.py +30 -0
  233. metadata/generated/schema/configuration/loginConfiguration.py +21 -14
  234. metadata/generated/schema/configuration/logoConfiguration.py +16 -13
  235. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +80 -49
  236. metadata/generated/schema/configuration/profilerConfiguration.py +18 -11
  237. metadata/generated/schema/configuration/slackAppConfiguration.py +27 -11
  238. metadata/generated/schema/configuration/taskNotificationConfiguration.py +11 -6
  239. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +21 -14
  240. metadata/generated/schema/configuration/themeConfiguration.py +42 -23
  241. metadata/generated/schema/configuration/uiThemePreference.py +19 -12
  242. metadata/generated/schema/dataInsight/__init__.py +1 -1
  243. metadata/generated/schema/dataInsight/custom/__init__.py +3 -0
  244. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +111 -0
  245. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +21 -0
  246. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +25 -0
  247. metadata/generated/schema/dataInsight/custom/formulaHolder.py +24 -0
  248. metadata/generated/schema/dataInsight/custom/lineChart.py +76 -0
  249. metadata/generated/schema/dataInsight/custom/summaryCard.py +49 -0
  250. metadata/generated/schema/dataInsight/dataInsightChart.py +79 -48
  251. metadata/generated/schema/dataInsight/dataInsightChartResult.py +40 -31
  252. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  253. metadata/generated/schema/dataInsight/kpi/basic.py +43 -32
  254. metadata/generated/schema/dataInsight/kpi/kpi.py +81 -52
  255. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  256. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +19 -12
  257. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +19 -12
  258. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +32 -19
  259. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +32 -18
  260. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +16 -8
  261. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +28 -19
  262. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +18 -13
  263. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +17 -9
  264. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +35 -17
  265. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +31 -17
  266. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +33 -17
  267. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +31 -17
  268. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +26 -14
  269. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +27 -14
  270. metadata/generated/schema/dataInsight/type/unusedAssets.py +17 -9
  271. metadata/generated/schema/email/__init__.py +1 -1
  272. metadata/generated/schema/email/emailRequest.py +26 -18
  273. metadata/generated/schema/email/emailTemplate.py +25 -0
  274. metadata/generated/schema/email/emailTemplatePlaceholder.py +20 -0
  275. metadata/generated/schema/email/smtpSettings.py +34 -18
  276. metadata/generated/schema/email/templateValidationReponse.py +29 -0
  277. metadata/generated/schema/entity/__init__.py +1 -1
  278. metadata/generated/schema/entity/applications/__init__.py +1 -1
  279. metadata/generated/schema/entity/applications/app.py +164 -107
  280. metadata/generated/schema/entity/applications/appRunRecord.py +42 -30
  281. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  282. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +24 -7
  283. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  284. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  285. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +28 -18
  286. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +22 -13
  287. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +23 -14
  288. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +28 -18
  289. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +20 -13
  290. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +19 -12
  291. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +11 -7
  292. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +19 -12
  293. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +11 -7
  294. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +11 -7
  295. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +20 -13
  296. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +11 -7
  297. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +68 -39
  298. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +47 -19
  299. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  300. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +65 -6
  301. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +17 -11
  302. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +34 -24
  303. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  304. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +19 -16
  305. metadata/generated/schema/entity/applications/createAppRequest.py +35 -23
  306. metadata/generated/schema/entity/applications/jobStatus.py +10 -6
  307. metadata/generated/schema/entity/applications/liveExecutionContext.py +15 -6
  308. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  309. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +144 -89
  310. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +95 -64
  311. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +8 -3
  312. metadata/generated/schema/entity/automations/__init__.py +1 -1
  313. metadata/generated/schema/entity/automations/testServiceConnection.py +47 -29
  314. metadata/generated/schema/entity/automations/workflow.py +82 -48
  315. metadata/generated/schema/entity/bot.py +64 -37
  316. metadata/generated/schema/entity/classification/__init__.py +1 -1
  317. metadata/generated/schema/entity/classification/classification.py +90 -50
  318. metadata/generated/schema/entity/classification/tag.py +104 -53
  319. metadata/generated/schema/entity/data/__init__.py +1 -1
  320. metadata/generated/schema/entity/data/apiCollection.py +145 -0
  321. metadata/generated/schema/entity/data/apiEndpoint.py +173 -0
  322. metadata/generated/schema/entity/data/chart.py +120 -72
  323. metadata/generated/schema/entity/data/container.py +178 -116
  324. metadata/generated/schema/entity/data/dashboard.py +141 -89
  325. metadata/generated/schema/entity/data/dashboardDataModel.py +129 -77
  326. metadata/generated/schema/entity/data/database.py +167 -104
  327. metadata/generated/schema/entity/data/databaseSchema.py +157 -98
  328. metadata/generated/schema/entity/data/glossary.py +118 -64
  329. metadata/generated/schema/entity/data/glossaryTerm.py +173 -105
  330. metadata/generated/schema/entity/data/metrics.py +93 -55
  331. metadata/generated/schema/entity/data/mlmodel.py +213 -148
  332. metadata/generated/schema/entity/data/pipeline.py +227 -150
  333. metadata/generated/schema/entity/data/query.py +126 -80
  334. metadata/generated/schema/entity/data/report.py +90 -49
  335. metadata/generated/schema/entity/data/searchIndex.py +171 -114
  336. metadata/generated/schema/entity/data/storedProcedure.py +140 -86
  337. metadata/generated/schema/entity/data/table.py +636 -436
  338. metadata/generated/schema/entity/data/topic.py +175 -111
  339. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  340. metadata/generated/schema/entity/docStore/document.py +52 -31
  341. metadata/generated/schema/entity/domains/__init__.py +1 -1
  342. metadata/generated/schema/entity/domains/dataProduct.py +72 -43
  343. metadata/generated/schema/entity/domains/domain.py +69 -45
  344. metadata/generated/schema/entity/events/__init__.py +1 -1
  345. metadata/generated/schema/entity/events/webhook.py +46 -25
  346. metadata/generated/schema/entity/feed/__init__.py +1 -1
  347. metadata/generated/schema/entity/feed/assets.py +7 -5
  348. metadata/generated/schema/entity/feed/customProperty.py +14 -7
  349. metadata/generated/schema/entity/feed/description.py +21 -13
  350. metadata/generated/schema/entity/feed/domain.py +15 -10
  351. metadata/generated/schema/entity/feed/entityInfo.py +14 -8
  352. metadata/generated/schema/entity/feed/owner.py +16 -11
  353. metadata/generated/schema/entity/feed/suggestion.py +112 -35
  354. metadata/generated/schema/entity/feed/tag.py +15 -10
  355. metadata/generated/schema/entity/feed/testCaseResult.py +19 -13
  356. metadata/generated/schema/entity/feed/thread.py +226 -148
  357. metadata/generated/schema/entity/policies/__init__.py +1 -1
  358. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  359. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +25 -12
  360. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +42 -28
  361. metadata/generated/schema/entity/policies/accessControl/rule.py +36 -23
  362. metadata/generated/schema/entity/policies/filters.py +14 -12
  363. metadata/generated/schema/entity/policies/policy.py +107 -67
  364. metadata/generated/schema/entity/services/__init__.py +1 -1
  365. metadata/generated/schema/entity/services/apiService.py +114 -0
  366. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  367. metadata/generated/schema/entity/services/connections/apiService/__init__.py +5 -0
  368. metadata/generated/schema/entity/services/connections/apiService/restConnection.py +38 -0
  369. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -3
  370. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +21 -13
  371. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +30 -19
  372. metadata/generated/schema/entity/services/connections/common/sslConfig.py +14 -8
  373. metadata/generated/schema/entity/services/connections/connectionBasicType.py +71 -55
  374. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  375. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +23 -15
  376. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +44 -30
  377. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +50 -36
  378. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +51 -37
  379. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +31 -24
  380. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +35 -27
  381. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +35 -27
  382. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +97 -63
  383. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  384. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +25 -18
  385. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +23 -14
  386. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +25 -18
  387. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +25 -18
  388. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +27 -19
  389. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +53 -36
  390. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +38 -23
  391. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +30 -21
  392. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +30 -22
  393. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +46 -35
  394. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  395. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +70 -52
  396. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +131 -89
  397. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +83 -56
  398. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +31 -22
  399. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +109 -72
  400. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +11 -7
  402. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +10 -8
  403. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +11 -7
  404. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +10 -8
  405. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +46 -32
  406. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +23 -15
  407. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +87 -60
  408. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +11 -7
  410. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +11 -7
  411. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +11 -7
  412. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +59 -43
  413. metadata/generated/schema/entity/services/connections/database/db2Connection.py +64 -48
  414. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +41 -85
  415. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +3 -0
  416. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +109 -0
  417. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +42 -0
  418. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +52 -33
  419. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +73 -52
  420. metadata/generated/schema/entity/services/connections/database/druidConnection.py +66 -47
  421. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +38 -27
  422. metadata/generated/schema/entity/services/connections/database/glueConnection.py +34 -24
  423. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +85 -56
  424. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +108 -75
  425. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  426. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +11 -6
  427. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +7 -5
  428. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +14 -9
  429. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +36 -25
  430. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +7 -5
  431. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +67 -46
  432. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +23 -15
  433. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +102 -67
  434. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +75 -52
  435. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +56 -38
  436. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +89 -60
  437. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +88 -60
  438. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +105 -74
  439. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +78 -54
  440. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +97 -65
  441. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +90 -61
  442. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +78 -55
  443. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +75 -47
  444. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +84 -0
  445. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  446. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +15 -9
  447. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +33 -24
  448. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +53 -38
  449. metadata/generated/schema/entity/services/connections/database/sasConnection.py +45 -31
  450. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +82 -54
  451. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +121 -80
  452. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +82 -56
  453. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +168 -0
  454. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +92 -62
  455. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +108 -72
  456. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +89 -60
  457. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +71 -50
  458. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  459. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +23 -15
  460. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +101 -61
  461. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +18 -13
  462. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +15 -10
  463. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +85 -54
  464. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  465. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  466. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +117 -74
  467. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +72 -0
  468. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +43 -28
  469. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +40 -27
  470. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +50 -40
  471. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +151 -92
  472. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  473. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +23 -15
  474. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +29 -20
  475. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +20 -13
  476. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +15 -10
  477. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  478. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +25 -18
  479. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +34 -26
  480. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -7
  481. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +23 -15
  482. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +29 -20
  483. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +37 -27
  484. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +43 -0
  485. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +44 -28
  486. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +37 -27
  487. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +35 -0
  488. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +18 -13
  489. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +62 -40
  490. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +71 -45
  491. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +71 -41
  492. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -7
  493. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +28 -18
  494. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  495. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +23 -15
  496. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  497. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +22 -15
  498. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +14 -11
  499. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +48 -31
  500. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +54 -35
  501. metadata/generated/schema/entity/services/connections/serviceConnection.py +38 -22
  502. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  503. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +27 -19
  504. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +23 -15
  505. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +34 -20
  506. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +32 -22
  507. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +96 -57
  508. metadata/generated/schema/entity/services/connections/testConnectionResult.py +40 -26
  509. metadata/generated/schema/entity/services/dashboardService.py +93 -62
  510. metadata/generated/schema/entity/services/databaseService.py +101 -63
  511. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  512. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +198 -122
  513. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +24 -13
  514. metadata/generated/schema/entity/services/ingestionPipelines/status.py +47 -37
  515. metadata/generated/schema/entity/services/messagingService.py +103 -69
  516. metadata/generated/schema/entity/services/metadataService.py +96 -56
  517. metadata/generated/schema/entity/services/mlmodelService.py +96 -63
  518. metadata/generated/schema/entity/services/pipelineService.py +102 -63
  519. metadata/generated/schema/entity/services/searchService.py +92 -62
  520. metadata/generated/schema/entity/services/serviceType.py +2 -1
  521. metadata/generated/schema/entity/services/storageService.py +93 -62
  522. metadata/generated/schema/entity/teams/__init__.py +1 -1
  523. metadata/generated/schema/entity/teams/persona.py +59 -35
  524. metadata/generated/schema/entity/teams/role.py +82 -51
  525. metadata/generated/schema/entity/teams/team.py +128 -82
  526. metadata/generated/schema/entity/teams/teamHierarchy.py +52 -35
  527. metadata/generated/schema/entity/teams/user.py +122 -81
  528. metadata/generated/schema/entity/type.py +94 -48
  529. metadata/generated/schema/entity/utils/__init__.py +1 -1
  530. metadata/generated/schema/entity/utils/entitiesCount.py +25 -18
  531. metadata/generated/schema/entity/utils/servicesCount.py +25 -22
  532. metadata/generated/schema/entity/utils/supersetApiConnection.py +19 -14
  533. metadata/generated/schema/events/__init__.py +1 -1
  534. metadata/generated/schema/events/alertMetrics.py +22 -14
  535. metadata/generated/schema/events/api/__init__.py +1 -1
  536. metadata/generated/schema/events/api/createEventSubscription.py +63 -35
  537. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +27 -0
  538. metadata/generated/schema/events/emailAlertConfig.py +20 -12
  539. metadata/generated/schema/events/eventFilterRule.py +40 -24
  540. metadata/generated/schema/events/eventSubscription.py +220 -142
  541. metadata/generated/schema/events/eventSubscriptionOffset.py +12 -8
  542. metadata/generated/schema/events/failedEvent.py +25 -13
  543. metadata/generated/schema/events/filterResourceDescriptor.py +24 -14
  544. metadata/generated/schema/events/subscriptionResourceDescriptor.py +20 -11
  545. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  546. metadata/generated/schema/metadataIngestion/application.py +39 -25
  547. metadata/generated/schema/metadataIngestion/applicationPipeline.py +23 -16
  548. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +125 -67
  549. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +9 -6
  550. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +182 -106
  551. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +117 -73
  552. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +75 -47
  553. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +51 -32
  554. metadata/generated/schema/metadataIngestion/dbtPipeline.py +75 -51
  555. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  556. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +37 -25
  557. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +23 -14
  558. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +36 -27
  559. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +37 -25
  560. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +31 -20
  561. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +31 -20
  562. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +37 -25
  563. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +45 -22
  564. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +65 -43
  565. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +35 -17
  566. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +99 -50
  567. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +51 -27
  568. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  569. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +60 -33
  570. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +59 -38
  571. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +22 -14
  572. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +15 -10
  573. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +15 -10
  574. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +14 -9
  575. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +14 -9
  576. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +14 -10
  577. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +57 -28
  578. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +32 -17
  579. metadata/generated/schema/metadataIngestion/workflow.py +114 -62
  580. metadata/generated/schema/monitoring/__init__.py +1 -1
  581. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  582. metadata/generated/schema/security/__init__.py +1 -1
  583. metadata/generated/schema/security/client/__init__.py +1 -1
  584. metadata/generated/schema/security/client/auth0SSOClientConfig.py +10 -9
  585. metadata/generated/schema/security/client/azureSSOClientConfig.py +12 -11
  586. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +11 -10
  587. metadata/generated/schema/security/client/googleSSOClientConfig.py +17 -12
  588. metadata/generated/schema/security/client/oidcClientConfig.py +45 -30
  589. metadata/generated/schema/security/client/oktaSSOClientConfig.py +14 -11
  590. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +9 -8
  591. metadata/generated/schema/security/client/samlSSOClientConfig.py +104 -67
  592. metadata/generated/schema/security/credentials/__init__.py +1 -1
  593. metadata/generated/schema/security/credentials/accessTokenAuth.py +17 -13
  594. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +10 -8
  595. metadata/generated/schema/security/credentials/awsCredentials.py +57 -38
  596. metadata/generated/schema/security/credentials/azureCredentials.py +47 -29
  597. metadata/generated/schema/security/credentials/basicAuth.py +13 -11
  598. metadata/generated/schema/security/credentials/bitbucketCredentials.py +22 -14
  599. metadata/generated/schema/security/credentials/gcpCredentials.py +66 -33
  600. metadata/generated/schema/security/credentials/gcpExternalAccount.py +55 -26
  601. metadata/generated/schema/security/credentials/gcpValues.py +70 -49
  602. metadata/generated/schema/security/credentials/gitCredentials.py +32 -25
  603. metadata/generated/schema/security/credentials/githubCredentials.py +13 -7
  604. metadata/generated/schema/security/credentials/gitlabCredentials.py +34 -0
  605. metadata/generated/schema/security/sasl/__init__.py +3 -0
  606. metadata/generated/schema/security/sasl/saslClientConfig.py +40 -0
  607. metadata/generated/schema/security/secrets/__init__.py +1 -1
  608. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  609. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +39 -24
  610. metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -1
  611. metadata/generated/schema/security/securityConfiguration.py +15 -9
  612. metadata/generated/schema/security/ssl/__init__.py +1 -1
  613. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +30 -20
  614. metadata/generated/schema/security/ssl/verifySSLConfig.py +16 -18
  615. metadata/generated/schema/settings/__init__.py +1 -1
  616. metadata/generated/schema/settings/settings.py +10 -5
  617. metadata/generated/schema/system/__init__.py +1 -1
  618. metadata/generated/schema/system/entityError.py +7 -5
  619. metadata/generated/schema/system/eventPublisherJob.py +58 -37
  620. metadata/generated/schema/system/indexingError.py +7 -5
  621. metadata/generated/schema/system/limitsResponse.py +20 -0
  622. metadata/generated/schema/system/ui/__init__.py +1 -1
  623. metadata/generated/schema/system/ui/knowledgePanel.py +12 -8
  624. metadata/generated/schema/system/ui/page.py +24 -18
  625. metadata/generated/schema/system/validationResponse.py +40 -28
  626. metadata/generated/schema/tests/__init__.py +1 -1
  627. metadata/generated/schema/tests/assigned.py +11 -7
  628. metadata/generated/schema/tests/basic.py +98 -55
  629. metadata/generated/schema/tests/customMetric.py +41 -29
  630. metadata/generated/schema/tests/dataQualityReport.py +40 -0
  631. metadata/generated/schema/tests/resolved.py +17 -13
  632. metadata/generated/schema/tests/testCase.py +137 -76
  633. metadata/generated/schema/tests/testCaseResolutionStatus.py +49 -33
  634. metadata/generated/schema/tests/testDefinition.py +129 -67
  635. metadata/generated/schema/tests/testSuite.py +137 -75
  636. metadata/generated/schema/type/__init__.py +1 -1
  637. metadata/generated/schema/type/apiSchema.py +27 -0
  638. metadata/generated/schema/type/auditLog.py +29 -20
  639. metadata/generated/schema/type/basic.py +177 -110
  640. metadata/generated/schema/type/bulkOperationResult.py +45 -30
  641. metadata/generated/schema/type/changeEvent.py +72 -40
  642. metadata/generated/schema/type/changeEventType.py +1 -1
  643. metadata/generated/schema/type/collectionDescriptor.py +24 -15
  644. metadata/generated/schema/type/csvDocumentation.py +11 -8
  645. metadata/generated/schema/type/csvErrorType.py +1 -1
  646. metadata/generated/schema/type/csvFile.py +25 -17
  647. metadata/generated/schema/type/csvImportResult.py +34 -21
  648. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  649. metadata/generated/schema/type/customProperties/complexTypes.py +110 -64
  650. metadata/generated/schema/type/customProperties/enumConfig.py +7 -5
  651. metadata/generated/schema/type/customProperty.py +34 -25
  652. metadata/generated/schema/type/dailyCount.py +10 -7
  653. metadata/generated/schema/type/databaseConnectionConfig.py +69 -42
  654. metadata/generated/schema/type/entityHierarchy.py +35 -26
  655. metadata/generated/schema/type/entityHistory.py +71 -45
  656. metadata/generated/schema/type/entityLineage.py +89 -57
  657. metadata/generated/schema/type/entityReference.py +51 -30
  658. metadata/generated/schema/type/entityReferenceList.py +11 -8
  659. metadata/generated/schema/type/entityRelationship.py +68 -39
  660. metadata/generated/schema/type/entityUsage.py +15 -10
  661. metadata/generated/schema/type/filterPattern.py +21 -12
  662. metadata/generated/schema/type/function.py +30 -21
  663. metadata/generated/schema/type/include.py +1 -1
  664. metadata/generated/schema/type/jdbcConnection.py +24 -19
  665. metadata/generated/schema/type/lifeCycle.py +47 -26
  666. metadata/generated/schema/type/paging.py +32 -21
  667. metadata/generated/schema/type/profile.py +13 -11
  668. metadata/generated/schema/type/queryParserData.py +47 -32
  669. metadata/generated/schema/type/reaction.py +13 -8
  670. metadata/generated/schema/type/schedule.py +18 -11
  671. metadata/generated/schema/type/schema.py +64 -44
  672. metadata/generated/schema/type/tableQuery.py +53 -40
  673. metadata/generated/schema/type/tableUsageCount.py +46 -33
  674. metadata/generated/schema/type/tagLabel.py +36 -24
  675. metadata/generated/schema/type/usageDetails.py +40 -24
  676. metadata/generated/schema/type/usageRequest.py +10 -7
  677. metadata/generated/schema/type/votes.py +21 -12
  678. metadata/great_expectations/action.py +14 -12
  679. metadata/great_expectations/utils/ometa_config_handler.py +1 -1
  680. metadata/ingestion/api/common.py +3 -4
  681. metadata/ingestion/api/delete.py +1 -1
  682. metadata/ingestion/api/models.py +6 -8
  683. metadata/ingestion/api/parser.py +13 -17
  684. metadata/ingestion/api/status.py +35 -13
  685. metadata/ingestion/api/topology_runner.py +2 -1
  686. metadata/ingestion/bulksink/metadata_usage.py +17 -14
  687. metadata/ingestion/connections/builders.py +12 -12
  688. metadata/ingestion/connections/secrets.py +8 -6
  689. metadata/ingestion/connections/test_connections.py +7 -2
  690. metadata/ingestion/lineage/models.py +5 -4
  691. metadata/ingestion/lineage/sql_lineage.py +43 -13
  692. metadata/ingestion/models/custom_properties.py +9 -9
  693. metadata/ingestion/models/custom_pydantic.py +70 -44
  694. metadata/ingestion/models/lf_tags_model.py +3 -3
  695. metadata/ingestion/models/ometa_classification.py +1 -1
  696. metadata/ingestion/models/ometa_lineage.py +24 -0
  697. metadata/ingestion/models/patch_request.py +96 -56
  698. metadata/ingestion/models/table_metadata.py +2 -2
  699. metadata/ingestion/models/topology.py +18 -12
  700. metadata/ingestion/ometa/client.py +34 -8
  701. metadata/ingestion/ometa/client_utils.py +6 -7
  702. metadata/ingestion/ometa/mixins/custom_property_mixin.py +2 -4
  703. metadata/ingestion/ometa/mixins/dashboard_mixin.py +2 -2
  704. metadata/ingestion/ometa/mixins/data_insight_mixin.py +10 -6
  705. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +5 -3
  706. metadata/ingestion/ometa/mixins/lineage_mixin.py +36 -15
  707. metadata/ingestion/ometa/mixins/mlmodel_mixin.py +2 -2
  708. metadata/ingestion/ometa/mixins/patch_mixin.py +32 -30
  709. metadata/ingestion/ometa/mixins/pipeline_mixin.py +3 -3
  710. metadata/ingestion/ometa/mixins/query_mixin.py +4 -4
  711. metadata/ingestion/ometa/mixins/role_policy_mixin.py +13 -13
  712. metadata/ingestion/ometa/mixins/search_index_mixin.py +5 -5
  713. metadata/ingestion/ometa/mixins/server_mixin.py +4 -5
  714. metadata/ingestion/ometa/mixins/service_mixin.py +2 -2
  715. metadata/ingestion/ometa/mixins/suggestions_mixin.py +2 -2
  716. metadata/ingestion/ometa/mixins/table_mixin.py +20 -20
  717. metadata/ingestion/ometa/mixins/tests_mixin.py +21 -18
  718. metadata/ingestion/ometa/mixins/topic_mixin.py +2 -2
  719. metadata/ingestion/ometa/mixins/user_mixin.py +44 -23
  720. metadata/ingestion/ometa/models.py +1 -1
  721. metadata/ingestion/ometa/ometa_api.py +6 -7
  722. metadata/ingestion/ometa/routes.py +18 -0
  723. metadata/ingestion/ometa/ttl_cache.py +47 -0
  724. metadata/ingestion/ometa/utils.py +3 -3
  725. metadata/ingestion/processor/query_parser.py +3 -3
  726. metadata/ingestion/sink/file.py +2 -2
  727. metadata/ingestion/sink/metadata_rest.py +59 -29
  728. metadata/ingestion/source/dashboard/dashboard_service.py +45 -45
  729. metadata/ingestion/source/dashboard/domodashboard/metadata.py +31 -19
  730. metadata/ingestion/source/dashboard/lightdash/client.py +1 -1
  731. metadata/ingestion/source/dashboard/lightdash/metadata.py +26 -14
  732. metadata/ingestion/source/dashboard/lightdash/models.py +11 -11
  733. metadata/ingestion/source/dashboard/looker/bulk_parser.py +3 -3
  734. metadata/ingestion/source/dashboard/looker/connection.py +1 -1
  735. metadata/ingestion/source/dashboard/looker/metadata.py +67 -42
  736. metadata/ingestion/source/dashboard/looker/parser.py +3 -3
  737. metadata/ingestion/source/dashboard/looker/utils.py +8 -5
  738. metadata/ingestion/source/dashboard/metabase/client.py +19 -2
  739. metadata/ingestion/source/dashboard/metabase/metadata.py +45 -14
  740. metadata/ingestion/source/dashboard/metabase/models.py +46 -28
  741. metadata/ingestion/source/dashboard/mode/client.py +1 -1
  742. metadata/ingestion/source/dashboard/mode/metadata.py +24 -14
  743. metadata/ingestion/source/dashboard/mstr/client.py +1 -1
  744. metadata/ingestion/source/dashboard/mstr/metadata.py +18 -11
  745. metadata/ingestion/source/dashboard/mstr/models.py +4 -4
  746. metadata/ingestion/source/dashboard/powerbi/client.py +2 -3
  747. metadata/ingestion/source/dashboard/powerbi/metadata.py +52 -31
  748. metadata/ingestion/source/dashboard/powerbi/models.py +25 -22
  749. metadata/ingestion/source/dashboard/qlikcloud/client.py +2 -2
  750. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +28 -16
  751. metadata/ingestion/source/dashboard/qlikcloud/models.py +4 -4
  752. metadata/ingestion/source/dashboard/qliksense/client.py +3 -3
  753. metadata/ingestion/source/dashboard/qliksense/metadata.py +35 -19
  754. metadata/ingestion/source/dashboard/qliksense/models.py +8 -8
  755. metadata/ingestion/source/dashboard/quicksight/metadata.py +26 -18
  756. metadata/ingestion/source/dashboard/quicksight/models.py +16 -16
  757. metadata/ingestion/source/dashboard/redash/client.py +1 -1
  758. metadata/ingestion/source/dashboard/redash/metadata.py +31 -19
  759. metadata/ingestion/source/dashboard/superset/api_source.py +46 -18
  760. metadata/ingestion/source/dashboard/superset/client.py +2 -2
  761. metadata/ingestion/source/dashboard/superset/db_source.py +44 -18
  762. metadata/ingestion/source/dashboard/superset/metadata.py +2 -2
  763. metadata/ingestion/source/dashboard/superset/mixin.py +12 -13
  764. metadata/ingestion/source/dashboard/superset/models.py +68 -68
  765. metadata/ingestion/source/dashboard/tableau/connection.py +2 -2
  766. metadata/ingestion/source/dashboard/tableau/metadata.py +398 -94
  767. metadata/ingestion/source/dashboard/tableau/models.py +33 -35
  768. metadata/ingestion/source/dashboard/tableau/queries.py +14 -0
  769. metadata/ingestion/source/database/athena/connection.py +34 -5
  770. metadata/ingestion/source/database/athena/metadata.py +43 -168
  771. metadata/ingestion/source/database/athena/models.py +10 -10
  772. metadata/ingestion/source/database/athena/query_parser.py +2 -2
  773. metadata/ingestion/source/database/athena/utils.py +191 -0
  774. metadata/ingestion/source/database/azuresql/metadata.py +4 -4
  775. metadata/ingestion/source/database/azuresql/query_parser.py +2 -2
  776. metadata/ingestion/source/database/bigquery/connection.py +17 -5
  777. metadata/ingestion/source/database/bigquery/helper.py +1 -1
  778. metadata/ingestion/source/database/bigquery/metadata.py +91 -52
  779. metadata/ingestion/source/database/bigquery/query_parser.py +3 -3
  780. metadata/ingestion/source/database/bigtable/connection.py +2 -2
  781. metadata/ingestion/source/database/bigtable/metadata.py +2 -2
  782. metadata/ingestion/source/database/clickhouse/connection.py +3 -3
  783. metadata/ingestion/source/database/clickhouse/metadata.py +11 -8
  784. metadata/ingestion/source/database/clickhouse/query_parser.py +4 -4
  785. metadata/ingestion/source/database/column_type_parser.py +19 -11
  786. metadata/ingestion/source/database/common_db_source.py +106 -48
  787. metadata/ingestion/source/database/common_nosql_source.py +22 -17
  788. metadata/ingestion/source/database/couchbase/connection.py +1 -1
  789. metadata/ingestion/source/database/couchbase/metadata.py +3 -3
  790. metadata/ingestion/source/database/database_service.py +30 -17
  791. metadata/ingestion/source/database/databricks/connection.py +1 -1
  792. metadata/ingestion/source/database/databricks/lineage.py +4 -3
  793. metadata/ingestion/source/database/databricks/metadata.py +93 -24
  794. metadata/ingestion/source/database/databricks/queries.py +5 -1
  795. metadata/ingestion/source/database/databricks/query_parser.py +2 -2
  796. metadata/ingestion/source/database/databricks/usage.py +4 -3
  797. metadata/ingestion/source/database/datalake/clients/azure_blob.py +72 -0
  798. metadata/ingestion/source/database/datalake/clients/base.py +47 -0
  799. metadata/ingestion/source/database/datalake/clients/gcs.py +136 -0
  800. metadata/ingestion/source/database/datalake/clients/s3.py +79 -0
  801. metadata/ingestion/source/database/datalake/connection.py +12 -70
  802. metadata/ingestion/source/database/datalake/metadata.py +75 -219
  803. metadata/ingestion/source/database/db2/metadata.py +13 -13
  804. metadata/ingestion/source/database/dbt/dbt_config.py +3 -2
  805. metadata/ingestion/source/database/dbt/dbt_service.py +15 -6
  806. metadata/ingestion/source/database/dbt/metadata.py +128 -83
  807. metadata/ingestion/source/database/dbt/models.py +9 -9
  808. metadata/ingestion/source/database/deltalake/clients/__init__.py +0 -0
  809. metadata/ingestion/source/database/deltalake/clients/base.py +82 -0
  810. metadata/ingestion/source/database/deltalake/clients/pyspark.py +304 -0
  811. metadata/ingestion/source/database/deltalake/clients/s3.py +200 -0
  812. metadata/ingestion/source/database/deltalake/connection.py +56 -84
  813. metadata/ingestion/source/database/deltalake/metadata.py +67 -193
  814. metadata/ingestion/source/database/domodatabase/metadata.py +83 -38
  815. metadata/ingestion/source/database/domodatabase/models.py +6 -6
  816. metadata/ingestion/source/database/doris/metadata.py +3 -3
  817. metadata/ingestion/source/database/druid/metadata.py +2 -2
  818. metadata/ingestion/source/database/dynamodb/metadata.py +5 -5
  819. metadata/ingestion/source/database/dynamodb/models.py +2 -2
  820. metadata/ingestion/source/database/extended_sample_data.py +41 -41
  821. metadata/ingestion/source/database/external_table_lineage_mixin.py +58 -3
  822. metadata/ingestion/source/database/glue/metadata.py +23 -18
  823. metadata/ingestion/source/database/glue/models.py +6 -6
  824. metadata/ingestion/source/database/greenplum/metadata.py +4 -4
  825. metadata/ingestion/source/database/hive/connection.py +2 -2
  826. metadata/ingestion/source/database/hive/metadata.py +2 -2
  827. metadata/ingestion/source/database/iceberg/catalog/hive.py +1 -1
  828. metadata/ingestion/source/database/iceberg/catalog/rest.py +1 -1
  829. metadata/ingestion/source/database/iceberg/fs/__init__.py +1 -1
  830. metadata/ingestion/source/database/iceberg/fs/s3.py +1 -1
  831. metadata/ingestion/source/database/iceberg/helper.py +4 -4
  832. metadata/ingestion/source/database/iceberg/metadata.py +28 -22
  833. metadata/ingestion/source/database/iceberg/models.py +6 -6
  834. metadata/ingestion/source/database/impala/connection.py +3 -3
  835. metadata/ingestion/source/database/impala/metadata.py +2 -2
  836. metadata/ingestion/source/database/incremental_metadata_extraction.py +1 -1
  837. metadata/ingestion/source/database/life_cycle_query_mixin.py +12 -8
  838. metadata/ingestion/source/database/lineage_source.py +25 -13
  839. metadata/ingestion/source/database/mariadb/metadata.py +2 -2
  840. metadata/ingestion/source/database/mongodb/connection.py +1 -1
  841. metadata/ingestion/source/database/mongodb/metadata.py +2 -2
  842. metadata/ingestion/source/database/mssql/constants.py +23 -0
  843. metadata/ingestion/source/database/mssql/lineage.py +25 -0
  844. metadata/ingestion/source/database/mssql/metadata.py +30 -10
  845. metadata/ingestion/source/database/mssql/queries.py +2 -0
  846. metadata/ingestion/source/database/mssql/query_parser.py +2 -2
  847. metadata/ingestion/source/database/mssql/usage.py +42 -0
  848. metadata/ingestion/source/database/mssql/utils.py +16 -1
  849. metadata/ingestion/source/database/mysql/metadata.py +2 -2
  850. metadata/ingestion/source/database/oracle/metadata.py +16 -13
  851. metadata/ingestion/source/database/oracle/models.py +1 -1
  852. metadata/ingestion/source/database/oracle/query_parser.py +2 -2
  853. metadata/ingestion/source/database/pinotdb/metadata.py +2 -2
  854. metadata/ingestion/source/database/postgres/lineage.py +2 -1
  855. metadata/ingestion/source/database/postgres/metadata.py +28 -6
  856. metadata/ingestion/source/database/postgres/pgspider/lineage.py +2 -2
  857. metadata/ingestion/source/database/postgres/queries.py +7 -0
  858. metadata/ingestion/source/database/postgres/query_parser.py +4 -4
  859. metadata/ingestion/source/database/postgres/usage.py +15 -2
  860. metadata/ingestion/source/database/presto/connection.py +2 -2
  861. metadata/ingestion/source/database/presto/metadata.py +2 -2
  862. metadata/ingestion/source/database/query/lineage.py +1 -1
  863. metadata/ingestion/source/database/query/usage.py +5 -3
  864. metadata/ingestion/source/database/query_parser_source.py +1 -1
  865. metadata/ingestion/source/database/redshift/metadata.py +7 -7
  866. metadata/ingestion/source/database/redshift/models.py +1 -1
  867. metadata/ingestion/source/database/redshift/queries.py +26 -2
  868. metadata/ingestion/source/database/redshift/query_parser.py +2 -2
  869. metadata/ingestion/source/database/salesforce/connection.py +1 -3
  870. metadata/ingestion/source/database/salesforce/metadata.py +22 -17
  871. metadata/ingestion/source/database/sample_data.py +198 -98
  872. metadata/ingestion/source/database/sample_usage.py +5 -7
  873. metadata/ingestion/source/database/saperp/__init__.py +0 -0
  874. metadata/ingestion/source/database/saperp/client.py +170 -0
  875. metadata/ingestion/source/database/saperp/connection.py +50 -0
  876. metadata/ingestion/source/database/saperp/constants.py +27 -0
  877. metadata/ingestion/source/database/saperp/metadata.py +322 -0
  878. metadata/ingestion/source/database/saperp/models.py +100 -0
  879. metadata/ingestion/source/database/saphana/metadata.py +2 -2
  880. metadata/ingestion/source/database/sas/client.py +2 -2
  881. metadata/ingestion/source/database/sas/metadata.py +35 -44
  882. metadata/ingestion/source/database/singlestore/metadata.py +2 -2
  883. metadata/ingestion/source/database/snowflake/connection.py +3 -3
  884. metadata/ingestion/source/database/snowflake/metadata.py +39 -12
  885. metadata/ingestion/source/database/snowflake/models.py +7 -7
  886. metadata/ingestion/source/database/snowflake/query_parser.py +2 -2
  887. metadata/ingestion/source/database/sql_column_handler.py +4 -3
  888. metadata/ingestion/source/database/sqlite/metadata.py +2 -2
  889. metadata/ingestion/source/database/stored_procedures_mixin.py +10 -11
  890. metadata/ingestion/source/database/teradata/metadata.py +4 -4
  891. metadata/ingestion/source/database/trino/connection.py +5 -7
  892. metadata/ingestion/source/database/trino/metadata.py +2 -2
  893. metadata/ingestion/source/database/trino/query_parser.py +2 -2
  894. metadata/ingestion/source/database/unitycatalog/client.py +21 -0
  895. metadata/ingestion/source/database/unitycatalog/lineage.py +7 -7
  896. metadata/ingestion/source/database/unitycatalog/metadata.py +82 -42
  897. metadata/ingestion/source/database/unitycatalog/models.py +16 -16
  898. metadata/ingestion/source/database/unitycatalog/query_parser.py +2 -2
  899. metadata/ingestion/source/database/usage_source.py +51 -34
  900. metadata/ingestion/source/database/vertica/metadata.py +3 -3
  901. metadata/ingestion/source/database/vertica/query_parser.py +4 -4
  902. metadata/ingestion/source/messaging/common_broker_source.py +18 -4
  903. metadata/ingestion/source/messaging/kafka/connection.py +1 -1
  904. metadata/ingestion/source/messaging/kafka/metadata.py +7 -9
  905. metadata/ingestion/source/messaging/kinesis/metadata.py +10 -5
  906. metadata/ingestion/source/messaging/kinesis/models.py +3 -5
  907. metadata/ingestion/source/messaging/messaging_service.py +12 -7
  908. metadata/ingestion/source/messaging/redpanda/metadata.py +2 -2
  909. metadata/ingestion/source/metadata/alationsink/__init__.py +0 -0
  910. metadata/ingestion/source/metadata/alationsink/client.py +213 -0
  911. metadata/ingestion/source/metadata/alationsink/connection.py +53 -0
  912. metadata/ingestion/source/metadata/alationsink/constants.py +60 -0
  913. metadata/ingestion/source/metadata/alationsink/metadata.py +397 -0
  914. metadata/ingestion/source/metadata/alationsink/models.py +148 -0
  915. metadata/ingestion/source/metadata/amundsen/connection.py +1 -1
  916. metadata/ingestion/source/metadata/amundsen/metadata.py +14 -10
  917. metadata/ingestion/source/metadata/atlas/client.py +1 -1
  918. metadata/ingestion/source/metadata/atlas/metadata.py +11 -12
  919. metadata/ingestion/source/mlmodel/mlflow/metadata.py +12 -6
  920. metadata/ingestion/source/mlmodel/mlmodel_service.py +13 -6
  921. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +34 -21
  922. metadata/ingestion/source/pipeline/airbyte/client.py +1 -1
  923. metadata/ingestion/source/pipeline/airbyte/metadata.py +15 -11
  924. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +3 -4
  925. metadata/ingestion/source/pipeline/airflow/metadata.py +28 -17
  926. metadata/ingestion/source/pipeline/airflow/models.py +21 -24
  927. metadata/ingestion/source/pipeline/dagster/client.py +3 -3
  928. metadata/ingestion/source/pipeline/dagster/metadata.py +31 -17
  929. metadata/ingestion/source/pipeline/dagster/models.py +14 -14
  930. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +120 -126
  931. metadata/ingestion/source/pipeline/databrickspipeline/models.py +72 -0
  932. metadata/ingestion/source/pipeline/dbtcloud/__init__.py +0 -0
  933. metadata/ingestion/source/pipeline/dbtcloud/client.py +143 -0
  934. metadata/ingestion/source/pipeline/dbtcloud/connection.py +60 -0
  935. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +345 -0
  936. metadata/ingestion/source/pipeline/dbtcloud/models.py +72 -0
  937. metadata/ingestion/source/pipeline/dbtcloud/queries.py +45 -0
  938. metadata/ingestion/source/pipeline/domopipeline/metadata.py +21 -12
  939. metadata/ingestion/source/pipeline/fivetran/client.py +20 -6
  940. metadata/ingestion/source/pipeline/fivetran/metadata.py +124 -50
  941. metadata/ingestion/source/pipeline/fivetran/models.py +20 -0
  942. metadata/ingestion/source/pipeline/flink/client.py +53 -0
  943. metadata/ingestion/source/pipeline/flink/connection.py +52 -0
  944. metadata/ingestion/source/pipeline/flink/metadata.py +188 -0
  945. metadata/ingestion/source/pipeline/flink/models.py +44 -0
  946. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +23 -11
  947. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +7 -8
  948. metadata/ingestion/source/pipeline/nifi/metadata.py +17 -8
  949. metadata/ingestion/source/pipeline/openlineage/connection.py +12 -3
  950. metadata/ingestion/source/pipeline/openlineage/metadata.py +3 -27
  951. metadata/ingestion/source/pipeline/pipeline_service.py +52 -8
  952. metadata/ingestion/source/pipeline/spline/client.py +1 -1
  953. metadata/ingestion/source/pipeline/spline/metadata.py +3 -3
  954. metadata/ingestion/source/pipeline/spline/models.py +12 -12
  955. metadata/ingestion/source/search/elasticsearch/connection.py +2 -2
  956. metadata/ingestion/source/search/elasticsearch/metadata.py +5 -4
  957. metadata/ingestion/source/search/search_service.py +13 -6
  958. metadata/ingestion/source/storage/gcs/client.py +58 -0
  959. metadata/ingestion/source/storage/gcs/connection.py +159 -0
  960. metadata/ingestion/source/storage/gcs/metadata.py +465 -0
  961. metadata/ingestion/source/storage/gcs/models.py +86 -0
  962. metadata/ingestion/source/storage/s3/metadata.py +268 -18
  963. metadata/ingestion/source/storage/s3/models.py +30 -11
  964. metadata/ingestion/source/storage/storage_service.py +43 -7
  965. metadata/ingestion/stage/table_usage.py +3 -3
  966. metadata/mixins/pandas/pandas_mixin.py +2 -2
  967. metadata/mixins/sqalchemy/sqa_mixin.py +1 -1
  968. metadata/parsers/avro_parser.py +12 -10
  969. metadata/parsers/json_schema_parser.py +4 -4
  970. metadata/parsers/protobuf_parser.py +4 -5
  971. metadata/pii/processor.py +4 -6
  972. metadata/profiler/adaptors/dynamodb.py +2 -2
  973. metadata/profiler/adaptors/mongodb.py +5 -17
  974. metadata/profiler/api/models.py +9 -9
  975. metadata/profiler/interface/nosql/profiler_interface.py +5 -8
  976. metadata/profiler/interface/pandas/profiler_interface.py +8 -10
  977. metadata/profiler/interface/profiler_interface.py +3 -3
  978. metadata/profiler/interface/profiler_interface_factory.py +46 -51
  979. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +28 -17
  980. metadata/profiler/interface/sqlalchemy/profiler_interface.py +3 -5
  981. metadata/profiler/metrics/composed/null_ratio.py +5 -4
  982. metadata/profiler/metrics/static/mean.py +9 -0
  983. metadata/profiler/metrics/system/queries/snowflake.py +1 -1
  984. metadata/profiler/metrics/window/percentille_mixin.py +1 -1
  985. metadata/profiler/orm/converter/base.py +21 -13
  986. metadata/profiler/orm/converter/common.py +3 -4
  987. metadata/profiler/orm/functions/datetime.py +1 -1
  988. metadata/profiler/orm/functions/sum.py +8 -1
  989. metadata/profiler/orm/functions/table_metric_computer.py +4 -5
  990. metadata/profiler/orm/registry.py +5 -0
  991. metadata/profiler/orm/types/custom_hex_byte_string.py +5 -3
  992. metadata/profiler/orm/types/undetermined_type.py +38 -0
  993. metadata/profiler/processor/core.py +19 -13
  994. metadata/profiler/processor/default.py +4 -1
  995. metadata/profiler/processor/processor.py +4 -4
  996. metadata/profiler/processor/sample_data_handler.py +5 -5
  997. metadata/profiler/processor/sampler/nosql/sampler.py +9 -4
  998. metadata/profiler/processor/sampler/sampler_factory.py +36 -6
  999. metadata/profiler/processor/sampler/sqlalchemy/azuresql/sampler.py +40 -0
  1000. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +33 -62
  1001. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +60 -0
  1002. metadata/profiler/source/base/profiler_source.py +12 -14
  1003. metadata/profiler/source/bigquery/profiler_source.py +2 -2
  1004. metadata/profiler/source/bigquery/type_mapper.py +2 -2
  1005. metadata/profiler/source/metadata.py +14 -16
  1006. metadata/profiler/source/metadata_ext.py +3 -3
  1007. metadata/readers/dataframe/avro.py +1 -3
  1008. metadata/readers/dataframe/models.py +29 -12
  1009. metadata/readers/dataframe/parquet.py +3 -3
  1010. metadata/readers/dataframe/reader_factory.py +6 -0
  1011. metadata/readers/file/api_reader.py +5 -2
  1012. metadata/readers/file/bitbucket.py +4 -4
  1013. metadata/readers/file/credentials.py +5 -5
  1014. metadata/readers/file/github.py +6 -6
  1015. metadata/readers/file/gitlab.py +172 -0
  1016. metadata/utils/class_helper.py +3 -5
  1017. metadata/utils/credentials.py +2 -2
  1018. metadata/utils/datalake/datalake_utils.py +4 -2
  1019. metadata/utils/entity_link.py +8 -0
  1020. metadata/utils/execution_time_tracker.py +2 -2
  1021. metadata/utils/filters.py +25 -17
  1022. metadata/utils/fqn.py +59 -14
  1023. metadata/utils/helpers.py +15 -12
  1024. metadata/utils/importer.py +3 -2
  1025. metadata/utils/life_cycle_utils.py +3 -3
  1026. metadata/utils/logger.py +9 -24
  1027. metadata/utils/lru_cache.py +3 -0
  1028. metadata/utils/metadata_service_helper.py +3 -1
  1029. metadata/utils/secrets/gcp_secrets_manager.py +128 -0
  1030. metadata/utils/secrets/secrets_manager_factory.py +3 -0
  1031. metadata/utils/source_hash.py +1 -1
  1032. metadata/utils/ssl_manager.py +20 -26
  1033. metadata/utils/ssl_registry.py +5 -1
  1034. metadata/utils/storage_metadata_config.py +6 -5
  1035. metadata/utils/tag_utils.py +21 -13
  1036. metadata/utils/time_utils.py +45 -5
  1037. metadata/workflow/application.py +4 -4
  1038. metadata/workflow/application_output_handler.py +6 -19
  1039. metadata/workflow/base.py +42 -5
  1040. metadata/workflow/data_quality.py +5 -5
  1041. metadata/workflow/ingestion.py +4 -6
  1042. metadata/workflow/metadata.py +4 -4
  1043. metadata/workflow/output_handler.py +35 -214
  1044. metadata/workflow/profiler.py +5 -5
  1045. metadata/workflow/usage.py +8 -6
  1046. metadata/workflow/workflow_init_error_handler.py +154 -0
  1047. metadata/workflow/workflow_output_handler.py +189 -98
  1048. metadata/workflow/workflow_status_mixin.py +20 -11
  1049. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/METADATA +387 -312
  1050. openmetadata_ingestion-1.5.0.0rc2.dist-info/RECORD +1598 -0
  1051. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/top_level.txt +1 -0
  1052. metadata/cli/insight.py +0 -51
  1053. metadata/data_insight/processor/kpi/kpi_runner.py +0 -156
  1054. metadata/data_insight/processor/kpi/run_result_registry.py +0 -162
  1055. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +0 -253
  1056. metadata/data_insight/processor/reports/data_processor.py +0 -72
  1057. metadata/data_insight/processor/reports/entity_report_data_processor.py +0 -262
  1058. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +0 -384
  1059. metadata/data_insight/producer/cost_analysis_producer.py +0 -126
  1060. metadata/data_insight/producer/entity_producer.py +0 -63
  1061. metadata/data_insight/producer/producer_factory.py +0 -53
  1062. metadata/data_insight/producer/web_analytics_producer.py +0 -84
  1063. metadata/data_insight/source/metadata.py +0 -175
  1064. metadata/workflow/data_insight.py +0 -78
  1065. openmetadata_ingestion-1.4.8.0.dist-info/RECORD +0 -1509
  1066. {metadata/cli → _openmetadata_testutils}/__init__.py +0 -0
  1067. {metadata/data_insight/processor → _openmetadata_testutils/helpers}/__init__.py +0 -0
  1068. {metadata/data_insight/processor/kpi → _openmetadata_testutils/kafka}/__init__.py +0 -0
  1069. {metadata/data_insight/processor/reports → _openmetadata_testutils/postgres}/__init__.py +0 -0
  1070. {metadata/data_insight/producer → _openmetadata_testutils/pydantic}/__init__.py +0 -0
  1071. /metadata/{data_insight/source → ingestion/source/database/datalake/clients}/__init__.py +0 -0
  1072. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/LICENSE +0 -0
  1073. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/WHEEL +0 -0
  1074. {openmetadata_ingestion-1.4.8.0.dist-info → openmetadata_ingestion-1.5.0.0rc2.dist-info}/entry_points.txt +0 -0
@@ -14,26 +14,23 @@ Interfaces with database for all database engine
14
14
  supporting sqlalchemy abstraction layer
15
15
  """
16
16
 
17
- from datetime import datetime, timezone
18
- from typing import Optional, Union
17
+ from typing import Union
19
18
 
20
19
  from sqlalchemy.orm import DeclarativeMeta
21
20
  from sqlalchemy.orm.util import AliasedClass
22
21
 
22
+ from metadata.data_quality.builders.i_validator_builder import IValidatorBuilder
23
+ from metadata.data_quality.builders.sqa_validator_builder import SQAValidatorBuilder
23
24
  from metadata.data_quality.interface.test_suite_interface import TestSuiteInterface
24
- from metadata.data_quality.validations.validator import Validator
25
25
  from metadata.generated.schema.entity.data.table import Table
26
26
  from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
27
- from metadata.generated.schema.tests.basic import TestCaseResult
28
27
  from metadata.generated.schema.tests.testCase import TestCase
29
- from metadata.generated.schema.tests.testDefinition import TestDefinition
30
28
  from metadata.ingestion.connections.session import create_and_bind_session
31
29
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
32
30
  from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
33
31
  from metadata.profiler.processor.runner import QueryRunner
34
32
  from metadata.profiler.processor.sampler.sqlalchemy.sampler import SQASampler
35
33
  from metadata.utils.constants import TEN_MIN
36
- from metadata.utils.importer import import_test_case_class
37
34
  from metadata.utils.logger import test_suite_logger
38
35
  from metadata.utils.ssl_manager import get_ssl_connection
39
36
  from metadata.utils.timeout import cls_timeout
@@ -144,37 +141,7 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteInterface):
144
141
  )
145
142
  )
146
143
 
147
- def run_test_case(
148
- self,
149
- test_case: TestCase,
150
- ) -> Optional[TestCaseResult]:
151
- """Run table tests where platformsTest=OpenMetadata
152
-
153
- Args:
154
- test_case: test case object to execute
155
-
156
- Returns:
157
- TestCaseResult object
158
- """
159
-
160
- try:
161
- TestHandler = import_test_case_class( # pylint: disable=invalid-name
162
- self.ometa_client.get_by_id(
163
- TestDefinition, test_case.testDefinition.id
164
- ).entityType.value,
165
- "sqlalchemy",
166
- test_case.testDefinition.fullyQualifiedName,
167
- )
168
-
169
- test_handler = TestHandler(
170
- self.runner,
171
- test_case=test_case,
172
- execution_date=int(datetime.now(tz=timezone.utc).timestamp() * 1000),
173
- )
174
-
175
- return Validator(validator_obj=test_handler).validate()
176
- except Exception as err:
177
- logger.error(
178
- f"Error executing {test_case.testDefinition.fullyQualifiedName} - {err}"
179
- )
180
- raise RuntimeError(err)
144
+ def _get_validator_builder(
145
+ self, test_case: TestCase, entity_type: str
146
+ ) -> IValidatorBuilder:
147
+ return SQAValidatorBuilder(self.runner, test_case, entity_type)
@@ -15,20 +15,34 @@ supporting sqlalchemy abstraction layer
15
15
  """
16
16
 
17
17
  from abc import ABC, abstractmethod
18
- from typing import Optional
18
+ from typing import Optional, Type
19
19
 
20
+ from metadata.data_quality.builders.i_validator_builder import IValidatorBuilder
21
+ from metadata.data_quality.validations.base_test_handler import BaseTestValidator
22
+ from metadata.data_quality.validations.runtime_param_setter.param_setter import (
23
+ RuntimeParameterSetter,
24
+ )
25
+ from metadata.data_quality.validations.runtime_param_setter.param_setter_factory import (
26
+ RuntimeParameterSetterFactory,
27
+ )
20
28
  from metadata.generated.schema.entity.data.table import Table
21
29
  from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
22
30
  from metadata.generated.schema.tests.basic import TestCaseResult
23
31
  from metadata.generated.schema.tests.testCase import TestCase
32
+ from metadata.generated.schema.tests.testDefinition import TestDefinition
24
33
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
25
34
  from metadata.profiler.api.models import ProfileSampleConfig
35
+ from metadata.utils.logger import test_suite_logger
26
36
  from metadata.utils.partition import get_partition_details
27
37
 
38
+ logger = test_suite_logger()
39
+
28
40
 
29
41
  class TestSuiteInterface(ABC):
30
42
  """Abstract interface for the processor"""
31
43
 
44
+ runtime_params_setter_fact = RuntimeParameterSetterFactory
45
+
32
46
  @abstractmethod
33
47
  def __init__(
34
48
  self,
@@ -41,10 +55,78 @@ class TestSuiteInterface(ABC):
41
55
  self.service_connection_config = service_connection_config
42
56
  self.table_entity = table_entity
43
57
 
58
+ @property
59
+ def sampler(self):
60
+ """Get the sampler object
61
+
62
+ Note: Overriden in the implementation class. This should be removed from the interface. It has been
63
+ implemented as the RuntimeParameterSetter takes the sampler as an argument, though we may want to
64
+ remove that dependency.
65
+ """
66
+ return None
67
+
44
68
  @abstractmethod
69
+ def _get_validator_builder(
70
+ self, test_case: TestCase, entity_type: str
71
+ ) -> IValidatorBuilder:
72
+ """get the builder class for the validator. Define this in the implementation class
73
+
74
+ Args:
75
+ test_case (TestCase): test case object
76
+ entity_type (str): type of the entity
77
+
78
+ Returns:
79
+ IValidatorBuilder: a validator builder
80
+ """
81
+ raise NotImplementedError
82
+
83
+ @classmethod
84
+ def _get_runtime_params_setter_fact(cls) -> RuntimeParameterSetterFactory:
85
+ """Get the runtime parameter setter factory."""
86
+ return cls.runtime_params_setter_fact()
87
+
88
+ @classmethod
89
+ def _set_runtime_params_setter_fact(
90
+ cls, class_fact: Type[RuntimeParameterSetterFactory]
91
+ ):
92
+ """Set the runtime parameter setter factory.
93
+ Use this method to set the runtime parameter setter factory and override the default.
94
+
95
+ Args:
96
+ class_fact (Type[RuntimeParameterSetterFactory]): the runtime parameter setter factory class
97
+ """
98
+ cls.runtime_params_setter_fact = class_fact
99
+
45
100
  def run_test_case(self, test_case: TestCase) -> Optional[TestCaseResult]:
46
101
  """run column data quality tests"""
47
- raise NotImplementedError
102
+ runtime_params_setter_fact: RuntimeParameterSetterFactory = (
103
+ self._get_runtime_params_setter_fact()
104
+ ) # type: ignore
105
+ runtime_params_setter: Optional[
106
+ RuntimeParameterSetter
107
+ ] = runtime_params_setter_fact.get_runtime_param_setter(
108
+ test_case.testDefinition.fullyQualifiedName, # type: ignore
109
+ self.ometa_client,
110
+ self.service_connection_config,
111
+ self.table_entity,
112
+ self.sampler,
113
+ )
114
+
115
+ # get `column` or `table` type for validator import
116
+ entity_type: str = self.ometa_client.get_by_id(
117
+ TestDefinition, test_case.testDefinition.id
118
+ ).entityType.value
119
+
120
+ validator_builder = self._get_validator_builder(test_case, entity_type)
121
+ validator_builder.set_runtime_params(runtime_params_setter)
122
+ validator: BaseTestValidator = validator_builder.validator
123
+ try:
124
+ return validator.run_validation()
125
+ except Exception as err:
126
+ logger.error(
127
+ f"Error executing {test_case.testDefinition.fullyQualifiedName} - {err}"
128
+ )
129
+ raise RuntimeError(err)
48
130
 
49
131
  def _get_sample_query(self) -> Optional[str]:
50
132
  """Get the sampling query for the data quality tests
@@ -14,7 +14,7 @@ This Processor is in charge of executing the test cases
14
14
  """
15
15
  import traceback
16
16
  from copy import deepcopy
17
- from typing import List, Optional, cast
17
+ from typing import List, Optional
18
18
 
19
19
  from metadata.data_quality.api.models import (
20
20
  TableAndTests,
@@ -36,7 +36,11 @@ from metadata.generated.schema.metadataIngestion.workflow import (
36
36
  OpenMetadataWorkflowConfig,
37
37
  )
38
38
  from metadata.generated.schema.tests.testCase import TestCase
39
- from metadata.generated.schema.tests.testDefinition import TestDefinition, TestPlatform
39
+ from metadata.generated.schema.tests.testDefinition import (
40
+ EntityType,
41
+ TestDefinition,
42
+ TestPlatform,
43
+ )
40
44
  from metadata.generated.schema.tests.testSuite import TestSuite
41
45
  from metadata.generated.schema.type.basic import EntityLink, FullyQualifiedEntityName
42
46
  from metadata.ingestion.api.models import Either
@@ -60,8 +64,8 @@ class TestCaseRunner(Processor):
60
64
  self.metadata = metadata
61
65
 
62
66
  self.processor_config: TestSuiteProcessorConfig = (
63
- TestSuiteProcessorConfig.parse_obj(
64
- self.config.processor.dict().get("config")
67
+ TestSuiteProcessorConfig.model_validate(
68
+ self.config.processor.model_dump().get("config")
65
69
  )
66
70
  )
67
71
 
@@ -82,20 +86,14 @@ class TestCaseRunner(Processor):
82
86
  test_suite_fqn=fqn.build(
83
87
  None,
84
88
  TestSuite,
85
- table_fqn=record.table.fullyQualifiedName.__root__,
89
+ table_fqn=record.table.fullyQualifiedName.root,
86
90
  ),
87
- table_fqn=record.table.fullyQualifiedName.__root__,
91
+ table_fqn=record.table.fullyQualifiedName.root,
88
92
  )
89
-
90
- if not test_cases:
91
- return Either(
92
- left=StackTraceError(
93
- name="No test Cases",
94
- error=f"No tests cases found for table {record.table.fullyQualifiedName.__root__}",
95
- )
96
- )
97
-
98
93
  openmetadata_test_cases = self.filter_for_om_test_cases(test_cases)
94
+ openmetadata_test_cases = self.filter_incompatible_test_cases(
95
+ record.table, openmetadata_test_cases
96
+ )
99
97
 
100
98
  test_suite_runner = test_suite_source_factory.create(
101
99
  record.service_type.lower(),
@@ -104,6 +102,12 @@ class TestCaseRunner(Processor):
104
102
  record.table,
105
103
  ).get_data_quality_runner()
106
104
 
105
+ logger.debug(
106
+ f"Found {len(openmetadata_test_cases)} test cases for table {record.table.fullyQualifiedName.root}"
107
+ )
108
+ if len(openmetadata_test_cases) == 0:
109
+ logger.warning("No test cases found for the table")
110
+
107
111
  test_results = [
108
112
  test_case_result
109
113
  for test_case in openmetadata_test_cases
@@ -113,17 +117,14 @@ class TestCaseRunner(Processor):
113
117
  return Either(right=TestCaseResults(test_results=test_results))
114
118
 
115
119
  def get_test_cases(
116
- self, test_cases: Optional[List[TestCase]], test_suite_fqn: str, table_fqn: str
120
+ self, test_cases: List[TestCase], test_suite_fqn: str, table_fqn: str
117
121
  ) -> List[TestCase]:
118
122
  """
119
123
  Based on the test suite test cases that we already know, pick up
120
124
  the rest from the YAML config, compare and create the new ones
121
125
  """
122
126
  if self.processor_config.testCases is not None:
123
- cli_test_cases = self.get_test_case_from_cli_config() # type: ignore
124
- cli_test_cases = cast(
125
- List[TestCaseDefinition], cli_test_cases
126
- ) # satisfy type checker
127
+ cli_test_cases = self.get_test_case_from_cli_config()
127
128
  return self.compare_and_create_test_cases(
128
129
  cli_test_cases_definitions=cli_test_cases,
129
130
  test_cases=test_cases,
@@ -135,15 +136,13 @@ class TestCaseRunner(Processor):
135
136
 
136
137
  def get_test_case_from_cli_config(
137
138
  self,
138
- ) -> Optional[List[TestCaseDefinition]]:
139
+ ) -> List[TestCaseDefinition]:
139
140
  """Get all the test cases names defined in the CLI config file"""
140
- if self.processor_config.testCases is not None:
141
- return list(self.processor_config.testCases)
142
- return None
141
+ return list(self.processor_config.testCases or [])
143
142
 
144
143
  def compare_and_create_test_cases(
145
144
  self,
146
- cli_test_cases_definitions: Optional[List[TestCaseDefinition]],
145
+ cli_test_cases_definitions: List[TestCaseDefinition],
147
146
  test_cases: List[TestCase],
148
147
  table_fqn: str,
149
148
  test_suite_fqn: str,
@@ -162,9 +161,7 @@ class TestCaseRunner(Processor):
162
161
  return test_cases
163
162
  test_cases = deepcopy(test_cases) or []
164
163
  test_case_names = (
165
- {test_case.name.__root__ for test_case in test_cases}
166
- if test_cases
167
- else set()
164
+ {test_case.name.root for test_case in test_cases} if test_cases else set()
168
165
  )
169
166
 
170
167
  # we'll check the test cases defined in the CLI config file and not present in the platform
@@ -196,20 +193,22 @@ class TestCaseRunner(Processor):
196
193
  description=test_case_to_create.description,
197
194
  displayName=test_case_to_create.displayName,
198
195
  testDefinition=FullyQualifiedEntityName(
199
- __root__=test_case_to_create.testDefinitionName
196
+ test_case_to_create.testDefinitionName
200
197
  ),
201
198
  entityLink=EntityLink(
202
- __root__=entity_link.get_entity_link(
199
+ entity_link.get_entity_link(
203
200
  Table,
204
201
  fqn=table_fqn,
205
202
  column_name=test_case_to_create.columnName,
206
203
  )
207
204
  ),
208
205
  testSuite=test_suite_fqn,
209
- parameterValues=list(test_case_to_create.parameterValues)
210
- if test_case_to_create.parameterValues
211
- else None,
212
- owner=None,
206
+ parameterValues=(
207
+ list(test_case_to_create.parameterValues)
208
+ if test_case_to_create.parameterValues
209
+ else None
210
+ ),
211
+ owners=None,
213
212
  computePassedFailedRowCount=test_case_to_create.computePassedFailedRowCount,
214
213
  )
215
214
  )
@@ -245,11 +244,11 @@ class TestCaseRunner(Processor):
245
244
  test_case_to_update.name for test_case_to_update in test_cases_to_update
246
245
  }
247
246
  for indx, test_case in enumerate(deepcopy(test_cases)):
248
- if test_case.name.__root__ in test_cases_to_update_names:
247
+ if test_case.name.root in test_cases_to_update_names:
249
248
  test_case_definition = next(
250
249
  test_case_to_update
251
250
  for test_case_to_update in test_cases_to_update
252
- if test_case_to_update.name == test_case.name.__root__
251
+ if test_case_to_update.name == test_case.name.root
253
252
  )
254
253
  updated_test_case = self.metadata.patch_test_case_definition(
255
254
  test_case=test_case,
@@ -281,7 +280,7 @@ class TestCaseRunner(Processor):
281
280
  )
282
281
  if TestPlatform.OpenMetadata not in test_definition.testPlatforms:
283
282
  logger.debug(
284
- f"Test case {test_case.name.__root__} is not an OpenMetadata test case."
283
+ f"Test case {test_case.name.root} is not an OpenMetadata test case."
285
284
  )
286
285
  continue
287
286
  om_test_cases.append(test_case)
@@ -294,15 +293,15 @@ class TestCaseRunner(Processor):
294
293
  """Execute the test case and return the result, if any"""
295
294
  try:
296
295
  test_result = test_suite_runner.run_and_handle(test_case)
297
- self.status.scanned(test_case.fullyQualifiedName.__root__)
296
+ self.status.scanned(test_case.fullyQualifiedName.root)
298
297
  return test_result
299
298
  except Exception as exc:
300
- error = f"Could not run test case {test_case.name.__root__}: {exc}"
299
+ error = f"Could not run test case {test_case.name.root}: {exc}"
301
300
  logger.debug(traceback.format_exc())
302
301
  logger.error(error)
303
302
  self.status.failed(
304
303
  StackTraceError(
305
- name=test_case.name.__root__,
304
+ name=test_case.name.root,
306
305
  error=error,
307
306
  stackTrace=traceback.format_exc(),
308
307
  )
@@ -321,3 +320,40 @@ class TestCaseRunner(Processor):
321
320
 
322
321
  def close(self) -> None:
323
322
  """Nothing to close"""
323
+
324
+ def filter_incompatible_test_cases(
325
+ self, table: Table, test_cases: List[TestCase]
326
+ ) -> List[TestCase]:
327
+ """Filter out test cases that are defined for incompatible columns. An example of this is a
328
+ test case that checks for a column value to be between two values, but the column is of type
329
+ VARCHAR and not a numeric type. Incompatible test cases will be logged as failures.
330
+
331
+ Args:
332
+ table: Table entity the test cases are run against
333
+ test_cases: List of test cases
334
+
335
+ Returns:
336
+ List of test cases that are compatible with the table columns
337
+ """
338
+ result: List[TestCase] = []
339
+ for tc in test_cases:
340
+ test_definition: TestDefinition = self.metadata.get_by_id(
341
+ TestDefinition, tc.testDefinition.id, nullable=False
342
+ )
343
+ if test_definition.entityType != EntityType.COLUMN:
344
+ result.append(tc)
345
+ continue
346
+ column_name = entity_link.get_decoded_column(tc.entityLink.root)
347
+ column = next(c for c in table.columns if c.name.root == column_name)
348
+
349
+ if column.dataType not in test_definition.supportedDataTypes:
350
+ self.status.failed(
351
+ StackTraceError(
352
+ name="Incompatible Column for Test Case",
353
+ error=f"Test case {tc.name.root} of type {test_definition.name.root}"
354
+ f" is not compatible with column {column.name.root} of type {column.dataType.value}",
355
+ )
356
+ )
357
+ else:
358
+ result.append(tc)
359
+ return result
@@ -71,7 +71,7 @@ class BaseTestSuiteRunner:
71
71
  DatabaseService.__config__
72
72
  """
73
73
  config_copy = deepcopy(
74
- config.source.serviceConnection.__root__.config # type: ignore
74
+ config.source.serviceConnection.root.config # type: ignore
75
75
  )
76
76
  if hasattr(
77
77
  config_copy, # type: ignore
@@ -31,8 +31,8 @@ class DataTestsRunner:
31
31
  def run_and_handle(self, test_case: TestCase):
32
32
  """run and handle test case validation"""
33
33
  logger.info(
34
- f"Executing test case {test_case.name.__root__} "
35
- f"for entity {self.test_runner_interface.table_entity.fullyQualifiedName.__root__}"
34
+ f"Executing test case {test_case.name.root} "
35
+ f"for entity {self.test_runner_interface.table_entity.fullyQualifiedName.root}"
36
36
  )
37
37
  test_result = self.test_runner_interface.run_test_case(
38
38
  test_case,
@@ -72,7 +72,7 @@ class TestSuiteSource(Source):
72
72
  """
73
73
  table: Table = self.metadata.get_by_name(
74
74
  entity=Table,
75
- fqn=self.source_config.entityFullyQualifiedName.__root__,
75
+ fqn=self.source_config.entityFullyQualifiedName.root,
76
76
  fields=["tableProfilerConfig", "testSuite"],
77
77
  )
78
78
 
@@ -80,19 +80,21 @@ class TestSuiteSource(Source):
80
80
 
81
81
  def _get_test_cases_from_test_suite(
82
82
  self, test_suite: Optional[TestSuite]
83
- ) -> Optional[List[TestCase]]:
83
+ ) -> List[TestCase]:
84
84
  """Return test cases if the test suite exists and has them"""
85
85
  if test_suite:
86
86
  test_cases = self.metadata.list_all_entities(
87
87
  entity=TestCase,
88
88
  fields=["testSuite", "entityLink", "testDefinition"],
89
- params={"testSuiteId": test_suite.id.__root__},
89
+ params={"testSuiteId": test_suite.id.root},
90
90
  )
91
91
  test_cases = cast(List[TestCase], test_cases) # satisfy type checker
92
-
92
+ if self.source_config.testCases is not None:
93
+ test_cases = [
94
+ t for t in test_cases if t.name in self.source_config.testCases
95
+ ]
93
96
  return test_cases
94
-
95
- return None
97
+ return []
96
98
 
97
99
  def prepare(self):
98
100
  """Nothing to prepare"""
@@ -110,7 +112,7 @@ class TestSuiteSource(Source):
110
112
  yield Either(
111
113
  left=StackTraceError(
112
114
  name="Missing Table",
113
- error=f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.__root__}."
115
+ error=f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.root}."
114
116
  " Make sure the table exists in OpenMetadata and/or the JWT Token provided is valid.",
115
117
  )
116
118
  )
@@ -125,31 +127,31 @@ class TestSuiteSource(Source):
125
127
  name=fqn.build(
126
128
  None,
127
129
  TestSuite,
128
- table_fqn=self.source_config.entityFullyQualifiedName.__root__,
130
+ table_fqn=self.source_config.entityFullyQualifiedName.root,
129
131
  ),
130
- displayName=f"{self.source_config.entityFullyQualifiedName.__root__} Test Suite",
132
+ displayName=f"{self.source_config.entityFullyQualifiedName.root} Test Suite",
131
133
  description="Test Suite created from YAML processor config file",
132
- owner=None,
133
- executableEntityReference=self.source_config.entityFullyQualifiedName.__root__,
134
+ owners=None,
135
+ executableEntityReference=self.source_config.entityFullyQualifiedName.root,
134
136
  )
135
137
  yield Either(
136
138
  right=TableAndTests(
137
139
  executable_test_suite=executable_test_suite,
138
- service_type=self.config.source.serviceConnection.__root__.config.type.value,
140
+ service_type=self.config.source.serviceConnection.root.config.type.value,
139
141
  )
140
142
  )
141
143
 
142
144
  test_suite: Optional[TestSuite] = None
143
145
  if table.testSuite:
144
146
  test_suite = self.metadata.get_by_id(
145
- entity=TestSuite, entity_id=table.testSuite.id.__root__
147
+ entity=TestSuite, entity_id=table.testSuite.id.root
146
148
  )
147
149
 
148
150
  if test_suite and not test_suite.executable:
149
151
  yield Either(
150
152
  left=StackTraceError(
151
153
  name="Non-executable Test Suite",
152
- error=f"The table {self.source_config.entityFullyQualifiedName.__root__} "
154
+ error=f"The table {self.source_config.entityFullyQualifiedName.root} "
153
155
  "has a test suite that is not executable.",
154
156
  )
155
157
  )
@@ -161,7 +163,7 @@ class TestSuiteSource(Source):
161
163
  right=TableAndTests(
162
164
  table=table,
163
165
  test_cases=test_suite_cases,
164
- service_type=self.config.source.serviceConnection.__root__.config.type.value,
166
+ service_type=self.config.source.serviceConnection.root.config.type.value,
165
167
  )
166
168
  )
167
169
 
@@ -18,8 +18,11 @@ from __future__ import annotations
18
18
  import reprlib
19
19
  from abc import ABC, abstractmethod
20
20
  from datetime import datetime
21
- from typing import Callable, List, Optional, TypeVar, Union
21
+ from typing import TYPE_CHECKING, Callable, List, Optional, Type, TypeVar, Union
22
22
 
23
+ from metadata.data_quality.validations.runtime_param_setter.param_setter import (
24
+ RuntimeParameterSetter,
25
+ )
23
26
  from metadata.generated.schema.tests.basic import (
24
27
  TestCaseResult,
25
28
  TestCaseStatus,
@@ -28,16 +31,24 @@ from metadata.generated.schema.tests.basic import (
28
31
  from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
29
32
  from metadata.profiler.processor.runner import QueryRunner
30
33
 
34
+ if TYPE_CHECKING:
35
+ from pandas import DataFrame
36
+
31
37
  T = TypeVar("T", bound=Callable)
32
38
  R = TypeVar("R")
33
39
 
34
40
 
35
41
  class BaseTestValidator(ABC):
36
- """Abstract class for test case handlers"""
42
+ """Abstract class for test case handlers
43
+ The runtime_parameter_setter is run after the test case is created to set the runtime parameters.
44
+ This can be useful to resolve complex test parameters based on the parameters gibven by the user.
45
+ """
46
+
47
+ runtime_parameter_setter: Optional[Type[RuntimeParameterSetter]] = None
37
48
 
38
49
  def __init__(
39
50
  self,
40
- runner: QueryRunner,
51
+ runner: Union[QueryRunner, List["DataFrame"]],
41
52
  test_case: TestCase,
42
53
  execution_date: Union[datetime, float],
43
54
  ) -> None:
@@ -94,6 +105,8 @@ class BaseTestValidator(ABC):
94
105
  row_count: Optional[int] = None,
95
106
  failed_rows: Optional[int] = None,
96
107
  passed_rows: Optional[int] = None,
108
+ min_bound: Optional[float] = None,
109
+ max_bound: Optional[float] = None,
97
110
  ) -> TestCaseResult:
98
111
  """Returns a TestCaseResult object with the given args
99
112
 
@@ -111,6 +124,9 @@ class BaseTestValidator(ABC):
111
124
  result=result,
112
125
  testResultValue=test_result_value,
113
126
  sampleData=None,
127
+ # if users don't set the min/max bound, we'll change the inf/-inf (used for computation) to None
128
+ minBound=None if min_bound == float("-inf") else min_bound,
129
+ maxBound=None if max_bound == float("inf") else max_bound,
114
130
  )
115
131
 
116
132
  if (row_count is not None and row_count != 0) and (
@@ -149,7 +165,7 @@ class BaseTestValidator(ABC):
149
165
  """
150
166
  return TestCaseStatus.Success if condition else TestCaseStatus.Failed
151
167
 
152
- def get_min_bound(self, param_name: str):
168
+ def get_min_bound(self, param_name: str) -> Optional[float]:
153
169
  """get min value for max value in column test case"""
154
170
  return self.get_test_case_param_value(
155
171
  self.test_case.parameterValues, # type: ignore
@@ -158,7 +174,7 @@ class BaseTestValidator(ABC):
158
174
  default=float("-inf"),
159
175
  )
160
176
 
161
- def get_max_bound(self, param_name: str):
177
+ def get_max_bound(self, param_name: str) -> Optional[float]:
162
178
  """get max value for max value in column test case"""
163
179
  return self.get_test_case_param_value(
164
180
  self.test_case.parameterValues, # type: ignore
@@ -166,3 +182,7 @@ class BaseTestValidator(ABC):
166
182
  float,
167
183
  default=float("inf"),
168
184
  )
185
+
186
+ def get_predicted_value(self) -> Optional[str]:
187
+ """Get predicted value"""
188
+ return None
@@ -83,6 +83,8 @@ class BaseColumnValueLengthsToBeBetweenValidator(BaseTestValidator):
83
83
  ],
84
84
  row_count=row_count,
85
85
  failed_rows=failed_rows,
86
+ min_bound=min_bound,
87
+ max_bound=max_bound,
86
88
  )
87
89
 
88
90
  @abstractmethod
@@ -65,6 +65,8 @@ class BaseColumnValueMaxToBeBetweenValidator(BaseTestValidator):
65
65
  self.get_test_case_status(min_bound <= res <= max_bound),
66
66
  f"Found max={res} vs. the expected min={min_bound}, max={max_bound}.",
67
67
  [TestResultValue(name=MAX, value=str(res))],
68
+ min_bound=min_bound,
69
+ max_bound=max_bound,
68
70
  )
69
71
 
70
72
  @abstractmethod
@@ -65,6 +65,8 @@ class BaseColumnValueMeanToBeBetweenValidator(BaseTestValidator):
65
65
  self.get_test_case_status(min_bound <= res <= max_bound),
66
66
  f"Found mean={res} vs. the expected min={min_bound}, max={max_bound}.",
67
67
  [TestResultValue(name=MEAN, value=str(res))],
68
+ min_bound=min_bound,
69
+ max_bound=max_bound,
68
70
  )
69
71
 
70
72
  @abstractmethod
@@ -65,6 +65,8 @@ class BaseColumnValueMedianToBeBetweenValidator(BaseTestValidator):
65
65
  self.get_test_case_status(min_bound <= res <= max_bound),
66
66
  f"Found median={res} vs. the expected min={min_bound}, max={max_bound}.",
67
67
  [TestResultValue(name=MEDIAN, value=str(res))],
68
+ min_bound=min_bound,
69
+ max_bound=max_bound,
68
70
  )
69
71
 
70
72
  @abstractmethod
@@ -65,6 +65,8 @@ class BaseColumnValueMinToBeBetweenValidator(BaseTestValidator):
65
65
  self.get_test_case_status(min_bound <= res <= max_bound),
66
66
  f"Found min={res} vs. the expected min={min_bound}, max={max_bound}.",
67
67
  [TestResultValue(name=MIN, value=str(res))],
68
+ min_bound=min_bound,
69
+ max_bound=max_bound,
68
70
  )
69
71
 
70
72
  @abstractmethod