openmetadata-ingestion 1.6.8.0__py3-none-any.whl → 1.7.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1550) hide show
  1. airflow_provider_openmetadata/__init__.py +3 -3
  2. airflow_provider_openmetadata/hooks/openmetadata.py +3 -3
  3. airflow_provider_openmetadata/lineage/backend.py +3 -3
  4. airflow_provider_openmetadata/lineage/callback.py +3 -3
  5. airflow_provider_openmetadata/lineage/config/commons.py +3 -3
  6. airflow_provider_openmetadata/lineage/config/loader.py +3 -3
  7. airflow_provider_openmetadata/lineage/operator.py +3 -3
  8. airflow_provider_openmetadata/lineage/runner.py +3 -3
  9. airflow_provider_openmetadata/lineage/status.py +3 -3
  10. metadata/__main__.py +3 -3
  11. metadata/__version__.py +3 -3
  12. metadata/antlr/split_listener.py +3 -3
  13. metadata/automations/execute_runner.py +49 -0
  14. metadata/automations/extended_runner.py +13 -0
  15. metadata/automations/runner.py +37 -38
  16. metadata/cli/app.py +3 -3
  17. metadata/cli/classify.py +3 -3
  18. metadata/cli/dataquality.py +3 -3
  19. metadata/cli/ingest.py +3 -3
  20. metadata/cli/lineage.py +3 -3
  21. metadata/cli/profile.py +3 -3
  22. metadata/cli/restore.py +3 -3
  23. metadata/cli/usage.py +3 -3
  24. metadata/clients/aws_client.py +3 -3
  25. metadata/clients/azure_client.py +3 -3
  26. metadata/clients/domo_client.py +3 -3
  27. metadata/cmd.py +3 -3
  28. metadata/config/common.py +3 -3
  29. metadata/data_quality/api/models.py +3 -3
  30. metadata/data_quality/builders/validator_builder.py +3 -3
  31. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +3 -3
  32. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +3 -3
  33. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +3 -3
  34. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +3 -3
  35. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +3 -3
  36. metadata/data_quality/interface/test_suite_interface.py +3 -3
  37. metadata/data_quality/processor/test_case_runner.py +3 -3
  38. metadata/data_quality/runner/base_test_suite_source.py +3 -3
  39. metadata/data_quality/runner/core.py +3 -3
  40. metadata/data_quality/source/test_suite.py +3 -3
  41. metadata/data_quality/validations/base_test_handler.py +3 -3
  42. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +3 -3
  43. metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +3 -3
  44. metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +3 -3
  45. metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +3 -3
  46. metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +3 -3
  47. metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +3 -3
  48. metadata/data_quality/validations/column/base/columnValuesMissingCount.py +3 -3
  49. metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +3 -3
  50. metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py +3 -3
  51. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +3 -3
  52. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +3 -3
  53. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +3 -3
  54. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +3 -3
  55. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +3 -3
  56. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +3 -3
  57. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +3 -3
  58. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +3 -3
  59. metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py +3 -3
  60. metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py +3 -3
  61. metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py +3 -3
  62. metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py +3 -3
  63. metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py +3 -3
  64. metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py +3 -3
  65. metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py +3 -3
  66. metadata/data_quality/validations/column/pandas/columnValuesToBeAtExpectedLocation.py +3 -3
  67. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +3 -3
  68. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +3 -3
  69. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +3 -3
  70. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +3 -3
  71. metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py +3 -3
  72. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +3 -3
  73. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +3 -3
  74. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +3 -3
  75. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +3 -3
  76. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +3 -3
  77. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +3 -3
  78. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +3 -3
  79. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +3 -3
  80. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +3 -3
  81. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +3 -3
  82. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +3 -3
  83. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +3 -3
  84. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +3 -3
  85. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +3 -3
  86. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +3 -3
  87. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +3 -3
  88. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +3 -3
  89. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +3 -3
  90. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +3 -3
  91. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +3 -3
  92. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -2
  93. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +2 -2
  94. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +6 -3
  95. metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py +3 -3
  96. metadata/data_quality/validations/table/base/tableColumnCountToEqual.py +3 -3
  97. metadata/data_quality/validations/table/base/tableColumnNameToExist.py +3 -3
  98. metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +3 -3
  99. metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +14 -4
  100. metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +3 -3
  101. metadata/data_quality/validations/table/base/tableRowCountToEqual.py +3 -3
  102. metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py +3 -3
  103. metadata/data_quality/validations/table/pandas/tableColumnCountToBeBetween.py +3 -3
  104. metadata/data_quality/validations/table/pandas/tableColumnCountToEqual.py +3 -3
  105. metadata/data_quality/validations/table/pandas/tableColumnNameToExist.py +3 -3
  106. metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +3 -3
  107. metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py +3 -3
  108. metadata/data_quality/validations/table/pandas/tableRowCountToBeBetween.py +3 -3
  109. metadata/data_quality/validations/table/pandas/tableRowCountToEqual.py +3 -3
  110. metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +3 -3
  111. metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToBeBetween.py +3 -3
  112. metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToEqual.py +3 -3
  113. metadata/data_quality/validations/table/sqlalchemy/tableColumnNameToExist.py +3 -3
  114. metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +3 -3
  115. metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +3 -3
  116. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +2 -2
  117. metadata/data_quality/validations/table/sqlalchemy/tableRowCountToBeBetween.py +3 -3
  118. metadata/data_quality/validations/table/sqlalchemy/tableRowCountToEqual.py +3 -3
  119. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +3 -3
  120. metadata/examples/workflows/cassandra.yaml +35 -0
  121. metadata/examples/workflows/cockroach.yaml +24 -0
  122. metadata/examples/workflows/nifi.yaml +51 -0
  123. metadata/examples/workflows/opensearch.yaml +20 -0
  124. metadata/examples/workflows/wherescape.yaml +2 -2
  125. metadata/generated/schema/analytics/__init__.py +1 -1
  126. metadata/generated/schema/analytics/basic.py +1 -1
  127. metadata/generated/schema/analytics/reportData.py +1 -1
  128. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  129. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  130. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  131. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  132. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  133. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  134. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  135. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  136. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  137. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  138. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  139. metadata/generated/schema/api/__init__.py +1 -1
  140. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  141. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  142. metadata/generated/schema/api/analytics/__init__.py +1 -1
  143. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  144. metadata/generated/schema/api/automations/__init__.py +1 -1
  145. metadata/generated/schema/api/automations/createWorkflow.py +14 -4
  146. metadata/generated/schema/api/bulkAssets.py +1 -1
  147. metadata/generated/schema/api/classification/__init__.py +1 -1
  148. metadata/generated/schema/api/classification/createClassification.py +2 -2
  149. metadata/generated/schema/api/classification/createTag.py +2 -2
  150. metadata/generated/schema/api/classification/loadTags.py +1 -1
  151. metadata/generated/schema/api/createBot.py +1 -1
  152. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  153. metadata/generated/schema/api/createType.py +1 -1
  154. metadata/generated/schema/api/data/__init__.py +1 -1
  155. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  156. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  157. metadata/generated/schema/api/data/createChart.py +1 -1
  158. metadata/generated/schema/api/data/createContainer.py +1 -1
  159. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  160. metadata/generated/schema/api/data/createDashboard.py +1 -1
  161. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  162. metadata/generated/schema/api/data/createDatabase.py +1 -1
  163. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  164. metadata/generated/schema/api/data/createGlossary.py +1 -1
  165. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  166. metadata/generated/schema/api/data/createMetric.py +1 -1
  167. metadata/generated/schema/api/data/createMlModel.py +1 -1
  168. metadata/generated/schema/api/data/createPipeline.py +5 -1
  169. metadata/generated/schema/api/data/createQuery.py +1 -1
  170. metadata/generated/schema/api/data/createQueryCostRecord.py +35 -0
  171. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  172. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  173. metadata/generated/schema/api/data/createTable.py +1 -1
  174. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  175. metadata/generated/schema/api/data/createTopic.py +1 -1
  176. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  177. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  178. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  179. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  180. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  181. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  182. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  183. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  184. metadata/generated/schema/api/docStore/__init__.py +1 -1
  185. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  186. metadata/generated/schema/api/domains/__init__.py +1 -1
  187. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  188. metadata/generated/schema/api/domains/createDomain.py +8 -1
  189. metadata/generated/schema/api/feed/__init__.py +1 -1
  190. metadata/generated/schema/api/feed/closeTask.py +1 -1
  191. metadata/generated/schema/api/feed/createPost.py +1 -1
  192. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  193. metadata/generated/schema/api/feed/createThread.py +1 -1
  194. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  195. metadata/generated/schema/api/feed/threadCount.py +25 -1
  196. metadata/generated/schema/api/governance/__init__.py +1 -1
  197. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  198. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  199. metadata/generated/schema/api/lineage/__init__.py +1 -1
  200. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  201. metadata/generated/schema/api/lineage/esLineageData.py +85 -0
  202. metadata/generated/schema/api/lineage/lineageDirection.py +12 -0
  203. metadata/generated/schema/api/lineage/nodeInformation.py +19 -0
  204. metadata/generated/schema/api/lineage/searchLineageRequest.py +45 -0
  205. metadata/generated/schema/api/lineage/searchLineageResult.py +46 -0
  206. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  207. metadata/generated/schema/api/policies/__init__.py +1 -1
  208. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  209. metadata/generated/schema/api/search/__init__.py +3 -0
  210. metadata/generated/schema/api/search/previewSearchRequest.py +47 -0
  211. metadata/generated/schema/api/services/__init__.py +1 -1
  212. metadata/generated/schema/api/services/createApiService.py +9 -2
  213. metadata/generated/schema/api/services/createDashboardService.py +9 -2
  214. metadata/generated/schema/api/services/createDatabaseService.py +9 -2
  215. metadata/generated/schema/api/services/createMessagingService.py +9 -2
  216. metadata/generated/schema/api/services/createMetadataService.py +9 -2
  217. metadata/generated/schema/api/services/createMlModelService.py +9 -2
  218. metadata/generated/schema/api/services/createPipelineService.py +9 -2
  219. metadata/generated/schema/api/services/createSearchService.py +9 -2
  220. metadata/generated/schema/api/services/createStorageService.py +9 -2
  221. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  222. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  223. metadata/generated/schema/api/setOwner.py +1 -1
  224. metadata/generated/schema/api/teams/__init__.py +1 -1
  225. metadata/generated/schema/api/teams/createPersona.py +1 -1
  226. metadata/generated/schema/api/teams/createRole.py +1 -1
  227. metadata/generated/schema/api/teams/createTeam.py +1 -1
  228. metadata/generated/schema/api/teams/createUser.py +1 -1
  229. metadata/generated/schema/api/tests/__init__.py +1 -1
  230. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  231. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  232. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  233. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  234. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  235. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  236. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  237. metadata/generated/schema/api/voteRequest.py +1 -1
  238. metadata/generated/schema/auth/__init__.py +1 -1
  239. metadata/generated/schema/auth/basicAuth.py +1 -1
  240. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  241. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  242. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  243. metadata/generated/schema/auth/emailRequest.py +1 -1
  244. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  245. metadata/generated/schema/auth/generateToken.py +1 -1
  246. metadata/generated/schema/auth/jwtAuth.py +1 -1
  247. metadata/generated/schema/auth/loginRequest.py +1 -1
  248. metadata/generated/schema/auth/logoutRequest.py +1 -1
  249. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  250. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  251. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  252. metadata/generated/schema/auth/refreshToken.py +1 -1
  253. metadata/generated/schema/auth/registrationRequest.py +1 -1
  254. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  255. metadata/generated/schema/auth/revokeToken.py +1 -1
  256. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  257. metadata/generated/schema/auth/ssoAuth.py +1 -1
  258. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  259. metadata/generated/schema/configuration/__init__.py +1 -1
  260. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  261. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  262. metadata/generated/schema/configuration/authConfig.py +1 -1
  263. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  264. metadata/generated/schema/configuration/authorizerConfiguration.py +4 -1
  265. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  266. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  267. metadata/generated/schema/configuration/elasticSearchConfiguration.py +55 -1
  268. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  269. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  270. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  271. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  272. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  273. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  274. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  275. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  276. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  277. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  278. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  279. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  280. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  281. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  282. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  283. metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +21 -0
  284. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  285. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  286. metadata/generated/schema/configuration/searchSettings.py +463 -4
  287. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  288. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  289. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  290. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  291. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  292. metadata/generated/schema/configuration/workflowSettings.py +8 -1
  293. metadata/generated/schema/dataInsight/__init__.py +1 -1
  294. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  295. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  296. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  297. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  298. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  299. metadata/generated/schema/dataInsight/custom/lineChart.py +15 -1
  300. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  301. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  302. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  303. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  304. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  305. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  306. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  307. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  308. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  309. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  310. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  311. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  312. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  313. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  314. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  315. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  316. metadata/generated/schema/email/__init__.py +1 -1
  317. metadata/generated/schema/email/emailRequest.py +1 -1
  318. metadata/generated/schema/email/emailTemplate.py +1 -1
  319. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  320. metadata/generated/schema/email/smtpSettings.py +1 -2
  321. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  322. metadata/generated/schema/entity/__init__.py +1 -1
  323. metadata/generated/schema/entity/applications/__init__.py +1 -1
  324. metadata/generated/schema/entity/applications/app.py +14 -1
  325. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  326. metadata/generated/schema/entity/applications/appRunRecord.py +44 -7
  327. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  328. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +9 -3
  329. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  330. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  331. metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
  332. metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +41 -0
  333. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  334. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  335. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  336. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  337. metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +71 -0
  338. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  339. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  340. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  341. metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
  342. metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +33 -0
  343. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  344. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  345. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  346. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  347. metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +45 -0
  348. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  349. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +13 -1
  350. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  351. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  352. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  353. metadata/generated/schema/entity/applications/configuration/internal/autoPilotAppConfig.py +48 -0
  354. metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +46 -0
  355. metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +46 -0
  356. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +109 -2
  357. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  358. metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +1 -1
  359. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  360. metadata/generated/schema/entity/applications/configuration/private/__init__.py +3 -0
  361. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  362. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +3 -22
  363. metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +3 -0
  364. metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +24 -0
  365. metadata/generated/schema/entity/applications/configuration/private/limits.py +33 -0
  366. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  367. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  368. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  369. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  370. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +13 -1
  371. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +13 -1
  372. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  373. metadata/generated/schema/entity/automations/__init__.py +1 -1
  374. metadata/generated/schema/entity/automations/testServiceConnection.py +8 -1
  375. metadata/generated/schema/entity/automations/workflow.py +15 -4
  376. metadata/generated/schema/entity/bot.py +1 -1
  377. metadata/generated/schema/entity/classification/__init__.py +1 -1
  378. metadata/generated/schema/entity/classification/classification.py +1 -1
  379. metadata/generated/schema/entity/classification/tag.py +1 -1
  380. metadata/generated/schema/entity/data/__init__.py +1 -1
  381. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  382. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  383. metadata/generated/schema/entity/data/chart.py +1 -1
  384. metadata/generated/schema/entity/data/container.py +1 -1
  385. metadata/generated/schema/entity/data/dashboard.py +1 -1
  386. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  387. metadata/generated/schema/entity/data/database.py +5 -1
  388. metadata/generated/schema/entity/data/databaseSchema.py +7 -5
  389. metadata/generated/schema/entity/data/glossary.py +1 -1
  390. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  391. metadata/generated/schema/entity/data/metric.py +1 -1
  392. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  393. metadata/generated/schema/entity/data/pipeline.py +14 -1
  394. metadata/generated/schema/entity/data/query.py +1 -1
  395. metadata/generated/schema/entity/data/queryCostRecord.py +47 -0
  396. metadata/generated/schema/entity/data/queryCostSearchResult.py +56 -0
  397. metadata/generated/schema/entity/data/report.py +1 -1
  398. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  399. metadata/generated/schema/entity/data/storedProcedure.py +3 -1
  400. metadata/generated/schema/entity/data/table.py +6 -1
  401. metadata/generated/schema/entity/data/topic.py +1 -1
  402. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  403. metadata/generated/schema/entity/docStore/document.py +1 -1
  404. metadata/generated/schema/entity/domains/__init__.py +1 -1
  405. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  406. metadata/generated/schema/entity/domains/domain.py +8 -1
  407. metadata/generated/schema/entity/events/__init__.py +1 -1
  408. metadata/generated/schema/entity/events/webhook.py +1 -1
  409. metadata/generated/schema/entity/feed/__init__.py +1 -1
  410. metadata/generated/schema/entity/feed/assets.py +1 -1
  411. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  412. metadata/generated/schema/entity/feed/description.py +1 -1
  413. metadata/generated/schema/entity/feed/domain.py +1 -1
  414. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  415. metadata/generated/schema/entity/feed/owner.py +1 -1
  416. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  417. metadata/generated/schema/entity/feed/tag.py +1 -1
  418. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  419. metadata/generated/schema/entity/feed/thread.py +1 -1
  420. metadata/generated/schema/entity/policies/__init__.py +1 -1
  421. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  422. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  423. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  424. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  425. metadata/generated/schema/entity/policies/filters.py +1 -1
  426. metadata/generated/schema/entity/policies/policy.py +1 -1
  427. metadata/generated/schema/entity/services/__init__.py +1 -1
  428. metadata/generated/schema/entity/services/apiService.py +8 -1
  429. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  430. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  431. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  432. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  433. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  434. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  435. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  436. metadata/generated/schema/entity/services/connections/connectionBasicType.py +7 -1
  437. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  438. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +34 -1
  439. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +34 -1
  440. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +34 -1
  441. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +34 -1
  442. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +34 -1
  443. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +34 -1
  444. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +34 -1
  445. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +34 -1
  446. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +34 -1
  447. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  448. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  449. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  450. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  451. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  452. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +50 -2
  453. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +34 -1
  454. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +34 -1
  455. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +34 -1
  456. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +35 -1
  457. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +34 -1
  458. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +34 -1
  459. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  460. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +26 -1
  461. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +26 -1
  462. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +36 -1
  463. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +26 -1
  464. metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +3 -0
  465. metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +61 -0
  466. metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +106 -0
  467. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +26 -1
  468. metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +136 -0
  469. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  470. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  471. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  472. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  473. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  474. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  475. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +26 -1
  476. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +26 -1
  477. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +27 -3
  478. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  479. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  480. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  481. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  482. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +26 -1
  483. metadata/generated/schema/entity/services/connections/database/db2Connection.py +26 -1
  484. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +26 -1
  485. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  486. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  487. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  488. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +26 -1
  489. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +26 -1
  490. metadata/generated/schema/entity/services/connections/database/druidConnection.py +26 -1
  491. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +26 -1
  492. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +26 -1
  493. metadata/generated/schema/entity/services/connections/database/glueConnection.py +26 -1
  494. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +26 -1
  495. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +26 -1
  496. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  497. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  498. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  499. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  500. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  501. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  502. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  503. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +26 -1
  504. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +26 -1
  505. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +26 -1
  506. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +26 -1
  507. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +29 -1
  508. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +29 -1
  509. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +26 -1
  510. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +26 -1
  511. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +26 -1
  512. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +26 -1
  513. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +30 -1
  514. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +26 -1
  515. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +26 -1
  516. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  517. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  518. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  519. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +26 -1
  520. metadata/generated/schema/entity/services/connections/database/sasConnection.py +27 -1
  521. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +26 -1
  522. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +47 -2
  523. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +26 -1
  524. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +26 -1
  525. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +26 -1
  526. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +26 -1
  527. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +26 -1
  528. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +26 -1
  529. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  530. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +10 -1
  531. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +10 -1
  532. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +10 -1
  533. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +10 -1
  534. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +10 -1
  535. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  536. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  537. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +26 -1
  538. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +26 -1
  539. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +26 -1
  540. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +26 -1
  541. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +26 -1
  542. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +26 -2
  543. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  544. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +10 -1
  545. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +10 -1
  546. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +10 -1
  547. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +10 -1
  548. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +10 -1
  549. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  550. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +10 -1
  551. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +10 -1
  552. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -1
  553. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +10 -1
  554. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +10 -1
  555. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +10 -1
  556. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +10 -1
  557. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +11 -1
  558. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +10 -1
  559. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +10 -1
  560. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +10 -1
  561. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +10 -1
  562. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +11 -1
  563. metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
  564. metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
  565. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +10 -1
  566. metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
  567. metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
  568. metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
  569. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +10 -1
  570. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +10 -1
  571. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -1
  572. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +10 -1
  573. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +10 -1
  574. metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py +1 -1
  575. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  576. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +10 -1
  577. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  578. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  579. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  580. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +10 -1
  581. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +37 -29
  582. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  583. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  584. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +10 -1
  585. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +10 -1
  586. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +10 -1
  587. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +10 -1
  588. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  589. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  590. metadata/generated/schema/entity/services/dashboardService.py +8 -1
  591. metadata/generated/schema/entity/services/databaseService.py +14 -1
  592. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  593. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +8 -1
  594. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  595. metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +47 -0
  596. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  597. metadata/generated/schema/entity/services/messagingService.py +8 -1
  598. metadata/generated/schema/entity/services/metadataService.py +8 -1
  599. metadata/generated/schema/entity/services/mlmodelService.py +8 -1
  600. metadata/generated/schema/entity/services/pipelineService.py +8 -1
  601. metadata/generated/schema/entity/services/searchService.py +9 -2
  602. metadata/generated/schema/entity/services/serviceType.py +1 -1
  603. metadata/generated/schema/entity/services/storageService.py +8 -1
  604. metadata/generated/schema/entity/teams/__init__.py +1 -1
  605. metadata/generated/schema/entity/teams/persona.py +5 -1
  606. metadata/generated/schema/entity/teams/role.py +1 -1
  607. metadata/generated/schema/entity/teams/team.py +1 -1
  608. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  609. metadata/generated/schema/entity/teams/user.py +5 -1
  610. metadata/generated/schema/entity/type.py +1 -1
  611. metadata/generated/schema/entity/utils/__init__.py +1 -1
  612. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  613. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  614. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  615. metadata/generated/schema/events/__init__.py +1 -1
  616. metadata/generated/schema/events/alertMetrics.py +1 -1
  617. metadata/generated/schema/events/api/__init__.py +1 -1
  618. metadata/generated/schema/events/api/createEventSubscription.py +9 -1
  619. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  620. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  621. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  622. metadata/generated/schema/events/api/typedEvent.py +1 -1
  623. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  624. metadata/generated/schema/events/eventFilterRule.py +1 -1
  625. metadata/generated/schema/events/eventSubscription.py +9 -2
  626. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  627. metadata/generated/schema/events/failedEvent.py +1 -1
  628. metadata/generated/schema/events/failedEventResponse.py +1 -1
  629. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  630. metadata/generated/schema/events/statusContext.py +1 -1
  631. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  632. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  633. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  634. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  635. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  636. metadata/generated/schema/governance/workflows/elements/edge.py +2 -2
  637. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +4 -1
  638. metadata/generated/schema/governance/workflows/elements/nodeType.py +2 -1
  639. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  640. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  641. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +28 -8
  642. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +86 -0
  643. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +73 -0
  644. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +27 -6
  645. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +28 -7
  646. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  647. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +9 -4
  648. metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +3 -0
  649. metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +30 -0
  650. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  651. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +9 -4
  652. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  653. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +24 -7
  654. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  655. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +8 -4
  656. metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +20 -0
  657. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +18 -5
  658. metadata/generated/schema/governance/workflows/workflowDefinition.py +30 -7
  659. metadata/generated/schema/governance/workflows/workflowInstance.py +11 -8
  660. metadata/generated/schema/governance/workflows/workflowInstanceState.py +4 -8
  661. metadata/generated/schema/jobs/__init__.py +1 -1
  662. metadata/generated/schema/jobs/backgroundJob.py +1 -1
  663. metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
  664. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  665. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  666. metadata/generated/schema/metadataIngestion/application.py +1 -1
  667. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  668. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  669. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  670. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +3 -3
  671. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +4 -4
  672. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +5 -1
  673. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +18 -2
  674. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +9 -1
  675. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  676. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  677. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  678. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  679. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  680. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  681. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  682. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  683. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  684. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  685. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  686. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  687. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  688. metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +71 -0
  689. metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +3 -0
  690. metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +24 -0
  691. metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +28 -0
  692. metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +28 -0
  693. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  694. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  695. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  696. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  697. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  698. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  699. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  700. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  701. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  702. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  703. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  704. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  705. metadata/generated/schema/metadataIngestion/workflow.py +3 -1
  706. metadata/generated/schema/monitoring/__init__.py +1 -1
  707. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  708. metadata/generated/schema/search/__init__.py +3 -0
  709. metadata/generated/schema/search/aggregationRequest.py +80 -0
  710. metadata/generated/schema/search/searchRequest.py +106 -0
  711. metadata/generated/schema/security/__init__.py +1 -1
  712. metadata/generated/schema/security/client/__init__.py +1 -1
  713. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  714. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  715. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  716. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  717. metadata/generated/schema/security/client/oidcClientConfig.py +7 -1
  718. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  719. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  720. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  721. metadata/generated/schema/security/credentials/__init__.py +1 -1
  722. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  723. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  724. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  725. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  726. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  727. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  728. metadata/generated/schema/security/credentials/gcpCredentials.py +42 -22
  729. metadata/generated/schema/security/credentials/gcpExternalAccount.py +2 -2
  730. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  731. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  732. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  733. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  734. metadata/generated/schema/security/sasl/__init__.py +1 -1
  735. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  736. metadata/generated/schema/security/secrets/__init__.py +1 -1
  737. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  738. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  739. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  740. metadata/generated/schema/security/securityConfiguration.py +1 -1
  741. metadata/generated/schema/security/ssl/__init__.py +1 -1
  742. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  743. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  744. metadata/generated/schema/settings/__init__.py +1 -1
  745. metadata/generated/schema/settings/settings.py +4 -1
  746. metadata/generated/schema/system/__init__.py +1 -1
  747. metadata/generated/schema/system/entityError.py +1 -1
  748. metadata/generated/schema/system/eventPublisherJob.py +18 -7
  749. metadata/generated/schema/system/indexingError.py +1 -1
  750. metadata/generated/schema/system/limitsResponse.py +1 -1
  751. metadata/generated/schema/system/ui/__init__.py +1 -1
  752. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  753. metadata/generated/schema/system/ui/navigationItem.py +38 -0
  754. metadata/generated/schema/system/ui/page.py +23 -15
  755. metadata/generated/schema/system/ui/tab.py +36 -0
  756. metadata/generated/schema/system/ui/uiCustomization.py +45 -0
  757. metadata/generated/schema/system/validationResponse.py +1 -1
  758. metadata/generated/schema/tests/__init__.py +1 -1
  759. metadata/generated/schema/tests/assigned.py +1 -1
  760. metadata/generated/schema/tests/basic.py +1 -1
  761. metadata/generated/schema/tests/customMetric.py +1 -1
  762. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  763. metadata/generated/schema/tests/resolved.py +1 -1
  764. metadata/generated/schema/tests/testCase.py +4 -1
  765. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  766. metadata/generated/schema/tests/testDefinition.py +1 -1
  767. metadata/generated/schema/tests/testSuite.py +1 -1
  768. metadata/generated/schema/type/__init__.py +1 -1
  769. metadata/generated/schema/type/apiSchema.py +1 -1
  770. metadata/generated/schema/type/assetCertification.py +1 -1
  771. metadata/generated/schema/type/auditLog.py +1 -1
  772. metadata/generated/schema/type/basic.py +3 -3
  773. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  774. metadata/generated/schema/type/changeEvent.py +1 -1
  775. metadata/generated/schema/type/changeEventType.py +1 -1
  776. metadata/generated/schema/type/changeSummaryMap.py +37 -0
  777. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  778. metadata/generated/schema/type/csvDocumentation.py +1 -1
  779. metadata/generated/schema/type/csvErrorType.py +1 -1
  780. metadata/generated/schema/type/csvFile.py +1 -1
  781. metadata/generated/schema/type/csvImportResult.py +1 -1
  782. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  783. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  784. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  785. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  786. metadata/generated/schema/type/customProperty.py +1 -1
  787. metadata/generated/schema/type/dailyCount.py +1 -1
  788. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  789. metadata/generated/schema/type/entityHierarchy.py +1 -1
  790. metadata/generated/schema/type/entityHistory.py +6 -3
  791. metadata/generated/schema/type/entityLineage.py +27 -1
  792. metadata/generated/schema/type/entityReference.py +1 -1
  793. metadata/generated/schema/type/entityReferenceList.py +1 -1
  794. metadata/generated/schema/type/entityRelationship.py +1 -1
  795. metadata/generated/schema/type/entityUsage.py +1 -1
  796. metadata/generated/schema/type/filterPattern.py +3 -3
  797. metadata/generated/schema/type/function.py +1 -1
  798. metadata/generated/schema/type/include.py +1 -1
  799. metadata/generated/schema/type/jdbcConnection.py +1 -1
  800. metadata/generated/schema/type/lifeCycle.py +1 -1
  801. metadata/generated/schema/type/paging.py +1 -1
  802. metadata/generated/schema/type/profile.py +1 -1
  803. metadata/generated/schema/type/queryParserData.py +4 -1
  804. metadata/generated/schema/type/reaction.py +1 -1
  805. metadata/generated/schema/type/schedule.py +1 -1
  806. metadata/generated/schema/type/schema.py +1 -1
  807. metadata/generated/schema/type/tableQuery.py +4 -1
  808. metadata/generated/schema/type/tableUsageCount.py +18 -1
  809. metadata/generated/schema/type/tagLabel.py +1 -1
  810. metadata/generated/schema/type/usageDetails.py +1 -1
  811. metadata/generated/schema/type/usageRequest.py +1 -1
  812. metadata/generated/schema/type/votes.py +1 -1
  813. metadata/great_expectations/action.py +2 -2
  814. metadata/great_expectations/utils/ometa_config_handler.py +2 -2
  815. metadata/ingestion/api/closeable.py +3 -3
  816. metadata/ingestion/api/common.py +3 -3
  817. metadata/ingestion/api/delete.py +3 -3
  818. metadata/ingestion/api/models.py +3 -3
  819. metadata/ingestion/api/parser.py +3 -3
  820. metadata/ingestion/api/status.py +3 -3
  821. metadata/ingestion/api/step.py +3 -3
  822. metadata/ingestion/api/steps.py +3 -3
  823. metadata/ingestion/api/topology_runner.py +6 -6
  824. metadata/ingestion/bulksink/metadata_usage.py +30 -9
  825. metadata/ingestion/connections/builders.py +3 -3
  826. metadata/ingestion/connections/headers.py +3 -3
  827. metadata/ingestion/connections/secrets.py +3 -3
  828. metadata/ingestion/connections/session.py +3 -3
  829. metadata/ingestion/connections/test_connections.py +3 -3
  830. metadata/ingestion/lineage/masker.py +16 -5
  831. metadata/ingestion/lineage/models.py +3 -3
  832. metadata/ingestion/lineage/parser.py +17 -4
  833. metadata/ingestion/lineage/sql_lineage.py +3 -3
  834. metadata/ingestion/models/custom_basemodel_validation.py +2 -2
  835. metadata/ingestion/models/custom_properties.py +3 -3
  836. metadata/ingestion/models/custom_pydantic.py +46 -8
  837. metadata/ingestion/models/custom_types.py +2 -2
  838. metadata/ingestion/models/data_insight.py +3 -3
  839. metadata/ingestion/models/delete_entity.py +3 -3
  840. metadata/ingestion/models/encoders.py +3 -3
  841. metadata/ingestion/models/entity_interface.py +3 -3
  842. metadata/ingestion/models/lf_tags_model.py +3 -3
  843. metadata/ingestion/models/life_cycle.py +3 -3
  844. metadata/ingestion/models/ometa_classification.py +3 -3
  845. metadata/ingestion/models/ometa_lineage.py +3 -3
  846. metadata/ingestion/models/ometa_topic_data.py +3 -3
  847. metadata/ingestion/models/patch_request.py +3 -3
  848. metadata/ingestion/models/pipeline_status.py +3 -3
  849. metadata/ingestion/models/profile_data.py +3 -3
  850. metadata/ingestion/models/search_index_data.py +3 -3
  851. metadata/ingestion/models/table_metadata.py +3 -3
  852. metadata/ingestion/models/tests_data.py +3 -3
  853. metadata/ingestion/models/topology.py +3 -3
  854. metadata/ingestion/models/user.py +3 -3
  855. metadata/ingestion/ometa/auth_provider.py +3 -3
  856. metadata/ingestion/ometa/client.py +23 -3
  857. metadata/ingestion/ometa/client_utils.py +3 -3
  858. metadata/ingestion/ometa/credentials.py +3 -3
  859. metadata/ingestion/ometa/mixins/custom_property_mixin.py +3 -3
  860. metadata/ingestion/ometa/mixins/dashboard_mixin.py +3 -3
  861. metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -3
  862. metadata/ingestion/ometa/mixins/domain_mixin.py +80 -0
  863. metadata/ingestion/ometa/mixins/es_mixin.py +45 -5
  864. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +3 -3
  865. metadata/ingestion/ometa/mixins/lineage_mixin.py +23 -44
  866. metadata/ingestion/ometa/mixins/mlmodel_mixin.py +3 -3
  867. metadata/ingestion/ometa/mixins/patch_mixin.py +18 -8
  868. metadata/ingestion/ometa/mixins/patch_mixin_utils.py +2 -2
  869. metadata/ingestion/ometa/mixins/pipeline_mixin.py +21 -3
  870. metadata/ingestion/ometa/mixins/query_mixin.py +46 -3
  871. metadata/ingestion/ometa/mixins/role_policy_mixin.py +2 -2
  872. metadata/ingestion/ometa/mixins/search_index_mixin.py +3 -3
  873. metadata/ingestion/ometa/mixins/server_mixin.py +3 -3
  874. metadata/ingestion/ometa/mixins/service_mixin.py +3 -3
  875. metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
  876. metadata/ingestion/ometa/mixins/table_mixin.py +3 -3
  877. metadata/ingestion/ometa/mixins/tests_mixin.py +3 -3
  878. metadata/ingestion/ometa/mixins/topic_mixin.py +3 -3
  879. metadata/ingestion/ometa/mixins/user_mixin.py +3 -3
  880. metadata/ingestion/ometa/mixins/version_mixin.py +3 -3
  881. metadata/ingestion/ometa/models.py +3 -3
  882. metadata/ingestion/ometa/ometa_api.py +31 -6
  883. metadata/ingestion/ometa/routes.py +10 -3
  884. metadata/ingestion/ometa/ttl_cache.py +3 -3
  885. metadata/ingestion/ometa/utils.py +16 -3
  886. metadata/ingestion/processor/query_parser.py +4 -3
  887. metadata/ingestion/sink/file.py +3 -3
  888. metadata/ingestion/sink/metadata_rest.py +24 -4
  889. metadata/ingestion/source/api/api_service.py +2 -2
  890. metadata/ingestion/source/api/rest/connection.py +2 -2
  891. metadata/ingestion/source/api/rest/metadata.py +2 -2
  892. metadata/ingestion/source/api/rest/models.py +2 -2
  893. metadata/ingestion/source/connections.py +3 -3
  894. metadata/ingestion/source/dashboard/dashboard_service.py +12 -4
  895. metadata/ingestion/source/dashboard/domodashboard/connection.py +3 -3
  896. metadata/ingestion/source/dashboard/domodashboard/metadata.py +6 -4
  897. metadata/ingestion/source/dashboard/lightdash/client.py +3 -3
  898. metadata/ingestion/source/dashboard/lightdash/connection.py +3 -3
  899. metadata/ingestion/source/dashboard/lightdash/metadata.py +4 -4
  900. metadata/ingestion/source/dashboard/looker/bulk_parser.py +3 -3
  901. metadata/ingestion/source/dashboard/looker/columns.py +3 -3
  902. metadata/ingestion/source/dashboard/looker/connection.py +3 -3
  903. metadata/ingestion/source/dashboard/looker/links.py +3 -3
  904. metadata/ingestion/source/dashboard/looker/metadata.py +11 -5
  905. metadata/ingestion/source/dashboard/looker/models.py +3 -3
  906. metadata/ingestion/source/dashboard/looker/parser.py +3 -3
  907. metadata/ingestion/source/dashboard/looker/utils.py +3 -3
  908. metadata/ingestion/source/dashboard/metabase/client.py +68 -9
  909. metadata/ingestion/source/dashboard/metabase/connection.py +3 -3
  910. metadata/ingestion/source/dashboard/metabase/metadata.py +91 -40
  911. metadata/ingestion/source/dashboard/metabase/models.py +5 -4
  912. metadata/ingestion/source/dashboard/microstrategy/client.py +2 -2
  913. metadata/ingestion/source/dashboard/microstrategy/connection.py +3 -3
  914. metadata/ingestion/source/dashboard/microstrategy/metadata.py +6 -4
  915. metadata/ingestion/source/dashboard/microstrategy/models.py +3 -3
  916. metadata/ingestion/source/dashboard/mode/client.py +3 -3
  917. metadata/ingestion/source/dashboard/mode/connection.py +3 -3
  918. metadata/ingestion/source/dashboard/mode/metadata.py +19 -12
  919. metadata/ingestion/source/dashboard/powerbi/client.py +3 -3
  920. metadata/ingestion/source/dashboard/powerbi/connection.py +3 -3
  921. metadata/ingestion/source/dashboard/powerbi/file_client.py +3 -3
  922. metadata/ingestion/source/dashboard/powerbi/metadata.py +9 -10
  923. metadata/ingestion/source/dashboard/powerbi/models.py +2 -2
  924. metadata/ingestion/source/dashboard/qlikcloud/client.py +28 -5
  925. metadata/ingestion/source/dashboard/qlikcloud/connection.py +3 -3
  926. metadata/ingestion/source/dashboard/qlikcloud/constants.py +3 -3
  927. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +72 -18
  928. metadata/ingestion/source/dashboard/qlikcloud/models.py +47 -4
  929. metadata/ingestion/source/dashboard/qliksense/client.py +3 -3
  930. metadata/ingestion/source/dashboard/qliksense/connection.py +3 -3
  931. metadata/ingestion/source/dashboard/qliksense/constants.py +3 -3
  932. metadata/ingestion/source/dashboard/qliksense/metadata.py +29 -21
  933. metadata/ingestion/source/dashboard/qliksense/models.py +3 -3
  934. metadata/ingestion/source/dashboard/quicksight/connection.py +3 -3
  935. metadata/ingestion/source/dashboard/quicksight/metadata.py +35 -32
  936. metadata/ingestion/source/dashboard/quicksight/models.py +3 -3
  937. metadata/ingestion/source/dashboard/redash/client.py +3 -3
  938. metadata/ingestion/source/dashboard/redash/connection.py +3 -3
  939. metadata/ingestion/source/dashboard/redash/metadata.py +15 -13
  940. metadata/ingestion/source/dashboard/sigma/client.py +3 -3
  941. metadata/ingestion/source/dashboard/sigma/connection.py +3 -3
  942. metadata/ingestion/source/dashboard/sigma/metadata.py +15 -17
  943. metadata/ingestion/source/dashboard/sigma/models.py +2 -2
  944. metadata/ingestion/source/dashboard/superset/api_source.py +28 -29
  945. metadata/ingestion/source/dashboard/superset/client.py +3 -3
  946. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  947. metadata/ingestion/source/dashboard/superset/db_source.py +25 -23
  948. metadata/ingestion/source/dashboard/superset/metadata.py +3 -3
  949. metadata/ingestion/source/dashboard/superset/mixin.py +210 -56
  950. metadata/ingestion/source/dashboard/superset/models.py +6 -3
  951. metadata/ingestion/source/dashboard/superset/queries.py +10 -7
  952. metadata/ingestion/source/dashboard/superset/utils.py +36 -0
  953. metadata/ingestion/source/dashboard/tableau/__init__.py +3 -3
  954. metadata/ingestion/source/dashboard/tableau/client.py +59 -4
  955. metadata/ingestion/source/dashboard/tableau/connection.py +3 -3
  956. metadata/ingestion/source/dashboard/tableau/metadata.py +243 -128
  957. metadata/ingestion/source/dashboard/tableau/models.py +3 -3
  958. metadata/ingestion/source/dashboard/tableau/queries.py +3 -3
  959. metadata/ingestion/source/database/athena/client.py +3 -3
  960. metadata/ingestion/source/database/athena/connection.py +3 -3
  961. metadata/ingestion/source/database/athena/lineage.py +3 -3
  962. metadata/ingestion/source/database/athena/metadata.py +3 -3
  963. metadata/ingestion/source/database/athena/models.py +3 -3
  964. metadata/ingestion/source/database/athena/query_parser.py +3 -3
  965. metadata/ingestion/source/database/athena/usage.py +3 -3
  966. metadata/ingestion/source/database/athena/utils.py +3 -3
  967. metadata/ingestion/source/database/azuresql/connection.py +3 -3
  968. metadata/ingestion/source/database/azuresql/lineage.py +3 -3
  969. metadata/ingestion/source/database/azuresql/metadata.py +3 -3
  970. metadata/ingestion/source/database/azuresql/queries.py +3 -3
  971. metadata/ingestion/source/database/azuresql/query_parser.py +3 -3
  972. metadata/ingestion/source/database/azuresql/usage.py +3 -3
  973. metadata/ingestion/source/database/bigquery/connection.py +3 -3
  974. metadata/ingestion/source/database/bigquery/helper.py +12 -4
  975. metadata/ingestion/source/database/bigquery/incremental_table_processor.py +3 -3
  976. metadata/ingestion/source/database/bigquery/lineage.py +3 -3
  977. metadata/ingestion/source/database/bigquery/metadata.py +85 -13
  978. metadata/ingestion/source/database/bigquery/models.py +3 -3
  979. metadata/ingestion/source/database/bigquery/queries.py +5 -4
  980. metadata/ingestion/source/database/bigquery/query_parser.py +4 -3
  981. metadata/ingestion/source/database/bigquery/usage.py +3 -3
  982. metadata/ingestion/source/database/bigtable/client.py +2 -2
  983. metadata/ingestion/source/database/bigtable/connection.py +2 -2
  984. metadata/ingestion/source/database/bigtable/metadata.py +13 -5
  985. metadata/ingestion/source/database/bigtable/models.py +2 -2
  986. metadata/ingestion/source/database/cassandra/__init__.py +0 -0
  987. metadata/ingestion/source/database/cassandra/connection.py +138 -0
  988. metadata/ingestion/source/database/cassandra/helpers.py +100 -0
  989. metadata/ingestion/source/database/cassandra/metadata.py +138 -0
  990. metadata/ingestion/source/database/cassandra/queries.py +45 -0
  991. metadata/ingestion/source/database/cassandra/service_spec.py +10 -0
  992. metadata/ingestion/source/database/clickhouse/connection.py +3 -3
  993. metadata/ingestion/source/database/clickhouse/lineage.py +3 -3
  994. metadata/ingestion/source/database/clickhouse/metadata.py +3 -3
  995. metadata/ingestion/source/database/clickhouse/queries.py +3 -3
  996. metadata/ingestion/source/database/clickhouse/query_parser.py +3 -3
  997. metadata/ingestion/source/database/clickhouse/usage.py +3 -3
  998. metadata/ingestion/source/database/clickhouse/utils.py +3 -3
  999. metadata/ingestion/source/database/cockroach/__init__.py +0 -0
  1000. metadata/ingestion/source/database/cockroach/connection.py +71 -0
  1001. metadata/ingestion/source/database/cockroach/metadata.py +206 -0
  1002. metadata/ingestion/source/database/cockroach/queries.py +90 -0
  1003. metadata/ingestion/source/database/cockroach/service_spec.py +4 -0
  1004. metadata/ingestion/source/database/column_helpers.py +3 -3
  1005. metadata/ingestion/source/database/column_type_parser.py +3 -3
  1006. metadata/ingestion/source/database/common_db_source.py +12 -3
  1007. metadata/ingestion/source/database/common_nosql_source.py +81 -32
  1008. metadata/ingestion/source/database/common_pg_mappings.py +49 -0
  1009. metadata/ingestion/source/database/couchbase/connection.py +3 -3
  1010. metadata/ingestion/source/database/couchbase/metadata.py +11 -5
  1011. metadata/ingestion/source/database/couchbase/models.py +2 -2
  1012. metadata/ingestion/source/database/couchbase/queries.py +3 -3
  1013. metadata/ingestion/source/database/database_service.py +3 -3
  1014. metadata/ingestion/source/database/databricks/client.py +3 -3
  1015. metadata/ingestion/source/database/databricks/connection.py +3 -3
  1016. metadata/ingestion/source/database/databricks/lineage.py +3 -3
  1017. metadata/ingestion/source/database/databricks/metadata.py +11 -3
  1018. metadata/ingestion/source/database/databricks/queries.py +3 -3
  1019. metadata/ingestion/source/database/databricks/query_parser.py +3 -3
  1020. metadata/ingestion/source/database/databricks/usage.py +3 -3
  1021. metadata/ingestion/source/database/datalake/clients/azure_blob.py +3 -3
  1022. metadata/ingestion/source/database/datalake/clients/base.py +3 -3
  1023. metadata/ingestion/source/database/datalake/clients/gcs.py +3 -3
  1024. metadata/ingestion/source/database/datalake/clients/s3.py +3 -3
  1025. metadata/ingestion/source/database/datalake/columns.py +3 -3
  1026. metadata/ingestion/source/database/datalake/connection.py +3 -3
  1027. metadata/ingestion/source/database/datalake/metadata.py +18 -12
  1028. metadata/ingestion/source/database/db2/connection.py +3 -3
  1029. metadata/ingestion/source/database/db2/lineage.py +3 -3
  1030. metadata/ingestion/source/database/db2/metadata.py +3 -3
  1031. metadata/ingestion/source/database/db2/service_spec.py +4 -1
  1032. metadata/ingestion/source/database/db2/utils.py +3 -3
  1033. metadata/ingestion/source/database/dbt/constants.py +3 -3
  1034. metadata/ingestion/source/database/dbt/dbt_config.py +4 -3
  1035. metadata/ingestion/source/database/dbt/dbt_service.py +3 -3
  1036. metadata/ingestion/source/database/dbt/dbt_utils.py +4 -4
  1037. metadata/ingestion/source/database/dbt/metadata.py +8 -4
  1038. metadata/ingestion/source/database/dbt/models.py +3 -3
  1039. metadata/ingestion/source/database/deltalake/clients/base.py +3 -3
  1040. metadata/ingestion/source/database/deltalake/clients/pyspark.py +3 -3
  1041. metadata/ingestion/source/database/deltalake/clients/s3.py +3 -3
  1042. metadata/ingestion/source/database/deltalake/connection.py +3 -3
  1043. metadata/ingestion/source/database/deltalake/metadata.py +3 -3
  1044. metadata/ingestion/source/database/domodatabase/connection.py +3 -3
  1045. metadata/ingestion/source/database/domodatabase/metadata.py +3 -3
  1046. metadata/ingestion/source/database/domodatabase/models.py +3 -3
  1047. metadata/ingestion/source/database/doris/connection.py +3 -3
  1048. metadata/ingestion/source/database/doris/metadata.py +10 -5
  1049. metadata/ingestion/source/database/doris/utils.py +3 -3
  1050. metadata/ingestion/source/database/druid/connection.py +3 -3
  1051. metadata/ingestion/source/database/druid/lineage.py +3 -3
  1052. metadata/ingestion/source/database/druid/metadata.py +3 -3
  1053. metadata/ingestion/source/database/druid/service_spec.py +5 -1
  1054. metadata/ingestion/source/database/dynamodb/connection.py +3 -3
  1055. metadata/ingestion/source/database/dynamodb/metadata.py +9 -6
  1056. metadata/ingestion/source/database/dynamodb/models.py +2 -2
  1057. metadata/ingestion/source/database/extended_sample_data.py +3 -3
  1058. metadata/ingestion/source/database/external_table_lineage_mixin.py +3 -3
  1059. metadata/ingestion/source/database/glue/connection.py +3 -3
  1060. metadata/ingestion/source/database/glue/metadata.py +3 -3
  1061. metadata/ingestion/source/database/glue/models.py +3 -3
  1062. metadata/ingestion/source/database/greenplum/connection.py +3 -3
  1063. metadata/ingestion/source/database/greenplum/lineage.py +3 -3
  1064. metadata/ingestion/source/database/greenplum/metadata.py +9 -40
  1065. metadata/ingestion/source/database/greenplum/queries.py +3 -3
  1066. metadata/ingestion/source/database/greenplum/service_spec.py +5 -1
  1067. metadata/ingestion/source/database/greenplum/utils.py +3 -3
  1068. metadata/ingestion/source/database/hive/connection.py +3 -3
  1069. metadata/ingestion/source/database/hive/lineage.py +3 -3
  1070. metadata/ingestion/source/database/hive/metadata.py +3 -3
  1071. metadata/ingestion/source/database/hive/metastore_dialects/mixin.py +3 -3
  1072. metadata/ingestion/source/database/hive/metastore_dialects/mysql/__init__.py +3 -3
  1073. metadata/ingestion/source/database/hive/metastore_dialects/mysql/dialect.py +38 -16
  1074. metadata/ingestion/source/database/hive/metastore_dialects/postgres/__init__.py +3 -3
  1075. metadata/ingestion/source/database/hive/metastore_dialects/postgres/dialect.py +41 -17
  1076. metadata/ingestion/source/database/hive/queries.py +3 -3
  1077. metadata/ingestion/source/database/hive/service_spec.py +5 -1
  1078. metadata/ingestion/source/database/hive/utils.py +3 -3
  1079. metadata/ingestion/source/database/iceberg/catalog/__init__.py +3 -3
  1080. metadata/ingestion/source/database/iceberg/catalog/base.py +3 -3
  1081. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +3 -3
  1082. metadata/ingestion/source/database/iceberg/catalog/glue.py +3 -3
  1083. metadata/ingestion/source/database/iceberg/catalog/hive.py +3 -3
  1084. metadata/ingestion/source/database/iceberg/catalog/rest.py +3 -3
  1085. metadata/ingestion/source/database/iceberg/connection.py +3 -3
  1086. metadata/ingestion/source/database/iceberg/fs/__init__.py +3 -3
  1087. metadata/ingestion/source/database/iceberg/fs/azure.py +3 -3
  1088. metadata/ingestion/source/database/iceberg/fs/base.py +3 -3
  1089. metadata/ingestion/source/database/iceberg/fs/s3.py +3 -3
  1090. metadata/ingestion/source/database/iceberg/helper.py +3 -3
  1091. metadata/ingestion/source/database/iceberg/metadata.py +3 -3
  1092. metadata/ingestion/source/database/iceberg/models.py +3 -3
  1093. metadata/ingestion/source/database/impala/connection.py +3 -3
  1094. metadata/ingestion/source/database/impala/lineage.py +3 -3
  1095. metadata/ingestion/source/database/impala/metadata.py +3 -3
  1096. metadata/ingestion/source/database/impala/queries.py +3 -3
  1097. metadata/ingestion/source/database/impala/service_spec.py +5 -1
  1098. metadata/ingestion/source/database/incremental_metadata_extraction.py +3 -3
  1099. metadata/ingestion/source/database/life_cycle_query_mixin.py +14 -10
  1100. metadata/ingestion/source/database/lineage_source.py +173 -47
  1101. metadata/ingestion/source/database/mariadb/connection.py +3 -3
  1102. metadata/ingestion/source/database/mariadb/lineage.py +3 -3
  1103. metadata/ingestion/source/database/mariadb/metadata.py +3 -3
  1104. metadata/ingestion/source/database/mariadb/service_spec.py +9 -1
  1105. metadata/ingestion/source/database/mongodb/connection.py +3 -3
  1106. metadata/ingestion/source/database/mongodb/metadata.py +11 -6
  1107. metadata/ingestion/source/database/mssql/connection.py +3 -3
  1108. metadata/ingestion/source/database/mssql/constants.py +3 -3
  1109. metadata/ingestion/source/database/mssql/lineage.py +3 -3
  1110. metadata/ingestion/source/database/mssql/metadata.py +71 -5
  1111. metadata/ingestion/source/database/mssql/models.py +3 -3
  1112. metadata/ingestion/source/database/mssql/queries.py +46 -3
  1113. metadata/ingestion/source/database/mssql/query_parser.py +3 -3
  1114. metadata/ingestion/source/database/mssql/usage.py +3 -3
  1115. metadata/ingestion/source/database/mssql/utils.py +3 -3
  1116. metadata/ingestion/source/database/multi_db_source.py +3 -3
  1117. metadata/ingestion/source/database/mysql/connection.py +3 -3
  1118. metadata/ingestion/source/database/mysql/lineage.py +3 -3
  1119. metadata/ingestion/source/database/mysql/metadata.py +3 -3
  1120. metadata/ingestion/source/database/mysql/queries.py +3 -3
  1121. metadata/ingestion/source/database/mysql/query_parser.py +3 -3
  1122. metadata/ingestion/source/database/mysql/usage.py +3 -3
  1123. metadata/ingestion/source/database/mysql/utils.py +8 -6
  1124. metadata/ingestion/source/database/oracle/connection.py +27 -5
  1125. metadata/ingestion/source/database/oracle/lineage.py +3 -3
  1126. metadata/ingestion/source/database/oracle/metadata.py +39 -22
  1127. metadata/ingestion/source/database/oracle/models.py +5 -4
  1128. metadata/ingestion/source/database/oracle/queries.py +25 -4
  1129. metadata/ingestion/source/database/oracle/query_parser.py +3 -3
  1130. metadata/ingestion/source/database/oracle/usage.py +3 -3
  1131. metadata/ingestion/source/database/oracle/utils.py +3 -3
  1132. metadata/ingestion/source/database/pinotdb/connection.py +3 -3
  1133. metadata/ingestion/source/database/pinotdb/lineage.py +3 -3
  1134. metadata/ingestion/source/database/pinotdb/metadata.py +3 -3
  1135. metadata/ingestion/source/database/pinotdb/service_spec.py +5 -1
  1136. metadata/ingestion/source/database/postgres/connection.py +3 -3
  1137. metadata/ingestion/source/database/postgres/converter_orm.py +3 -3
  1138. metadata/ingestion/source/database/postgres/lineage.py +3 -3
  1139. metadata/ingestion/source/database/postgres/metadata.py +35 -59
  1140. metadata/ingestion/source/database/postgres/models.py +3 -2
  1141. metadata/ingestion/source/database/postgres/pgspider/lineage.py +2 -2
  1142. metadata/ingestion/source/database/postgres/pgspider/queries.py +2 -2
  1143. metadata/ingestion/source/database/postgres/queries.py +21 -4
  1144. metadata/ingestion/source/database/postgres/query_parser.py +3 -3
  1145. metadata/ingestion/source/database/postgres/types/money.py +3 -3
  1146. metadata/ingestion/source/database/postgres/usage.py +3 -3
  1147. metadata/ingestion/source/database/postgres/utils.py +3 -3
  1148. metadata/ingestion/source/database/presto/connection.py +3 -3
  1149. metadata/ingestion/source/database/presto/metadata.py +3 -3
  1150. metadata/ingestion/source/database/presto/queries.py +3 -3
  1151. metadata/ingestion/source/database/query/lineage.py +3 -3
  1152. metadata/ingestion/source/database/query/usage.py +3 -3
  1153. metadata/ingestion/source/database/query_parser_source.py +14 -9
  1154. metadata/ingestion/source/database/redshift/connection.py +3 -3
  1155. metadata/ingestion/source/database/redshift/incremental_table_processor.py +3 -3
  1156. metadata/ingestion/source/database/redshift/lineage.py +3 -3
  1157. metadata/ingestion/source/database/redshift/metadata.py +3 -3
  1158. metadata/ingestion/source/database/redshift/models.py +3 -3
  1159. metadata/ingestion/source/database/redshift/queries.py +3 -3
  1160. metadata/ingestion/source/database/redshift/query_parser.py +3 -3
  1161. metadata/ingestion/source/database/redshift/usage.py +3 -3
  1162. metadata/ingestion/source/database/redshift/utils.py +3 -3
  1163. metadata/ingestion/source/database/salesforce/connection.py +3 -3
  1164. metadata/ingestion/source/database/salesforce/metadata.py +3 -3
  1165. metadata/ingestion/source/database/sample_data.py +14 -3
  1166. metadata/ingestion/source/database/sample_usage.py +5 -3
  1167. metadata/ingestion/source/database/saperp/client.py +3 -3
  1168. metadata/ingestion/source/database/saperp/connection.py +3 -3
  1169. metadata/ingestion/source/database/saperp/constants.py +2 -2
  1170. metadata/ingestion/source/database/saperp/metadata.py +3 -3
  1171. metadata/ingestion/source/database/saperp/models.py +3 -3
  1172. metadata/ingestion/source/database/saphana/cdata_parser.py +3 -3
  1173. metadata/ingestion/source/database/saphana/connection.py +3 -3
  1174. metadata/ingestion/source/database/saphana/lineage.py +4 -4
  1175. metadata/ingestion/source/database/saphana/metadata.py +3 -3
  1176. metadata/ingestion/source/database/saphana/models.py +3 -3
  1177. metadata/ingestion/source/database/saphana/queries.py +3 -3
  1178. metadata/ingestion/source/database/sas/client.py +3 -3
  1179. metadata/ingestion/source/database/sas/connection.py +3 -3
  1180. metadata/ingestion/source/database/sas/extension_attr.py +3 -3
  1181. metadata/ingestion/source/database/sas/metadata.py +3 -3
  1182. metadata/ingestion/source/database/singlestore/connection.py +3 -3
  1183. metadata/ingestion/source/database/singlestore/lineage.py +3 -3
  1184. metadata/ingestion/source/database/singlestore/metadata.py +3 -3
  1185. metadata/ingestion/source/database/singlestore/service_spec.py +6 -1
  1186. metadata/ingestion/source/database/snowflake/connection.py +9 -3
  1187. metadata/ingestion/source/database/snowflake/constants.py +48 -0
  1188. metadata/ingestion/source/database/snowflake/lineage.py +3 -3
  1189. metadata/ingestion/source/database/snowflake/metadata.py +119 -6
  1190. metadata/ingestion/source/database/snowflake/models.py +6 -6
  1191. metadata/ingestion/source/database/snowflake/profiler/profiler.py +3 -3
  1192. metadata/ingestion/source/database/snowflake/queries.py +30 -4
  1193. metadata/ingestion/source/database/snowflake/query_parser.py +5 -3
  1194. metadata/ingestion/source/database/snowflake/usage.py +3 -3
  1195. metadata/ingestion/source/database/snowflake/utils.py +113 -29
  1196. metadata/ingestion/source/database/sql_column_handler.py +98 -63
  1197. metadata/ingestion/source/database/sqlalchemy_source.py +3 -3
  1198. metadata/ingestion/source/database/sqlite/connection.py +3 -3
  1199. metadata/ingestion/source/database/sqlite/lineage.py +3 -3
  1200. metadata/ingestion/source/database/sqlite/metadata.py +3 -3
  1201. metadata/ingestion/source/database/sqlite/service_spec.py +5 -1
  1202. metadata/ingestion/source/database/stored_procedures_mixin.py +6 -4
  1203. metadata/ingestion/source/database/teradata/connection.py +3 -3
  1204. metadata/ingestion/source/database/teradata/lineage.py +3 -3
  1205. metadata/ingestion/source/database/teradata/metadata.py +3 -3
  1206. metadata/ingestion/source/database/teradata/queries.py +3 -3
  1207. metadata/ingestion/source/database/teradata/service_spec.py +5 -1
  1208. metadata/ingestion/source/database/teradata/utils.py +3 -3
  1209. metadata/ingestion/source/database/trino/connection.py +3 -3
  1210. metadata/ingestion/source/database/trino/lineage.py +98 -3
  1211. metadata/ingestion/source/database/trino/metadata.py +3 -3
  1212. metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py +3 -3
  1213. metadata/ingestion/source/database/trino/queries.py +3 -3
  1214. metadata/ingestion/source/database/trino/query_parser.py +3 -3
  1215. metadata/ingestion/source/database/trino/usage.py +3 -3
  1216. metadata/ingestion/source/database/unitycatalog/client.py +3 -3
  1217. metadata/ingestion/source/database/unitycatalog/connection.py +26 -3
  1218. metadata/ingestion/source/database/unitycatalog/lineage.py +5 -3
  1219. metadata/ingestion/source/database/unitycatalog/metadata.py +5 -3
  1220. metadata/ingestion/source/database/unitycatalog/models.py +3 -3
  1221. metadata/ingestion/source/database/unitycatalog/query_parser.py +3 -3
  1222. metadata/ingestion/source/database/unitycatalog/usage.py +3 -3
  1223. metadata/ingestion/source/database/usage_source.py +5 -3
  1224. metadata/ingestion/source/database/vertica/connection.py +3 -3
  1225. metadata/ingestion/source/database/vertica/lineage.py +3 -3
  1226. metadata/ingestion/source/database/vertica/metadata.py +3 -3
  1227. metadata/ingestion/source/database/vertica/queries.py +3 -3
  1228. metadata/ingestion/source/database/vertica/query_parser.py +3 -3
  1229. metadata/ingestion/source/database/vertica/usage.py +3 -3
  1230. metadata/ingestion/source/messaging/common_broker_source.py +3 -3
  1231. metadata/ingestion/source/messaging/kafka/connection.py +3 -3
  1232. metadata/ingestion/source/messaging/kafka/metadata.py +3 -3
  1233. metadata/ingestion/source/messaging/kinesis/connection.py +3 -3
  1234. metadata/ingestion/source/messaging/kinesis/metadata.py +4 -5
  1235. metadata/ingestion/source/messaging/kinesis/models.py +2 -2
  1236. metadata/ingestion/source/messaging/messaging_service.py +5 -3
  1237. metadata/ingestion/source/messaging/redpanda/connection.py +3 -3
  1238. metadata/ingestion/source/messaging/redpanda/metadata.py +3 -3
  1239. metadata/ingestion/source/metadata/alationsink/client.py +2 -2
  1240. metadata/ingestion/source/metadata/alationsink/connection.py +2 -2
  1241. metadata/ingestion/source/metadata/alationsink/constants.py +2 -2
  1242. metadata/ingestion/source/metadata/alationsink/metadata.py +34 -20
  1243. metadata/ingestion/source/metadata/alationsink/models.py +2 -2
  1244. metadata/ingestion/source/metadata/amundsen/client.py +3 -3
  1245. metadata/ingestion/source/metadata/amundsen/connection.py +3 -3
  1246. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  1247. metadata/ingestion/source/metadata/amundsen/queries.py +3 -3
  1248. metadata/ingestion/source/metadata/atlas/client.py +3 -3
  1249. metadata/ingestion/source/metadata/atlas/connection.py +3 -3
  1250. metadata/ingestion/source/metadata/atlas/metadata.py +5 -4
  1251. metadata/ingestion/source/mlmodel/mlflow/connection.py +3 -3
  1252. metadata/ingestion/source/mlmodel/mlflow/metadata.py +3 -3
  1253. metadata/ingestion/source/mlmodel/mlmodel_service.py +5 -3
  1254. metadata/ingestion/source/mlmodel/sagemaker/connection.py +3 -3
  1255. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +3 -3
  1256. metadata/ingestion/source/models.py +3 -3
  1257. metadata/ingestion/source/pipeline/airbyte/client.py +3 -3
  1258. metadata/ingestion/source/pipeline/airbyte/connection.py +3 -3
  1259. metadata/ingestion/source/pipeline/airbyte/metadata.py +3 -3
  1260. metadata/ingestion/source/pipeline/airflow/connection.py +3 -3
  1261. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +3 -3
  1262. metadata/ingestion/source/pipeline/airflow/metadata.py +28 -3
  1263. metadata/ingestion/source/pipeline/airflow/models.py +4 -3
  1264. metadata/ingestion/source/pipeline/airflow/utils.py +3 -3
  1265. metadata/ingestion/source/pipeline/dagster/client.py +3 -3
  1266. metadata/ingestion/source/pipeline/dagster/connection.py +3 -3
  1267. metadata/ingestion/source/pipeline/dagster/metadata.py +3 -3
  1268. metadata/ingestion/source/pipeline/dagster/models.py +3 -3
  1269. metadata/ingestion/source/pipeline/dagster/queries.py +3 -3
  1270. metadata/ingestion/source/pipeline/databrickspipeline/connection.py +3 -3
  1271. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +3 -3
  1272. metadata/ingestion/source/pipeline/databrickspipeline/models.py +3 -3
  1273. metadata/ingestion/source/pipeline/dbtcloud/client.py +3 -3
  1274. metadata/ingestion/source/pipeline/dbtcloud/connection.py +3 -3
  1275. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +3 -3
  1276. metadata/ingestion/source/pipeline/dbtcloud/models.py +3 -3
  1277. metadata/ingestion/source/pipeline/dbtcloud/queries.py +3 -3
  1278. metadata/ingestion/source/pipeline/domopipeline/connection.py +3 -3
  1279. metadata/ingestion/source/pipeline/domopipeline/metadata.py +3 -3
  1280. metadata/ingestion/source/pipeline/fivetran/client.py +3 -3
  1281. metadata/ingestion/source/pipeline/fivetran/connection.py +3 -3
  1282. metadata/ingestion/source/pipeline/fivetran/metadata.py +3 -3
  1283. metadata/ingestion/source/pipeline/flink/client.py +3 -3
  1284. metadata/ingestion/source/pipeline/flink/connection.py +3 -3
  1285. metadata/ingestion/source/pipeline/flink/metadata.py +3 -3
  1286. metadata/ingestion/source/pipeline/flink/models.py +3 -3
  1287. metadata/ingestion/source/pipeline/gluepipeline/connection.py +3 -3
  1288. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +3 -3
  1289. metadata/ingestion/source/pipeline/gluepipeline/models.py +3 -3
  1290. metadata/ingestion/source/pipeline/kafkaconnect/client.py +3 -3
  1291. metadata/ingestion/source/pipeline/kafkaconnect/connection.py +3 -3
  1292. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +3 -3
  1293. metadata/ingestion/source/pipeline/kafkaconnect/models.py +3 -3
  1294. metadata/ingestion/source/pipeline/nifi/client.py +71 -77
  1295. metadata/ingestion/source/pipeline/nifi/connection.py +4 -21
  1296. metadata/ingestion/source/pipeline/nifi/metadata.py +3 -3
  1297. metadata/ingestion/source/pipeline/openlineage/connection.py +3 -3
  1298. metadata/ingestion/source/pipeline/openlineage/metadata.py +4 -3
  1299. metadata/ingestion/source/pipeline/openlineage/models.py +3 -3
  1300. metadata/ingestion/source/pipeline/pipeline_service.py +124 -5
  1301. metadata/ingestion/source/pipeline/spline/client.py +3 -3
  1302. metadata/ingestion/source/pipeline/spline/connection.py +3 -3
  1303. metadata/ingestion/source/pipeline/spline/metadata.py +3 -3
  1304. metadata/ingestion/source/pipeline/spline/models.py +3 -3
  1305. metadata/ingestion/source/pipeline/spline/utils.py +3 -3
  1306. metadata/ingestion/source/search/elasticsearch/connection.py +3 -3
  1307. metadata/ingestion/source/search/elasticsearch/metadata.py +3 -3
  1308. metadata/ingestion/source/search/elasticsearch/parser.py +3 -3
  1309. metadata/ingestion/source/search/opensearch/connection.py +215 -0
  1310. metadata/ingestion/source/search/opensearch/metadata.py +270 -0
  1311. metadata/ingestion/source/search/opensearch/parser.py +71 -0
  1312. metadata/ingestion/source/search/opensearch/service_spec.py +4 -0
  1313. metadata/ingestion/source/search/search_service.py +5 -3
  1314. metadata/ingestion/source/sqa_types.py +3 -3
  1315. metadata/ingestion/source/storage/gcs/client.py +2 -2
  1316. metadata/ingestion/source/storage/gcs/connection.py +2 -2
  1317. metadata/ingestion/source/storage/gcs/metadata.py +2 -2
  1318. metadata/ingestion/source/storage/gcs/models.py +2 -2
  1319. metadata/ingestion/source/storage/s3/connection.py +3 -3
  1320. metadata/ingestion/source/storage/s3/metadata.py +3 -3
  1321. metadata/ingestion/source/storage/s3/models.py +3 -3
  1322. metadata/ingestion/source/storage/storage_service.py +5 -3
  1323. metadata/ingestion/stage/table_usage.py +56 -4
  1324. metadata/mixins/pandas/pandas_mixin.py +3 -3
  1325. metadata/mixins/sqalchemy/sqa_mixin.py +3 -3
  1326. metadata/parsers/avro_parser.py +3 -3
  1327. metadata/parsers/json_schema_parser.py +3 -3
  1328. metadata/parsers/protobuf_parser.py +3 -3
  1329. metadata/parsers/schema_parsers.py +3 -3
  1330. metadata/pii/constants.py +3 -3
  1331. metadata/pii/models.py +3 -3
  1332. metadata/pii/ner.py +3 -3
  1333. metadata/pii/processor.py +3 -3
  1334. metadata/pii/scanners/base.py +3 -3
  1335. metadata/pii/scanners/column_name_scanner.py +3 -3
  1336. metadata/pii/scanners/ner_scanner.py +3 -3
  1337. metadata/profiler/adaptors/adaptor_factory.py +2 -2
  1338. metadata/profiler/adaptors/dynamodb.py +2 -2
  1339. metadata/profiler/adaptors/factory.py +2 -2
  1340. metadata/profiler/adaptors/mongodb.py +2 -2
  1341. metadata/profiler/adaptors/nosql_adaptor.py +2 -2
  1342. metadata/profiler/api/models.py +3 -3
  1343. metadata/profiler/config.py +3 -3
  1344. metadata/profiler/factory.py +3 -3
  1345. metadata/profiler/interface/nosql/profiler_interface.py +3 -3
  1346. metadata/profiler/interface/pandas/profiler_interface.py +3 -3
  1347. metadata/profiler/interface/profiler_interface.py +3 -3
  1348. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -3
  1349. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +3 -3
  1350. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +3 -3
  1351. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +3 -3
  1352. metadata/profiler/interface/sqlalchemy/profiler_interface.py +3 -3
  1353. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +3 -3
  1354. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +3 -3
  1355. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +3 -3
  1356. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +3 -3
  1357. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +3 -3
  1358. metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py +6 -7
  1359. metadata/profiler/metrics/composed/distinct_ratio.py +3 -3
  1360. metadata/profiler/metrics/composed/duplicate_count.py +3 -3
  1361. metadata/profiler/metrics/composed/ilike_ratio.py +3 -3
  1362. metadata/profiler/metrics/composed/iqr.py +3 -3
  1363. metadata/profiler/metrics/composed/like_ratio.py +3 -3
  1364. metadata/profiler/metrics/composed/non_parametric_skew.py +3 -3
  1365. metadata/profiler/metrics/composed/null_ratio.py +3 -3
  1366. metadata/profiler/metrics/composed/unique_ratio.py +3 -3
  1367. metadata/profiler/metrics/core.py +3 -3
  1368. metadata/profiler/metrics/hybrid/histogram.py +3 -3
  1369. metadata/profiler/metrics/registry.py +3 -3
  1370. metadata/profiler/metrics/static/column_count.py +3 -3
  1371. metadata/profiler/metrics/static/column_names.py +3 -3
  1372. metadata/profiler/metrics/static/count.py +3 -3
  1373. metadata/profiler/metrics/static/count_in_set.py +3 -3
  1374. metadata/profiler/metrics/static/distinct_count.py +3 -3
  1375. metadata/profiler/metrics/static/ilike_count.py +3 -3
  1376. metadata/profiler/metrics/static/like_count.py +3 -3
  1377. metadata/profiler/metrics/static/max.py +3 -3
  1378. metadata/profiler/metrics/static/max_length.py +3 -3
  1379. metadata/profiler/metrics/static/mean.py +3 -3
  1380. metadata/profiler/metrics/static/min.py +3 -3
  1381. metadata/profiler/metrics/static/min_length.py +3 -3
  1382. metadata/profiler/metrics/static/not_like_count.py +3 -3
  1383. metadata/profiler/metrics/static/not_regexp_match_count.py +3 -3
  1384. metadata/profiler/metrics/static/null_count.py +3 -3
  1385. metadata/profiler/metrics/static/null_missing_count.py +3 -3
  1386. metadata/profiler/metrics/static/regexp_match_count.py +3 -3
  1387. metadata/profiler/metrics/static/row_count.py +3 -3
  1388. metadata/profiler/metrics/static/stddev.py +3 -3
  1389. metadata/profiler/metrics/static/sum.py +3 -3
  1390. metadata/profiler/metrics/static/unique_count.py +3 -3
  1391. metadata/profiler/metrics/system/dml_operation.py +3 -3
  1392. metadata/profiler/metrics/system/system.py +3 -3
  1393. metadata/profiler/metrics/window/first_quartile.py +3 -3
  1394. metadata/profiler/metrics/window/median.py +3 -3
  1395. metadata/profiler/metrics/window/third_quartile.py +3 -3
  1396. metadata/profiler/orm/converter/azuresql/converter.py +39 -0
  1397. metadata/profiler/orm/converter/base.py +3 -3
  1398. metadata/profiler/orm/converter/bigquery/converter.py +3 -3
  1399. metadata/profiler/orm/converter/common.py +3 -3
  1400. metadata/profiler/orm/converter/converter_registry.py +5 -3
  1401. metadata/profiler/orm/converter/mariadb/converter.py +2 -2
  1402. metadata/profiler/orm/converter/mssql/converter.py +3 -3
  1403. metadata/profiler/orm/converter/redshift/converter.py +3 -3
  1404. metadata/profiler/orm/converter/snowflake/converter.py +3 -3
  1405. metadata/profiler/orm/functions/concat.py +3 -3
  1406. metadata/profiler/orm/functions/conn_test.py +3 -3
  1407. metadata/profiler/orm/functions/count.py +3 -3
  1408. metadata/profiler/orm/functions/datetime.py +3 -3
  1409. metadata/profiler/orm/functions/length.py +5 -3
  1410. metadata/profiler/orm/functions/md5.py +3 -3
  1411. metadata/profiler/orm/functions/median.py +22 -3
  1412. metadata/profiler/orm/functions/modulo.py +4 -3
  1413. metadata/profiler/orm/functions/random_num.py +4 -3
  1414. metadata/profiler/orm/functions/substr.py +3 -3
  1415. metadata/profiler/orm/functions/sum.py +4 -3
  1416. metadata/profiler/orm/functions/table_metric_computer.py +3 -3
  1417. metadata/profiler/orm/functions/unique_count.py +3 -3
  1418. metadata/profiler/orm/registry.py +5 -3
  1419. metadata/profiler/orm/types/bytea_to_string.py +3 -3
  1420. metadata/profiler/orm/types/custom_array.py +3 -3
  1421. metadata/profiler/orm/types/custom_datetimerange.py +3 -3
  1422. metadata/profiler/orm/types/custom_hex_byte_string.py +3 -3
  1423. metadata/profiler/orm/types/custom_image.py +3 -3
  1424. metadata/profiler/orm/types/custom_ip.py +3 -3
  1425. metadata/profiler/orm/types/custom_time.py +2 -2
  1426. metadata/profiler/orm/types/custom_timestamp.py +3 -3
  1427. metadata/profiler/orm/types/undetermined_type.py +3 -3
  1428. metadata/profiler/orm/types/uuid.py +3 -3
  1429. metadata/profiler/processor/core.py +4 -3
  1430. metadata/profiler/processor/default.py +3 -3
  1431. metadata/profiler/processor/handle_partition.py +3 -3
  1432. metadata/profiler/processor/metric_filter.py +4 -4
  1433. metadata/profiler/processor/models.py +3 -3
  1434. metadata/profiler/processor/processor.py +3 -3
  1435. metadata/profiler/processor/runner.py +3 -3
  1436. metadata/profiler/processor/sample_data_handler.py +3 -3
  1437. metadata/profiler/registry.py +3 -3
  1438. metadata/profiler/source/database/base/profiler_source.py +4 -3
  1439. metadata/profiler/source/database/bigquery/profiler_source.py +3 -3
  1440. metadata/profiler/source/database/bigquery/type_mapper.py +3 -3
  1441. metadata/profiler/source/fetcher/config.py +3 -3
  1442. metadata/profiler/source/fetcher/entity_fetcher.py +3 -3
  1443. metadata/profiler/source/fetcher/fetcher_strategy.py +7 -3
  1444. metadata/profiler/source/fetcher/profiler_source_factory.py +3 -3
  1445. metadata/profiler/source/metadata.py +3 -3
  1446. metadata/profiler/source/metadata_ext.py +3 -3
  1447. metadata/profiler/source/model.py +3 -3
  1448. metadata/profiler/source/profiler_source_interface.py +3 -3
  1449. metadata/readers/dataframe/avro.py +3 -3
  1450. metadata/readers/dataframe/base.py +3 -3
  1451. metadata/readers/dataframe/common.py +3 -3
  1452. metadata/readers/dataframe/dsv.py +3 -3
  1453. metadata/readers/dataframe/json.py +3 -3
  1454. metadata/readers/dataframe/models.py +3 -3
  1455. metadata/readers/dataframe/parquet.py +3 -3
  1456. metadata/readers/dataframe/reader_factory.py +3 -3
  1457. metadata/readers/file/adls.py +3 -3
  1458. metadata/readers/file/api_reader.py +3 -3
  1459. metadata/readers/file/base.py +3 -3
  1460. metadata/readers/file/bitbucket.py +3 -3
  1461. metadata/readers/file/config_source_factory.py +3 -3
  1462. metadata/readers/file/credentials.py +3 -3
  1463. metadata/readers/file/gcs.py +3 -3
  1464. metadata/readers/file/github.py +3 -3
  1465. metadata/readers/file/gitlab.py +3 -3
  1466. metadata/readers/file/local.py +3 -3
  1467. metadata/readers/file/s3.py +3 -3
  1468. metadata/readers/models.py +3 -3
  1469. metadata/sampler/config.py +4 -3
  1470. metadata/sampler/models.py +6 -3
  1471. metadata/sampler/nosql/sampler.py +7 -3
  1472. metadata/sampler/pandas/sampler.py +4 -3
  1473. metadata/sampler/partition.py +12 -5
  1474. metadata/sampler/processor.py +3 -3
  1475. metadata/sampler/sampler_interface.py +3 -4
  1476. metadata/sampler/sqlalchemy/azuresql/sampler.py +33 -6
  1477. metadata/sampler/sqlalchemy/bigquery/sampler.py +3 -3
  1478. metadata/sampler/sqlalchemy/mssql/sampler.py +5 -4
  1479. metadata/sampler/sqlalchemy/postgres/sampler.py +3 -3
  1480. metadata/sampler/sqlalchemy/sampler.py +54 -45
  1481. metadata/sampler/sqlalchemy/snowflake/sampler.py +5 -4
  1482. metadata/sampler/sqlalchemy/trino/sampler.py +12 -11
  1483. metadata/timer/repeated_timer.py +3 -3
  1484. metadata/utils/bigquery_utils.py +3 -3
  1485. metadata/utils/class_helper.py +3 -3
  1486. metadata/utils/client_version.py +3 -3
  1487. metadata/utils/constants.py +4 -3
  1488. metadata/utils/constraints.py +2 -2
  1489. metadata/utils/credentials.py +10 -3
  1490. metadata/utils/custom_thread_pool.py +3 -3
  1491. metadata/utils/datalake/datalake_utils.py +3 -3
  1492. metadata/utils/db_utils.py +3 -3
  1493. metadata/utils/deprecation.py +3 -3
  1494. metadata/utils/dispatch.py +3 -3
  1495. metadata/utils/elasticsearch.py +7 -3
  1496. metadata/utils/entity_link.py +3 -3
  1497. metadata/utils/entity_utils.py +3 -3
  1498. metadata/utils/execution_time_tracker.py +3 -3
  1499. metadata/utils/filters.py +3 -3
  1500. metadata/utils/fqn.py +3 -3
  1501. metadata/utils/helpers.py +94 -3
  1502. metadata/utils/importer.py +3 -3
  1503. metadata/utils/life_cycle_utils.py +3 -3
  1504. metadata/utils/logger.py +12 -3
  1505. metadata/utils/lru_cache.py +3 -3
  1506. metadata/utils/messaging_utils.py +2 -2
  1507. metadata/utils/profiler_utils.py +3 -3
  1508. metadata/utils/s3_utils.py +3 -3
  1509. metadata/utils/secrets/aws_based_secrets_manager.py +2 -2
  1510. metadata/utils/secrets/aws_secrets_manager.py +2 -2
  1511. metadata/utils/secrets/aws_ssm_secrets_manager.py +2 -2
  1512. metadata/utils/secrets/azure_kv_secrets_manager.py +2 -2
  1513. metadata/utils/secrets/db_secrets_manager.py +2 -2
  1514. metadata/utils/secrets/external_secrets_manager.py +2 -2
  1515. metadata/utils/secrets/secrets_manager.py +2 -2
  1516. metadata/utils/secrets/secrets_manager_factory.py +2 -2
  1517. metadata/utils/singleton.py +3 -3
  1518. metadata/utils/source_hash.py +3 -3
  1519. metadata/utils/sqa_like_column.py +3 -3
  1520. metadata/utils/sqa_utils.py +3 -3
  1521. metadata/utils/sqlalchemy_utils.py +3 -3
  1522. metadata/utils/ssl_manager.py +38 -3
  1523. metadata/utils/ssl_registry.py +3 -3
  1524. metadata/utils/storage_metadata_config.py +3 -3
  1525. metadata/utils/stored_procedures.py +3 -3
  1526. metadata/utils/tag_utils.py +75 -30
  1527. metadata/utils/test_utils.py +2 -2
  1528. metadata/utils/time_utils.py +3 -3
  1529. metadata/utils/timeout.py +3 -3
  1530. metadata/utils/uuid_encoder.py +3 -3
  1531. metadata/workflow/application.py +3 -3
  1532. metadata/workflow/application_output_handler.py +3 -3
  1533. metadata/workflow/base.py +3 -3
  1534. metadata/workflow/classification.py +6 -3
  1535. metadata/workflow/data_quality.py +3 -3
  1536. metadata/workflow/ingestion.py +3 -3
  1537. metadata/workflow/metadata.py +3 -3
  1538. metadata/workflow/output_handler.py +3 -3
  1539. metadata/workflow/profiler.py +16 -8
  1540. metadata/workflow/usage.py +3 -3
  1541. metadata/workflow/workflow_init_error_handler.py +3 -3
  1542. metadata/workflow/workflow_output_handler.py +3 -3
  1543. metadata/workflow/workflow_status_mixin.py +9 -3
  1544. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/METADATA +441 -373
  1545. openmetadata_ingestion-1.7.0.0rc2.dist-info/RECORD +1895 -0
  1546. openmetadata_ingestion-1.6.8.0.dist-info/RECORD +0 -1825
  1547. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/LICENSE +0 -0
  1548. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/WHEEL +0 -0
  1549. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/entry_points.txt +0 -0
  1550. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -12,11 +12,19 @@
12
12
  Client to interact with Nifi apis
13
13
  """
14
14
  import traceback
15
- from typing import Any, Iterable, List, Optional
16
-
17
- import requests
18
- from requests import HTTPError
19
-
15
+ from typing import Dict, Iterable, List
16
+
17
+ from metadata.generated.schema.entity.services.connections.pipeline.nifi.basicAuth import (
18
+ NifiBasicAuth,
19
+ )
20
+ from metadata.generated.schema.entity.services.connections.pipeline.nifi.clientCertificateAuth import (
21
+ NifiClientCertificateAuth,
22
+ )
23
+ from metadata.generated.schema.entity.services.connections.pipeline.nifiConnection import (
24
+ NifiConnection,
25
+ )
26
+ from metadata.ingestion.ometa.client import REST, ClientConfig, HTTPError
27
+ from metadata.utils.constants import AUTHORIZATION_HEADER, NO_ACCESS_TOKEN
20
28
  from metadata.utils.helpers import clean_uri
21
29
  from metadata.utils.logger import ingestion_logger
22
30
 
@@ -26,6 +34,8 @@ IDENTIFIER = "identifier"
26
34
  PROCESS_GROUPS_STARTER = "/process-groups/"
27
35
  RESOURCES = "resources"
28
36
  REQUESTS_TIMEOUT = 60 * 5
37
+ CONTENT_HEADER = {"Content-type": "application/x-www-form-urlencoded"}
38
+ NIFI_API_BASE_ENDPOINT = "/nifi-api"
29
39
 
30
40
 
31
41
  class NifiClient:
@@ -33,39 +43,43 @@ class NifiClient:
33
43
  Wrapper on top of Nifi REST API
34
44
  """
35
45
 
36
- # pylint: disable=too-many-arguments
37
- def __init__(
38
- self,
39
- host_port: str,
40
- username: Optional[str] = None,
41
- password: Optional[str] = None,
42
- ca_file_path: Optional[str] = None,
43
- client_cert_path: Optional[str] = None,
44
- client_key_path: Optional[str] = None,
45
- verify: bool = False,
46
- ):
47
- self._token = None
48
- self._resources = None
49
-
50
- self.content_headers = {"Content-Type": "application/x-www-form-urlencoded"}
51
- self.api_endpoint = clean_uri(host_port) + "/nifi-api"
52
- self.username = username
53
- self.password = password
54
-
55
- if all(setting for setting in [self.username, self.password]):
56
- self.data = {"username": self.username, "password": self.password}
57
- self.verify = verify
58
- self.headers = {
59
- "Authorization": f"Bearer {self.token}",
60
- **self.content_headers,
46
+ client: REST
47
+
48
+ def __init__(self, connection: NifiConnection):
49
+ self.connection = connection
50
+ self._token, self._resources, self.data = None, None, None
51
+ self.api_endpoint = clean_uri(self.connection.hostPort) + NIFI_API_BASE_ENDPOINT
52
+
53
+ client_config = ClientConfig(
54
+ api_version="",
55
+ timeout=REQUESTS_TIMEOUT,
56
+ base_url=self.api_endpoint,
57
+ extra_headers=CONTENT_HEADER,
58
+ )
59
+
60
+ if isinstance(self.connection.nifiConfig, NifiBasicAuth):
61
+ self.verify = self.connection.nifiConfig.verifySSL
62
+ self.data = {
63
+ "username": self.connection.nifiConfig.username,
64
+ "password": self.connection.nifiConfig.password.get_secret_value()
65
+ if self.connection.nifiConfig.password
66
+ else None,
61
67
  }
62
- self.client_cert = None
63
- else:
64
- self.data = None
65
- self.verify = ca_file_path if ca_file_path else False
66
- self.client_cert = (client_cert_path, client_key_path)
67
- self.headers = self.content_headers
68
- access = self.get("access")
68
+ client_config.verify = self.connection.nifiConfig.verifySSL
69
+ client_config.auth_header = AUTHORIZATION_HEADER
70
+ client_config.access_token = self.token
71
+
72
+ self.client = REST(client_config)
73
+ elif isinstance(self.connection.nifiConfig, NifiClientCertificateAuth):
74
+ ca_path = self.connection.nifiConfig.certificateAuthorityPath
75
+ cc_path = self.connection.nifiConfig.clientCertificatePath
76
+ ck_path = self.connection.nifiConfig.clientkeyPath
77
+
78
+ client_config.verify = ca_path if ca_path else False
79
+ client_config.cert = (cc_path, ck_path)
80
+
81
+ self.client = REST(client_config)
82
+ access = self.client.get("access")
69
83
  logger.debug(access)
70
84
 
71
85
  @property
@@ -76,13 +90,19 @@ class NifiClient:
76
90
  """
77
91
  if not self._token:
78
92
  try:
79
- res = requests.post(
80
- f"{self.api_endpoint}/access/token",
81
- verify=self.verify,
82
- headers=self.content_headers,
83
- data=self.data,
84
- timeout=REQUESTS_TIMEOUT,
93
+ client = REST(
94
+ ClientConfig(
95
+ base_url=self.api_endpoint,
96
+ verify=self.verify,
97
+ extra_headers=CONTENT_HEADER,
98
+ timeout=REQUESTS_TIMEOUT,
99
+ api_version="",
100
+ auth_token_mode=None,
101
+ auth_token=lambda: (NO_ACCESS_TOKEN, 0),
102
+ )
85
103
  )
104
+
105
+ res = client.post("access/token", data=self.data)
86
106
  self._token = res.text
87
107
 
88
108
  if res.status_code not in (200, 201):
@@ -100,6 +120,7 @@ class NifiClient:
100
120
 
101
121
  except Exception as err:
102
122
  logger.error(f"Fetching token failed due to - {err}")
123
+ logger.debug(traceback.format_exc())
103
124
  raise err
104
125
 
105
126
  return self._token
@@ -110,7 +131,7 @@ class NifiClient:
110
131
  This can be expensive. Only query it once.
111
132
  """
112
133
  if not self._resources:
113
- self._resources = self.get(RESOURCES) # API endpoint
134
+ self._resources = self.client.get(RESOURCES) # API endpoint
114
135
 
115
136
  # Get the first `resources` key from the dict
116
137
  try:
@@ -118,33 +139,6 @@ class NifiClient:
118
139
  except AttributeError:
119
140
  return []
120
141
 
121
- def get(self, path: str) -> Optional[Any]:
122
- """
123
- GET call wrapper
124
- """
125
- try:
126
- res = requests.get(
127
- f"{self.api_endpoint}/{path}",
128
- verify=self.verify,
129
- headers=self.headers,
130
- timeout=REQUESTS_TIMEOUT,
131
- cert=self.client_cert,
132
- )
133
-
134
- return res.json()
135
-
136
- except HTTPError as err:
137
- logger.warning(f"Connection error calling the Nifi API - {err}")
138
- raise err
139
-
140
- except ValueError as err:
141
- logger.warning(f"Cannot pick up the JSON from API response - {err}")
142
- raise err
143
-
144
- except Exception as err:
145
- logger.warning(f"Unknown error calling Nifi API - {err}")
146
- raise err
147
-
148
142
  def _get_process_group_ids(self) -> List[str]:
149
143
  return [
150
144
  elem.get(IDENTIFIER).replace(PROCESS_GROUPS_STARTER, "")
@@ -152,10 +146,10 @@ class NifiClient:
152
146
  if elem.get(IDENTIFIER).startswith(PROCESS_GROUPS_STARTER)
153
147
  ]
154
148
 
155
- def get_process_group(self, id_: str) -> dict:
156
- return self.get(f"flow/process-groups/{id_}")
149
+ def get_process_group(self, id_: str) -> Dict:
150
+ return self.client.get(f"flow/process-groups/{id_}")
157
151
 
158
- def list_process_groups(self) -> Iterable[dict]:
152
+ def list_process_groups(self) -> Iterable[Dict]:
159
153
  """
160
154
  This will call the API endpoints
161
155
  one at a time.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,9 +17,6 @@ from typing import Optional
17
17
  from metadata.generated.schema.entity.automations.workflow import (
18
18
  Workflow as AutomationWorkflow,
19
19
  )
20
- from metadata.generated.schema.entity.services.connections.pipeline.nifi.basicAuth import (
21
- NifiBasicAuth,
22
- )
23
20
  from metadata.generated.schema.entity.services.connections.pipeline.nifiConnection import (
24
21
  NifiConnection,
25
22
  )
@@ -36,22 +33,8 @@ def get_connection(connection: NifiConnection) -> NifiClient:
36
33
  """
37
34
  Create connection
38
35
  """
39
- if isinstance(connection.nifiConfig, NifiBasicAuth):
40
- return NifiClient(
41
- host_port=connection.hostPort,
42
- username=connection.nifiConfig.username,
43
- password=connection.nifiConfig.password.get_secret_value()
44
- if connection.nifiConfig.password
45
- else None,
46
- verify=connection.nifiConfig.verifySSL,
47
- )
48
36
 
49
- return NifiClient(
50
- host_port=connection.hostPort,
51
- ca_file_path=connection.nifiConfig.certificateAuthorityPath,
52
- client_cert_path=connection.nifiConfig.clientCertificatePath,
53
- client_key_path=connection.nifiConfig.clientkeyPath,
54
- )
37
+ return NifiClient(connection=connection)
55
38
 
56
39
 
57
40
  def test_connection(
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.