openmetadata-ingestion 1.6.8.0__py3-none-any.whl → 1.7.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (1550) hide show
  1. airflow_provider_openmetadata/__init__.py +3 -3
  2. airflow_provider_openmetadata/hooks/openmetadata.py +3 -3
  3. airflow_provider_openmetadata/lineage/backend.py +3 -3
  4. airflow_provider_openmetadata/lineage/callback.py +3 -3
  5. airflow_provider_openmetadata/lineage/config/commons.py +3 -3
  6. airflow_provider_openmetadata/lineage/config/loader.py +3 -3
  7. airflow_provider_openmetadata/lineage/operator.py +3 -3
  8. airflow_provider_openmetadata/lineage/runner.py +3 -3
  9. airflow_provider_openmetadata/lineage/status.py +3 -3
  10. metadata/__main__.py +3 -3
  11. metadata/__version__.py +3 -3
  12. metadata/antlr/split_listener.py +3 -3
  13. metadata/automations/execute_runner.py +49 -0
  14. metadata/automations/extended_runner.py +13 -0
  15. metadata/automations/runner.py +37 -38
  16. metadata/cli/app.py +3 -3
  17. metadata/cli/classify.py +3 -3
  18. metadata/cli/dataquality.py +3 -3
  19. metadata/cli/ingest.py +3 -3
  20. metadata/cli/lineage.py +3 -3
  21. metadata/cli/profile.py +3 -3
  22. metadata/cli/restore.py +3 -3
  23. metadata/cli/usage.py +3 -3
  24. metadata/clients/aws_client.py +3 -3
  25. metadata/clients/azure_client.py +3 -3
  26. metadata/clients/domo_client.py +3 -3
  27. metadata/cmd.py +3 -3
  28. metadata/config/common.py +3 -3
  29. metadata/data_quality/api/models.py +3 -3
  30. metadata/data_quality/builders/validator_builder.py +3 -3
  31. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +3 -3
  32. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +3 -3
  33. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +3 -3
  34. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +3 -3
  35. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +3 -3
  36. metadata/data_quality/interface/test_suite_interface.py +3 -3
  37. metadata/data_quality/processor/test_case_runner.py +3 -3
  38. metadata/data_quality/runner/base_test_suite_source.py +3 -3
  39. metadata/data_quality/runner/core.py +3 -3
  40. metadata/data_quality/source/test_suite.py +3 -3
  41. metadata/data_quality/validations/base_test_handler.py +3 -3
  42. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +3 -3
  43. metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +3 -3
  44. metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +3 -3
  45. metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +3 -3
  46. metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +3 -3
  47. metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +3 -3
  48. metadata/data_quality/validations/column/base/columnValuesMissingCount.py +3 -3
  49. metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +3 -3
  50. metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py +3 -3
  51. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +3 -3
  52. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +3 -3
  53. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +3 -3
  54. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +3 -3
  55. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +3 -3
  56. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +3 -3
  57. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +3 -3
  58. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +3 -3
  59. metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py +3 -3
  60. metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py +3 -3
  61. metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py +3 -3
  62. metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py +3 -3
  63. metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py +3 -3
  64. metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py +3 -3
  65. metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py +3 -3
  66. metadata/data_quality/validations/column/pandas/columnValuesToBeAtExpectedLocation.py +3 -3
  67. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +3 -3
  68. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +3 -3
  69. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +3 -3
  70. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +3 -3
  71. metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py +3 -3
  72. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +3 -3
  73. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +3 -3
  74. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +3 -3
  75. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +3 -3
  76. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +3 -3
  77. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +3 -3
  78. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +3 -3
  79. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +3 -3
  80. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +3 -3
  81. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +3 -3
  82. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +3 -3
  83. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +3 -3
  84. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +3 -3
  85. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +3 -3
  86. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +3 -3
  87. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +3 -3
  88. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +3 -3
  89. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +3 -3
  90. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +3 -3
  91. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +3 -3
  92. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -2
  93. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +2 -2
  94. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +6 -3
  95. metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py +3 -3
  96. metadata/data_quality/validations/table/base/tableColumnCountToEqual.py +3 -3
  97. metadata/data_quality/validations/table/base/tableColumnNameToExist.py +3 -3
  98. metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +3 -3
  99. metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +14 -4
  100. metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +3 -3
  101. metadata/data_quality/validations/table/base/tableRowCountToEqual.py +3 -3
  102. metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py +3 -3
  103. metadata/data_quality/validations/table/pandas/tableColumnCountToBeBetween.py +3 -3
  104. metadata/data_quality/validations/table/pandas/tableColumnCountToEqual.py +3 -3
  105. metadata/data_quality/validations/table/pandas/tableColumnNameToExist.py +3 -3
  106. metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +3 -3
  107. metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py +3 -3
  108. metadata/data_quality/validations/table/pandas/tableRowCountToBeBetween.py +3 -3
  109. metadata/data_quality/validations/table/pandas/tableRowCountToEqual.py +3 -3
  110. metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +3 -3
  111. metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToBeBetween.py +3 -3
  112. metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToEqual.py +3 -3
  113. metadata/data_quality/validations/table/sqlalchemy/tableColumnNameToExist.py +3 -3
  114. metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +3 -3
  115. metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +3 -3
  116. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +2 -2
  117. metadata/data_quality/validations/table/sqlalchemy/tableRowCountToBeBetween.py +3 -3
  118. metadata/data_quality/validations/table/sqlalchemy/tableRowCountToEqual.py +3 -3
  119. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +3 -3
  120. metadata/examples/workflows/cassandra.yaml +35 -0
  121. metadata/examples/workflows/cockroach.yaml +24 -0
  122. metadata/examples/workflows/nifi.yaml +51 -0
  123. metadata/examples/workflows/opensearch.yaml +20 -0
  124. metadata/examples/workflows/wherescape.yaml +2 -2
  125. metadata/generated/schema/analytics/__init__.py +1 -1
  126. metadata/generated/schema/analytics/basic.py +1 -1
  127. metadata/generated/schema/analytics/reportData.py +1 -1
  128. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  129. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  130. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  131. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  132. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  133. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  134. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  135. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  136. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  137. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  138. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  139. metadata/generated/schema/api/__init__.py +1 -1
  140. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  141. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  142. metadata/generated/schema/api/analytics/__init__.py +1 -1
  143. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  144. metadata/generated/schema/api/automations/__init__.py +1 -1
  145. metadata/generated/schema/api/automations/createWorkflow.py +14 -4
  146. metadata/generated/schema/api/bulkAssets.py +1 -1
  147. metadata/generated/schema/api/classification/__init__.py +1 -1
  148. metadata/generated/schema/api/classification/createClassification.py +2 -2
  149. metadata/generated/schema/api/classification/createTag.py +2 -2
  150. metadata/generated/schema/api/classification/loadTags.py +1 -1
  151. metadata/generated/schema/api/createBot.py +1 -1
  152. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  153. metadata/generated/schema/api/createType.py +1 -1
  154. metadata/generated/schema/api/data/__init__.py +1 -1
  155. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  156. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  157. metadata/generated/schema/api/data/createChart.py +1 -1
  158. metadata/generated/schema/api/data/createContainer.py +1 -1
  159. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  160. metadata/generated/schema/api/data/createDashboard.py +1 -1
  161. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  162. metadata/generated/schema/api/data/createDatabase.py +1 -1
  163. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  164. metadata/generated/schema/api/data/createGlossary.py +1 -1
  165. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  166. metadata/generated/schema/api/data/createMetric.py +1 -1
  167. metadata/generated/schema/api/data/createMlModel.py +1 -1
  168. metadata/generated/schema/api/data/createPipeline.py +5 -1
  169. metadata/generated/schema/api/data/createQuery.py +1 -1
  170. metadata/generated/schema/api/data/createQueryCostRecord.py +35 -0
  171. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  172. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  173. metadata/generated/schema/api/data/createTable.py +1 -1
  174. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  175. metadata/generated/schema/api/data/createTopic.py +1 -1
  176. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  177. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  178. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  179. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  180. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  181. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  182. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  183. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  184. metadata/generated/schema/api/docStore/__init__.py +1 -1
  185. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  186. metadata/generated/schema/api/domains/__init__.py +1 -1
  187. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  188. metadata/generated/schema/api/domains/createDomain.py +8 -1
  189. metadata/generated/schema/api/feed/__init__.py +1 -1
  190. metadata/generated/schema/api/feed/closeTask.py +1 -1
  191. metadata/generated/schema/api/feed/createPost.py +1 -1
  192. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  193. metadata/generated/schema/api/feed/createThread.py +1 -1
  194. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  195. metadata/generated/schema/api/feed/threadCount.py +25 -1
  196. metadata/generated/schema/api/governance/__init__.py +1 -1
  197. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  198. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  199. metadata/generated/schema/api/lineage/__init__.py +1 -1
  200. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  201. metadata/generated/schema/api/lineage/esLineageData.py +85 -0
  202. metadata/generated/schema/api/lineage/lineageDirection.py +12 -0
  203. metadata/generated/schema/api/lineage/nodeInformation.py +19 -0
  204. metadata/generated/schema/api/lineage/searchLineageRequest.py +45 -0
  205. metadata/generated/schema/api/lineage/searchLineageResult.py +46 -0
  206. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  207. metadata/generated/schema/api/policies/__init__.py +1 -1
  208. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  209. metadata/generated/schema/api/search/__init__.py +3 -0
  210. metadata/generated/schema/api/search/previewSearchRequest.py +47 -0
  211. metadata/generated/schema/api/services/__init__.py +1 -1
  212. metadata/generated/schema/api/services/createApiService.py +9 -2
  213. metadata/generated/schema/api/services/createDashboardService.py +9 -2
  214. metadata/generated/schema/api/services/createDatabaseService.py +9 -2
  215. metadata/generated/schema/api/services/createMessagingService.py +9 -2
  216. metadata/generated/schema/api/services/createMetadataService.py +9 -2
  217. metadata/generated/schema/api/services/createMlModelService.py +9 -2
  218. metadata/generated/schema/api/services/createPipelineService.py +9 -2
  219. metadata/generated/schema/api/services/createSearchService.py +9 -2
  220. metadata/generated/schema/api/services/createStorageService.py +9 -2
  221. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  222. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  223. metadata/generated/schema/api/setOwner.py +1 -1
  224. metadata/generated/schema/api/teams/__init__.py +1 -1
  225. metadata/generated/schema/api/teams/createPersona.py +1 -1
  226. metadata/generated/schema/api/teams/createRole.py +1 -1
  227. metadata/generated/schema/api/teams/createTeam.py +1 -1
  228. metadata/generated/schema/api/teams/createUser.py +1 -1
  229. metadata/generated/schema/api/tests/__init__.py +1 -1
  230. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  231. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  232. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  233. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  234. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  235. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  236. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  237. metadata/generated/schema/api/voteRequest.py +1 -1
  238. metadata/generated/schema/auth/__init__.py +1 -1
  239. metadata/generated/schema/auth/basicAuth.py +1 -1
  240. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  241. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  242. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  243. metadata/generated/schema/auth/emailRequest.py +1 -1
  244. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  245. metadata/generated/schema/auth/generateToken.py +1 -1
  246. metadata/generated/schema/auth/jwtAuth.py +1 -1
  247. metadata/generated/schema/auth/loginRequest.py +1 -1
  248. metadata/generated/schema/auth/logoutRequest.py +1 -1
  249. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  250. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  251. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  252. metadata/generated/schema/auth/refreshToken.py +1 -1
  253. metadata/generated/schema/auth/registrationRequest.py +1 -1
  254. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  255. metadata/generated/schema/auth/revokeToken.py +1 -1
  256. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  257. metadata/generated/schema/auth/ssoAuth.py +1 -1
  258. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  259. metadata/generated/schema/configuration/__init__.py +1 -1
  260. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  261. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  262. metadata/generated/schema/configuration/authConfig.py +1 -1
  263. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  264. metadata/generated/schema/configuration/authorizerConfiguration.py +4 -1
  265. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  266. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  267. metadata/generated/schema/configuration/elasticSearchConfiguration.py +55 -1
  268. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  269. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  270. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  271. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  272. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  273. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  274. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  275. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  276. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  277. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  278. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  279. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  280. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  281. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  282. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  283. metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +21 -0
  284. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  285. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  286. metadata/generated/schema/configuration/searchSettings.py +463 -4
  287. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  288. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  289. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  290. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  291. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  292. metadata/generated/schema/configuration/workflowSettings.py +8 -1
  293. metadata/generated/schema/dataInsight/__init__.py +1 -1
  294. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  295. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  296. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  297. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  298. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  299. metadata/generated/schema/dataInsight/custom/lineChart.py +15 -1
  300. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  301. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  302. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  303. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  304. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  305. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  306. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  307. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  308. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  309. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  310. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  311. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  312. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  313. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  314. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  315. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  316. metadata/generated/schema/email/__init__.py +1 -1
  317. metadata/generated/schema/email/emailRequest.py +1 -1
  318. metadata/generated/schema/email/emailTemplate.py +1 -1
  319. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  320. metadata/generated/schema/email/smtpSettings.py +1 -2
  321. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  322. metadata/generated/schema/entity/__init__.py +1 -1
  323. metadata/generated/schema/entity/applications/__init__.py +1 -1
  324. metadata/generated/schema/entity/applications/app.py +14 -1
  325. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  326. metadata/generated/schema/entity/applications/appRunRecord.py +44 -7
  327. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  328. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +9 -3
  329. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  330. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  331. metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
  332. metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +41 -0
  333. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  334. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  335. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  336. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  337. metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +71 -0
  338. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  339. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  340. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  341. metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
  342. metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +33 -0
  343. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  344. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  345. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  346. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  347. metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +45 -0
  348. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  349. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +13 -1
  350. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  351. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  352. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  353. metadata/generated/schema/entity/applications/configuration/internal/autoPilotAppConfig.py +48 -0
  354. metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +46 -0
  355. metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +46 -0
  356. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +109 -2
  357. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  358. metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +1 -1
  359. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  360. metadata/generated/schema/entity/applications/configuration/private/__init__.py +3 -0
  361. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  362. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +3 -22
  363. metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +3 -0
  364. metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +24 -0
  365. metadata/generated/schema/entity/applications/configuration/private/limits.py +33 -0
  366. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  367. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  368. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  369. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  370. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +13 -1
  371. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +13 -1
  372. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  373. metadata/generated/schema/entity/automations/__init__.py +1 -1
  374. metadata/generated/schema/entity/automations/testServiceConnection.py +8 -1
  375. metadata/generated/schema/entity/automations/workflow.py +15 -4
  376. metadata/generated/schema/entity/bot.py +1 -1
  377. metadata/generated/schema/entity/classification/__init__.py +1 -1
  378. metadata/generated/schema/entity/classification/classification.py +1 -1
  379. metadata/generated/schema/entity/classification/tag.py +1 -1
  380. metadata/generated/schema/entity/data/__init__.py +1 -1
  381. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  382. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  383. metadata/generated/schema/entity/data/chart.py +1 -1
  384. metadata/generated/schema/entity/data/container.py +1 -1
  385. metadata/generated/schema/entity/data/dashboard.py +1 -1
  386. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  387. metadata/generated/schema/entity/data/database.py +5 -1
  388. metadata/generated/schema/entity/data/databaseSchema.py +7 -5
  389. metadata/generated/schema/entity/data/glossary.py +1 -1
  390. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  391. metadata/generated/schema/entity/data/metric.py +1 -1
  392. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  393. metadata/generated/schema/entity/data/pipeline.py +14 -1
  394. metadata/generated/schema/entity/data/query.py +1 -1
  395. metadata/generated/schema/entity/data/queryCostRecord.py +47 -0
  396. metadata/generated/schema/entity/data/queryCostSearchResult.py +56 -0
  397. metadata/generated/schema/entity/data/report.py +1 -1
  398. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  399. metadata/generated/schema/entity/data/storedProcedure.py +3 -1
  400. metadata/generated/schema/entity/data/table.py +6 -1
  401. metadata/generated/schema/entity/data/topic.py +1 -1
  402. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  403. metadata/generated/schema/entity/docStore/document.py +1 -1
  404. metadata/generated/schema/entity/domains/__init__.py +1 -1
  405. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  406. metadata/generated/schema/entity/domains/domain.py +8 -1
  407. metadata/generated/schema/entity/events/__init__.py +1 -1
  408. metadata/generated/schema/entity/events/webhook.py +1 -1
  409. metadata/generated/schema/entity/feed/__init__.py +1 -1
  410. metadata/generated/schema/entity/feed/assets.py +1 -1
  411. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  412. metadata/generated/schema/entity/feed/description.py +1 -1
  413. metadata/generated/schema/entity/feed/domain.py +1 -1
  414. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  415. metadata/generated/schema/entity/feed/owner.py +1 -1
  416. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  417. metadata/generated/schema/entity/feed/tag.py +1 -1
  418. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  419. metadata/generated/schema/entity/feed/thread.py +1 -1
  420. metadata/generated/schema/entity/policies/__init__.py +1 -1
  421. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  422. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  423. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  424. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  425. metadata/generated/schema/entity/policies/filters.py +1 -1
  426. metadata/generated/schema/entity/policies/policy.py +1 -1
  427. metadata/generated/schema/entity/services/__init__.py +1 -1
  428. metadata/generated/schema/entity/services/apiService.py +8 -1
  429. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  430. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  431. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  432. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  433. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  434. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  435. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  436. metadata/generated/schema/entity/services/connections/connectionBasicType.py +7 -1
  437. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  438. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +34 -1
  439. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +34 -1
  440. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +34 -1
  441. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +34 -1
  442. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +34 -1
  443. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +34 -1
  444. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +34 -1
  445. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +34 -1
  446. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +34 -1
  447. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  448. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  449. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  450. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  451. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  452. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +50 -2
  453. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +34 -1
  454. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +34 -1
  455. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +34 -1
  456. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +35 -1
  457. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +34 -1
  458. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +34 -1
  459. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  460. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +26 -1
  461. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +26 -1
  462. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +36 -1
  463. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +26 -1
  464. metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +3 -0
  465. metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +61 -0
  466. metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +106 -0
  467. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +26 -1
  468. metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +136 -0
  469. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  470. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  471. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  472. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  473. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  474. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  475. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +26 -1
  476. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +26 -1
  477. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +27 -3
  478. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  479. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  480. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  481. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  482. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +26 -1
  483. metadata/generated/schema/entity/services/connections/database/db2Connection.py +26 -1
  484. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +26 -1
  485. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  486. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  487. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  488. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +26 -1
  489. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +26 -1
  490. metadata/generated/schema/entity/services/connections/database/druidConnection.py +26 -1
  491. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +26 -1
  492. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +26 -1
  493. metadata/generated/schema/entity/services/connections/database/glueConnection.py +26 -1
  494. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +26 -1
  495. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +26 -1
  496. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  497. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  498. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  499. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  500. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  501. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  502. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  503. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +26 -1
  504. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +26 -1
  505. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +26 -1
  506. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +26 -1
  507. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +29 -1
  508. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +29 -1
  509. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +26 -1
  510. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +26 -1
  511. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +26 -1
  512. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +26 -1
  513. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +30 -1
  514. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +26 -1
  515. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +26 -1
  516. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  517. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  518. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  519. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +26 -1
  520. metadata/generated/schema/entity/services/connections/database/sasConnection.py +27 -1
  521. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +26 -1
  522. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +47 -2
  523. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +26 -1
  524. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +26 -1
  525. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +26 -1
  526. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +26 -1
  527. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +26 -1
  528. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +26 -1
  529. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  530. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +10 -1
  531. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +10 -1
  532. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +10 -1
  533. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +10 -1
  534. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +10 -1
  535. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  536. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  537. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +26 -1
  538. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +26 -1
  539. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +26 -1
  540. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +26 -1
  541. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +26 -1
  542. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +26 -2
  543. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  544. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +10 -1
  545. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +10 -1
  546. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +10 -1
  547. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +10 -1
  548. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +10 -1
  549. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  550. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +10 -1
  551. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +10 -1
  552. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -1
  553. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +10 -1
  554. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +10 -1
  555. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +10 -1
  556. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +10 -1
  557. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +11 -1
  558. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +10 -1
  559. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +10 -1
  560. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +10 -1
  561. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +10 -1
  562. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +11 -1
  563. metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
  564. metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
  565. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +10 -1
  566. metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
  567. metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
  568. metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
  569. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +10 -1
  570. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +10 -1
  571. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -1
  572. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +10 -1
  573. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +10 -1
  574. metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py +1 -1
  575. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  576. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +10 -1
  577. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  578. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  579. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  580. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +10 -1
  581. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +37 -29
  582. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  583. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  584. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +10 -1
  585. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +10 -1
  586. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +10 -1
  587. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +10 -1
  588. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  589. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  590. metadata/generated/schema/entity/services/dashboardService.py +8 -1
  591. metadata/generated/schema/entity/services/databaseService.py +14 -1
  592. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  593. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +8 -1
  594. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  595. metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +47 -0
  596. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  597. metadata/generated/schema/entity/services/messagingService.py +8 -1
  598. metadata/generated/schema/entity/services/metadataService.py +8 -1
  599. metadata/generated/schema/entity/services/mlmodelService.py +8 -1
  600. metadata/generated/schema/entity/services/pipelineService.py +8 -1
  601. metadata/generated/schema/entity/services/searchService.py +9 -2
  602. metadata/generated/schema/entity/services/serviceType.py +1 -1
  603. metadata/generated/schema/entity/services/storageService.py +8 -1
  604. metadata/generated/schema/entity/teams/__init__.py +1 -1
  605. metadata/generated/schema/entity/teams/persona.py +5 -1
  606. metadata/generated/schema/entity/teams/role.py +1 -1
  607. metadata/generated/schema/entity/teams/team.py +1 -1
  608. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  609. metadata/generated/schema/entity/teams/user.py +5 -1
  610. metadata/generated/schema/entity/type.py +1 -1
  611. metadata/generated/schema/entity/utils/__init__.py +1 -1
  612. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  613. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  614. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  615. metadata/generated/schema/events/__init__.py +1 -1
  616. metadata/generated/schema/events/alertMetrics.py +1 -1
  617. metadata/generated/schema/events/api/__init__.py +1 -1
  618. metadata/generated/schema/events/api/createEventSubscription.py +9 -1
  619. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  620. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  621. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  622. metadata/generated/schema/events/api/typedEvent.py +1 -1
  623. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  624. metadata/generated/schema/events/eventFilterRule.py +1 -1
  625. metadata/generated/schema/events/eventSubscription.py +9 -2
  626. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  627. metadata/generated/schema/events/failedEvent.py +1 -1
  628. metadata/generated/schema/events/failedEventResponse.py +1 -1
  629. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  630. metadata/generated/schema/events/statusContext.py +1 -1
  631. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  632. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  633. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  634. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  635. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  636. metadata/generated/schema/governance/workflows/elements/edge.py +2 -2
  637. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +4 -1
  638. metadata/generated/schema/governance/workflows/elements/nodeType.py +2 -1
  639. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  640. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  641. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +28 -8
  642. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +86 -0
  643. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +73 -0
  644. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +27 -6
  645. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +28 -7
  646. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  647. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +9 -4
  648. metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +3 -0
  649. metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +30 -0
  650. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  651. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +9 -4
  652. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  653. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +24 -7
  654. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  655. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +8 -4
  656. metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +20 -0
  657. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +18 -5
  658. metadata/generated/schema/governance/workflows/workflowDefinition.py +30 -7
  659. metadata/generated/schema/governance/workflows/workflowInstance.py +11 -8
  660. metadata/generated/schema/governance/workflows/workflowInstanceState.py +4 -8
  661. metadata/generated/schema/jobs/__init__.py +1 -1
  662. metadata/generated/schema/jobs/backgroundJob.py +1 -1
  663. metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
  664. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  665. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  666. metadata/generated/schema/metadataIngestion/application.py +1 -1
  667. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  668. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  669. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  670. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +3 -3
  671. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +4 -4
  672. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +5 -1
  673. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +18 -2
  674. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +9 -1
  675. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  676. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  677. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  678. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  679. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  680. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  681. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  682. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  683. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  684. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  685. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  686. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  687. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  688. metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +71 -0
  689. metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +3 -0
  690. metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +24 -0
  691. metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +28 -0
  692. metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +28 -0
  693. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  694. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  695. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  696. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  697. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  698. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  699. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  700. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  701. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  702. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  703. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  704. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  705. metadata/generated/schema/metadataIngestion/workflow.py +3 -1
  706. metadata/generated/schema/monitoring/__init__.py +1 -1
  707. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  708. metadata/generated/schema/search/__init__.py +3 -0
  709. metadata/generated/schema/search/aggregationRequest.py +80 -0
  710. metadata/generated/schema/search/searchRequest.py +106 -0
  711. metadata/generated/schema/security/__init__.py +1 -1
  712. metadata/generated/schema/security/client/__init__.py +1 -1
  713. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  714. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  715. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  716. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  717. metadata/generated/schema/security/client/oidcClientConfig.py +7 -1
  718. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  719. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  720. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  721. metadata/generated/schema/security/credentials/__init__.py +1 -1
  722. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  723. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  724. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  725. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  726. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  727. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  728. metadata/generated/schema/security/credentials/gcpCredentials.py +42 -22
  729. metadata/generated/schema/security/credentials/gcpExternalAccount.py +2 -2
  730. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  731. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  732. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  733. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  734. metadata/generated/schema/security/sasl/__init__.py +1 -1
  735. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  736. metadata/generated/schema/security/secrets/__init__.py +1 -1
  737. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  738. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  739. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  740. metadata/generated/schema/security/securityConfiguration.py +1 -1
  741. metadata/generated/schema/security/ssl/__init__.py +1 -1
  742. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  743. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  744. metadata/generated/schema/settings/__init__.py +1 -1
  745. metadata/generated/schema/settings/settings.py +4 -1
  746. metadata/generated/schema/system/__init__.py +1 -1
  747. metadata/generated/schema/system/entityError.py +1 -1
  748. metadata/generated/schema/system/eventPublisherJob.py +18 -7
  749. metadata/generated/schema/system/indexingError.py +1 -1
  750. metadata/generated/schema/system/limitsResponse.py +1 -1
  751. metadata/generated/schema/system/ui/__init__.py +1 -1
  752. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  753. metadata/generated/schema/system/ui/navigationItem.py +38 -0
  754. metadata/generated/schema/system/ui/page.py +23 -15
  755. metadata/generated/schema/system/ui/tab.py +36 -0
  756. metadata/generated/schema/system/ui/uiCustomization.py +45 -0
  757. metadata/generated/schema/system/validationResponse.py +1 -1
  758. metadata/generated/schema/tests/__init__.py +1 -1
  759. metadata/generated/schema/tests/assigned.py +1 -1
  760. metadata/generated/schema/tests/basic.py +1 -1
  761. metadata/generated/schema/tests/customMetric.py +1 -1
  762. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  763. metadata/generated/schema/tests/resolved.py +1 -1
  764. metadata/generated/schema/tests/testCase.py +4 -1
  765. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  766. metadata/generated/schema/tests/testDefinition.py +1 -1
  767. metadata/generated/schema/tests/testSuite.py +1 -1
  768. metadata/generated/schema/type/__init__.py +1 -1
  769. metadata/generated/schema/type/apiSchema.py +1 -1
  770. metadata/generated/schema/type/assetCertification.py +1 -1
  771. metadata/generated/schema/type/auditLog.py +1 -1
  772. metadata/generated/schema/type/basic.py +3 -3
  773. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  774. metadata/generated/schema/type/changeEvent.py +1 -1
  775. metadata/generated/schema/type/changeEventType.py +1 -1
  776. metadata/generated/schema/type/changeSummaryMap.py +37 -0
  777. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  778. metadata/generated/schema/type/csvDocumentation.py +1 -1
  779. metadata/generated/schema/type/csvErrorType.py +1 -1
  780. metadata/generated/schema/type/csvFile.py +1 -1
  781. metadata/generated/schema/type/csvImportResult.py +1 -1
  782. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  783. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  784. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  785. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  786. metadata/generated/schema/type/customProperty.py +1 -1
  787. metadata/generated/schema/type/dailyCount.py +1 -1
  788. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  789. metadata/generated/schema/type/entityHierarchy.py +1 -1
  790. metadata/generated/schema/type/entityHistory.py +6 -3
  791. metadata/generated/schema/type/entityLineage.py +27 -1
  792. metadata/generated/schema/type/entityReference.py +1 -1
  793. metadata/generated/schema/type/entityReferenceList.py +1 -1
  794. metadata/generated/schema/type/entityRelationship.py +1 -1
  795. metadata/generated/schema/type/entityUsage.py +1 -1
  796. metadata/generated/schema/type/filterPattern.py +3 -3
  797. metadata/generated/schema/type/function.py +1 -1
  798. metadata/generated/schema/type/include.py +1 -1
  799. metadata/generated/schema/type/jdbcConnection.py +1 -1
  800. metadata/generated/schema/type/lifeCycle.py +1 -1
  801. metadata/generated/schema/type/paging.py +1 -1
  802. metadata/generated/schema/type/profile.py +1 -1
  803. metadata/generated/schema/type/queryParserData.py +4 -1
  804. metadata/generated/schema/type/reaction.py +1 -1
  805. metadata/generated/schema/type/schedule.py +1 -1
  806. metadata/generated/schema/type/schema.py +1 -1
  807. metadata/generated/schema/type/tableQuery.py +4 -1
  808. metadata/generated/schema/type/tableUsageCount.py +18 -1
  809. metadata/generated/schema/type/tagLabel.py +1 -1
  810. metadata/generated/schema/type/usageDetails.py +1 -1
  811. metadata/generated/schema/type/usageRequest.py +1 -1
  812. metadata/generated/schema/type/votes.py +1 -1
  813. metadata/great_expectations/action.py +2 -2
  814. metadata/great_expectations/utils/ometa_config_handler.py +2 -2
  815. metadata/ingestion/api/closeable.py +3 -3
  816. metadata/ingestion/api/common.py +3 -3
  817. metadata/ingestion/api/delete.py +3 -3
  818. metadata/ingestion/api/models.py +3 -3
  819. metadata/ingestion/api/parser.py +3 -3
  820. metadata/ingestion/api/status.py +3 -3
  821. metadata/ingestion/api/step.py +3 -3
  822. metadata/ingestion/api/steps.py +3 -3
  823. metadata/ingestion/api/topology_runner.py +6 -6
  824. metadata/ingestion/bulksink/metadata_usage.py +30 -9
  825. metadata/ingestion/connections/builders.py +3 -3
  826. metadata/ingestion/connections/headers.py +3 -3
  827. metadata/ingestion/connections/secrets.py +3 -3
  828. metadata/ingestion/connections/session.py +3 -3
  829. metadata/ingestion/connections/test_connections.py +3 -3
  830. metadata/ingestion/lineage/masker.py +16 -5
  831. metadata/ingestion/lineage/models.py +3 -3
  832. metadata/ingestion/lineage/parser.py +17 -4
  833. metadata/ingestion/lineage/sql_lineage.py +3 -3
  834. metadata/ingestion/models/custom_basemodel_validation.py +2 -2
  835. metadata/ingestion/models/custom_properties.py +3 -3
  836. metadata/ingestion/models/custom_pydantic.py +46 -8
  837. metadata/ingestion/models/custom_types.py +2 -2
  838. metadata/ingestion/models/data_insight.py +3 -3
  839. metadata/ingestion/models/delete_entity.py +3 -3
  840. metadata/ingestion/models/encoders.py +3 -3
  841. metadata/ingestion/models/entity_interface.py +3 -3
  842. metadata/ingestion/models/lf_tags_model.py +3 -3
  843. metadata/ingestion/models/life_cycle.py +3 -3
  844. metadata/ingestion/models/ometa_classification.py +3 -3
  845. metadata/ingestion/models/ometa_lineage.py +3 -3
  846. metadata/ingestion/models/ometa_topic_data.py +3 -3
  847. metadata/ingestion/models/patch_request.py +3 -3
  848. metadata/ingestion/models/pipeline_status.py +3 -3
  849. metadata/ingestion/models/profile_data.py +3 -3
  850. metadata/ingestion/models/search_index_data.py +3 -3
  851. metadata/ingestion/models/table_metadata.py +3 -3
  852. metadata/ingestion/models/tests_data.py +3 -3
  853. metadata/ingestion/models/topology.py +3 -3
  854. metadata/ingestion/models/user.py +3 -3
  855. metadata/ingestion/ometa/auth_provider.py +3 -3
  856. metadata/ingestion/ometa/client.py +23 -3
  857. metadata/ingestion/ometa/client_utils.py +3 -3
  858. metadata/ingestion/ometa/credentials.py +3 -3
  859. metadata/ingestion/ometa/mixins/custom_property_mixin.py +3 -3
  860. metadata/ingestion/ometa/mixins/dashboard_mixin.py +3 -3
  861. metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -3
  862. metadata/ingestion/ometa/mixins/domain_mixin.py +80 -0
  863. metadata/ingestion/ometa/mixins/es_mixin.py +45 -5
  864. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +3 -3
  865. metadata/ingestion/ometa/mixins/lineage_mixin.py +23 -44
  866. metadata/ingestion/ometa/mixins/mlmodel_mixin.py +3 -3
  867. metadata/ingestion/ometa/mixins/patch_mixin.py +18 -8
  868. metadata/ingestion/ometa/mixins/patch_mixin_utils.py +2 -2
  869. metadata/ingestion/ometa/mixins/pipeline_mixin.py +21 -3
  870. metadata/ingestion/ometa/mixins/query_mixin.py +46 -3
  871. metadata/ingestion/ometa/mixins/role_policy_mixin.py +2 -2
  872. metadata/ingestion/ometa/mixins/search_index_mixin.py +3 -3
  873. metadata/ingestion/ometa/mixins/server_mixin.py +3 -3
  874. metadata/ingestion/ometa/mixins/service_mixin.py +3 -3
  875. metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
  876. metadata/ingestion/ometa/mixins/table_mixin.py +3 -3
  877. metadata/ingestion/ometa/mixins/tests_mixin.py +3 -3
  878. metadata/ingestion/ometa/mixins/topic_mixin.py +3 -3
  879. metadata/ingestion/ometa/mixins/user_mixin.py +3 -3
  880. metadata/ingestion/ometa/mixins/version_mixin.py +3 -3
  881. metadata/ingestion/ometa/models.py +3 -3
  882. metadata/ingestion/ometa/ometa_api.py +31 -6
  883. metadata/ingestion/ometa/routes.py +10 -3
  884. metadata/ingestion/ometa/ttl_cache.py +3 -3
  885. metadata/ingestion/ometa/utils.py +16 -3
  886. metadata/ingestion/processor/query_parser.py +4 -3
  887. metadata/ingestion/sink/file.py +3 -3
  888. metadata/ingestion/sink/metadata_rest.py +24 -4
  889. metadata/ingestion/source/api/api_service.py +2 -2
  890. metadata/ingestion/source/api/rest/connection.py +2 -2
  891. metadata/ingestion/source/api/rest/metadata.py +2 -2
  892. metadata/ingestion/source/api/rest/models.py +2 -2
  893. metadata/ingestion/source/connections.py +3 -3
  894. metadata/ingestion/source/dashboard/dashboard_service.py +12 -4
  895. metadata/ingestion/source/dashboard/domodashboard/connection.py +3 -3
  896. metadata/ingestion/source/dashboard/domodashboard/metadata.py +6 -4
  897. metadata/ingestion/source/dashboard/lightdash/client.py +3 -3
  898. metadata/ingestion/source/dashboard/lightdash/connection.py +3 -3
  899. metadata/ingestion/source/dashboard/lightdash/metadata.py +4 -4
  900. metadata/ingestion/source/dashboard/looker/bulk_parser.py +3 -3
  901. metadata/ingestion/source/dashboard/looker/columns.py +3 -3
  902. metadata/ingestion/source/dashboard/looker/connection.py +3 -3
  903. metadata/ingestion/source/dashboard/looker/links.py +3 -3
  904. metadata/ingestion/source/dashboard/looker/metadata.py +11 -5
  905. metadata/ingestion/source/dashboard/looker/models.py +3 -3
  906. metadata/ingestion/source/dashboard/looker/parser.py +3 -3
  907. metadata/ingestion/source/dashboard/looker/utils.py +3 -3
  908. metadata/ingestion/source/dashboard/metabase/client.py +68 -9
  909. metadata/ingestion/source/dashboard/metabase/connection.py +3 -3
  910. metadata/ingestion/source/dashboard/metabase/metadata.py +91 -40
  911. metadata/ingestion/source/dashboard/metabase/models.py +5 -4
  912. metadata/ingestion/source/dashboard/microstrategy/client.py +2 -2
  913. metadata/ingestion/source/dashboard/microstrategy/connection.py +3 -3
  914. metadata/ingestion/source/dashboard/microstrategy/metadata.py +6 -4
  915. metadata/ingestion/source/dashboard/microstrategy/models.py +3 -3
  916. metadata/ingestion/source/dashboard/mode/client.py +3 -3
  917. metadata/ingestion/source/dashboard/mode/connection.py +3 -3
  918. metadata/ingestion/source/dashboard/mode/metadata.py +19 -12
  919. metadata/ingestion/source/dashboard/powerbi/client.py +3 -3
  920. metadata/ingestion/source/dashboard/powerbi/connection.py +3 -3
  921. metadata/ingestion/source/dashboard/powerbi/file_client.py +3 -3
  922. metadata/ingestion/source/dashboard/powerbi/metadata.py +9 -10
  923. metadata/ingestion/source/dashboard/powerbi/models.py +2 -2
  924. metadata/ingestion/source/dashboard/qlikcloud/client.py +28 -5
  925. metadata/ingestion/source/dashboard/qlikcloud/connection.py +3 -3
  926. metadata/ingestion/source/dashboard/qlikcloud/constants.py +3 -3
  927. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +72 -18
  928. metadata/ingestion/source/dashboard/qlikcloud/models.py +47 -4
  929. metadata/ingestion/source/dashboard/qliksense/client.py +3 -3
  930. metadata/ingestion/source/dashboard/qliksense/connection.py +3 -3
  931. metadata/ingestion/source/dashboard/qliksense/constants.py +3 -3
  932. metadata/ingestion/source/dashboard/qliksense/metadata.py +29 -21
  933. metadata/ingestion/source/dashboard/qliksense/models.py +3 -3
  934. metadata/ingestion/source/dashboard/quicksight/connection.py +3 -3
  935. metadata/ingestion/source/dashboard/quicksight/metadata.py +35 -32
  936. metadata/ingestion/source/dashboard/quicksight/models.py +3 -3
  937. metadata/ingestion/source/dashboard/redash/client.py +3 -3
  938. metadata/ingestion/source/dashboard/redash/connection.py +3 -3
  939. metadata/ingestion/source/dashboard/redash/metadata.py +15 -13
  940. metadata/ingestion/source/dashboard/sigma/client.py +3 -3
  941. metadata/ingestion/source/dashboard/sigma/connection.py +3 -3
  942. metadata/ingestion/source/dashboard/sigma/metadata.py +15 -17
  943. metadata/ingestion/source/dashboard/sigma/models.py +2 -2
  944. metadata/ingestion/source/dashboard/superset/api_source.py +28 -29
  945. metadata/ingestion/source/dashboard/superset/client.py +3 -3
  946. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  947. metadata/ingestion/source/dashboard/superset/db_source.py +25 -23
  948. metadata/ingestion/source/dashboard/superset/metadata.py +3 -3
  949. metadata/ingestion/source/dashboard/superset/mixin.py +210 -56
  950. metadata/ingestion/source/dashboard/superset/models.py +6 -3
  951. metadata/ingestion/source/dashboard/superset/queries.py +10 -7
  952. metadata/ingestion/source/dashboard/superset/utils.py +36 -0
  953. metadata/ingestion/source/dashboard/tableau/__init__.py +3 -3
  954. metadata/ingestion/source/dashboard/tableau/client.py +59 -4
  955. metadata/ingestion/source/dashboard/tableau/connection.py +3 -3
  956. metadata/ingestion/source/dashboard/tableau/metadata.py +243 -128
  957. metadata/ingestion/source/dashboard/tableau/models.py +3 -3
  958. metadata/ingestion/source/dashboard/tableau/queries.py +3 -3
  959. metadata/ingestion/source/database/athena/client.py +3 -3
  960. metadata/ingestion/source/database/athena/connection.py +3 -3
  961. metadata/ingestion/source/database/athena/lineage.py +3 -3
  962. metadata/ingestion/source/database/athena/metadata.py +3 -3
  963. metadata/ingestion/source/database/athena/models.py +3 -3
  964. metadata/ingestion/source/database/athena/query_parser.py +3 -3
  965. metadata/ingestion/source/database/athena/usage.py +3 -3
  966. metadata/ingestion/source/database/athena/utils.py +3 -3
  967. metadata/ingestion/source/database/azuresql/connection.py +3 -3
  968. metadata/ingestion/source/database/azuresql/lineage.py +3 -3
  969. metadata/ingestion/source/database/azuresql/metadata.py +3 -3
  970. metadata/ingestion/source/database/azuresql/queries.py +3 -3
  971. metadata/ingestion/source/database/azuresql/query_parser.py +3 -3
  972. metadata/ingestion/source/database/azuresql/usage.py +3 -3
  973. metadata/ingestion/source/database/bigquery/connection.py +3 -3
  974. metadata/ingestion/source/database/bigquery/helper.py +12 -4
  975. metadata/ingestion/source/database/bigquery/incremental_table_processor.py +3 -3
  976. metadata/ingestion/source/database/bigquery/lineage.py +3 -3
  977. metadata/ingestion/source/database/bigquery/metadata.py +85 -13
  978. metadata/ingestion/source/database/bigquery/models.py +3 -3
  979. metadata/ingestion/source/database/bigquery/queries.py +5 -4
  980. metadata/ingestion/source/database/bigquery/query_parser.py +4 -3
  981. metadata/ingestion/source/database/bigquery/usage.py +3 -3
  982. metadata/ingestion/source/database/bigtable/client.py +2 -2
  983. metadata/ingestion/source/database/bigtable/connection.py +2 -2
  984. metadata/ingestion/source/database/bigtable/metadata.py +13 -5
  985. metadata/ingestion/source/database/bigtable/models.py +2 -2
  986. metadata/ingestion/source/database/cassandra/__init__.py +0 -0
  987. metadata/ingestion/source/database/cassandra/connection.py +138 -0
  988. metadata/ingestion/source/database/cassandra/helpers.py +100 -0
  989. metadata/ingestion/source/database/cassandra/metadata.py +138 -0
  990. metadata/ingestion/source/database/cassandra/queries.py +45 -0
  991. metadata/ingestion/source/database/cassandra/service_spec.py +10 -0
  992. metadata/ingestion/source/database/clickhouse/connection.py +3 -3
  993. metadata/ingestion/source/database/clickhouse/lineage.py +3 -3
  994. metadata/ingestion/source/database/clickhouse/metadata.py +3 -3
  995. metadata/ingestion/source/database/clickhouse/queries.py +3 -3
  996. metadata/ingestion/source/database/clickhouse/query_parser.py +3 -3
  997. metadata/ingestion/source/database/clickhouse/usage.py +3 -3
  998. metadata/ingestion/source/database/clickhouse/utils.py +3 -3
  999. metadata/ingestion/source/database/cockroach/__init__.py +0 -0
  1000. metadata/ingestion/source/database/cockroach/connection.py +71 -0
  1001. metadata/ingestion/source/database/cockroach/metadata.py +206 -0
  1002. metadata/ingestion/source/database/cockroach/queries.py +90 -0
  1003. metadata/ingestion/source/database/cockroach/service_spec.py +4 -0
  1004. metadata/ingestion/source/database/column_helpers.py +3 -3
  1005. metadata/ingestion/source/database/column_type_parser.py +3 -3
  1006. metadata/ingestion/source/database/common_db_source.py +12 -3
  1007. metadata/ingestion/source/database/common_nosql_source.py +81 -32
  1008. metadata/ingestion/source/database/common_pg_mappings.py +49 -0
  1009. metadata/ingestion/source/database/couchbase/connection.py +3 -3
  1010. metadata/ingestion/source/database/couchbase/metadata.py +11 -5
  1011. metadata/ingestion/source/database/couchbase/models.py +2 -2
  1012. metadata/ingestion/source/database/couchbase/queries.py +3 -3
  1013. metadata/ingestion/source/database/database_service.py +3 -3
  1014. metadata/ingestion/source/database/databricks/client.py +3 -3
  1015. metadata/ingestion/source/database/databricks/connection.py +3 -3
  1016. metadata/ingestion/source/database/databricks/lineage.py +3 -3
  1017. metadata/ingestion/source/database/databricks/metadata.py +11 -3
  1018. metadata/ingestion/source/database/databricks/queries.py +3 -3
  1019. metadata/ingestion/source/database/databricks/query_parser.py +3 -3
  1020. metadata/ingestion/source/database/databricks/usage.py +3 -3
  1021. metadata/ingestion/source/database/datalake/clients/azure_blob.py +3 -3
  1022. metadata/ingestion/source/database/datalake/clients/base.py +3 -3
  1023. metadata/ingestion/source/database/datalake/clients/gcs.py +3 -3
  1024. metadata/ingestion/source/database/datalake/clients/s3.py +3 -3
  1025. metadata/ingestion/source/database/datalake/columns.py +3 -3
  1026. metadata/ingestion/source/database/datalake/connection.py +3 -3
  1027. metadata/ingestion/source/database/datalake/metadata.py +18 -12
  1028. metadata/ingestion/source/database/db2/connection.py +3 -3
  1029. metadata/ingestion/source/database/db2/lineage.py +3 -3
  1030. metadata/ingestion/source/database/db2/metadata.py +3 -3
  1031. metadata/ingestion/source/database/db2/service_spec.py +4 -1
  1032. metadata/ingestion/source/database/db2/utils.py +3 -3
  1033. metadata/ingestion/source/database/dbt/constants.py +3 -3
  1034. metadata/ingestion/source/database/dbt/dbt_config.py +4 -3
  1035. metadata/ingestion/source/database/dbt/dbt_service.py +3 -3
  1036. metadata/ingestion/source/database/dbt/dbt_utils.py +4 -4
  1037. metadata/ingestion/source/database/dbt/metadata.py +8 -4
  1038. metadata/ingestion/source/database/dbt/models.py +3 -3
  1039. metadata/ingestion/source/database/deltalake/clients/base.py +3 -3
  1040. metadata/ingestion/source/database/deltalake/clients/pyspark.py +3 -3
  1041. metadata/ingestion/source/database/deltalake/clients/s3.py +3 -3
  1042. metadata/ingestion/source/database/deltalake/connection.py +3 -3
  1043. metadata/ingestion/source/database/deltalake/metadata.py +3 -3
  1044. metadata/ingestion/source/database/domodatabase/connection.py +3 -3
  1045. metadata/ingestion/source/database/domodatabase/metadata.py +3 -3
  1046. metadata/ingestion/source/database/domodatabase/models.py +3 -3
  1047. metadata/ingestion/source/database/doris/connection.py +3 -3
  1048. metadata/ingestion/source/database/doris/metadata.py +10 -5
  1049. metadata/ingestion/source/database/doris/utils.py +3 -3
  1050. metadata/ingestion/source/database/druid/connection.py +3 -3
  1051. metadata/ingestion/source/database/druid/lineage.py +3 -3
  1052. metadata/ingestion/source/database/druid/metadata.py +3 -3
  1053. metadata/ingestion/source/database/druid/service_spec.py +5 -1
  1054. metadata/ingestion/source/database/dynamodb/connection.py +3 -3
  1055. metadata/ingestion/source/database/dynamodb/metadata.py +9 -6
  1056. metadata/ingestion/source/database/dynamodb/models.py +2 -2
  1057. metadata/ingestion/source/database/extended_sample_data.py +3 -3
  1058. metadata/ingestion/source/database/external_table_lineage_mixin.py +3 -3
  1059. metadata/ingestion/source/database/glue/connection.py +3 -3
  1060. metadata/ingestion/source/database/glue/metadata.py +3 -3
  1061. metadata/ingestion/source/database/glue/models.py +3 -3
  1062. metadata/ingestion/source/database/greenplum/connection.py +3 -3
  1063. metadata/ingestion/source/database/greenplum/lineage.py +3 -3
  1064. metadata/ingestion/source/database/greenplum/metadata.py +9 -40
  1065. metadata/ingestion/source/database/greenplum/queries.py +3 -3
  1066. metadata/ingestion/source/database/greenplum/service_spec.py +5 -1
  1067. metadata/ingestion/source/database/greenplum/utils.py +3 -3
  1068. metadata/ingestion/source/database/hive/connection.py +3 -3
  1069. metadata/ingestion/source/database/hive/lineage.py +3 -3
  1070. metadata/ingestion/source/database/hive/metadata.py +3 -3
  1071. metadata/ingestion/source/database/hive/metastore_dialects/mixin.py +3 -3
  1072. metadata/ingestion/source/database/hive/metastore_dialects/mysql/__init__.py +3 -3
  1073. metadata/ingestion/source/database/hive/metastore_dialects/mysql/dialect.py +38 -16
  1074. metadata/ingestion/source/database/hive/metastore_dialects/postgres/__init__.py +3 -3
  1075. metadata/ingestion/source/database/hive/metastore_dialects/postgres/dialect.py +41 -17
  1076. metadata/ingestion/source/database/hive/queries.py +3 -3
  1077. metadata/ingestion/source/database/hive/service_spec.py +5 -1
  1078. metadata/ingestion/source/database/hive/utils.py +3 -3
  1079. metadata/ingestion/source/database/iceberg/catalog/__init__.py +3 -3
  1080. metadata/ingestion/source/database/iceberg/catalog/base.py +3 -3
  1081. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +3 -3
  1082. metadata/ingestion/source/database/iceberg/catalog/glue.py +3 -3
  1083. metadata/ingestion/source/database/iceberg/catalog/hive.py +3 -3
  1084. metadata/ingestion/source/database/iceberg/catalog/rest.py +3 -3
  1085. metadata/ingestion/source/database/iceberg/connection.py +3 -3
  1086. metadata/ingestion/source/database/iceberg/fs/__init__.py +3 -3
  1087. metadata/ingestion/source/database/iceberg/fs/azure.py +3 -3
  1088. metadata/ingestion/source/database/iceberg/fs/base.py +3 -3
  1089. metadata/ingestion/source/database/iceberg/fs/s3.py +3 -3
  1090. metadata/ingestion/source/database/iceberg/helper.py +3 -3
  1091. metadata/ingestion/source/database/iceberg/metadata.py +3 -3
  1092. metadata/ingestion/source/database/iceberg/models.py +3 -3
  1093. metadata/ingestion/source/database/impala/connection.py +3 -3
  1094. metadata/ingestion/source/database/impala/lineage.py +3 -3
  1095. metadata/ingestion/source/database/impala/metadata.py +3 -3
  1096. metadata/ingestion/source/database/impala/queries.py +3 -3
  1097. metadata/ingestion/source/database/impala/service_spec.py +5 -1
  1098. metadata/ingestion/source/database/incremental_metadata_extraction.py +3 -3
  1099. metadata/ingestion/source/database/life_cycle_query_mixin.py +14 -10
  1100. metadata/ingestion/source/database/lineage_source.py +173 -47
  1101. metadata/ingestion/source/database/mariadb/connection.py +3 -3
  1102. metadata/ingestion/source/database/mariadb/lineage.py +3 -3
  1103. metadata/ingestion/source/database/mariadb/metadata.py +3 -3
  1104. metadata/ingestion/source/database/mariadb/service_spec.py +9 -1
  1105. metadata/ingestion/source/database/mongodb/connection.py +3 -3
  1106. metadata/ingestion/source/database/mongodb/metadata.py +11 -6
  1107. metadata/ingestion/source/database/mssql/connection.py +3 -3
  1108. metadata/ingestion/source/database/mssql/constants.py +3 -3
  1109. metadata/ingestion/source/database/mssql/lineage.py +3 -3
  1110. metadata/ingestion/source/database/mssql/metadata.py +71 -5
  1111. metadata/ingestion/source/database/mssql/models.py +3 -3
  1112. metadata/ingestion/source/database/mssql/queries.py +46 -3
  1113. metadata/ingestion/source/database/mssql/query_parser.py +3 -3
  1114. metadata/ingestion/source/database/mssql/usage.py +3 -3
  1115. metadata/ingestion/source/database/mssql/utils.py +3 -3
  1116. metadata/ingestion/source/database/multi_db_source.py +3 -3
  1117. metadata/ingestion/source/database/mysql/connection.py +3 -3
  1118. metadata/ingestion/source/database/mysql/lineage.py +3 -3
  1119. metadata/ingestion/source/database/mysql/metadata.py +3 -3
  1120. metadata/ingestion/source/database/mysql/queries.py +3 -3
  1121. metadata/ingestion/source/database/mysql/query_parser.py +3 -3
  1122. metadata/ingestion/source/database/mysql/usage.py +3 -3
  1123. metadata/ingestion/source/database/mysql/utils.py +8 -6
  1124. metadata/ingestion/source/database/oracle/connection.py +27 -5
  1125. metadata/ingestion/source/database/oracle/lineage.py +3 -3
  1126. metadata/ingestion/source/database/oracle/metadata.py +39 -22
  1127. metadata/ingestion/source/database/oracle/models.py +5 -4
  1128. metadata/ingestion/source/database/oracle/queries.py +25 -4
  1129. metadata/ingestion/source/database/oracle/query_parser.py +3 -3
  1130. metadata/ingestion/source/database/oracle/usage.py +3 -3
  1131. metadata/ingestion/source/database/oracle/utils.py +3 -3
  1132. metadata/ingestion/source/database/pinotdb/connection.py +3 -3
  1133. metadata/ingestion/source/database/pinotdb/lineage.py +3 -3
  1134. metadata/ingestion/source/database/pinotdb/metadata.py +3 -3
  1135. metadata/ingestion/source/database/pinotdb/service_spec.py +5 -1
  1136. metadata/ingestion/source/database/postgres/connection.py +3 -3
  1137. metadata/ingestion/source/database/postgres/converter_orm.py +3 -3
  1138. metadata/ingestion/source/database/postgres/lineage.py +3 -3
  1139. metadata/ingestion/source/database/postgres/metadata.py +35 -59
  1140. metadata/ingestion/source/database/postgres/models.py +3 -2
  1141. metadata/ingestion/source/database/postgres/pgspider/lineage.py +2 -2
  1142. metadata/ingestion/source/database/postgres/pgspider/queries.py +2 -2
  1143. metadata/ingestion/source/database/postgres/queries.py +21 -4
  1144. metadata/ingestion/source/database/postgres/query_parser.py +3 -3
  1145. metadata/ingestion/source/database/postgres/types/money.py +3 -3
  1146. metadata/ingestion/source/database/postgres/usage.py +3 -3
  1147. metadata/ingestion/source/database/postgres/utils.py +3 -3
  1148. metadata/ingestion/source/database/presto/connection.py +3 -3
  1149. metadata/ingestion/source/database/presto/metadata.py +3 -3
  1150. metadata/ingestion/source/database/presto/queries.py +3 -3
  1151. metadata/ingestion/source/database/query/lineage.py +3 -3
  1152. metadata/ingestion/source/database/query/usage.py +3 -3
  1153. metadata/ingestion/source/database/query_parser_source.py +14 -9
  1154. metadata/ingestion/source/database/redshift/connection.py +3 -3
  1155. metadata/ingestion/source/database/redshift/incremental_table_processor.py +3 -3
  1156. metadata/ingestion/source/database/redshift/lineage.py +3 -3
  1157. metadata/ingestion/source/database/redshift/metadata.py +3 -3
  1158. metadata/ingestion/source/database/redshift/models.py +3 -3
  1159. metadata/ingestion/source/database/redshift/queries.py +3 -3
  1160. metadata/ingestion/source/database/redshift/query_parser.py +3 -3
  1161. metadata/ingestion/source/database/redshift/usage.py +3 -3
  1162. metadata/ingestion/source/database/redshift/utils.py +3 -3
  1163. metadata/ingestion/source/database/salesforce/connection.py +3 -3
  1164. metadata/ingestion/source/database/salesforce/metadata.py +3 -3
  1165. metadata/ingestion/source/database/sample_data.py +14 -3
  1166. metadata/ingestion/source/database/sample_usage.py +5 -3
  1167. metadata/ingestion/source/database/saperp/client.py +3 -3
  1168. metadata/ingestion/source/database/saperp/connection.py +3 -3
  1169. metadata/ingestion/source/database/saperp/constants.py +2 -2
  1170. metadata/ingestion/source/database/saperp/metadata.py +3 -3
  1171. metadata/ingestion/source/database/saperp/models.py +3 -3
  1172. metadata/ingestion/source/database/saphana/cdata_parser.py +3 -3
  1173. metadata/ingestion/source/database/saphana/connection.py +3 -3
  1174. metadata/ingestion/source/database/saphana/lineage.py +4 -4
  1175. metadata/ingestion/source/database/saphana/metadata.py +3 -3
  1176. metadata/ingestion/source/database/saphana/models.py +3 -3
  1177. metadata/ingestion/source/database/saphana/queries.py +3 -3
  1178. metadata/ingestion/source/database/sas/client.py +3 -3
  1179. metadata/ingestion/source/database/sas/connection.py +3 -3
  1180. metadata/ingestion/source/database/sas/extension_attr.py +3 -3
  1181. metadata/ingestion/source/database/sas/metadata.py +3 -3
  1182. metadata/ingestion/source/database/singlestore/connection.py +3 -3
  1183. metadata/ingestion/source/database/singlestore/lineage.py +3 -3
  1184. metadata/ingestion/source/database/singlestore/metadata.py +3 -3
  1185. metadata/ingestion/source/database/singlestore/service_spec.py +6 -1
  1186. metadata/ingestion/source/database/snowflake/connection.py +9 -3
  1187. metadata/ingestion/source/database/snowflake/constants.py +48 -0
  1188. metadata/ingestion/source/database/snowflake/lineage.py +3 -3
  1189. metadata/ingestion/source/database/snowflake/metadata.py +119 -6
  1190. metadata/ingestion/source/database/snowflake/models.py +6 -6
  1191. metadata/ingestion/source/database/snowflake/profiler/profiler.py +3 -3
  1192. metadata/ingestion/source/database/snowflake/queries.py +30 -4
  1193. metadata/ingestion/source/database/snowflake/query_parser.py +5 -3
  1194. metadata/ingestion/source/database/snowflake/usage.py +3 -3
  1195. metadata/ingestion/source/database/snowflake/utils.py +113 -29
  1196. metadata/ingestion/source/database/sql_column_handler.py +98 -63
  1197. metadata/ingestion/source/database/sqlalchemy_source.py +3 -3
  1198. metadata/ingestion/source/database/sqlite/connection.py +3 -3
  1199. metadata/ingestion/source/database/sqlite/lineage.py +3 -3
  1200. metadata/ingestion/source/database/sqlite/metadata.py +3 -3
  1201. metadata/ingestion/source/database/sqlite/service_spec.py +5 -1
  1202. metadata/ingestion/source/database/stored_procedures_mixin.py +6 -4
  1203. metadata/ingestion/source/database/teradata/connection.py +3 -3
  1204. metadata/ingestion/source/database/teradata/lineage.py +3 -3
  1205. metadata/ingestion/source/database/teradata/metadata.py +3 -3
  1206. metadata/ingestion/source/database/teradata/queries.py +3 -3
  1207. metadata/ingestion/source/database/teradata/service_spec.py +5 -1
  1208. metadata/ingestion/source/database/teradata/utils.py +3 -3
  1209. metadata/ingestion/source/database/trino/connection.py +3 -3
  1210. metadata/ingestion/source/database/trino/lineage.py +98 -3
  1211. metadata/ingestion/source/database/trino/metadata.py +3 -3
  1212. metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py +3 -3
  1213. metadata/ingestion/source/database/trino/queries.py +3 -3
  1214. metadata/ingestion/source/database/trino/query_parser.py +3 -3
  1215. metadata/ingestion/source/database/trino/usage.py +3 -3
  1216. metadata/ingestion/source/database/unitycatalog/client.py +3 -3
  1217. metadata/ingestion/source/database/unitycatalog/connection.py +26 -3
  1218. metadata/ingestion/source/database/unitycatalog/lineage.py +5 -3
  1219. metadata/ingestion/source/database/unitycatalog/metadata.py +5 -3
  1220. metadata/ingestion/source/database/unitycatalog/models.py +3 -3
  1221. metadata/ingestion/source/database/unitycatalog/query_parser.py +3 -3
  1222. metadata/ingestion/source/database/unitycatalog/usage.py +3 -3
  1223. metadata/ingestion/source/database/usage_source.py +5 -3
  1224. metadata/ingestion/source/database/vertica/connection.py +3 -3
  1225. metadata/ingestion/source/database/vertica/lineage.py +3 -3
  1226. metadata/ingestion/source/database/vertica/metadata.py +3 -3
  1227. metadata/ingestion/source/database/vertica/queries.py +3 -3
  1228. metadata/ingestion/source/database/vertica/query_parser.py +3 -3
  1229. metadata/ingestion/source/database/vertica/usage.py +3 -3
  1230. metadata/ingestion/source/messaging/common_broker_source.py +3 -3
  1231. metadata/ingestion/source/messaging/kafka/connection.py +3 -3
  1232. metadata/ingestion/source/messaging/kafka/metadata.py +3 -3
  1233. metadata/ingestion/source/messaging/kinesis/connection.py +3 -3
  1234. metadata/ingestion/source/messaging/kinesis/metadata.py +4 -5
  1235. metadata/ingestion/source/messaging/kinesis/models.py +2 -2
  1236. metadata/ingestion/source/messaging/messaging_service.py +5 -3
  1237. metadata/ingestion/source/messaging/redpanda/connection.py +3 -3
  1238. metadata/ingestion/source/messaging/redpanda/metadata.py +3 -3
  1239. metadata/ingestion/source/metadata/alationsink/client.py +2 -2
  1240. metadata/ingestion/source/metadata/alationsink/connection.py +2 -2
  1241. metadata/ingestion/source/metadata/alationsink/constants.py +2 -2
  1242. metadata/ingestion/source/metadata/alationsink/metadata.py +34 -20
  1243. metadata/ingestion/source/metadata/alationsink/models.py +2 -2
  1244. metadata/ingestion/source/metadata/amundsen/client.py +3 -3
  1245. metadata/ingestion/source/metadata/amundsen/connection.py +3 -3
  1246. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  1247. metadata/ingestion/source/metadata/amundsen/queries.py +3 -3
  1248. metadata/ingestion/source/metadata/atlas/client.py +3 -3
  1249. metadata/ingestion/source/metadata/atlas/connection.py +3 -3
  1250. metadata/ingestion/source/metadata/atlas/metadata.py +5 -4
  1251. metadata/ingestion/source/mlmodel/mlflow/connection.py +3 -3
  1252. metadata/ingestion/source/mlmodel/mlflow/metadata.py +3 -3
  1253. metadata/ingestion/source/mlmodel/mlmodel_service.py +5 -3
  1254. metadata/ingestion/source/mlmodel/sagemaker/connection.py +3 -3
  1255. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +3 -3
  1256. metadata/ingestion/source/models.py +3 -3
  1257. metadata/ingestion/source/pipeline/airbyte/client.py +3 -3
  1258. metadata/ingestion/source/pipeline/airbyte/connection.py +3 -3
  1259. metadata/ingestion/source/pipeline/airbyte/metadata.py +3 -3
  1260. metadata/ingestion/source/pipeline/airflow/connection.py +3 -3
  1261. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +3 -3
  1262. metadata/ingestion/source/pipeline/airflow/metadata.py +28 -3
  1263. metadata/ingestion/source/pipeline/airflow/models.py +4 -3
  1264. metadata/ingestion/source/pipeline/airflow/utils.py +3 -3
  1265. metadata/ingestion/source/pipeline/dagster/client.py +3 -3
  1266. metadata/ingestion/source/pipeline/dagster/connection.py +3 -3
  1267. metadata/ingestion/source/pipeline/dagster/metadata.py +3 -3
  1268. metadata/ingestion/source/pipeline/dagster/models.py +3 -3
  1269. metadata/ingestion/source/pipeline/dagster/queries.py +3 -3
  1270. metadata/ingestion/source/pipeline/databrickspipeline/connection.py +3 -3
  1271. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +3 -3
  1272. metadata/ingestion/source/pipeline/databrickspipeline/models.py +3 -3
  1273. metadata/ingestion/source/pipeline/dbtcloud/client.py +3 -3
  1274. metadata/ingestion/source/pipeline/dbtcloud/connection.py +3 -3
  1275. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +3 -3
  1276. metadata/ingestion/source/pipeline/dbtcloud/models.py +3 -3
  1277. metadata/ingestion/source/pipeline/dbtcloud/queries.py +3 -3
  1278. metadata/ingestion/source/pipeline/domopipeline/connection.py +3 -3
  1279. metadata/ingestion/source/pipeline/domopipeline/metadata.py +3 -3
  1280. metadata/ingestion/source/pipeline/fivetran/client.py +3 -3
  1281. metadata/ingestion/source/pipeline/fivetran/connection.py +3 -3
  1282. metadata/ingestion/source/pipeline/fivetran/metadata.py +3 -3
  1283. metadata/ingestion/source/pipeline/flink/client.py +3 -3
  1284. metadata/ingestion/source/pipeline/flink/connection.py +3 -3
  1285. metadata/ingestion/source/pipeline/flink/metadata.py +3 -3
  1286. metadata/ingestion/source/pipeline/flink/models.py +3 -3
  1287. metadata/ingestion/source/pipeline/gluepipeline/connection.py +3 -3
  1288. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +3 -3
  1289. metadata/ingestion/source/pipeline/gluepipeline/models.py +3 -3
  1290. metadata/ingestion/source/pipeline/kafkaconnect/client.py +3 -3
  1291. metadata/ingestion/source/pipeline/kafkaconnect/connection.py +3 -3
  1292. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +3 -3
  1293. metadata/ingestion/source/pipeline/kafkaconnect/models.py +3 -3
  1294. metadata/ingestion/source/pipeline/nifi/client.py +71 -77
  1295. metadata/ingestion/source/pipeline/nifi/connection.py +4 -21
  1296. metadata/ingestion/source/pipeline/nifi/metadata.py +3 -3
  1297. metadata/ingestion/source/pipeline/openlineage/connection.py +3 -3
  1298. metadata/ingestion/source/pipeline/openlineage/metadata.py +4 -3
  1299. metadata/ingestion/source/pipeline/openlineage/models.py +3 -3
  1300. metadata/ingestion/source/pipeline/pipeline_service.py +124 -5
  1301. metadata/ingestion/source/pipeline/spline/client.py +3 -3
  1302. metadata/ingestion/source/pipeline/spline/connection.py +3 -3
  1303. metadata/ingestion/source/pipeline/spline/metadata.py +3 -3
  1304. metadata/ingestion/source/pipeline/spline/models.py +3 -3
  1305. metadata/ingestion/source/pipeline/spline/utils.py +3 -3
  1306. metadata/ingestion/source/search/elasticsearch/connection.py +3 -3
  1307. metadata/ingestion/source/search/elasticsearch/metadata.py +3 -3
  1308. metadata/ingestion/source/search/elasticsearch/parser.py +3 -3
  1309. metadata/ingestion/source/search/opensearch/connection.py +215 -0
  1310. metadata/ingestion/source/search/opensearch/metadata.py +270 -0
  1311. metadata/ingestion/source/search/opensearch/parser.py +71 -0
  1312. metadata/ingestion/source/search/opensearch/service_spec.py +4 -0
  1313. metadata/ingestion/source/search/search_service.py +5 -3
  1314. metadata/ingestion/source/sqa_types.py +3 -3
  1315. metadata/ingestion/source/storage/gcs/client.py +2 -2
  1316. metadata/ingestion/source/storage/gcs/connection.py +2 -2
  1317. metadata/ingestion/source/storage/gcs/metadata.py +2 -2
  1318. metadata/ingestion/source/storage/gcs/models.py +2 -2
  1319. metadata/ingestion/source/storage/s3/connection.py +3 -3
  1320. metadata/ingestion/source/storage/s3/metadata.py +3 -3
  1321. metadata/ingestion/source/storage/s3/models.py +3 -3
  1322. metadata/ingestion/source/storage/storage_service.py +5 -3
  1323. metadata/ingestion/stage/table_usage.py +56 -4
  1324. metadata/mixins/pandas/pandas_mixin.py +3 -3
  1325. metadata/mixins/sqalchemy/sqa_mixin.py +3 -3
  1326. metadata/parsers/avro_parser.py +3 -3
  1327. metadata/parsers/json_schema_parser.py +3 -3
  1328. metadata/parsers/protobuf_parser.py +3 -3
  1329. metadata/parsers/schema_parsers.py +3 -3
  1330. metadata/pii/constants.py +3 -3
  1331. metadata/pii/models.py +3 -3
  1332. metadata/pii/ner.py +3 -3
  1333. metadata/pii/processor.py +3 -3
  1334. metadata/pii/scanners/base.py +3 -3
  1335. metadata/pii/scanners/column_name_scanner.py +3 -3
  1336. metadata/pii/scanners/ner_scanner.py +3 -3
  1337. metadata/profiler/adaptors/adaptor_factory.py +2 -2
  1338. metadata/profiler/adaptors/dynamodb.py +2 -2
  1339. metadata/profiler/adaptors/factory.py +2 -2
  1340. metadata/profiler/adaptors/mongodb.py +2 -2
  1341. metadata/profiler/adaptors/nosql_adaptor.py +2 -2
  1342. metadata/profiler/api/models.py +3 -3
  1343. metadata/profiler/config.py +3 -3
  1344. metadata/profiler/factory.py +3 -3
  1345. metadata/profiler/interface/nosql/profiler_interface.py +3 -3
  1346. metadata/profiler/interface/pandas/profiler_interface.py +3 -3
  1347. metadata/profiler/interface/profiler_interface.py +3 -3
  1348. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -3
  1349. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +3 -3
  1350. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +3 -3
  1351. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +3 -3
  1352. metadata/profiler/interface/sqlalchemy/profiler_interface.py +3 -3
  1353. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +3 -3
  1354. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +3 -3
  1355. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +3 -3
  1356. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +3 -3
  1357. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +3 -3
  1358. metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py +6 -7
  1359. metadata/profiler/metrics/composed/distinct_ratio.py +3 -3
  1360. metadata/profiler/metrics/composed/duplicate_count.py +3 -3
  1361. metadata/profiler/metrics/composed/ilike_ratio.py +3 -3
  1362. metadata/profiler/metrics/composed/iqr.py +3 -3
  1363. metadata/profiler/metrics/composed/like_ratio.py +3 -3
  1364. metadata/profiler/metrics/composed/non_parametric_skew.py +3 -3
  1365. metadata/profiler/metrics/composed/null_ratio.py +3 -3
  1366. metadata/profiler/metrics/composed/unique_ratio.py +3 -3
  1367. metadata/profiler/metrics/core.py +3 -3
  1368. metadata/profiler/metrics/hybrid/histogram.py +3 -3
  1369. metadata/profiler/metrics/registry.py +3 -3
  1370. metadata/profiler/metrics/static/column_count.py +3 -3
  1371. metadata/profiler/metrics/static/column_names.py +3 -3
  1372. metadata/profiler/metrics/static/count.py +3 -3
  1373. metadata/profiler/metrics/static/count_in_set.py +3 -3
  1374. metadata/profiler/metrics/static/distinct_count.py +3 -3
  1375. metadata/profiler/metrics/static/ilike_count.py +3 -3
  1376. metadata/profiler/metrics/static/like_count.py +3 -3
  1377. metadata/profiler/metrics/static/max.py +3 -3
  1378. metadata/profiler/metrics/static/max_length.py +3 -3
  1379. metadata/profiler/metrics/static/mean.py +3 -3
  1380. metadata/profiler/metrics/static/min.py +3 -3
  1381. metadata/profiler/metrics/static/min_length.py +3 -3
  1382. metadata/profiler/metrics/static/not_like_count.py +3 -3
  1383. metadata/profiler/metrics/static/not_regexp_match_count.py +3 -3
  1384. metadata/profiler/metrics/static/null_count.py +3 -3
  1385. metadata/profiler/metrics/static/null_missing_count.py +3 -3
  1386. metadata/profiler/metrics/static/regexp_match_count.py +3 -3
  1387. metadata/profiler/metrics/static/row_count.py +3 -3
  1388. metadata/profiler/metrics/static/stddev.py +3 -3
  1389. metadata/profiler/metrics/static/sum.py +3 -3
  1390. metadata/profiler/metrics/static/unique_count.py +3 -3
  1391. metadata/profiler/metrics/system/dml_operation.py +3 -3
  1392. metadata/profiler/metrics/system/system.py +3 -3
  1393. metadata/profiler/metrics/window/first_quartile.py +3 -3
  1394. metadata/profiler/metrics/window/median.py +3 -3
  1395. metadata/profiler/metrics/window/third_quartile.py +3 -3
  1396. metadata/profiler/orm/converter/azuresql/converter.py +39 -0
  1397. metadata/profiler/orm/converter/base.py +3 -3
  1398. metadata/profiler/orm/converter/bigquery/converter.py +3 -3
  1399. metadata/profiler/orm/converter/common.py +3 -3
  1400. metadata/profiler/orm/converter/converter_registry.py +5 -3
  1401. metadata/profiler/orm/converter/mariadb/converter.py +2 -2
  1402. metadata/profiler/orm/converter/mssql/converter.py +3 -3
  1403. metadata/profiler/orm/converter/redshift/converter.py +3 -3
  1404. metadata/profiler/orm/converter/snowflake/converter.py +3 -3
  1405. metadata/profiler/orm/functions/concat.py +3 -3
  1406. metadata/profiler/orm/functions/conn_test.py +3 -3
  1407. metadata/profiler/orm/functions/count.py +3 -3
  1408. metadata/profiler/orm/functions/datetime.py +3 -3
  1409. metadata/profiler/orm/functions/length.py +5 -3
  1410. metadata/profiler/orm/functions/md5.py +3 -3
  1411. metadata/profiler/orm/functions/median.py +22 -3
  1412. metadata/profiler/orm/functions/modulo.py +4 -3
  1413. metadata/profiler/orm/functions/random_num.py +4 -3
  1414. metadata/profiler/orm/functions/substr.py +3 -3
  1415. metadata/profiler/orm/functions/sum.py +4 -3
  1416. metadata/profiler/orm/functions/table_metric_computer.py +3 -3
  1417. metadata/profiler/orm/functions/unique_count.py +3 -3
  1418. metadata/profiler/orm/registry.py +5 -3
  1419. metadata/profiler/orm/types/bytea_to_string.py +3 -3
  1420. metadata/profiler/orm/types/custom_array.py +3 -3
  1421. metadata/profiler/orm/types/custom_datetimerange.py +3 -3
  1422. metadata/profiler/orm/types/custom_hex_byte_string.py +3 -3
  1423. metadata/profiler/orm/types/custom_image.py +3 -3
  1424. metadata/profiler/orm/types/custom_ip.py +3 -3
  1425. metadata/profiler/orm/types/custom_time.py +2 -2
  1426. metadata/profiler/orm/types/custom_timestamp.py +3 -3
  1427. metadata/profiler/orm/types/undetermined_type.py +3 -3
  1428. metadata/profiler/orm/types/uuid.py +3 -3
  1429. metadata/profiler/processor/core.py +4 -3
  1430. metadata/profiler/processor/default.py +3 -3
  1431. metadata/profiler/processor/handle_partition.py +3 -3
  1432. metadata/profiler/processor/metric_filter.py +4 -4
  1433. metadata/profiler/processor/models.py +3 -3
  1434. metadata/profiler/processor/processor.py +3 -3
  1435. metadata/profiler/processor/runner.py +3 -3
  1436. metadata/profiler/processor/sample_data_handler.py +3 -3
  1437. metadata/profiler/registry.py +3 -3
  1438. metadata/profiler/source/database/base/profiler_source.py +4 -3
  1439. metadata/profiler/source/database/bigquery/profiler_source.py +3 -3
  1440. metadata/profiler/source/database/bigquery/type_mapper.py +3 -3
  1441. metadata/profiler/source/fetcher/config.py +3 -3
  1442. metadata/profiler/source/fetcher/entity_fetcher.py +3 -3
  1443. metadata/profiler/source/fetcher/fetcher_strategy.py +7 -3
  1444. metadata/profiler/source/fetcher/profiler_source_factory.py +3 -3
  1445. metadata/profiler/source/metadata.py +3 -3
  1446. metadata/profiler/source/metadata_ext.py +3 -3
  1447. metadata/profiler/source/model.py +3 -3
  1448. metadata/profiler/source/profiler_source_interface.py +3 -3
  1449. metadata/readers/dataframe/avro.py +3 -3
  1450. metadata/readers/dataframe/base.py +3 -3
  1451. metadata/readers/dataframe/common.py +3 -3
  1452. metadata/readers/dataframe/dsv.py +3 -3
  1453. metadata/readers/dataframe/json.py +3 -3
  1454. metadata/readers/dataframe/models.py +3 -3
  1455. metadata/readers/dataframe/parquet.py +3 -3
  1456. metadata/readers/dataframe/reader_factory.py +3 -3
  1457. metadata/readers/file/adls.py +3 -3
  1458. metadata/readers/file/api_reader.py +3 -3
  1459. metadata/readers/file/base.py +3 -3
  1460. metadata/readers/file/bitbucket.py +3 -3
  1461. metadata/readers/file/config_source_factory.py +3 -3
  1462. metadata/readers/file/credentials.py +3 -3
  1463. metadata/readers/file/gcs.py +3 -3
  1464. metadata/readers/file/github.py +3 -3
  1465. metadata/readers/file/gitlab.py +3 -3
  1466. metadata/readers/file/local.py +3 -3
  1467. metadata/readers/file/s3.py +3 -3
  1468. metadata/readers/models.py +3 -3
  1469. metadata/sampler/config.py +4 -3
  1470. metadata/sampler/models.py +6 -3
  1471. metadata/sampler/nosql/sampler.py +7 -3
  1472. metadata/sampler/pandas/sampler.py +4 -3
  1473. metadata/sampler/partition.py +12 -5
  1474. metadata/sampler/processor.py +3 -3
  1475. metadata/sampler/sampler_interface.py +3 -4
  1476. metadata/sampler/sqlalchemy/azuresql/sampler.py +33 -6
  1477. metadata/sampler/sqlalchemy/bigquery/sampler.py +3 -3
  1478. metadata/sampler/sqlalchemy/mssql/sampler.py +5 -4
  1479. metadata/sampler/sqlalchemy/postgres/sampler.py +3 -3
  1480. metadata/sampler/sqlalchemy/sampler.py +54 -45
  1481. metadata/sampler/sqlalchemy/snowflake/sampler.py +5 -4
  1482. metadata/sampler/sqlalchemy/trino/sampler.py +12 -11
  1483. metadata/timer/repeated_timer.py +3 -3
  1484. metadata/utils/bigquery_utils.py +3 -3
  1485. metadata/utils/class_helper.py +3 -3
  1486. metadata/utils/client_version.py +3 -3
  1487. metadata/utils/constants.py +4 -3
  1488. metadata/utils/constraints.py +2 -2
  1489. metadata/utils/credentials.py +10 -3
  1490. metadata/utils/custom_thread_pool.py +3 -3
  1491. metadata/utils/datalake/datalake_utils.py +3 -3
  1492. metadata/utils/db_utils.py +3 -3
  1493. metadata/utils/deprecation.py +3 -3
  1494. metadata/utils/dispatch.py +3 -3
  1495. metadata/utils/elasticsearch.py +7 -3
  1496. metadata/utils/entity_link.py +3 -3
  1497. metadata/utils/entity_utils.py +3 -3
  1498. metadata/utils/execution_time_tracker.py +3 -3
  1499. metadata/utils/filters.py +3 -3
  1500. metadata/utils/fqn.py +3 -3
  1501. metadata/utils/helpers.py +94 -3
  1502. metadata/utils/importer.py +3 -3
  1503. metadata/utils/life_cycle_utils.py +3 -3
  1504. metadata/utils/logger.py +12 -3
  1505. metadata/utils/lru_cache.py +3 -3
  1506. metadata/utils/messaging_utils.py +2 -2
  1507. metadata/utils/profiler_utils.py +3 -3
  1508. metadata/utils/s3_utils.py +3 -3
  1509. metadata/utils/secrets/aws_based_secrets_manager.py +2 -2
  1510. metadata/utils/secrets/aws_secrets_manager.py +2 -2
  1511. metadata/utils/secrets/aws_ssm_secrets_manager.py +2 -2
  1512. metadata/utils/secrets/azure_kv_secrets_manager.py +2 -2
  1513. metadata/utils/secrets/db_secrets_manager.py +2 -2
  1514. metadata/utils/secrets/external_secrets_manager.py +2 -2
  1515. metadata/utils/secrets/secrets_manager.py +2 -2
  1516. metadata/utils/secrets/secrets_manager_factory.py +2 -2
  1517. metadata/utils/singleton.py +3 -3
  1518. metadata/utils/source_hash.py +3 -3
  1519. metadata/utils/sqa_like_column.py +3 -3
  1520. metadata/utils/sqa_utils.py +3 -3
  1521. metadata/utils/sqlalchemy_utils.py +3 -3
  1522. metadata/utils/ssl_manager.py +38 -3
  1523. metadata/utils/ssl_registry.py +3 -3
  1524. metadata/utils/storage_metadata_config.py +3 -3
  1525. metadata/utils/stored_procedures.py +3 -3
  1526. metadata/utils/tag_utils.py +75 -30
  1527. metadata/utils/test_utils.py +2 -2
  1528. metadata/utils/time_utils.py +3 -3
  1529. metadata/utils/timeout.py +3 -3
  1530. metadata/utils/uuid_encoder.py +3 -3
  1531. metadata/workflow/application.py +3 -3
  1532. metadata/workflow/application_output_handler.py +3 -3
  1533. metadata/workflow/base.py +3 -3
  1534. metadata/workflow/classification.py +6 -3
  1535. metadata/workflow/data_quality.py +3 -3
  1536. metadata/workflow/ingestion.py +3 -3
  1537. metadata/workflow/metadata.py +3 -3
  1538. metadata/workflow/output_handler.py +3 -3
  1539. metadata/workflow/profiler.py +16 -8
  1540. metadata/workflow/usage.py +3 -3
  1541. metadata/workflow/workflow_init_error_handler.py +3 -3
  1542. metadata/workflow/workflow_output_handler.py +3 -3
  1543. metadata/workflow/workflow_status_mixin.py +9 -3
  1544. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/METADATA +441 -373
  1545. openmetadata_ingestion-1.7.0.0rc2.dist-info/RECORD +1895 -0
  1546. openmetadata_ingestion-1.6.8.0.dist-info/RECORD +0 -1825
  1547. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/LICENSE +0 -0
  1548. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/WHEEL +0 -0
  1549. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/entry_points.txt +0 -0
  1550. {openmetadata_ingestion-1.6.8.0.dist-info → openmetadata_ingestion-1.7.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -24,6 +24,7 @@ from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
24
24
  from metadata.generated.schema.entity.data.storedProcedure import (
25
25
  Language,
26
26
  StoredProcedureCode,
27
+ StoredProcedureType,
27
28
  )
28
29
  from metadata.generated.schema.entity.data.table import TableType
29
30
  from metadata.generated.schema.entity.services.connections.database.oracleConnection import (
@@ -45,10 +46,11 @@ from metadata.ingestion.source.database.common_db_source import (
45
46
  TableNameAndType,
46
47
  )
47
48
  from metadata.ingestion.source.database.oracle.models import (
48
- FetchProcedureList,
49
- OracleStoredProcedure,
49
+ FetchObjectList,
50
+ OracleStoredObject,
50
51
  )
51
52
  from metadata.ingestion.source.database.oracle.queries import (
53
+ ORACLE_GET_STORED_PACKAGES,
52
54
  ORACLE_GET_STORED_PROCEDURES,
53
55
  )
54
56
  from metadata.ingestion.source.database.oracle.utils import (
@@ -181,41 +183,51 @@ class OracleSource(CommonDbSourceService):
181
183
  logger.warning(f"Failed to fetch Schema definition for {table_name}: {exc}")
182
184
  return None
183
185
 
184
- def process_result(self, data: FetchProcedureList):
186
+ def process_result(self, data: FetchObjectList):
185
187
  """Process data as per our stored procedure format"""
186
188
  result_dict = {}
187
189
 
188
190
  for row in data:
189
- owner, name, line, text = row
191
+
192
+ owner, name, line, text, procedure_type = row
190
193
  key = (owner, name)
191
194
  if key not in result_dict:
192
- result_dict[key] = {"lines": [], "text": ""}
195
+ result_dict[key] = {"lines": [], "text": "", "procedure_type": ""}
193
196
  result_dict[key]["lines"].append(line)
194
197
  result_dict[key]["text"] += text
198
+ result_dict[key]["procedure_type"] = procedure_type
195
199
 
196
200
  # Return the concatenated text for each procedure name, ordered by line
197
201
  return result_dict
198
202
 
199
- def get_stored_procedures(self) -> Iterable[OracleStoredProcedure]:
203
+ def _get_stored_procedures_internal(
204
+ self, query: str
205
+ ) -> Iterable[OracleStoredObject]:
206
+ results: FetchObjectList = self.engine.execute(
207
+ query.format(schema=self.context.get().database_schema.upper())
208
+ ).all()
209
+ results = self.process_result(data=results)
210
+ for row in results.items():
211
+ stored_procedure = OracleStoredObject(
212
+ name=row[0][1],
213
+ definition=row[1]["text"],
214
+ owner=row[0][0],
215
+ procedure_type=row[1]["procedure_type"],
216
+ )
217
+ yield stored_procedure
218
+
219
+ def get_stored_procedures(self) -> Iterable[OracleStoredObject]:
200
220
  """List Oracle Stored Procedures"""
201
221
  if self.source_config.includeStoredProcedures:
202
- results: FetchProcedureList = self.engine.execute(
203
- ORACLE_GET_STORED_PROCEDURES.format(
204
- schema=self.context.get().database_schema.upper()
205
- )
206
- ).all()
207
- results = self.process_result(data=results)
208
- for row in results.items():
209
- stored_procedure = OracleStoredProcedure(
210
- name=row[0][1], definition=row[1]["text"], owner=row[0][0]
211
- )
212
- yield stored_procedure
222
+ yield from self._get_stored_procedures_internal(
223
+ ORACLE_GET_STORED_PROCEDURES
224
+ )
225
+ yield from self._get_stored_procedures_internal(ORACLE_GET_STORED_PACKAGES)
213
226
 
214
227
  def yield_stored_procedure(
215
- self, stored_procedure: OracleStoredProcedure
228
+ self, stored_procedure: OracleStoredObject
216
229
  ) -> Iterable[Either[CreateStoredProcedureRequest]]:
217
230
  """Prepare the stored procedure payload"""
218
-
219
231
  try:
220
232
  stored_procedure_request = CreateStoredProcedureRequest(
221
233
  name=EntityName(stored_procedure.name),
@@ -223,6 +235,11 @@ class OracleSource(CommonDbSourceService):
223
235
  language=Language.SQL,
224
236
  code=stored_procedure.definition,
225
237
  ),
238
+ storedProcedureType=(
239
+ StoredProcedureType.StoredPackage
240
+ if stored_procedure.procedure_type == "StoredPackage"
241
+ else StoredProcedureType.StoredProcedure
242
+ ),
226
243
  owners=self.metadata.get_reference_by_name(
227
244
  name=stored_procedure.owner.lower(), is_owner=True
228
245
  ),
@@ -6,7 +6,7 @@ from typing import List, Optional
6
6
  from pydantic import BaseModel, Field
7
7
 
8
8
 
9
- class OracleStoredProcedure(BaseModel):
9
+ class OracleStoredObject(BaseModel):
10
10
  """Oracle Stored Procedure list query results"""
11
11
 
12
12
  name: str
@@ -15,9 +15,10 @@ class OracleStoredProcedure(BaseModel):
15
15
  None, description="Will only be informed for non-SQL routines."
16
16
  )
17
17
  owner: str
18
+ procedure_type: Optional[str] = Field(None, alias="procedure_type")
18
19
 
19
20
 
20
- class FetchProcedure(BaseModel):
21
+ class FetchObject(BaseModel):
21
22
  """Oracle Fetch Stored Procedure Raw Model"""
22
23
 
23
24
  owner: Optional[str] = None
@@ -26,5 +27,5 @@ class FetchProcedure(BaseModel):
26
27
  text: str
27
28
 
28
29
 
29
- class FetchProcedureList(BaseModel):
30
- __name__: List[FetchProcedure]
30
+ class FetchObjectList(BaseModel):
31
+ __name__: List[FetchObject]
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -87,13 +87,34 @@ SELECT
87
87
  OWNER,
88
88
  NAME,
89
89
  LINE,
90
- TEXT
90
+ TEXT,
91
+ 'StoredProcedure' as procedure_type
91
92
  FROM
92
93
  DBA_SOURCE
93
94
  WHERE
94
95
  type = 'PROCEDURE' and owner = '{schema}'
95
96
  """
96
97
  )
98
+ ORACLE_GET_SCHEMA = """
99
+ SELECT USERNAME AS SCHEMA_NAME
100
+ FROM ALL_USERS
101
+ WHERE ROWNUM = 1
102
+ ORDER BY USERNAME
103
+ """
104
+ ORACLE_GET_STORED_PACKAGES = textwrap.dedent(
105
+ """
106
+ SELECT
107
+ OWNER,
108
+ NAME,
109
+ LINE,
110
+ TEXT,
111
+ 'StoredPackage' as procedure_type
112
+
113
+ FROM
114
+ DBA_SOURCE
115
+ WHERE TYPE IN ('PACKAGE', 'PACKAGE BODY') AND owner = '{schema}'
116
+ """
117
+ )
97
118
  CHECK_ACCESS_TO_ALL = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2"
98
119
  ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
99
120
  """
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,4 +1,8 @@
1
+ from metadata.ingestion.source.database.pinotdb.lineage import PinotdbLineageSource
1
2
  from metadata.ingestion.source.database.pinotdb.metadata import PinotdbSource
2
3
  from metadata.utils.service_spec.default import DefaultDatabaseSpec
3
4
 
4
- ServiceSpec = DefaultDatabaseSpec(metadata_source_class=PinotdbSource)
5
+ ServiceSpec = DefaultDatabaseSpec(
6
+ metadata_source_class=PinotdbSource,
7
+ lineage_source_class=PinotdbLineageSource,
8
+ )
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,9 +15,8 @@ import traceback
15
15
  from collections import namedtuple
16
16
  from typing import Iterable, Optional, Tuple
17
17
 
18
- from sqlalchemy import String as SqlAlchemyString
19
18
  from sqlalchemy import sql
20
- from sqlalchemy.dialects.postgresql.base import PGDialect, ischema_names
19
+ from sqlalchemy.dialects.postgresql.base import PGDialect
21
20
  from sqlalchemy.engine import Inspector
22
21
 
23
22
  from metadata.generated.schema.api.data.createStoredProcedure import (
@@ -46,17 +45,22 @@ from metadata.ingestion.api.models import Either
46
45
  from metadata.ingestion.api.steps import InvalidSourceException
47
46
  from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
48
47
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
49
- from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type
50
48
  from metadata.ingestion.source.database.common_db_source import (
51
49
  CommonDbSourceService,
52
50
  TableNameAndType,
53
51
  )
52
+ from metadata.ingestion.source.database.common_pg_mappings import (
53
+ INTERVAL_TYPE_MAP,
54
+ RELKIND_MAP,
55
+ ischema_names,
56
+ )
54
57
  from metadata.ingestion.source.database.mssql.models import STORED_PROC_LANGUAGE_MAP
55
58
  from metadata.ingestion.source.database.multi_db_source import MultiDBSource
56
59
  from metadata.ingestion.source.database.postgres.models import PostgresStoredProcedure
57
60
  from metadata.ingestion.source.database.postgres.queries import (
58
61
  POSTGRES_GET_ALL_TABLE_PG_POLICY,
59
62
  POSTGRES_GET_DB_NAMES,
63
+ POSTGRES_GET_FUNCTIONS,
60
64
  POSTGRES_GET_STORED_PROCEDURES,
61
65
  POSTGRES_GET_TABLE_NAMES,
62
66
  POSTGRES_PARTITION_DETAILS,
@@ -95,43 +99,6 @@ TableKey = namedtuple("TableKey", ["schema", "table_name"])
95
99
  logger = ingestion_logger()
96
100
 
97
101
 
98
- INTERVAL_TYPE_MAP = {
99
- "list": PartitionIntervalTypes.COLUMN_VALUE,
100
- "hash": PartitionIntervalTypes.COLUMN_VALUE,
101
- "range": PartitionIntervalTypes.TIME_UNIT,
102
- }
103
-
104
- RELKIND_MAP = {
105
- "r": TableType.Regular,
106
- "p": TableType.Partitioned,
107
- "f": TableType.Foreign,
108
- }
109
-
110
- GEOMETRY = create_sqlalchemy_type("GEOMETRY")
111
- POINT = create_sqlalchemy_type("POINT")
112
- POLYGON = create_sqlalchemy_type("POLYGON")
113
-
114
- ischema_names.update(
115
- {
116
- "geometry": GEOMETRY,
117
- "point": POINT,
118
- "polygon": POLYGON,
119
- "box": create_sqlalchemy_type("BOX"),
120
- "bpchar": SqlAlchemyString,
121
- "circle": create_sqlalchemy_type("CIRCLE"),
122
- "line": create_sqlalchemy_type("LINE"),
123
- "lseg": create_sqlalchemy_type("LSEG"),
124
- "path": create_sqlalchemy_type("PATH"),
125
- "pg_lsn": create_sqlalchemy_type("PG_LSN"),
126
- "pg_snapshot": create_sqlalchemy_type("PG_SNAPSHOT"),
127
- "tsquery": create_sqlalchemy_type("TSQUERY"),
128
- "txid_snapshot": create_sqlalchemy_type("TXID_SNAPSHOT"),
129
- "xid": SqlAlchemyString,
130
- "xml": create_sqlalchemy_type("XML"),
131
- }
132
- )
133
-
134
-
135
102
  PGDialect.get_all_table_comments = get_all_table_comments
136
103
  PGDialect.get_table_comment = get_table_comment
137
104
  PGDialect._get_column_info = get_column_info # pylint: disable=protected-access
@@ -307,25 +274,33 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
307
274
  )
308
275
  )
309
276
 
277
+ def _get_stored_procedures_internal(
278
+ self, query: str
279
+ ) -> Iterable[PostgresStoredProcedure]:
280
+ results = self.engine.execute(query).all()
281
+ for row in results:
282
+ try:
283
+ stored_procedure = PostgresStoredProcedure.model_validate(
284
+ dict(row._mapping)
285
+ )
286
+ yield stored_procedure
287
+ except Exception as exc:
288
+ logger.error()
289
+ self.status.failed(
290
+ error=StackTraceError(
291
+ name=dict(row).get("name", "UNKNOWN"),
292
+ error=f"Error parsing Stored Procedure payload: {exc}",
293
+ stackTrace=traceback.format_exc(),
294
+ )
295
+ )
296
+
310
297
  def get_stored_procedures(self) -> Iterable[PostgresStoredProcedure]:
311
298
  """List stored procedures"""
312
299
  if self.source_config.includeStoredProcedures:
313
- results = self.engine.execute(POSTGRES_GET_STORED_PROCEDURES).all()
314
- for row in results:
315
- try:
316
- stored_procedure = PostgresStoredProcedure.model_validate(
317
- dict(row._mapping)
318
- )
319
- yield stored_procedure
320
- except Exception as exc:
321
- logger.error()
322
- self.status.failed(
323
- error=StackTraceError(
324
- name=dict(row).get("name", "UNKNOWN"),
325
- error=f"Error parsing Stored Procedure payload: {exc}",
326
- stackTrace=traceback.format_exc(),
327
- )
328
- )
300
+ yield from self._get_stored_procedures_internal(
301
+ POSTGRES_GET_STORED_PROCEDURES
302
+ )
303
+ yield from self._get_stored_procedures_internal(POSTGRES_GET_FUNCTIONS)
329
304
 
330
305
  def yield_stored_procedure(
331
306
  self, stored_procedure
@@ -346,6 +321,7 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
346
321
  database_name=self.context.get().database,
347
322
  schema_name=self.context.get().database_schema,
348
323
  ),
324
+ storedProcedureType=stored_procedure.procedure_type,
349
325
  )
350
326
  yield Either(right=stored_procedure_request)
351
327
  self.register_record_stored_proc_request(stored_procedure_request)
@@ -1,8 +1,8 @@
1
1
  # Copyright 2024 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -23,3 +23,4 @@ class PostgresStoredProcedure(BaseModel):
23
23
  schema: str = Field(alias="schema_name")
24
24
  definition: str
25
25
  language: Optional[str] = None
26
+ procedure_type: Optional[str] = Field(None, alias="procedure_type")
@@ -1,9 +1,9 @@
1
1
  # Copyright 2021 Collate
2
2
  # Portions Copyright(c) 2023, TOSHIBA CORPORATION
3
- # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
5
5
  # You may obtain a copy of the License at
6
- # http://www.apache.org/licenses/LICENSE-2.0
6
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
7
7
  # Unless required by applicable law or agreed to in writing, software
8
8
  # distributed under the License is distributed on an "AS IS" BASIS,
9
9
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
1
  # Copyright(c) 2023, TOSHIBA CORPORATION
2
- # Licensed under the Apache License, Version 2.0 (the "License");
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -217,8 +217,25 @@ POSTGRES_GET_STORED_PROCEDURES = """
217
217
  nspname AS schema_name,
218
218
  proargtypes AS argument_types,
219
219
  prorettype::regtype AS return_type,
220
- prosrc AS definition
220
+ prosrc AS definition,
221
+ 'StoredProcedure' as procedure_type
221
222
  FROM pg_proc
222
223
  JOIN pg_namespace ON pg_proc.pronamespace = pg_namespace.oid
223
224
  WHERE prokind = 'p';
224
225
  """
226
+
227
+ POSTGRES_GET_FUNCTIONS = """
228
+ SELECT
229
+ proname AS procedure_name,
230
+ nspname AS schema_name,
231
+ proargtypes AS argument_types,
232
+ prorettype :: regtype AS return_type,
233
+ prosrc AS definition,
234
+ 'Function' as procedure_type
235
+ FROM
236
+ pg_proc
237
+ JOIN pg_namespace ON pg_proc.pronamespace = pg_namespace.oid
238
+ WHERE
239
+ prokind = 'f'
240
+ and pg_namespace.nspname NOT IN ('pg_catalog', 'information_schema');
241
+ """
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.