openmetadata-ingestion 1.6.2.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1857) hide show
  1. _openmetadata_testutils/helpers/login_user.py +22 -0
  2. airflow_provider_openmetadata/__init__.py +3 -3
  3. airflow_provider_openmetadata/hooks/openmetadata.py +3 -3
  4. airflow_provider_openmetadata/lineage/backend.py +23 -4
  5. airflow_provider_openmetadata/lineage/callback.py +3 -3
  6. airflow_provider_openmetadata/lineage/config/commons.py +3 -3
  7. airflow_provider_openmetadata/lineage/config/loader.py +17 -3
  8. airflow_provider_openmetadata/lineage/operator.py +3 -3
  9. airflow_provider_openmetadata/lineage/runner.py +3 -3
  10. airflow_provider_openmetadata/lineage/status.py +3 -3
  11. metadata/__init__.py +33 -0
  12. metadata/__main__.py +3 -3
  13. metadata/__version__.py +3 -3
  14. metadata/antlr/split_listener.py +3 -3
  15. metadata/applications/example.py +74 -0
  16. metadata/automations/execute_runner.py +49 -0
  17. metadata/automations/extended_runner.py +13 -0
  18. metadata/automations/runner.py +48 -39
  19. metadata/cli/app.py +3 -3
  20. metadata/cli/classify.py +8 -8
  21. metadata/cli/common.py +25 -0
  22. metadata/cli/dataquality.py +8 -8
  23. metadata/cli/ingest.py +5 -7
  24. metadata/cli/ingest_dbt.py +367 -0
  25. metadata/cli/lineage.py +3 -3
  26. metadata/cli/profile.py +5 -7
  27. metadata/cli/restore.py +3 -3
  28. metadata/cli/usage.py +5 -7
  29. metadata/clients/aws_client.py +32 -11
  30. metadata/clients/azure_client.py +3 -3
  31. metadata/clients/domo_client.py +5 -5
  32. metadata/cmd.py +30 -3
  33. metadata/config/common.py +3 -3
  34. metadata/data_quality/api/models.py +3 -3
  35. metadata/data_quality/builders/validator_builder.py +3 -3
  36. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +3 -3
  37. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +3 -3
  38. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +3 -3
  39. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +3 -3
  40. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +3 -3
  41. metadata/data_quality/interface/test_suite_interface.py +12 -6
  42. metadata/data_quality/processor/test_case_runner.py +11 -22
  43. metadata/data_quality/runner/base_test_suite_source.py +16 -10
  44. metadata/data_quality/runner/core.py +3 -3
  45. metadata/data_quality/source/test_suite.py +49 -7
  46. metadata/data_quality/validations/base_test_handler.py +3 -3
  47. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +3 -3
  48. metadata/data_quality/validations/column/base/columnValueMaxToBeBetween.py +3 -3
  49. metadata/data_quality/validations/column/base/columnValueMeanToBeBetween.py +3 -3
  50. metadata/data_quality/validations/column/base/columnValueMedianToBeBetween.py +3 -3
  51. metadata/data_quality/validations/column/base/columnValueMinToBeBetween.py +3 -3
  52. metadata/data_quality/validations/column/base/columnValueStdDevToBeBetween.py +3 -3
  53. metadata/data_quality/validations/column/base/columnValuesMissingCount.py +3 -3
  54. metadata/data_quality/validations/column/base/columnValuesSumToBeBetween.py +3 -3
  55. metadata/data_quality/validations/column/base/columnValuesToBeAtExpectedLocation.py +3 -3
  56. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +3 -3
  57. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +3 -3
  58. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +3 -3
  59. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +3 -3
  60. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +3 -3
  61. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +3 -3
  62. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +3 -3
  63. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +3 -3
  64. metadata/data_quality/validations/column/pandas/columnValueMaxToBeBetween.py +3 -3
  65. metadata/data_quality/validations/column/pandas/columnValueMeanToBeBetween.py +3 -3
  66. metadata/data_quality/validations/column/pandas/columnValueMedianToBeBetween.py +3 -3
  67. metadata/data_quality/validations/column/pandas/columnValueMinToBeBetween.py +3 -3
  68. metadata/data_quality/validations/column/pandas/columnValueStdDevToBeBetween.py +3 -3
  69. metadata/data_quality/validations/column/pandas/columnValuesMissingCount.py +3 -3
  70. metadata/data_quality/validations/column/pandas/columnValuesSumToBeBetween.py +3 -3
  71. metadata/data_quality/validations/column/pandas/columnValuesToBeAtExpectedLocation.py +3 -3
  72. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +3 -3
  73. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +3 -3
  74. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +3 -3
  75. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +3 -3
  76. metadata/data_quality/validations/column/pandas/columnValuesToBeUnique.py +3 -3
  77. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +3 -3
  78. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +3 -3
  79. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +3 -3
  80. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +3 -3
  81. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +3 -3
  82. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +3 -3
  83. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +3 -3
  84. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +3 -3
  85. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +3 -3
  86. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +3 -3
  87. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +3 -3
  88. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +3 -3
  89. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +3 -3
  90. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +3 -3
  91. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +3 -3
  92. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +17 -5
  93. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +4 -4
  94. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +3 -3
  95. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +3 -3
  96. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +3 -3
  97. metadata/data_quality/validations/models.py +16 -3
  98. metadata/data_quality/validations/runtime_param_setter/base_diff_params_setter.py +219 -0
  99. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -2
  100. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +11 -2
  101. metadata/data_quality/validations/runtime_param_setter/table_custom_sql_query_params_setter.py +31 -0
  102. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +56 -85
  103. metadata/data_quality/validations/table/base/tableColumnCountToBeBetween.py +3 -3
  104. metadata/data_quality/validations/table/base/tableColumnCountToEqual.py +3 -3
  105. metadata/data_quality/validations/table/base/tableColumnNameToExist.py +3 -3
  106. metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +3 -3
  107. metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +144 -5
  108. metadata/data_quality/validations/table/base/tableRowCountToBeBetween.py +3 -3
  109. metadata/data_quality/validations/table/base/tableRowCountToEqual.py +3 -3
  110. metadata/data_quality/validations/table/base/tableRowInsertedCountToBeBetween.py +3 -3
  111. metadata/data_quality/validations/table/pandas/tableColumnCountToBeBetween.py +3 -3
  112. metadata/data_quality/validations/table/pandas/tableColumnCountToEqual.py +3 -3
  113. metadata/data_quality/validations/table/pandas/tableColumnNameToExist.py +3 -3
  114. metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +3 -3
  115. metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py +41 -3
  116. metadata/data_quality/validations/table/pandas/tableRowCountToBeBetween.py +3 -3
  117. metadata/data_quality/validations/table/pandas/tableRowCountToEqual.py +3 -3
  118. metadata/data_quality/validations/table/pandas/tableRowInsertedCountToBeBetween.py +20 -5
  119. metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToBeBetween.py +3 -3
  120. metadata/data_quality/validations/table/sqlalchemy/tableColumnCountToEqual.py +3 -3
  121. metadata/data_quality/validations/table/sqlalchemy/tableColumnNameToExist.py +3 -3
  122. metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +3 -3
  123. metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +323 -3
  124. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +34 -3
  125. metadata/data_quality/validations/table/sqlalchemy/tableRowCountToBeBetween.py +3 -3
  126. metadata/data_quality/validations/table/sqlalchemy/tableRowCountToEqual.py +3 -3
  127. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +3 -3
  128. metadata/examples/workflows/airbyte.yaml +8 -2
  129. metadata/examples/workflows/bigquery.yaml +1 -0
  130. metadata/examples/workflows/cassandra.yaml +35 -0
  131. metadata/examples/workflows/cockroach.yaml +24 -0
  132. metadata/examples/workflows/confluent_cdc.yaml +70 -0
  133. metadata/examples/workflows/confluent_cdc_cloud.yaml +86 -0
  134. metadata/examples/workflows/confluent_cdc_local.yaml +120 -0
  135. metadata/examples/workflows/databricks.yaml +9 -1
  136. metadata/examples/workflows/databricks_usage.yaml +7 -1
  137. metadata/examples/workflows/exasol.yaml +25 -0
  138. metadata/examples/workflows/grafana.yaml +115 -0
  139. metadata/examples/workflows/kinesisfirehose.yaml +99 -0
  140. metadata/examples/workflows/metabase.yaml +1 -0
  141. metadata/examples/workflows/my_confluent_cloud.yaml +49 -0
  142. metadata/examples/workflows/nifi.yaml +51 -0
  143. metadata/examples/workflows/opensearch.yaml +20 -0
  144. metadata/examples/workflows/rest.yaml +6 -2
  145. metadata/examples/workflows/tableau.yaml +1 -3
  146. metadata/examples/workflows/unity_catalog.yaml +7 -1
  147. metadata/examples/workflows/unity_catalog_usage.yaml +7 -1
  148. metadata/examples/workflows/wherescape.yaml +28 -0
  149. metadata/generated/antlr/EntityLinkLexer.py +419 -353
  150. metadata/generated/schema/analytics/__init__.py +1 -1
  151. metadata/generated/schema/analytics/basic.py +1 -1
  152. metadata/generated/schema/analytics/reportData.py +1 -1
  153. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  154. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  155. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  156. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  157. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  158. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  159. metadata/generated/schema/analytics/webAnalyticEvent.py +9 -5
  160. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  161. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  162. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  163. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  164. metadata/generated/schema/api/__init__.py +1 -1
  165. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +3 -16
  166. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -10
  167. metadata/generated/schema/api/analytics/__init__.py +1 -1
  168. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +6 -5
  169. metadata/generated/schema/api/automations/__init__.py +1 -1
  170. metadata/generated/schema/api/automations/createWorkflow.py +18 -7
  171. metadata/generated/schema/api/bulkAssets.py +1 -1
  172. metadata/generated/schema/api/classification/__init__.py +1 -1
  173. metadata/generated/schema/api/classification/createClassification.py +63 -7
  174. metadata/generated/schema/api/classification/createTag.py +37 -6
  175. metadata/generated/schema/api/classification/createTagWithRecognizers.py +67 -0
  176. metadata/generated/schema/api/classification/loadTags.py +1 -1
  177. metadata/generated/schema/api/configuration/__init__.py +3 -0
  178. metadata/generated/schema/api/configuration/rdfConfiguration.py +47 -0
  179. metadata/generated/schema/api/createBot.py +5 -5
  180. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  181. metadata/generated/schema/api/createType.py +6 -5
  182. metadata/generated/schema/api/data/__init__.py +1 -1
  183. metadata/generated/schema/api/data/createAPICollection.py +4 -4
  184. metadata/generated/schema/api/data/createAPIEndpoint.py +4 -4
  185. metadata/generated/schema/api/data/createChart.py +12 -4
  186. metadata/generated/schema/api/data/createContainer.py +4 -4
  187. metadata/generated/schema/api/data/createCustomProperty.py +5 -1
  188. metadata/generated/schema/api/data/createDashboard.py +4 -4
  189. metadata/generated/schema/api/data/createDashboardDataModel.py +8 -4
  190. metadata/generated/schema/api/data/createDataContract.py +102 -0
  191. metadata/generated/schema/api/data/createDatabase.py +4 -4
  192. metadata/generated/schema/api/data/createDatabaseSchema.py +4 -4
  193. metadata/generated/schema/api/data/createDirectory.py +105 -0
  194. metadata/generated/schema/api/data/createEntityProfile.py +36 -0
  195. metadata/generated/schema/api/data/createFile.py +104 -0
  196. metadata/generated/schema/api/data/createGlossary.py +4 -4
  197. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  198. metadata/generated/schema/api/data/createMetric.py +15 -4
  199. metadata/generated/schema/api/data/createMlModel.py +4 -4
  200. metadata/generated/schema/api/data/createPipeline.py +8 -4
  201. metadata/generated/schema/api/data/createQuery.py +5 -4
  202. metadata/generated/schema/api/data/createQueryCostRecord.py +35 -0
  203. metadata/generated/schema/api/data/createSearchIndex.py +4 -4
  204. metadata/generated/schema/api/data/createSpreadsheet.py +119 -0
  205. metadata/generated/schema/api/data/createStoredProcedure.py +4 -4
  206. metadata/generated/schema/api/data/createTable.py +5 -4
  207. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  208. metadata/generated/schema/api/data/createTopic.py +5 -4
  209. metadata/generated/schema/api/data/createWorksheet.py +105 -0
  210. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  211. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  212. metadata/generated/schema/api/data/updateColumn.py +49 -0
  213. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  214. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +5 -4
  215. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  216. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  217. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  218. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +5 -5
  219. metadata/generated/schema/api/docStore/__init__.py +1 -1
  220. metadata/generated/schema/api/docStore/createDocument.py +6 -5
  221. metadata/generated/schema/api/domains/__init__.py +1 -1
  222. metadata/generated/schema/api/domains/createDataProduct.py +15 -6
  223. metadata/generated/schema/api/domains/createDomain.py +13 -2
  224. metadata/generated/schema/api/entityRelationship/__init__.py +3 -0
  225. metadata/generated/schema/api/entityRelationship/entityRelationshipDirection.py +12 -0
  226. metadata/generated/schema/api/entityRelationship/esEntityRelationshipData.py +57 -0
  227. metadata/generated/schema/api/entityRelationship/relationshipRef.py +32 -0
  228. metadata/generated/schema/api/entityRelationship/searchEntityRelationshipRequest.py +47 -0
  229. metadata/generated/schema/api/entityRelationship/searchEntityRelationshipResult.py +47 -0
  230. metadata/generated/schema/api/entityRelationship/searchSchemaEntityRelationshipResult.py +20 -0
  231. metadata/generated/schema/api/events/__init__.py +3 -0
  232. metadata/generated/schema/api/events/createNotificationTemplate.py +61 -0
  233. metadata/generated/schema/api/events/notificationTemplateValidationRequest.py +30 -0
  234. metadata/generated/schema/api/events/notificationTemplateValidationResponse.py +36 -0
  235. metadata/generated/schema/api/feed/__init__.py +1 -1
  236. metadata/generated/schema/api/feed/closeTask.py +1 -1
  237. metadata/generated/schema/api/feed/createPost.py +1 -1
  238. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  239. metadata/generated/schema/api/feed/createThread.py +6 -2
  240. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  241. metadata/generated/schema/api/feed/threadCount.py +25 -1
  242. metadata/generated/schema/api/governance/__init__.py +1 -1
  243. metadata/generated/schema/api/governance/createWorkflowDefinition.py +20 -5
  244. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  245. metadata/generated/schema/api/lineage/__init__.py +1 -1
  246. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  247. metadata/generated/schema/api/lineage/entityCountLineageRequest.py +66 -0
  248. metadata/generated/schema/api/lineage/esLineageData.py +88 -0
  249. metadata/generated/schema/api/lineage/lineageDirection.py +12 -0
  250. metadata/generated/schema/api/lineage/lineagePaginationInfo.py +57 -0
  251. metadata/generated/schema/api/lineage/nodeInformation.py +30 -0
  252. metadata/generated/schema/api/lineage/searchLineageRequest.py +45 -0
  253. metadata/generated/schema/api/lineage/searchLineageResult.py +46 -0
  254. metadata/generated/schema/api/mcp/__init__.py +3 -0
  255. metadata/generated/schema/api/mcp/mcpSearchResponse.py +118 -0
  256. metadata/generated/schema/api/mcp/mcpToolDefinition.py +54 -0
  257. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  258. metadata/generated/schema/api/policies/__init__.py +1 -1
  259. metadata/generated/schema/api/policies/createPolicy.py +6 -5
  260. metadata/generated/schema/api/rdf/__init__.py +3 -0
  261. metadata/generated/schema/api/rdf/sparqlQuery.py +57 -0
  262. metadata/generated/schema/api/rdf/sparqlResponse.py +56 -0
  263. metadata/generated/schema/api/scim/__init__.py +3 -0
  264. metadata/generated/schema/api/scim/scimGroup.py +52 -0
  265. metadata/generated/schema/api/scim/scimPatchOp.py +32 -0
  266. metadata/generated/schema/api/scim/scimUser.py +83 -0
  267. metadata/generated/schema/api/search/__init__.py +3 -0
  268. metadata/generated/schema/api/search/previewSearchRequest.py +47 -0
  269. metadata/generated/schema/api/services/__init__.py +1 -1
  270. metadata/generated/schema/api/services/createApiService.py +12 -5
  271. metadata/generated/schema/api/services/createDashboardService.py +12 -5
  272. metadata/generated/schema/api/services/createDatabaseService.py +12 -5
  273. metadata/generated/schema/api/services/createDriveService.py +65 -0
  274. metadata/generated/schema/api/services/createMessagingService.py +12 -5
  275. metadata/generated/schema/api/services/createMetadataService.py +13 -5
  276. metadata/generated/schema/api/services/createMlModelService.py +12 -5
  277. metadata/generated/schema/api/services/createPipelineService.py +12 -5
  278. metadata/generated/schema/api/services/createSearchService.py +12 -5
  279. metadata/generated/schema/api/services/createSecurityService.py +67 -0
  280. metadata/generated/schema/api/services/createStorageService.py +12 -5
  281. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  282. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +28 -5
  283. metadata/generated/schema/api/setOwner.py +1 -1
  284. metadata/generated/schema/api/teams/__init__.py +1 -1
  285. metadata/generated/schema/api/teams/createPersona.py +12 -4
  286. metadata/generated/schema/api/teams/createRole.py +5 -4
  287. metadata/generated/schema/api/teams/createTeam.py +8 -1
  288. metadata/generated/schema/api/teams/createUser.py +11 -1
  289. metadata/generated/schema/api/tests/__init__.py +1 -1
  290. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  291. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  292. metadata/generated/schema/api/tests/createTestCase.py +13 -6
  293. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  294. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  295. metadata/generated/schema/api/tests/createTestDefinition.py +5 -4
  296. metadata/generated/schema/api/tests/createTestSuite.py +17 -5
  297. metadata/generated/schema/api/tests/moveGlossaryTermRequest.py +27 -0
  298. metadata/generated/schema/api/validateGlossaryTagsRequest.py +31 -0
  299. metadata/generated/schema/api/voteRequest.py +1 -1
  300. metadata/generated/schema/auth/__init__.py +1 -1
  301. metadata/generated/schema/auth/basicAuth.py +1 -1
  302. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  303. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  304. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  305. metadata/generated/schema/auth/emailRequest.py +1 -1
  306. metadata/generated/schema/auth/emailVerificationToken.py +3 -4
  307. metadata/generated/schema/auth/generateToken.py +1 -1
  308. metadata/generated/schema/auth/jwtAuth.py +1 -1
  309. metadata/generated/schema/auth/loginRequest.py +1 -1
  310. metadata/generated/schema/auth/logoutRequest.py +1 -1
  311. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  312. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  313. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  314. metadata/generated/schema/auth/refreshToken.py +1 -1
  315. metadata/generated/schema/auth/registrationRequest.py +1 -1
  316. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  317. metadata/generated/schema/auth/revokeToken.py +1 -1
  318. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  319. metadata/generated/schema/auth/ssoAuth.py +1 -1
  320. metadata/generated/schema/auth/supportToken.py +35 -0
  321. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  322. metadata/generated/schema/configuration/__init__.py +1 -1
  323. metadata/generated/schema/configuration/aiPlatformConfiguration.py +58 -0
  324. metadata/generated/schema/configuration/appsPrivateConfiguration.py +4 -1
  325. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  326. metadata/generated/schema/configuration/authConfig.py +1 -1
  327. metadata/generated/schema/configuration/authenticationConfiguration.py +49 -13
  328. metadata/generated/schema/configuration/authorizerConfiguration.py +41 -9
  329. metadata/generated/schema/configuration/cacheConfiguration.py +137 -0
  330. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  331. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  332. metadata/generated/schema/configuration/elasticSearchConfiguration.py +80 -1
  333. metadata/generated/schema/configuration/entityRulesSettings.py +24 -0
  334. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  335. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  336. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  337. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  338. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  339. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  340. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  341. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  342. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  343. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  344. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  345. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  346. metadata/generated/schema/configuration/lineageSettings.py +9 -1
  347. metadata/generated/schema/configuration/logStorageConfiguration.py +105 -0
  348. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  349. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  350. metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +22 -0
  351. metadata/generated/schema/configuration/opertionalConfiguration.py +24 -0
  352. metadata/generated/schema/configuration/opsConfig.py +25 -0
  353. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +9 -2
  354. metadata/generated/schema/configuration/profilerConfiguration.py +2 -1
  355. metadata/generated/schema/configuration/searchSettings.py +508 -4
  356. metadata/generated/schema/configuration/securityConfiguration.py +26 -0
  357. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  358. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  359. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  360. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  361. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  362. metadata/generated/schema/configuration/workflowSettings.py +24 -3
  363. metadata/generated/schema/dataInsight/__init__.py +1 -1
  364. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  365. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +5 -1
  366. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  367. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  368. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  369. metadata/generated/schema/dataInsight/custom/lineChart.py +19 -1
  370. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  371. metadata/generated/schema/dataInsight/dataInsightChart.py +9 -5
  372. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  373. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  374. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  375. metadata/generated/schema/dataInsight/kpi/kpi.py +8 -4
  376. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  377. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  378. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  379. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  380. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  381. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  382. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  383. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  384. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  385. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  386. metadata/generated/schema/email/__init__.py +1 -1
  387. metadata/generated/schema/email/emailRequest.py +1 -1
  388. metadata/generated/schema/email/emailTemplate.py +1 -1
  389. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  390. metadata/generated/schema/email/smtpSettings.py +6 -5
  391. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  392. metadata/generated/schema/entity/__init__.py +1 -1
  393. metadata/generated/schema/entity/applications/__init__.py +1 -1
  394. metadata/generated/schema/entity/applications/app.py +40 -7
  395. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  396. metadata/generated/schema/entity/applications/appRunRecord.py +46 -6
  397. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  398. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +21 -6
  399. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  400. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  401. metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
  402. metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +41 -0
  403. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  404. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +5 -5
  405. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  406. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +7 -2
  407. metadata/generated/schema/entity/applications/configuration/external/automator/addTermsAction.py +48 -0
  408. metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +74 -0
  409. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  410. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +28 -2
  411. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  412. metadata/generated/schema/entity/applications/configuration/external/automator/propagationStopConfig.py +44 -0
  413. metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
  414. metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +33 -0
  415. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +10 -2
  416. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  417. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  418. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +27 -3
  419. metadata/generated/schema/entity/applications/configuration/external/automator/removeTermsAction.py +59 -0
  420. metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +45 -0
  421. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  422. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +26 -1
  423. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  424. metadata/generated/schema/entity/applications/configuration/external/metadataExporterAppConfig.py +103 -0
  425. metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/__init__.py +3 -0
  426. metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/bigQueryConnection.py +90 -0
  427. metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/databricksConnection.py +101 -0
  428. metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/redshiftConnection.py +76 -0
  429. metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/snowflakeConnection.py +115 -0
  430. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  431. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  432. metadata/generated/schema/entity/applications/configuration/internal/autoPilotAppConfig.py +48 -0
  433. metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +46 -0
  434. metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +46 -0
  435. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +111 -2
  436. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  437. metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +46 -0
  438. metadata/generated/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.py +18 -0
  439. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +9 -1
  440. metadata/generated/schema/entity/applications/configuration/private/__init__.py +3 -0
  441. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  442. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +3 -22
  443. metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +3 -0
  444. metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +24 -0
  445. metadata/generated/schema/entity/applications/configuration/private/limits.py +33 -0
  446. metadata/generated/schema/entity/applications/createAppRequest.py +14 -6
  447. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  448. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  449. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  450. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +32 -12
  451. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +35 -5
  452. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  453. metadata/generated/schema/entity/automations/__init__.py +1 -1
  454. metadata/generated/schema/entity/automations/queryRunnerRequest.py +46 -0
  455. metadata/generated/schema/entity/automations/response/__init__.py +3 -0
  456. metadata/generated/schema/entity/automations/response/queryRunnerResponse.py +51 -0
  457. metadata/generated/schema/entity/automations/testServiceConnection.py +12 -1
  458. metadata/generated/schema/entity/automations/testSparkEngineConnection.py +31 -0
  459. metadata/generated/schema/entity/automations/workflow.py +30 -9
  460. metadata/generated/schema/entity/bot.py +9 -5
  461. metadata/generated/schema/entity/classification/__init__.py +1 -1
  462. metadata/generated/schema/entity/classification/classification.py +69 -7
  463. metadata/generated/schema/entity/classification/tag.py +53 -8
  464. metadata/generated/schema/entity/data/__init__.py +1 -1
  465. metadata/generated/schema/entity/data/apiCollection.py +15 -4
  466. metadata/generated/schema/entity/data/apiEndpoint.py +15 -4
  467. metadata/generated/schema/entity/data/chart.py +26 -4
  468. metadata/generated/schema/entity/data/container.py +14 -4
  469. metadata/generated/schema/entity/data/dashboard.py +13 -4
  470. metadata/generated/schema/entity/data/dashboardDataModel.py +23 -4
  471. metadata/generated/schema/entity/data/dataContract.py +403 -0
  472. metadata/generated/schema/entity/data/database.py +21 -4
  473. metadata/generated/schema/entity/data/databaseSchema.py +25 -8
  474. metadata/generated/schema/entity/data/directory.py +178 -0
  475. metadata/generated/schema/entity/data/file.py +189 -0
  476. metadata/generated/schema/entity/data/glossary.py +13 -4
  477. metadata/generated/schema/entity/data/glossaryTerm.py +15 -15
  478. metadata/generated/schema/entity/data/metric.py +25 -5
  479. metadata/generated/schema/entity/data/mlmodel.py +13 -4
  480. metadata/generated/schema/entity/data/pipeline.py +156 -5
  481. metadata/generated/schema/entity/data/query.py +13 -4
  482. metadata/generated/schema/entity/data/queryCostRecord.py +47 -0
  483. metadata/generated/schema/entity/data/queryCostSearchResult.py +56 -0
  484. metadata/generated/schema/entity/data/report.py +13 -4
  485. metadata/generated/schema/entity/data/searchIndex.py +15 -4
  486. metadata/generated/schema/entity/data/spreadsheet.py +215 -0
  487. metadata/generated/schema/entity/data/storedProcedure.py +22 -4
  488. metadata/generated/schema/entity/data/table.py +179 -4
  489. metadata/generated/schema/entity/data/topic.py +13 -4
  490. metadata/generated/schema/entity/data/worksheet.py +173 -0
  491. metadata/generated/schema/entity/datacontract/__init__.py +3 -0
  492. metadata/generated/schema/entity/datacontract/dataContractResult.py +74 -0
  493. metadata/generated/schema/entity/datacontract/qualityValidation.py +31 -0
  494. metadata/generated/schema/entity/datacontract/schemaValidation.py +31 -0
  495. metadata/generated/schema/entity/datacontract/semanticsValidation.py +39 -0
  496. metadata/generated/schema/entity/datacontract/slaValidation.py +32 -0
  497. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  498. metadata/generated/schema/entity/docStore/document.py +9 -5
  499. metadata/generated/schema/entity/domains/__init__.py +1 -1
  500. metadata/generated/schema/entity/domains/dataProduct.py +171 -6
  501. metadata/generated/schema/entity/domains/domain.py +22 -3
  502. metadata/generated/schema/entity/events/__init__.py +1 -1
  503. metadata/generated/schema/entity/events/notificationTemplate.py +111 -0
  504. metadata/generated/schema/entity/events/webhook.py +7 -1
  505. metadata/generated/schema/entity/feed/__init__.py +1 -1
  506. metadata/generated/schema/entity/feed/assets.py +1 -1
  507. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  508. metadata/generated/schema/entity/feed/description.py +1 -1
  509. metadata/generated/schema/entity/feed/domain.py +8 -8
  510. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  511. metadata/generated/schema/entity/feed/owner.py +1 -1
  512. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  513. metadata/generated/schema/entity/feed/tag.py +1 -1
  514. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  515. metadata/generated/schema/entity/feed/thread.py +4 -3
  516. metadata/generated/schema/entity/policies/__init__.py +1 -1
  517. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  518. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +6 -1
  519. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  520. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  521. metadata/generated/schema/entity/policies/filters.py +1 -1
  522. metadata/generated/schema/entity/policies/policy.py +8 -4
  523. metadata/generated/schema/entity/services/__init__.py +1 -1
  524. metadata/generated/schema/entity/services/apiService.py +18 -3
  525. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  526. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  527. metadata/generated/schema/entity/services/connections/api/restConnection.py +15 -1
  528. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  529. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  530. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  531. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  532. metadata/generated/schema/entity/services/connections/connectionBasicType.py +7 -1
  533. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  534. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +39 -2
  535. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +34 -1
  536. metadata/generated/schema/entity/services/connections/dashboard/grafanaConnection.py +77 -0
  537. metadata/generated/schema/entity/services/connections/dashboard/hexConnection.py +76 -0
  538. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +34 -1
  539. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +46 -2
  540. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +50 -4
  541. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +34 -1
  542. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +34 -1
  543. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +50 -1
  544. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +34 -1
  545. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  546. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  547. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  548. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  549. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  550. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +51 -2
  551. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +34 -1
  552. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +34 -1
  553. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +34 -1
  554. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +40 -1
  555. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +34 -1
  556. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +51 -11
  557. metadata/generated/schema/entity/services/connections/dashboard/thoughtSpotConnection.py +77 -0
  558. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  559. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +26 -1
  560. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +26 -1
  561. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +48 -1
  562. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +26 -1
  563. metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +3 -0
  564. metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +61 -0
  565. metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +106 -0
  566. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +26 -1
  567. metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +136 -0
  568. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  569. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  570. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  571. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  572. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  573. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  574. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +26 -1
  575. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +31 -2
  576. metadata/generated/schema/entity/services/connections/database/databricks/__init__.py +3 -0
  577. metadata/generated/schema/entity/services/connections/database/databricks/azureAdSetup.py +37 -0
  578. metadata/generated/schema/entity/services/connections/database/databricks/databricksOAuth.py +30 -0
  579. metadata/generated/schema/entity/services/connections/database/databricks/personalAccessToken.py +23 -0
  580. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +48 -8
  581. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  582. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  583. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  584. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  585. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +26 -1
  586. metadata/generated/schema/entity/services/connections/database/db2Connection.py +34 -1
  587. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +26 -1
  588. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  589. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  590. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  591. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +26 -1
  592. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +26 -1
  593. metadata/generated/schema/entity/services/connections/database/druidConnection.py +26 -1
  594. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +26 -1
  595. metadata/generated/schema/entity/services/connections/database/epicConnection.py +80 -0
  596. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +29 -1
  597. metadata/generated/schema/entity/services/connections/database/glueConnection.py +26 -1
  598. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +26 -1
  599. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +41 -1
  600. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  601. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  602. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  603. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  604. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  605. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  606. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  607. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +26 -1
  608. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +26 -1
  609. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +26 -1
  610. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +26 -1
  611. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +54 -1
  612. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +37 -1
  613. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +26 -1
  614. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +26 -1
  615. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +26 -1
  616. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +26 -1
  617. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +30 -1
  618. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +50 -6
  619. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +26 -1
  620. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  621. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  622. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  623. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +26 -1
  624. metadata/generated/schema/entity/services/connections/database/sasConnection.py +32 -1
  625. metadata/generated/schema/entity/services/connections/database/serviceNowConnection.py +90 -0
  626. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +26 -1
  627. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +62 -1
  628. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +26 -1
  629. metadata/generated/schema/entity/services/connections/database/ssasConnection.py +47 -0
  630. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +52 -2
  631. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +26 -1
  632. metadata/generated/schema/entity/services/connections/database/timescaleConnection.py +162 -0
  633. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +34 -1
  634. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +39 -6
  635. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +26 -1
  636. metadata/generated/schema/entity/services/connections/drive/__init__.py +3 -0
  637. metadata/generated/schema/entity/services/connections/drive/customDriveConnection.py +78 -0
  638. metadata/generated/schema/entity/services/connections/drive/googleDriveConnection.py +115 -0
  639. metadata/generated/schema/entity/services/connections/drive/sharePointConnection.py +112 -0
  640. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  641. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +15 -2
  642. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +18 -1
  643. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +10 -1
  644. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +10 -1
  645. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +18 -1
  646. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  647. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  648. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +26 -1
  649. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +26 -1
  650. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +26 -1
  651. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +26 -1
  652. metadata/generated/schema/entity/services/connections/metadata/collibraConnection.py +73 -0
  653. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +26 -1
  654. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +26 -2
  655. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  656. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +15 -2
  657. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +10 -1
  658. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +10 -1
  659. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +10 -1
  660. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +15 -1
  661. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  662. metadata/generated/schema/entity/services/connections/pipeline/airbyte/__init__.py +3 -0
  663. metadata/generated/schema/entity/services/connections/pipeline/airbyte/basicAuth.py +26 -0
  664. metadata/generated/schema/entity/services/connections/pipeline/airbyte/oauthClientAuth.py +30 -0
  665. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +25 -10
  666. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +10 -1
  667. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -1
  668. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +15 -2
  669. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +10 -1
  670. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +28 -1
  671. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +15 -1
  672. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +24 -1
  673. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +10 -1
  674. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +11 -2
  675. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +15 -1
  676. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +10 -1
  677. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +16 -1
  678. metadata/generated/schema/entity/services/connections/pipeline/kinesisFirehoseConnection.py +58 -0
  679. metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +3 -0
  680. metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +39 -0
  681. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +13 -25
  682. metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +3 -0
  683. metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +40 -0
  684. metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +38 -0
  685. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +12 -2
  686. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +10 -1
  687. metadata/generated/schema/entity/services/connections/pipeline/snowplowConnection.py +93 -0
  688. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +16 -1
  689. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +10 -1
  690. metadata/generated/schema/entity/services/connections/pipeline/ssisConnection.py +56 -0
  691. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +10 -1
  692. metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py +51 -0
  693. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  694. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +15 -2
  695. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  696. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  697. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  698. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +10 -1
  699. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +37 -29
  700. metadata/generated/schema/entity/services/connections/security/__init__.py +3 -0
  701. metadata/generated/schema/entity/services/connections/security/ranger/__init__.py +3 -0
  702. metadata/generated/schema/entity/services/connections/security/ranger/basicAuth.py +26 -0
  703. metadata/generated/schema/entity/services/connections/security/rangerConnection.py +39 -0
  704. metadata/generated/schema/entity/services/connections/serviceConnection.py +7 -1
  705. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  706. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +10 -1
  707. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +15 -2
  708. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +10 -1
  709. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +19 -2
  710. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +9 -5
  711. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  712. metadata/generated/schema/entity/services/dashboardService.py +27 -3
  713. metadata/generated/schema/entity/services/databaseService.py +36 -3
  714. metadata/generated/schema/entity/services/driveService.py +143 -0
  715. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  716. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +50 -4
  717. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  718. metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +47 -0
  719. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  720. metadata/generated/schema/entity/services/messagingService.py +18 -3
  721. metadata/generated/schema/entity/services/metadataService.py +21 -3
  722. metadata/generated/schema/entity/services/mlmodelService.py +18 -3
  723. metadata/generated/schema/entity/services/pipelineService.py +30 -3
  724. metadata/generated/schema/entity/services/searchService.py +19 -4
  725. metadata/generated/schema/entity/services/securityService.py +135 -0
  726. metadata/generated/schema/entity/services/serviceType.py +3 -1
  727. metadata/generated/schema/entity/services/storageService.py +18 -3
  728. metadata/generated/schema/entity/teams/__init__.py +1 -1
  729. metadata/generated/schema/entity/teams/persona.py +20 -5
  730. metadata/generated/schema/entity/teams/role.py +9 -5
  731. metadata/generated/schema/entity/teams/team.py +12 -1
  732. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  733. metadata/generated/schema/entity/teams/user.py +39 -3
  734. metadata/generated/schema/entity/type.py +9 -11
  735. metadata/generated/schema/entity/utils/__init__.py +1 -1
  736. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  737. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  738. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  739. metadata/generated/schema/events/__init__.py +1 -1
  740. metadata/generated/schema/events/alertMetrics.py +1 -1
  741. metadata/generated/schema/events/api/__init__.py +1 -1
  742. metadata/generated/schema/events/api/createEventSubscription.py +13 -4
  743. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  744. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  745. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  746. metadata/generated/schema/events/api/typedEvent.py +1 -1
  747. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  748. metadata/generated/schema/events/eventFilterRule.py +1 -1
  749. metadata/generated/schema/events/eventSubscription.py +32 -6
  750. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  751. metadata/generated/schema/events/failedEvent.py +1 -1
  752. metadata/generated/schema/events/failedEventResponse.py +1 -1
  753. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  754. metadata/generated/schema/events/statusContext.py +1 -1
  755. metadata/generated/schema/events/subscriptionResourceDescriptor.py +2 -1
  756. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  757. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  758. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  759. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  760. metadata/generated/schema/governance/workflows/elements/edge.py +2 -2
  761. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +7 -1
  762. metadata/generated/schema/governance/workflows/elements/nodeType.py +2 -1
  763. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  764. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  765. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +28 -8
  766. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +86 -0
  767. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/dataCompletenessTask.py +124 -0
  768. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/rollbackEntityTask.py +54 -0
  769. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +73 -0
  770. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityAttributeTask.py +66 -0
  771. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +27 -6
  772. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +29 -9
  773. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  774. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +9 -4
  775. metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +3 -0
  776. metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +30 -0
  777. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  778. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +9 -4
  779. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  780. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +45 -7
  781. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  782. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +103 -8
  783. metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +20 -0
  784. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +94 -8
  785. metadata/generated/schema/governance/workflows/workflowDefinition.py +34 -7
  786. metadata/generated/schema/governance/workflows/workflowInstance.py +11 -8
  787. metadata/generated/schema/governance/workflows/workflowInstanceState.py +8 -8
  788. metadata/generated/schema/jobs/__init__.py +1 -1
  789. metadata/generated/schema/jobs/backgroundJob.py +12 -3
  790. metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
  791. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  792. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  793. metadata/generated/schema/metadataIngestion/application.py +8 -1
  794. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  795. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +4 -4
  796. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  797. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +3 -3
  798. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +20 -4
  799. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +29 -3
  800. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +42 -2
  801. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +9 -1
  802. metadata/generated/schema/metadataIngestion/dbtPipeline.py +16 -1
  803. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  804. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  805. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  806. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  807. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  808. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  809. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  810. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  811. metadata/generated/schema/metadataIngestion/driveServiceMetadataPipeline.py +165 -0
  812. metadata/generated/schema/metadataIngestion/engine/__init__.py +3 -0
  813. metadata/generated/schema/metadataIngestion/engine/nativeEngineConfig.py +23 -0
  814. metadata/generated/schema/metadataIngestion/engine/sparkEngineConfig.py +43 -0
  815. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  816. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  817. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +29 -2
  818. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  819. metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +71 -0
  820. metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +3 -0
  821. metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +24 -0
  822. metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +28 -0
  823. metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +28 -0
  824. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +2 -2
  825. metadata/generated/schema/metadataIngestion/securityServiceMetadataPipeline.py +27 -0
  826. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  827. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  828. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +17 -1
  829. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  830. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  831. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  832. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  833. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  834. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  835. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  836. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  837. metadata/generated/schema/metadataIngestion/workflow.py +19 -1
  838. metadata/generated/schema/monitoring/__init__.py +1 -1
  839. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  840. metadata/generated/schema/scim/__init__.py +3 -0
  841. metadata/generated/schema/scim/scimConfiguration.py +32 -0
  842. metadata/generated/schema/search/__init__.py +3 -0
  843. metadata/generated/schema/search/aggregationRequest.py +80 -0
  844. metadata/generated/schema/search/searchRequest.py +120 -0
  845. metadata/generated/schema/security/__init__.py +1 -1
  846. metadata/generated/schema/security/client/__init__.py +1 -1
  847. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  848. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  849. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  850. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  851. metadata/generated/schema/security/client/oidcClientConfig.py +22 -10
  852. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  853. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  854. metadata/generated/schema/security/client/samlSSOClientConfig.py +6 -6
  855. metadata/generated/schema/security/credentials/__init__.py +1 -1
  856. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  857. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  858. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  859. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  860. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  861. metadata/generated/schema/security/credentials/bitbucketCredentials.py +10 -2
  862. metadata/generated/schema/security/credentials/gcpCredentials.py +42 -22
  863. metadata/generated/schema/security/credentials/gcpExternalAccount.py +2 -2
  864. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  865. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  866. metadata/generated/schema/security/credentials/githubCredentials.py +10 -2
  867. metadata/generated/schema/security/credentials/gitlabCredentials.py +10 -2
  868. metadata/generated/schema/security/credentials/kubernetesCredentials.py +40 -0
  869. metadata/generated/schema/security/sasl/__init__.py +1 -1
  870. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  871. metadata/generated/schema/security/secrets/__init__.py +1 -1
  872. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  873. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  874. metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -1
  875. metadata/generated/schema/security/securityConfiguration.py +1 -1
  876. metadata/generated/schema/security/ssl/__init__.py +1 -1
  877. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  878. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  879. metadata/generated/schema/settings/__init__.py +1 -1
  880. metadata/generated/schema/settings/settings.py +11 -1
  881. metadata/generated/schema/system/__init__.py +1 -1
  882. metadata/generated/schema/system/entityError.py +1 -1
  883. metadata/generated/schema/system/eventPublisherJob.py +62 -11
  884. metadata/generated/schema/system/indexingError.py +2 -2
  885. metadata/generated/schema/system/limitsResponse.py +1 -1
  886. metadata/generated/schema/system/securityValidationResponse.py +62 -0
  887. metadata/generated/schema/system/ui/__init__.py +1 -1
  888. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  889. metadata/generated/schema/system/ui/navigationItem.py +38 -0
  890. metadata/generated/schema/system/ui/page.py +34 -18
  891. metadata/generated/schema/system/ui/tab.py +36 -0
  892. metadata/generated/schema/system/ui/uiCustomization.py +52 -0
  893. metadata/generated/schema/system/validationResponse.py +8 -1
  894. metadata/generated/schema/tests/__init__.py +1 -1
  895. metadata/generated/schema/tests/assigned.py +1 -1
  896. metadata/generated/schema/tests/basic.py +1 -1
  897. metadata/generated/schema/tests/customMetric.py +1 -1
  898. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  899. metadata/generated/schema/tests/resolved.py +1 -1
  900. metadata/generated/schema/tests/testCase.py +36 -5
  901. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  902. metadata/generated/schema/tests/testDefinition.py +9 -5
  903. metadata/generated/schema/tests/testSuite.py +26 -4
  904. metadata/generated/schema/type/__init__.py +1 -1
  905. metadata/generated/schema/type/apiSchema.py +1 -1
  906. metadata/generated/schema/type/assetCertification.py +1 -1
  907. metadata/generated/schema/type/auditLog.py +1 -1
  908. metadata/generated/schema/type/basic.py +61 -3
  909. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  910. metadata/generated/schema/type/changeEvent.py +9 -7
  911. metadata/generated/schema/type/changeEventType.py +1 -1
  912. metadata/generated/schema/type/changeSummaryMap.py +37 -0
  913. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  914. metadata/generated/schema/type/contextRecognizer.py +50 -0
  915. metadata/generated/schema/type/contractExecutionStatus.py +16 -0
  916. metadata/generated/schema/type/csvDocumentation.py +1 -1
  917. metadata/generated/schema/type/csvErrorType.py +1 -1
  918. metadata/generated/schema/type/csvFile.py +1 -1
  919. metadata/generated/schema/type/csvImportResult.py +1 -1
  920. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  921. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  922. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  923. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  924. metadata/generated/schema/type/customProperty.py +1 -1
  925. metadata/generated/schema/type/customRecognizer.py +39 -0
  926. metadata/generated/schema/type/dailyCount.py +1 -1
  927. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  928. metadata/generated/schema/type/denyListRecognizer.py +37 -0
  929. metadata/generated/schema/type/entityHierarchy.py +7 -6
  930. metadata/generated/schema/type/entityHistory.py +35 -1
  931. metadata/generated/schema/type/entityLineage.py +27 -1
  932. metadata/generated/schema/type/entityProfile.py +45 -0
  933. metadata/generated/schema/type/entityReference.py +1 -1
  934. metadata/generated/schema/type/entityReferenceList.py +1 -1
  935. metadata/generated/schema/type/{entityRelationship.py → entityRelationship/__init__.py} +2 -2
  936. metadata/generated/schema/type/entityRelationship/nodeInformation.py +25 -0
  937. metadata/generated/schema/type/entityUsage.py +1 -1
  938. metadata/generated/schema/type/filterPattern.py +3 -3
  939. metadata/generated/schema/type/function.py +1 -1
  940. metadata/generated/schema/type/include.py +1 -1
  941. metadata/generated/schema/type/jdbcConnection.py +1 -1
  942. metadata/generated/schema/type/layerPaging.py +24 -0
  943. metadata/generated/schema/type/lifeCycle.py +1 -1
  944. metadata/generated/schema/type/paging.py +1 -1
  945. metadata/generated/schema/type/patternRecognizer.py +39 -0
  946. metadata/generated/schema/type/personaPreferences.py +48 -0
  947. metadata/generated/schema/type/piiEntity.py +48 -0
  948. metadata/generated/schema/type/predefinedRecognizer.py +85 -0
  949. metadata/generated/schema/type/profile.py +1 -1
  950. metadata/generated/schema/type/queryParserData.py +4 -1
  951. metadata/generated/schema/type/reaction.py +1 -1
  952. metadata/generated/schema/type/recognizer.py +159 -0
  953. metadata/generated/schema/type/recognizerFeedback.py +120 -0
  954. metadata/generated/schema/type/recognizers/__init__.py +3 -0
  955. metadata/generated/schema/type/recognizers/patterns.py +29 -0
  956. metadata/generated/schema/type/recognizers/regexFlags.py +29 -0
  957. metadata/generated/schema/type/schedule.py +1 -1
  958. metadata/generated/schema/type/schema.py +1 -1
  959. metadata/generated/schema/type/status.py +16 -0
  960. metadata/generated/schema/type/tableQuery.py +4 -1
  961. metadata/generated/schema/type/tableUsageCount.py +18 -1
  962. metadata/generated/schema/type/tagLabel.py +9 -1
  963. metadata/generated/schema/type/usageDetails.py +1 -1
  964. metadata/generated/schema/type/usageRequest.py +1 -1
  965. metadata/generated/schema/type/votes.py +1 -1
  966. metadata/great_expectations/action.py +158 -21
  967. metadata/great_expectations/action1xx.py +471 -0
  968. metadata/great_expectations/utils/ometa_config_handler.py +2 -2
  969. metadata/ingestion/api/closeable.py +3 -3
  970. metadata/ingestion/api/common.py +3 -3
  971. metadata/ingestion/api/delete.py +3 -3
  972. metadata/ingestion/api/models.py +3 -3
  973. metadata/ingestion/api/parser.py +41 -12
  974. metadata/ingestion/api/status.py +9 -4
  975. metadata/ingestion/api/step.py +6 -4
  976. metadata/ingestion/api/steps.py +3 -3
  977. metadata/ingestion/api/topology_runner.py +54 -32
  978. metadata/ingestion/bulksink/metadata_usage.py +49 -11
  979. metadata/ingestion/connections/builders.py +22 -8
  980. metadata/ingestion/connections/connection.py +80 -0
  981. metadata/ingestion/connections/headers.py +3 -3
  982. metadata/ingestion/connections/query_logger.py +115 -0
  983. metadata/ingestion/connections/secrets.py +3 -3
  984. metadata/ingestion/connections/session.py +3 -3
  985. metadata/ingestion/connections/test_connections.py +4 -4
  986. metadata/ingestion/lineage/masker.py +45 -27
  987. metadata/ingestion/lineage/models.py +30 -3
  988. metadata/ingestion/lineage/parser.py +89 -36
  989. metadata/ingestion/lineage/sql_lineage.py +459 -112
  990. metadata/ingestion/models/custom_basemodel_validation.py +180 -49
  991. metadata/ingestion/models/custom_properties.py +6 -6
  992. metadata/ingestion/models/custom_pydantic.py +83 -27
  993. metadata/ingestion/models/custom_types.py +2 -2
  994. metadata/ingestion/models/data_insight.py +3 -3
  995. metadata/ingestion/models/delete_entity.py +3 -3
  996. metadata/ingestion/models/encoders.py +3 -3
  997. metadata/ingestion/models/entity_interface.py +3 -3
  998. metadata/ingestion/models/lf_tags_model.py +3 -3
  999. metadata/ingestion/models/life_cycle.py +3 -3
  1000. metadata/ingestion/models/ometa_classification.py +3 -3
  1001. metadata/ingestion/models/ometa_lineage.py +8 -4
  1002. metadata/ingestion/models/ometa_topic_data.py +3 -3
  1003. metadata/ingestion/models/patch_request.py +119 -11
  1004. metadata/ingestion/models/pipeline_status.py +3 -3
  1005. metadata/ingestion/models/profile_data.py +3 -3
  1006. metadata/ingestion/models/search_index_data.py +3 -3
  1007. metadata/ingestion/models/table_metadata.py +3 -3
  1008. metadata/ingestion/models/tests_data.py +3 -3
  1009. metadata/ingestion/models/topology.py +3 -3
  1010. metadata/ingestion/models/user.py +3 -3
  1011. metadata/ingestion/ometa/auth_provider.py +3 -3
  1012. metadata/ingestion/ometa/client.py +25 -5
  1013. metadata/ingestion/ometa/client_utils.py +3 -3
  1014. metadata/ingestion/ometa/credentials.py +3 -3
  1015. metadata/ingestion/ometa/mixins/csv_mixin.py +180 -0
  1016. metadata/ingestion/ometa/mixins/custom_property_mixin.py +3 -3
  1017. metadata/ingestion/ometa/mixins/dashboard_mixin.py +3 -3
  1018. metadata/ingestion/ometa/mixins/data_contract_mixin.py +205 -0
  1019. metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -3
  1020. metadata/ingestion/ometa/mixins/domain_mixin.py +128 -0
  1021. metadata/ingestion/ometa/mixins/es_mixin.py +134 -22
  1022. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +29 -3
  1023. metadata/ingestion/ometa/mixins/lineage_mixin.py +39 -12
  1024. metadata/ingestion/ometa/mixins/logs_mixin.py +316 -0
  1025. metadata/ingestion/ometa/mixins/mlmodel_mixin.py +3 -3
  1026. metadata/ingestion/ometa/mixins/patch_mixin.py +165 -65
  1027. metadata/ingestion/ometa/mixins/patch_mixin_utils.py +2 -2
  1028. metadata/ingestion/ometa/mixins/pipeline_mixin.py +21 -3
  1029. metadata/ingestion/ometa/mixins/profile_mixin.py +77 -0
  1030. metadata/ingestion/ometa/mixins/query_mixin.py +48 -3
  1031. metadata/ingestion/ometa/mixins/role_policy_mixin.py +2 -2
  1032. metadata/ingestion/ometa/mixins/search_index_mixin.py +3 -3
  1033. metadata/ingestion/ometa/mixins/server_mixin.py +3 -3
  1034. metadata/ingestion/ometa/mixins/service_mixin.py +3 -3
  1035. metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
  1036. metadata/ingestion/ometa/mixins/table_mixin.py +34 -4
  1037. metadata/ingestion/ometa/mixins/tag_glossary_mixin.py +68 -0
  1038. metadata/ingestion/ometa/mixins/tests_mixin.py +8 -8
  1039. metadata/ingestion/ometa/mixins/topic_mixin.py +3 -3
  1040. metadata/ingestion/ometa/mixins/user_mixin.py +60 -19
  1041. metadata/ingestion/ometa/mixins/version_mixin.py +3 -3
  1042. metadata/ingestion/ometa/models.py +3 -3
  1043. metadata/ingestion/ometa/ometa_api.py +94 -10
  1044. metadata/ingestion/ometa/routes.py +59 -3
  1045. metadata/ingestion/ometa/ttl_cache.py +3 -3
  1046. metadata/ingestion/ometa/utils.py +48 -4
  1047. metadata/ingestion/processor/query_parser.py +4 -3
  1048. metadata/ingestion/sink/file.py +3 -3
  1049. metadata/ingestion/sink/metadata_rest.py +139 -16
  1050. metadata/ingestion/source/api/api_service.py +4 -9
  1051. metadata/ingestion/source/api/rest/connection.py +22 -12
  1052. metadata/ingestion/source/api/rest/metadata.py +358 -67
  1053. metadata/ingestion/source/api/rest/models.py +5 -4
  1054. metadata/ingestion/source/api/rest/parser.py +103 -0
  1055. metadata/ingestion/source/connections.py +83 -17
  1056. metadata/ingestion/source/connections_utils.py +32 -0
  1057. metadata/ingestion/source/dashboard/dashboard_service.py +77 -25
  1058. metadata/ingestion/source/dashboard/domodashboard/connection.py +3 -3
  1059. metadata/ingestion/source/dashboard/domodashboard/metadata.py +35 -19
  1060. metadata/ingestion/source/dashboard/grafana/__init__.py +10 -0
  1061. metadata/ingestion/source/dashboard/grafana/client.py +223 -0
  1062. metadata/ingestion/source/dashboard/grafana/connection.py +66 -0
  1063. metadata/ingestion/source/dashboard/grafana/metadata.py +476 -0
  1064. metadata/ingestion/source/dashboard/grafana/models.py +159 -0
  1065. metadata/ingestion/source/dashboard/grafana/service_spec.py +17 -0
  1066. metadata/ingestion/source/dashboard/hex/client.py +104 -0
  1067. metadata/ingestion/source/dashboard/hex/connection.py +60 -0
  1068. metadata/ingestion/source/dashboard/hex/metadata.py +363 -0
  1069. metadata/ingestion/source/dashboard/hex/models.py +143 -0
  1070. metadata/ingestion/source/dashboard/hex/query_fetcher.py +503 -0
  1071. metadata/ingestion/source/dashboard/hex/service_spec.py +18 -0
  1072. metadata/ingestion/source/dashboard/hex/warehouse_queries.py +225 -0
  1073. metadata/ingestion/source/dashboard/lightdash/client.py +86 -12
  1074. metadata/ingestion/source/dashboard/lightdash/connection.py +3 -3
  1075. metadata/ingestion/source/dashboard/lightdash/metadata.py +49 -28
  1076. metadata/ingestion/source/dashboard/lightdash/models.py +11 -0
  1077. metadata/ingestion/source/dashboard/looker/bulk_parser.py +34 -17
  1078. metadata/ingestion/source/dashboard/looker/columns.py +3 -3
  1079. metadata/ingestion/source/dashboard/looker/connection.py +3 -3
  1080. metadata/ingestion/source/dashboard/looker/links.py +3 -3
  1081. metadata/ingestion/source/dashboard/looker/metadata.py +602 -100
  1082. metadata/ingestion/source/dashboard/looker/models.py +8 -3
  1083. metadata/ingestion/source/dashboard/looker/parser.py +6 -3
  1084. metadata/ingestion/source/dashboard/looker/utils.py +21 -6
  1085. metadata/ingestion/source/dashboard/metabase/client.py +99 -18
  1086. metadata/ingestion/source/dashboard/metabase/connection.py +3 -3
  1087. metadata/ingestion/source/dashboard/metabase/metadata.py +181 -48
  1088. metadata/ingestion/source/dashboard/metabase/models.py +54 -5
  1089. metadata/ingestion/source/dashboard/microstrategy/client.py +23 -2
  1090. metadata/ingestion/source/dashboard/microstrategy/connection.py +3 -3
  1091. metadata/ingestion/source/dashboard/microstrategy/helpers.py +67 -0
  1092. metadata/ingestion/source/dashboard/microstrategy/metadata.py +197 -6
  1093. metadata/ingestion/source/dashboard/microstrategy/models.py +16 -4
  1094. metadata/ingestion/source/dashboard/mode/client.py +5 -4
  1095. metadata/ingestion/source/dashboard/mode/connection.py +3 -3
  1096. metadata/ingestion/source/dashboard/mode/metadata.py +78 -25
  1097. metadata/ingestion/source/dashboard/powerbi/client.py +322 -65
  1098. metadata/ingestion/source/dashboard/powerbi/connection.py +3 -3
  1099. metadata/ingestion/source/dashboard/powerbi/databricks_parser.py +108 -0
  1100. metadata/ingestion/source/dashboard/powerbi/file_client.py +3 -3
  1101. metadata/ingestion/source/dashboard/powerbi/metadata.py +1207 -286
  1102. metadata/ingestion/source/dashboard/powerbi/models.py +102 -2
  1103. metadata/ingestion/source/dashboard/qlikcloud/client.py +63 -8
  1104. metadata/ingestion/source/dashboard/qlikcloud/connection.py +3 -3
  1105. metadata/ingestion/source/dashboard/qlikcloud/constants.py +9 -3
  1106. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +96 -21
  1107. metadata/ingestion/source/dashboard/qlikcloud/models.py +56 -4
  1108. metadata/ingestion/source/dashboard/qliksense/client.py +3 -3
  1109. metadata/ingestion/source/dashboard/qliksense/connection.py +3 -3
  1110. metadata/ingestion/source/dashboard/qliksense/constants.py +3 -3
  1111. metadata/ingestion/source/dashboard/qliksense/metadata.py +77 -28
  1112. metadata/ingestion/source/dashboard/qliksense/models.py +3 -3
  1113. metadata/ingestion/source/dashboard/quicksight/connection.py +3 -3
  1114. metadata/ingestion/source/dashboard/quicksight/metadata.py +124 -42
  1115. metadata/ingestion/source/dashboard/quicksight/models.py +3 -3
  1116. metadata/ingestion/source/dashboard/redash/client.py +10 -14
  1117. metadata/ingestion/source/dashboard/redash/connection.py +3 -3
  1118. metadata/ingestion/source/dashboard/redash/metadata.py +80 -25
  1119. metadata/ingestion/source/dashboard/sigma/client.py +72 -9
  1120. metadata/ingestion/source/dashboard/sigma/connection.py +7 -4
  1121. metadata/ingestion/source/dashboard/sigma/metadata.py +76 -26
  1122. metadata/ingestion/source/dashboard/sigma/models.py +8 -2
  1123. metadata/ingestion/source/dashboard/superset/api_source.py +74 -31
  1124. metadata/ingestion/source/dashboard/superset/client.py +6 -5
  1125. metadata/ingestion/source/dashboard/superset/connection.py +15 -17
  1126. metadata/ingestion/source/dashboard/superset/db_source.py +71 -27
  1127. metadata/ingestion/source/dashboard/superset/metadata.py +3 -3
  1128. metadata/ingestion/source/dashboard/superset/mixin.py +224 -59
  1129. metadata/ingestion/source/dashboard/superset/models.py +6 -3
  1130. metadata/ingestion/source/dashboard/superset/queries.py +10 -7
  1131. metadata/ingestion/source/dashboard/superset/utils.py +36 -0
  1132. metadata/ingestion/source/dashboard/tableau/__init__.py +3 -3
  1133. metadata/ingestion/source/dashboard/tableau/client.py +275 -109
  1134. metadata/ingestion/source/dashboard/tableau/connection.py +72 -54
  1135. metadata/ingestion/source/dashboard/tableau/metadata.py +472 -117
  1136. metadata/ingestion/source/dashboard/tableau/models.py +45 -18
  1137. metadata/ingestion/source/dashboard/tableau/queries.py +17 -3
  1138. metadata/ingestion/source/database/athena/client.py +3 -3
  1139. metadata/ingestion/source/database/athena/connection.py +3 -3
  1140. metadata/ingestion/source/database/athena/lineage.py +3 -3
  1141. metadata/ingestion/source/database/athena/metadata.py +52 -3
  1142. metadata/ingestion/source/database/athena/models.py +3 -3
  1143. metadata/ingestion/source/database/athena/query_parser.py +3 -3
  1144. metadata/ingestion/source/database/athena/usage.py +3 -3
  1145. metadata/ingestion/source/database/athena/utils.py +66 -9
  1146. metadata/ingestion/source/database/azuresql/connection.py +3 -3
  1147. metadata/ingestion/source/database/azuresql/lineage.py +3 -3
  1148. metadata/ingestion/source/database/azuresql/metadata.py +3 -3
  1149. metadata/ingestion/source/database/azuresql/queries.py +3 -3
  1150. metadata/ingestion/source/database/azuresql/query_parser.py +3 -3
  1151. metadata/ingestion/source/database/azuresql/usage.py +3 -3
  1152. metadata/ingestion/source/database/bigquery/connection.py +36 -7
  1153. metadata/ingestion/source/database/bigquery/helper.py +31 -19
  1154. metadata/ingestion/source/database/bigquery/incremental_table_processor.py +3 -3
  1155. metadata/ingestion/source/database/bigquery/lineage.py +8 -13
  1156. metadata/ingestion/source/database/bigquery/metadata.py +331 -66
  1157. metadata/ingestion/source/database/bigquery/models.py +3 -3
  1158. metadata/ingestion/source/database/bigquery/queries.py +26 -8
  1159. metadata/ingestion/source/database/bigquery/query_parser.py +4 -3
  1160. metadata/ingestion/source/database/bigquery/service_spec.py +4 -4
  1161. metadata/ingestion/source/database/bigquery/usage.py +3 -3
  1162. metadata/ingestion/source/database/bigtable/client.py +2 -2
  1163. metadata/ingestion/source/database/bigtable/connection.py +2 -2
  1164. metadata/ingestion/source/database/bigtable/metadata.py +13 -5
  1165. metadata/ingestion/source/database/bigtable/models.py +2 -2
  1166. metadata/ingestion/source/database/cassandra/connection.py +138 -0
  1167. metadata/ingestion/source/database/cassandra/helpers.py +100 -0
  1168. metadata/ingestion/source/database/cassandra/metadata.py +138 -0
  1169. metadata/ingestion/source/database/cassandra/queries.py +45 -0
  1170. metadata/ingestion/source/database/cassandra/service_spec.py +10 -0
  1171. metadata/ingestion/source/database/clickhouse/connection.py +3 -3
  1172. metadata/ingestion/source/database/clickhouse/lineage.py +3 -3
  1173. metadata/ingestion/source/database/clickhouse/metadata.py +3 -36
  1174. metadata/ingestion/source/database/clickhouse/queries.py +3 -3
  1175. metadata/ingestion/source/database/clickhouse/query_parser.py +3 -3
  1176. metadata/ingestion/source/database/clickhouse/usage.py +3 -3
  1177. metadata/ingestion/source/database/clickhouse/utils.py +3 -3
  1178. metadata/ingestion/source/database/cockroach/connection.py +71 -0
  1179. metadata/ingestion/source/database/cockroach/metadata.py +206 -0
  1180. metadata/ingestion/source/database/cockroach/queries.py +90 -0
  1181. metadata/ingestion/source/database/cockroach/service_spec.py +4 -0
  1182. metadata/ingestion/source/database/column_helpers.py +3 -3
  1183. metadata/ingestion/source/database/column_type_parser.py +11 -6
  1184. metadata/ingestion/source/database/common/data_diff/databricks_base.py +42 -0
  1185. metadata/ingestion/source/database/common_db_source.py +63 -59
  1186. metadata/ingestion/source/database/common_nosql_source.py +105 -52
  1187. metadata/ingestion/source/database/common_pg_mappings.py +56 -0
  1188. metadata/ingestion/source/database/couchbase/connection.py +3 -3
  1189. metadata/ingestion/source/database/couchbase/metadata.py +11 -5
  1190. metadata/ingestion/source/database/couchbase/models.py +2 -2
  1191. metadata/ingestion/source/database/couchbase/queries.py +3 -3
  1192. metadata/ingestion/source/database/database_service.py +154 -11
  1193. metadata/ingestion/source/database/databricks/auth.py +99 -0
  1194. metadata/ingestion/source/database/databricks/client.py +304 -19
  1195. metadata/ingestion/source/database/databricks/connection.py +154 -16
  1196. metadata/ingestion/source/database/databricks/data_diff/__init__.py +1 -0
  1197. metadata/ingestion/source/database/databricks/data_diff/data_diff.py +12 -0
  1198. metadata/ingestion/source/database/databricks/lineage.py +15 -27
  1199. metadata/ingestion/source/database/databricks/metadata.py +64 -17
  1200. metadata/ingestion/source/database/databricks/queries.py +133 -9
  1201. metadata/ingestion/source/database/databricks/query_parser.py +14 -18
  1202. metadata/ingestion/source/database/databricks/service_spec.py +6 -0
  1203. metadata/ingestion/source/database/databricks/usage.py +10 -40
  1204. metadata/ingestion/source/database/datalake/clients/azure_blob.py +3 -3
  1205. metadata/ingestion/source/database/datalake/clients/base.py +3 -3
  1206. metadata/ingestion/source/database/datalake/clients/gcs.py +3 -3
  1207. metadata/ingestion/source/database/datalake/clients/s3.py +3 -3
  1208. metadata/ingestion/source/database/datalake/columns.py +3 -3
  1209. metadata/ingestion/source/database/datalake/connection.py +54 -76
  1210. metadata/ingestion/source/database/datalake/metadata.py +41 -34
  1211. metadata/ingestion/source/database/datalake/service_spec.py +2 -0
  1212. metadata/ingestion/source/database/db2/connection.py +19 -4
  1213. metadata/ingestion/source/database/db2/lineage.py +3 -3
  1214. metadata/ingestion/source/database/db2/metadata.py +3 -3
  1215. metadata/ingestion/source/database/db2/service_spec.py +4 -1
  1216. metadata/ingestion/source/database/db2/utils.py +133 -3
  1217. metadata/ingestion/source/database/dbt/constants.py +20 -5
  1218. metadata/ingestion/source/database/dbt/dbt_config.py +93 -31
  1219. metadata/ingestion/source/database/dbt/dbt_service.py +34 -5
  1220. metadata/ingestion/source/database/dbt/dbt_utils.py +4 -4
  1221. metadata/ingestion/source/database/dbt/metadata.py +346 -116
  1222. metadata/ingestion/source/database/dbt/models.py +4 -3
  1223. metadata/ingestion/source/database/deltalake/clients/base.py +3 -3
  1224. metadata/ingestion/source/database/deltalake/clients/pyspark.py +4 -4
  1225. metadata/ingestion/source/database/deltalake/clients/s3.py +3 -3
  1226. metadata/ingestion/source/database/deltalake/connection.py +3 -3
  1227. metadata/ingestion/source/database/deltalake/metadata.py +21 -20
  1228. metadata/ingestion/source/database/domodatabase/connection.py +3 -3
  1229. metadata/ingestion/source/database/domodatabase/metadata.py +21 -20
  1230. metadata/ingestion/source/database/domodatabase/models.py +3 -3
  1231. metadata/ingestion/source/database/doris/connection.py +3 -3
  1232. metadata/ingestion/source/database/doris/metadata.py +14 -6
  1233. metadata/ingestion/source/database/doris/utils.py +3 -3
  1234. metadata/ingestion/source/database/druid/connection.py +3 -3
  1235. metadata/ingestion/source/database/druid/lineage.py +3 -3
  1236. metadata/ingestion/source/database/druid/metadata.py +3 -3
  1237. metadata/ingestion/source/database/druid/service_spec.py +5 -1
  1238. metadata/ingestion/source/database/dynamodb/connection.py +3 -3
  1239. metadata/ingestion/source/database/dynamodb/metadata.py +9 -6
  1240. metadata/ingestion/source/database/dynamodb/models.py +2 -2
  1241. metadata/ingestion/source/database/exasol/connection.py +20 -3
  1242. metadata/ingestion/source/database/exasol/lineage.py +24 -0
  1243. metadata/ingestion/source/database/exasol/metadata.py +6 -0
  1244. metadata/ingestion/source/database/exasol/queries.py +35 -0
  1245. metadata/ingestion/source/database/exasol/query_parser.py +33 -0
  1246. metadata/ingestion/source/database/exasol/service_spec.py +5 -1
  1247. metadata/ingestion/source/database/external_table_lineage_mixin.py +3 -3
  1248. metadata/ingestion/source/database/glue/connection.py +3 -3
  1249. metadata/ingestion/source/database/glue/metadata.py +100 -25
  1250. metadata/ingestion/source/database/glue/models.py +3 -3
  1251. metadata/ingestion/source/database/greenplum/connection.py +3 -3
  1252. metadata/ingestion/source/database/greenplum/lineage.py +3 -3
  1253. metadata/ingestion/source/database/greenplum/metadata.py +9 -40
  1254. metadata/ingestion/source/database/greenplum/queries.py +3 -3
  1255. metadata/ingestion/source/database/greenplum/service_spec.py +5 -1
  1256. metadata/ingestion/source/database/greenplum/utils.py +3 -3
  1257. metadata/ingestion/source/database/hive/connection.py +44 -5
  1258. metadata/ingestion/source/database/hive/custom_hive_connection.py +195 -0
  1259. metadata/ingestion/source/database/hive/lineage.py +3 -3
  1260. metadata/ingestion/source/database/hive/metadata.py +11 -4
  1261. metadata/ingestion/source/database/hive/metastore_dialects/mixin.py +3 -3
  1262. metadata/ingestion/source/database/hive/metastore_dialects/mysql/__init__.py +3 -3
  1263. metadata/ingestion/source/database/hive/metastore_dialects/mysql/dialect.py +38 -16
  1264. metadata/ingestion/source/database/hive/metastore_dialects/postgres/__init__.py +3 -3
  1265. metadata/ingestion/source/database/hive/metastore_dialects/postgres/dialect.py +41 -17
  1266. metadata/ingestion/source/database/hive/queries.py +3 -3
  1267. metadata/ingestion/source/database/hive/service_spec.py +5 -1
  1268. metadata/ingestion/source/database/hive/utils.py +3 -3
  1269. metadata/ingestion/source/database/iceberg/catalog/__init__.py +3 -3
  1270. metadata/ingestion/source/database/iceberg/catalog/base.py +3 -3
  1271. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +3 -3
  1272. metadata/ingestion/source/database/iceberg/catalog/glue.py +3 -3
  1273. metadata/ingestion/source/database/iceberg/catalog/hive.py +3 -3
  1274. metadata/ingestion/source/database/iceberg/catalog/rest.py +3 -3
  1275. metadata/ingestion/source/database/iceberg/connection.py +3 -3
  1276. metadata/ingestion/source/database/iceberg/fs/__init__.py +3 -3
  1277. metadata/ingestion/source/database/iceberg/fs/azure.py +3 -3
  1278. metadata/ingestion/source/database/iceberg/fs/base.py +3 -3
  1279. metadata/ingestion/source/database/iceberg/fs/s3.py +3 -3
  1280. metadata/ingestion/source/database/iceberg/helper.py +3 -3
  1281. metadata/ingestion/source/database/iceberg/metadata.py +64 -21
  1282. metadata/ingestion/source/database/iceberg/models.py +25 -16
  1283. metadata/ingestion/source/database/impala/connection.py +3 -3
  1284. metadata/ingestion/source/database/impala/lineage.py +3 -3
  1285. metadata/ingestion/source/database/impala/metadata.py +3 -3
  1286. metadata/ingestion/source/database/impala/queries.py +3 -3
  1287. metadata/ingestion/source/database/impala/service_spec.py +5 -1
  1288. metadata/ingestion/source/database/incremental_metadata_extraction.py +3 -3
  1289. metadata/ingestion/source/database/life_cycle_query_mixin.py +23 -10
  1290. metadata/ingestion/source/database/lineage_processors.py +386 -0
  1291. metadata/ingestion/source/database/lineage_source.py +417 -111
  1292. metadata/ingestion/source/database/mariadb/connection.py +3 -3
  1293. metadata/ingestion/source/database/mariadb/lineage.py +6 -6
  1294. metadata/ingestion/source/database/mariadb/metadata.py +89 -4
  1295. metadata/ingestion/source/database/mariadb/models.py +36 -0
  1296. metadata/ingestion/source/database/mariadb/queries.py +43 -0
  1297. metadata/ingestion/source/database/mariadb/service_spec.py +9 -1
  1298. metadata/ingestion/source/database/mongodb/connection.py +3 -3
  1299. metadata/ingestion/source/database/mongodb/metadata.py +11 -6
  1300. metadata/ingestion/source/database/mssql/connection.py +3 -3
  1301. metadata/ingestion/source/database/mssql/constants.py +3 -3
  1302. metadata/ingestion/source/database/mssql/lineage.py +6 -30
  1303. metadata/ingestion/source/database/mssql/metadata.py +71 -5
  1304. metadata/ingestion/source/database/mssql/models.py +3 -3
  1305. metadata/ingestion/source/database/mssql/queries.py +51 -8
  1306. metadata/ingestion/source/database/mssql/query_parser.py +3 -3
  1307. metadata/ingestion/source/database/mssql/usage.py +3 -3
  1308. metadata/ingestion/source/database/mssql/utils.py +3 -3
  1309. metadata/ingestion/source/database/multi_db_source.py +3 -3
  1310. metadata/ingestion/source/database/mysql/connection.py +55 -46
  1311. metadata/ingestion/source/database/mysql/lineage.py +7 -7
  1312. metadata/ingestion/source/database/mysql/metadata.py +3 -3
  1313. metadata/ingestion/source/database/mysql/queries.py +32 -3
  1314. metadata/ingestion/source/database/mysql/query_parser.py +34 -3
  1315. metadata/ingestion/source/database/mysql/service_spec.py +2 -0
  1316. metadata/ingestion/source/database/mysql/usage.py +3 -3
  1317. metadata/ingestion/source/database/mysql/utils.py +8 -6
  1318. metadata/ingestion/source/database/oracle/connection.py +165 -99
  1319. metadata/ingestion/source/database/oracle/lineage.py +6 -13
  1320. metadata/ingestion/source/database/oracle/metadata.py +40 -62
  1321. metadata/ingestion/source/database/oracle/models.py +5 -4
  1322. metadata/ingestion/source/database/oracle/queries.py +64 -6
  1323. metadata/ingestion/source/database/oracle/query_parser.py +3 -3
  1324. metadata/ingestion/source/database/oracle/service_spec.py +2 -0
  1325. metadata/ingestion/source/database/oracle/usage.py +3 -3
  1326. metadata/ingestion/source/database/oracle/utils.py +22 -17
  1327. metadata/ingestion/source/database/pinotdb/connection.py +3 -3
  1328. metadata/ingestion/source/database/pinotdb/lineage.py +3 -3
  1329. metadata/ingestion/source/database/pinotdb/metadata.py +3 -3
  1330. metadata/ingestion/source/database/pinotdb/service_spec.py +5 -1
  1331. metadata/ingestion/source/database/postgres/connection.py +60 -51
  1332. metadata/ingestion/source/database/postgres/converter_orm.py +3 -3
  1333. metadata/ingestion/source/database/postgres/lineage.py +5 -4
  1334. metadata/ingestion/source/database/postgres/metadata.py +51 -63
  1335. metadata/ingestion/source/database/postgres/models.py +4 -2
  1336. metadata/ingestion/source/database/postgres/pgspider/lineage.py +2 -2
  1337. metadata/ingestion/source/database/postgres/pgspider/queries.py +2 -2
  1338. metadata/ingestion/source/database/postgres/queries.py +55 -91
  1339. metadata/ingestion/source/database/postgres/query_parser.py +3 -3
  1340. metadata/ingestion/source/database/postgres/service_spec.py +2 -0
  1341. metadata/ingestion/source/database/postgres/types/money.py +3 -3
  1342. metadata/ingestion/source/database/postgres/usage.py +18 -3
  1343. metadata/ingestion/source/database/postgres/utils.py +58 -59
  1344. metadata/ingestion/source/database/presto/connection.py +3 -3
  1345. metadata/ingestion/source/database/presto/metadata.py +3 -3
  1346. metadata/ingestion/source/database/presto/queries.py +3 -3
  1347. metadata/ingestion/source/database/query/lineage.py +3 -3
  1348. metadata/ingestion/source/database/query/usage.py +3 -3
  1349. metadata/ingestion/source/database/query_parser_source.py +17 -15
  1350. metadata/ingestion/source/database/redshift/connection.py +4 -4
  1351. metadata/ingestion/source/database/redshift/incremental_table_processor.py +3 -3
  1352. metadata/ingestion/source/database/redshift/lineage.py +7 -13
  1353. metadata/ingestion/source/database/redshift/metadata.py +40 -22
  1354. metadata/ingestion/source/database/redshift/models.py +3 -3
  1355. metadata/ingestion/source/database/redshift/queries.py +8 -38
  1356. metadata/ingestion/source/database/redshift/query_parser.py +3 -3
  1357. metadata/ingestion/source/database/redshift/service_spec.py +4 -4
  1358. metadata/ingestion/source/database/redshift/usage.py +3 -3
  1359. metadata/ingestion/source/database/redshift/utils.py +31 -4
  1360. metadata/ingestion/source/database/salesforce/connection.py +17 -9
  1361. metadata/ingestion/source/database/salesforce/metadata.py +73 -34
  1362. metadata/ingestion/source/database/sample_data.py +692 -6
  1363. metadata/ingestion/source/database/sample_usage.py +5 -3
  1364. metadata/ingestion/source/database/saperp/client.py +3 -3
  1365. metadata/ingestion/source/database/saperp/connection.py +3 -3
  1366. metadata/ingestion/source/database/saperp/constants.py +2 -2
  1367. metadata/ingestion/source/database/saperp/metadata.py +3 -3
  1368. metadata/ingestion/source/database/saperp/models.py +3 -3
  1369. metadata/ingestion/source/database/saphana/cdata_parser.py +330 -120
  1370. metadata/ingestion/source/database/saphana/connection.py +3 -3
  1371. metadata/ingestion/source/database/saphana/lineage.py +20 -12
  1372. metadata/ingestion/source/database/saphana/metadata.py +3 -3
  1373. metadata/ingestion/source/database/saphana/models.py +3 -3
  1374. metadata/ingestion/source/database/saphana/queries.py +10 -3
  1375. metadata/ingestion/source/database/sas/client.py +5 -4
  1376. metadata/ingestion/source/database/sas/connection.py +3 -3
  1377. metadata/ingestion/source/database/sas/extension_attr.py +3 -3
  1378. metadata/ingestion/source/database/sas/metadata.py +22 -25
  1379. metadata/ingestion/source/database/singlestore/connection.py +3 -3
  1380. metadata/ingestion/source/database/singlestore/lineage.py +3 -3
  1381. metadata/ingestion/source/database/singlestore/metadata.py +3 -3
  1382. metadata/ingestion/source/database/singlestore/service_spec.py +6 -1
  1383. metadata/ingestion/source/database/snowflake/connection.py +207 -152
  1384. metadata/ingestion/source/database/snowflake/constants.py +69 -0
  1385. metadata/ingestion/source/database/snowflake/data_diff/data_diff.py +37 -0
  1386. metadata/ingestion/source/database/snowflake/lineage.py +55 -32
  1387. metadata/ingestion/source/database/snowflake/metadata.py +416 -63
  1388. metadata/ingestion/source/database/snowflake/models.py +6 -6
  1389. metadata/ingestion/source/database/snowflake/queries.py +76 -20
  1390. metadata/ingestion/source/database/snowflake/query_parser.py +18 -5
  1391. metadata/ingestion/source/database/snowflake/service_spec.py +10 -4
  1392. metadata/ingestion/source/database/snowflake/usage.py +100 -3
  1393. metadata/ingestion/source/database/snowflake/utils.py +143 -39
  1394. metadata/ingestion/source/database/sql_column_handler.py +112 -90
  1395. metadata/ingestion/source/database/sqlalchemy_source.py +3 -3
  1396. metadata/ingestion/source/database/sqlite/connection.py +3 -3
  1397. metadata/ingestion/source/database/sqlite/lineage.py +3 -3
  1398. metadata/ingestion/source/database/sqlite/metadata.py +3 -3
  1399. metadata/ingestion/source/database/sqlite/service_spec.py +5 -1
  1400. metadata/ingestion/source/database/stored_procedures_mixin.py +113 -176
  1401. metadata/ingestion/source/database/teradata/connection.py +3 -3
  1402. metadata/ingestion/source/database/teradata/lineage.py +3 -3
  1403. metadata/ingestion/source/database/teradata/metadata.py +3 -3
  1404. metadata/ingestion/source/database/teradata/queries.py +3 -3
  1405. metadata/ingestion/source/database/teradata/service_spec.py +5 -1
  1406. metadata/ingestion/source/database/teradata/utils.py +3 -3
  1407. metadata/ingestion/source/database/timescale/__init__.py +3 -0
  1408. metadata/ingestion/source/database/timescale/connection.py +103 -0
  1409. metadata/ingestion/source/database/timescale/lineage.py +124 -0
  1410. metadata/ingestion/source/database/timescale/metadata.py +228 -0
  1411. metadata/ingestion/source/database/timescale/models.py +60 -0
  1412. metadata/ingestion/source/database/timescale/queries.py +96 -0
  1413. metadata/ingestion/source/database/timescale/query_parser.py +106 -0
  1414. metadata/ingestion/source/database/timescale/service_spec.py +12 -0
  1415. metadata/ingestion/source/database/timescale/usage.py +43 -0
  1416. metadata/ingestion/source/database/timescale/utils.py +545 -0
  1417. metadata/ingestion/source/database/trino/connection.py +290 -123
  1418. metadata/ingestion/source/database/trino/lineage.py +98 -3
  1419. metadata/ingestion/source/database/trino/metadata.py +46 -7
  1420. metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py +29 -20
  1421. metadata/ingestion/source/database/trino/queries.py +8 -4
  1422. metadata/ingestion/source/database/trino/query_parser.py +18 -3
  1423. metadata/ingestion/source/database/trino/service_spec.py +2 -0
  1424. metadata/ingestion/source/database/trino/usage.py +3 -3
  1425. metadata/ingestion/source/database/unitycatalog/client.py +38 -4
  1426. metadata/ingestion/source/database/unitycatalog/connection.py +121 -17
  1427. metadata/ingestion/source/database/unitycatalog/data_diff/__init__.py +1 -0
  1428. metadata/ingestion/source/database/unitycatalog/data_diff/data_diff.py +12 -0
  1429. metadata/ingestion/source/database/unitycatalog/lineage.py +306 -63
  1430. metadata/ingestion/source/database/unitycatalog/metadata.py +160 -48
  1431. metadata/ingestion/source/database/unitycatalog/models.py +22 -6
  1432. metadata/ingestion/source/database/unitycatalog/queries.py +55 -0
  1433. metadata/ingestion/source/database/unitycatalog/query_parser.py +14 -3
  1434. metadata/ingestion/source/database/unitycatalog/service_spec.py +8 -0
  1435. metadata/ingestion/source/database/unitycatalog/usage.py +30 -5
  1436. metadata/ingestion/source/database/usage_source.py +11 -8
  1437. metadata/ingestion/source/database/vertica/connection.py +3 -3
  1438. metadata/ingestion/source/database/vertica/lineage.py +3 -3
  1439. metadata/ingestion/source/database/vertica/metadata.py +3 -3
  1440. metadata/ingestion/source/database/vertica/queries.py +8 -23
  1441. metadata/ingestion/source/database/vertica/query_parser.py +3 -3
  1442. metadata/ingestion/source/database/vertica/usage.py +3 -3
  1443. metadata/ingestion/source/drive/drive_service.py +648 -0
  1444. metadata/ingestion/source/drives/__init__.py +13 -0
  1445. metadata/ingestion/source/messaging/common_broker_source.py +3 -3
  1446. metadata/ingestion/source/messaging/kafka/connection.py +16 -7
  1447. metadata/ingestion/source/messaging/kafka/metadata.py +11 -13
  1448. metadata/ingestion/source/messaging/kinesis/connection.py +3 -3
  1449. metadata/ingestion/source/messaging/kinesis/metadata.py +7 -5
  1450. metadata/ingestion/source/messaging/kinesis/models.py +2 -2
  1451. metadata/ingestion/source/messaging/messaging_service.py +7 -10
  1452. metadata/ingestion/source/messaging/redpanda/connection.py +3 -3
  1453. metadata/ingestion/source/messaging/redpanda/metadata.py +3 -3
  1454. metadata/ingestion/source/metadata/alationsink/client.py +2 -2
  1455. metadata/ingestion/source/metadata/alationsink/connection.py +2 -2
  1456. metadata/ingestion/source/metadata/alationsink/constants.py +2 -2
  1457. metadata/ingestion/source/metadata/alationsink/metadata.py +36 -27
  1458. metadata/ingestion/source/metadata/alationsink/models.py +2 -2
  1459. metadata/ingestion/source/metadata/amundsen/client.py +9 -9
  1460. metadata/ingestion/source/metadata/amundsen/connection.py +3 -3
  1461. metadata/ingestion/source/metadata/amundsen/metadata.py +18 -20
  1462. metadata/ingestion/source/metadata/amundsen/queries.py +3 -3
  1463. metadata/ingestion/source/metadata/atlas/client.py +5 -4
  1464. metadata/ingestion/source/metadata/atlas/connection.py +3 -3
  1465. metadata/ingestion/source/metadata/atlas/metadata.py +7 -11
  1466. metadata/ingestion/source/mlmodel/mlflow/connection.py +3 -3
  1467. metadata/ingestion/source/mlmodel/mlflow/metadata.py +3 -3
  1468. metadata/ingestion/source/mlmodel/mlmodel_service.py +82 -11
  1469. metadata/ingestion/source/mlmodel/sagemaker/connection.py +3 -3
  1470. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +3 -3
  1471. metadata/ingestion/source/models.py +3 -3
  1472. metadata/ingestion/source/pipeline/airbyte/client.py +145 -7
  1473. metadata/ingestion/source/pipeline/airbyte/connection.py +21 -8
  1474. metadata/ingestion/source/pipeline/airbyte/constants.py +29 -0
  1475. metadata/ingestion/source/pipeline/airbyte/metadata.py +172 -31
  1476. metadata/ingestion/source/pipeline/airbyte/utils.py +99 -0
  1477. metadata/ingestion/source/pipeline/airflow/connection.py +14 -14
  1478. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +3 -3
  1479. metadata/ingestion/source/pipeline/airflow/metadata.py +112 -41
  1480. metadata/ingestion/source/pipeline/airflow/models.py +4 -3
  1481. metadata/ingestion/source/pipeline/airflow/utils.py +49 -7
  1482. metadata/ingestion/source/pipeline/dagster/client.py +3 -3
  1483. metadata/ingestion/source/pipeline/dagster/connection.py +3 -3
  1484. metadata/ingestion/source/pipeline/dagster/metadata.py +3 -3
  1485. metadata/ingestion/source/pipeline/dagster/models.py +3 -3
  1486. metadata/ingestion/source/pipeline/dagster/queries.py +3 -3
  1487. metadata/ingestion/source/pipeline/databrickspipeline/connection.py +30 -5
  1488. metadata/ingestion/source/pipeline/databrickspipeline/kafka_parser.py +603 -0
  1489. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +1138 -47
  1490. metadata/ingestion/source/pipeline/databrickspipeline/models.py +22 -6
  1491. metadata/ingestion/source/pipeline/dbtcloud/client.py +126 -68
  1492. metadata/ingestion/source/pipeline/dbtcloud/connection.py +3 -3
  1493. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +138 -87
  1494. metadata/ingestion/source/pipeline/dbtcloud/models.py +20 -3
  1495. metadata/ingestion/source/pipeline/dbtcloud/queries.py +10 -14
  1496. metadata/ingestion/source/pipeline/domopipeline/connection.py +3 -3
  1497. metadata/ingestion/source/pipeline/domopipeline/metadata.py +3 -3
  1498. metadata/ingestion/source/pipeline/fivetran/client.py +10 -6
  1499. metadata/ingestion/source/pipeline/fivetran/connection.py +3 -3
  1500. metadata/ingestion/source/pipeline/fivetran/metadata.py +127 -95
  1501. metadata/ingestion/source/pipeline/flink/client.py +3 -3
  1502. metadata/ingestion/source/pipeline/flink/connection.py +3 -3
  1503. metadata/ingestion/source/pipeline/flink/metadata.py +3 -3
  1504. metadata/ingestion/source/pipeline/flink/models.py +3 -3
  1505. metadata/ingestion/source/pipeline/gluepipeline/connection.py +3 -3
  1506. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +3 -3
  1507. metadata/ingestion/source/pipeline/gluepipeline/models.py +3 -3
  1508. metadata/ingestion/source/pipeline/kafkaconnect/client.py +391 -110
  1509. metadata/ingestion/source/pipeline/kafkaconnect/connection.py +3 -3
  1510. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +1683 -77
  1511. metadata/ingestion/source/pipeline/kafkaconnect/models.py +58 -21
  1512. metadata/ingestion/source/pipeline/nifi/client.py +71 -77
  1513. metadata/ingestion/source/pipeline/nifi/connection.py +4 -19
  1514. metadata/ingestion/source/pipeline/nifi/metadata.py +197 -10
  1515. metadata/ingestion/source/pipeline/openlineage/connection.py +3 -3
  1516. metadata/ingestion/source/pipeline/openlineage/metadata.py +10 -5
  1517. metadata/ingestion/source/pipeline/openlineage/models.py +6 -5
  1518. metadata/ingestion/source/pipeline/pipeline_service.py +146 -15
  1519. metadata/ingestion/source/pipeline/spline/client.py +3 -3
  1520. metadata/ingestion/source/pipeline/spline/connection.py +3 -3
  1521. metadata/ingestion/source/pipeline/spline/metadata.py +3 -3
  1522. metadata/ingestion/source/pipeline/spline/models.py +3 -3
  1523. metadata/ingestion/source/pipeline/spline/utils.py +3 -3
  1524. metadata/ingestion/source/search/elasticsearch/connection.py +4 -4
  1525. metadata/ingestion/source/search/elasticsearch/metadata.py +3 -3
  1526. metadata/ingestion/source/search/elasticsearch/parser.py +3 -3
  1527. metadata/ingestion/source/search/opensearch/connection.py +211 -0
  1528. metadata/ingestion/source/search/opensearch/metadata.py +270 -0
  1529. metadata/ingestion/source/search/opensearch/parser.py +71 -0
  1530. metadata/ingestion/source/search/opensearch/service_spec.py +4 -0
  1531. metadata/ingestion/source/search/search_service.py +7 -10
  1532. metadata/ingestion/source/security/security_service.py +128 -0
  1533. metadata/ingestion/source/sqa_types.py +3 -3
  1534. metadata/ingestion/source/storage/gcs/client.py +2 -2
  1535. metadata/ingestion/source/storage/gcs/connection.py +17 -7
  1536. metadata/ingestion/source/storage/gcs/metadata.py +287 -28
  1537. metadata/ingestion/source/storage/gcs/models.py +14 -8
  1538. metadata/ingestion/source/storage/s3/connection.py +3 -3
  1539. metadata/ingestion/source/storage/s3/metadata.py +75 -31
  1540. metadata/ingestion/source/storage/s3/models.py +3 -3
  1541. metadata/ingestion/source/storage/storage_service.py +9 -10
  1542. metadata/ingestion/stage/table_usage.py +56 -4
  1543. metadata/mixins/pandas/pandas_mixin.py +3 -3
  1544. metadata/mixins/sqalchemy/sqa_mixin.py +22 -11
  1545. metadata/parsers/avro_parser.py +3 -3
  1546. metadata/parsers/json_schema_parser.py +44 -4
  1547. metadata/parsers/protobuf_parser.py +3 -3
  1548. metadata/parsers/schema_parsers.py +3 -3
  1549. metadata/pii/algorithms/classifiers.py +242 -0
  1550. metadata/pii/algorithms/column_patterns.py +61 -0
  1551. metadata/pii/algorithms/feature_extraction.py +165 -0
  1552. metadata/pii/algorithms/patterns.py +171 -0
  1553. metadata/pii/algorithms/preprocessing.py +68 -0
  1554. metadata/pii/algorithms/presidio_patches.py +90 -0
  1555. metadata/pii/algorithms/presidio_recognizer_factory.py +302 -0
  1556. metadata/pii/algorithms/presidio_utils.py +292 -0
  1557. metadata/pii/algorithms/tags.py +111 -0
  1558. metadata/pii/algorithms/utils.py +42 -0
  1559. metadata/pii/base_processor.py +126 -0
  1560. metadata/pii/constants.py +11 -3
  1561. metadata/pii/models.py +3 -3
  1562. metadata/pii/ner.py +3 -3
  1563. metadata/pii/processor.py +52 -139
  1564. metadata/pii/processor_factory.py +28 -0
  1565. metadata/pii/scanners/base.py +3 -3
  1566. metadata/pii/scanners/column_name_scanner.py +5 -7
  1567. metadata/pii/scanners/custom_ner_scanner.py +290 -0
  1568. metadata/pii/scanners/ner_scanner.py +5 -12
  1569. metadata/pii/tag_analyzer.py +124 -0
  1570. metadata/pii/tag_processor.py +100 -0
  1571. metadata/profiler/adaptors/adaptor_factory.py +2 -2
  1572. metadata/profiler/adaptors/dynamodb.py +2 -2
  1573. metadata/profiler/adaptors/factory.py +2 -2
  1574. metadata/profiler/adaptors/mongodb.py +2 -2
  1575. metadata/profiler/adaptors/nosql_adaptor.py +2 -2
  1576. metadata/profiler/api/models.py +3 -3
  1577. metadata/profiler/config.py +3 -3
  1578. metadata/profiler/factory.py +3 -3
  1579. metadata/profiler/interface/nosql/profiler_interface.py +3 -3
  1580. metadata/profiler/interface/pandas/profiler_interface.py +3 -3
  1581. metadata/profiler/interface/profiler_interface.py +3 -3
  1582. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +52 -6
  1583. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +59 -5
  1584. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +3 -3
  1585. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +3 -3
  1586. metadata/profiler/interface/sqlalchemy/profiler_interface.py +86 -52
  1587. metadata/profiler/interface/sqlalchemy/redshift/profiler_interface.py +50 -0
  1588. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +3 -3
  1589. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +23 -3
  1590. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +3 -3
  1591. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +3 -3
  1592. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +19 -5
  1593. metadata/profiler/metrics/composed/distinct_ratio.py +3 -3
  1594. metadata/profiler/metrics/composed/duplicate_count.py +3 -3
  1595. metadata/profiler/metrics/composed/ilike_ratio.py +3 -3
  1596. metadata/profiler/metrics/composed/iqr.py +3 -3
  1597. metadata/profiler/metrics/composed/like_ratio.py +3 -3
  1598. metadata/profiler/metrics/composed/non_parametric_skew.py +3 -3
  1599. metadata/profiler/metrics/composed/null_ratio.py +3 -3
  1600. metadata/profiler/metrics/composed/unique_ratio.py +3 -3
  1601. metadata/profiler/metrics/core.py +3 -3
  1602. metadata/profiler/metrics/hybrid/cardinality_distribution.py +232 -0
  1603. metadata/profiler/metrics/hybrid/histogram.py +4 -4
  1604. metadata/profiler/metrics/registry.py +7 -3
  1605. metadata/profiler/metrics/static/column_count.py +3 -3
  1606. metadata/profiler/metrics/static/column_names.py +3 -3
  1607. metadata/profiler/metrics/static/count.py +3 -3
  1608. metadata/profiler/metrics/static/count_in_set.py +3 -3
  1609. metadata/profiler/metrics/static/distinct_count.py +3 -3
  1610. metadata/profiler/metrics/static/ilike_count.py +3 -3
  1611. metadata/profiler/metrics/static/like_count.py +3 -3
  1612. metadata/profiler/metrics/static/max.py +3 -3
  1613. metadata/profiler/metrics/static/max_length.py +3 -3
  1614. metadata/profiler/metrics/static/mean.py +3 -3
  1615. metadata/profiler/metrics/static/min.py +3 -3
  1616. metadata/profiler/metrics/static/min_length.py +3 -3
  1617. metadata/profiler/metrics/static/not_like_count.py +3 -3
  1618. metadata/profiler/metrics/static/not_regexp_match_count.py +7 -6
  1619. metadata/profiler/metrics/static/null_count.py +3 -3
  1620. metadata/profiler/metrics/static/null_missing_count.py +3 -3
  1621. metadata/profiler/metrics/static/regexp_match_count.py +6 -5
  1622. metadata/profiler/metrics/static/row_count.py +3 -3
  1623. metadata/profiler/metrics/static/stddev.py +3 -3
  1624. metadata/profiler/metrics/static/sum.py +3 -3
  1625. metadata/profiler/metrics/static/unique_count.py +3 -3
  1626. metadata/{ingestion/source/database/bigquery/profiler → profiler/metrics/system/bigquery}/system.py +44 -59
  1627. metadata/profiler/metrics/system/databricks/system.py +132 -0
  1628. metadata/profiler/metrics/system/dml_operation.py +5 -3
  1629. metadata/{ingestion/source/database/redshift/profiler → profiler/metrics/system/redshift}/system.py +39 -57
  1630. metadata/{ingestion/source/database/snowflake/profiler → profiler/metrics/system/snowflake}/system.py +29 -52
  1631. metadata/profiler/metrics/system/system.py +110 -41
  1632. metadata/profiler/metrics/window/first_quartile.py +6 -6
  1633. metadata/profiler/metrics/window/median.py +6 -6
  1634. metadata/profiler/metrics/window/third_quartile.py +6 -6
  1635. metadata/profiler/orm/converter/azuresql/converter.py +39 -0
  1636. metadata/profiler/orm/converter/base.py +3 -3
  1637. metadata/profiler/orm/converter/bigquery/converter.py +3 -3
  1638. metadata/profiler/orm/converter/common.py +3 -3
  1639. metadata/profiler/orm/converter/converter_registry.py +9 -3
  1640. metadata/profiler/orm/converter/mariadb/converter.py +28 -0
  1641. metadata/profiler/orm/converter/mssql/converter.py +3 -3
  1642. metadata/profiler/orm/converter/redshift/converter.py +3 -3
  1643. metadata/profiler/orm/converter/snowflake/converter.py +3 -3
  1644. metadata/profiler/orm/converter/trino/__init__.py +14 -0
  1645. metadata/profiler/orm/converter/trino/converter.py +39 -0
  1646. metadata/profiler/orm/functions/concat.py +3 -3
  1647. metadata/profiler/orm/functions/conn_test.py +8 -3
  1648. metadata/profiler/orm/functions/count.py +3 -3
  1649. metadata/profiler/orm/functions/datetime.py +3 -3
  1650. metadata/profiler/orm/functions/length.py +6 -3
  1651. metadata/profiler/orm/functions/md5.py +12 -3
  1652. metadata/profiler/orm/functions/median.py +22 -3
  1653. metadata/profiler/orm/functions/modulo.py +5 -3
  1654. metadata/profiler/orm/functions/random_num.py +8 -3
  1655. metadata/profiler/orm/functions/regexp.py +40 -0
  1656. metadata/profiler/orm/functions/substr.py +3 -3
  1657. metadata/profiler/orm/functions/sum.py +4 -3
  1658. metadata/profiler/orm/functions/table_metric_computer.py +28 -15
  1659. metadata/profiler/orm/functions/unique_count.py +3 -3
  1660. metadata/profiler/orm/registry.py +15 -3
  1661. metadata/profiler/orm/types/__init__.py +10 -0
  1662. metadata/profiler/orm/types/bytea_to_string.py +3 -3
  1663. metadata/profiler/orm/types/custom_array.py +3 -3
  1664. metadata/profiler/orm/types/custom_datetimerange.py +3 -3
  1665. metadata/profiler/orm/types/custom_hex_byte_string.py +3 -3
  1666. metadata/profiler/orm/types/custom_image.py +3 -3
  1667. metadata/profiler/orm/types/custom_ip.py +3 -3
  1668. metadata/profiler/orm/types/custom_time.py +54 -0
  1669. metadata/profiler/orm/types/custom_timestamp.py +3 -3
  1670. metadata/profiler/orm/types/trino.py +70 -0
  1671. metadata/profiler/orm/types/undetermined_type.py +3 -3
  1672. metadata/profiler/orm/types/uuid.py +3 -3
  1673. metadata/profiler/processor/core.py +9 -8
  1674. metadata/profiler/processor/default.py +33 -31
  1675. metadata/profiler/processor/handle_partition.py +3 -3
  1676. metadata/profiler/processor/metric_filter.py +20 -7
  1677. metadata/profiler/processor/models.py +17 -7
  1678. metadata/profiler/processor/processor.py +21 -6
  1679. metadata/profiler/processor/runner.py +32 -9
  1680. metadata/profiler/processor/sample_data_handler.py +3 -3
  1681. metadata/profiler/registry.py +3 -3
  1682. metadata/profiler/source/database/base/profiler_resolver.py +38 -0
  1683. metadata/profiler/source/database/base/profiler_source.py +60 -37
  1684. metadata/profiler/source/database/bigquery/profiler_source.py +5 -23
  1685. metadata/profiler/source/database/bigquery/type_mapper.py +3 -3
  1686. metadata/profiler/source/database/mssql/profiler_source.py +86 -0
  1687. metadata/profiler/source/fetcher/config.py +7 -3
  1688. metadata/profiler/source/fetcher/entity_fetcher.py +3 -3
  1689. metadata/profiler/source/fetcher/fetcher_strategy.py +26 -5
  1690. metadata/profiler/source/fetcher/profiler_source_factory.py +16 -3
  1691. metadata/profiler/source/metadata.py +3 -3
  1692. metadata/profiler/source/metadata_ext.py +3 -3
  1693. metadata/profiler/source/model.py +3 -3
  1694. metadata/profiler/source/profiler_source_interface.py +20 -16
  1695. metadata/readers/dataframe/avro.py +3 -3
  1696. metadata/readers/dataframe/base.py +5 -3
  1697. metadata/readers/dataframe/common.py +3 -3
  1698. metadata/readers/dataframe/dsv.py +53 -8
  1699. metadata/readers/dataframe/json.py +8 -4
  1700. metadata/readers/dataframe/mf4.py +75 -0
  1701. metadata/readers/dataframe/models.py +3 -3
  1702. metadata/readers/dataframe/parquet.py +226 -39
  1703. metadata/readers/dataframe/reader_factory.py +20 -4
  1704. metadata/readers/file/adls.py +3 -3
  1705. metadata/readers/file/api_reader.py +3 -3
  1706. metadata/readers/file/base.py +3 -3
  1707. metadata/readers/file/bitbucket.py +3 -3
  1708. metadata/readers/file/config_source_factory.py +3 -3
  1709. metadata/readers/file/credentials.py +3 -3
  1710. metadata/readers/file/gcs.py +3 -3
  1711. metadata/readers/file/github.py +3 -3
  1712. metadata/readers/file/gitlab.py +3 -3
  1713. metadata/readers/file/local.py +3 -3
  1714. metadata/readers/file/s3.py +3 -3
  1715. metadata/readers/models.py +3 -3
  1716. metadata/sampler/config.py +7 -6
  1717. metadata/sampler/models.py +9 -6
  1718. metadata/sampler/nosql/sampler.py +11 -7
  1719. metadata/sampler/pandas/sampler.py +12 -11
  1720. metadata/sampler/partition.py +14 -6
  1721. metadata/sampler/processor.py +30 -7
  1722. metadata/sampler/sampler_interface.py +28 -17
  1723. metadata/sampler/sqlalchemy/azuresql/sampler.py +32 -6
  1724. metadata/sampler/sqlalchemy/bigquery/sampler.py +30 -9
  1725. metadata/sampler/sqlalchemy/databricks/sampler.py +69 -0
  1726. metadata/sampler/sqlalchemy/mssql/sampler.py +16 -14
  1727. metadata/sampler/sqlalchemy/postgres/sampler.py +8 -8
  1728. metadata/sampler/sqlalchemy/sampler.py +155 -61
  1729. metadata/sampler/sqlalchemy/snowflake/sampler.py +12 -11
  1730. metadata/sampler/sqlalchemy/trino/sampler.py +12 -11
  1731. metadata/sampler/sqlalchemy/unitycatalog/sampler.py +27 -0
  1732. metadata/sdk/__init__.py +188 -0
  1733. metadata/sdk/api/__init__.py +7 -0
  1734. metadata/sdk/api/lineage.py +421 -0
  1735. metadata/sdk/api/search.py +382 -0
  1736. metadata/sdk/client.py +91 -0
  1737. metadata/sdk/config.py +121 -0
  1738. metadata/sdk/entities/__init__.py +64 -0
  1739. metadata/sdk/entities/apicollections.py +19 -0
  1740. metadata/sdk/entities/apiendpoints.py +19 -0
  1741. metadata/sdk/entities/base.py +487 -0
  1742. metadata/sdk/entities/charts.py +17 -0
  1743. metadata/sdk/entities/classifications.py +21 -0
  1744. metadata/sdk/entities/containers.py +17 -0
  1745. metadata/sdk/entities/custom_properties.py +144 -0
  1746. metadata/sdk/entities/dashboard_services.py +18 -0
  1747. metadata/sdk/entities/dashboarddatamodels.py +21 -0
  1748. metadata/sdk/entities/dashboards.py +17 -0
  1749. metadata/sdk/entities/database_services.py +18 -0
  1750. metadata/sdk/entities/databases.py +17 -0
  1751. metadata/sdk/entities/databaseschemas.py +19 -0
  1752. metadata/sdk/entities/datacontracts.py +19 -0
  1753. metadata/sdk/entities/dataproducts.py +19 -0
  1754. metadata/sdk/entities/domains.py +17 -0
  1755. metadata/sdk/entities/glossaries.py +17 -0
  1756. metadata/sdk/entities/glossary_terms.py +18 -0
  1757. metadata/sdk/entities/glossaryterms.py +19 -0
  1758. metadata/sdk/entities/metrics.py +49 -0
  1759. metadata/sdk/entities/mlmodels.py +17 -0
  1760. metadata/sdk/entities/pipelines.py +17 -0
  1761. metadata/sdk/entities/queries.py +17 -0
  1762. metadata/sdk/entities/searchindexes.py +19 -0
  1763. metadata/sdk/entities/storedprocedures.py +19 -0
  1764. metadata/sdk/entities/tables.py +68 -0
  1765. metadata/sdk/entities/tags.py +17 -0
  1766. metadata/sdk/entities/teams.py +17 -0
  1767. metadata/sdk/entities/testcases.py +17 -0
  1768. metadata/sdk/entities/testdefinitions.py +19 -0
  1769. metadata/sdk/entities/testsuites.py +17 -0
  1770. metadata/sdk/entities/users.py +17 -0
  1771. metadata/sdk/examples/builder_end_to_end.py +387 -0
  1772. metadata/sdk/types.py +15 -0
  1773. metadata/timer/repeated_timer.py +3 -3
  1774. metadata/utils/bigquery_utils.py +45 -7
  1775. metadata/utils/class_helper.py +4 -3
  1776. metadata/utils/client_version.py +3 -3
  1777. metadata/utils/constants.py +6 -3
  1778. metadata/utils/constraints.py +2 -2
  1779. metadata/utils/credentials.py +63 -7
  1780. metadata/utils/custom_thread_pool.py +3 -3
  1781. metadata/utils/datalake/datalake_utils.py +134 -6
  1782. metadata/utils/db_utils.py +42 -10
  1783. metadata/utils/dependency_injector/dependency_injector.py +403 -0
  1784. metadata/utils/deprecation.py +12 -9
  1785. metadata/utils/dispatch.py +9 -6
  1786. metadata/utils/elasticsearch.py +10 -3
  1787. metadata/utils/entity_link.py +3 -3
  1788. metadata/utils/entity_utils.py +9 -3
  1789. metadata/utils/execution_time_tracker.py +3 -3
  1790. metadata/utils/filters.py +106 -3
  1791. metadata/utils/fqn.py +190 -14
  1792. metadata/utils/helpers.py +121 -4
  1793. metadata/utils/importer.py +6 -5
  1794. metadata/utils/life_cycle_utils.py +3 -3
  1795. metadata/utils/logger.py +34 -11
  1796. metadata/utils/lru_cache.py +3 -3
  1797. metadata/utils/messaging_utils.py +2 -2
  1798. metadata/utils/metadata_service_helper.py +1 -1
  1799. metadata/utils/profiler_utils.py +3 -3
  1800. metadata/utils/s3_utils.py +3 -3
  1801. metadata/utils/secrets/aws_based_secrets_manager.py +2 -2
  1802. metadata/utils/secrets/aws_secrets_manager.py +2 -2
  1803. metadata/utils/secrets/aws_ssm_secrets_manager.py +2 -2
  1804. metadata/utils/secrets/azure_kv_secrets_manager.py +2 -2
  1805. metadata/utils/secrets/db_secrets_manager.py +2 -2
  1806. metadata/utils/secrets/external_secrets_manager.py +2 -2
  1807. metadata/utils/secrets/kubernetes_secrets_manager.py +176 -0
  1808. metadata/utils/secrets/secrets_manager.py +2 -2
  1809. metadata/utils/secrets/secrets_manager_factory.py +5 -2
  1810. metadata/utils/service_spec/default.py +4 -0
  1811. metadata/utils/service_spec/service_spec.py +67 -10
  1812. metadata/utils/singleton.py +3 -3
  1813. metadata/utils/source_hash.py +3 -3
  1814. metadata/utils/sqa_like_column.py +3 -3
  1815. metadata/utils/sqa_utils.py +21 -3
  1816. metadata/utils/sqlalchemy_utils.py +29 -5
  1817. metadata/utils/ssl_manager.py +186 -12
  1818. metadata/utils/ssl_registry.py +3 -3
  1819. metadata/utils/storage_metadata_config.py +3 -3
  1820. metadata/utils/stored_procedures.py +3 -3
  1821. metadata/utils/streamable_logger.py +569 -0
  1822. metadata/utils/tag_utils.py +75 -30
  1823. metadata/utils/test_utils.py +2 -2
  1824. metadata/utils/time_utils.py +3 -3
  1825. metadata/utils/timeout.py +3 -3
  1826. metadata/utils/uuid_encoder.py +3 -3
  1827. metadata/workflow/application.py +3 -3
  1828. metadata/workflow/application_output_handler.py +3 -3
  1829. metadata/workflow/base.py +60 -6
  1830. metadata/workflow/classification.py +8 -5
  1831. metadata/workflow/context/base.py +24 -0
  1832. metadata/workflow/context/context_manager.py +124 -0
  1833. metadata/workflow/context/workflow_context.py +29 -0
  1834. metadata/workflow/data_quality.py +3 -3
  1835. metadata/workflow/ingestion.py +18 -5
  1836. metadata/workflow/metadata.py +3 -3
  1837. metadata/workflow/output_handler.py +3 -3
  1838. metadata/workflow/profiler.py +24 -21
  1839. metadata/workflow/usage.py +3 -3
  1840. metadata/workflow/workflow_init_error_handler.py +3 -3
  1841. metadata/workflow/workflow_output_handler.py +3 -3
  1842. metadata/workflow/workflow_resource_metrics.py +111 -0
  1843. metadata/workflow/workflow_status_mixin.py +40 -19
  1844. {openmetadata_ingestion-1.6.2.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/METADATA +607 -376
  1845. openmetadata_ingestion-1.10.14.4.dist-info/RECORD +2162 -0
  1846. metadata/ingestion/source/database/bigquery/profiler/profiler.py +0 -30
  1847. metadata/ingestion/source/database/extended_sample_data.py +0 -534
  1848. metadata/ingestion/source/database/redshift/profiler/profiler.py +0 -14
  1849. metadata/ingestion/source/database/snowflake/profiler/profiler.py +0 -26
  1850. openmetadata_ingestion-1.6.2.0.dist-info/RECORD +0 -1814
  1851. /metadata/ingestion/source/database/{bigquery/profiler → cassandra}/__init__.py +0 -0
  1852. /metadata/ingestion/source/database/{redshift/profiler → cockroach}/__init__.py +0 -0
  1853. /metadata/ingestion/source/database/snowflake/{profiler → data_diff}/__init__.py +0 -0
  1854. {openmetadata_ingestion-1.6.2.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/LICENSE +0 -0
  1855. {openmetadata_ingestion-1.6.2.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/WHEEL +0 -0
  1856. {openmetadata_ingestion-1.6.2.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/entry_points.txt +0 -0
  1857. {openmetadata_ingestion-1.6.2.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,7 +15,7 @@ Validator for column values to be unique test case
15
15
 
16
16
  from typing import Optional
17
17
 
18
- from sqlalchemy import Column, inspect
18
+ from sqlalchemy import Column, inspect, literal_column
19
19
  from sqlalchemy.exc import SQLAlchemyError
20
20
 
21
21
  from metadata.data_quality.validations.column.base.columnValuesToBeUnique import (
@@ -25,6 +25,7 @@ from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
25
25
  SQAValidatorMixin,
26
26
  )
27
27
  from metadata.profiler.metrics.registry import Metrics
28
+ from metadata.profiler.orm.registry import Dialects
28
29
 
29
30
 
30
31
  class ColumnValuesToBeUniqueValidator(
@@ -57,7 +58,18 @@ class ColumnValuesToBeUniqueValidator(
57
58
  ) # type: ignore
58
59
 
59
60
  try:
60
- self.value = dict(self.runner.dispatch_query_select_first(count, unique_count.scalar_subquery().label("uniqueCount"))) # type: ignore
61
+ if self.runner.dialect == Dialects.Oracle:
62
+ query_group_by_ = [literal_column("2")]
63
+ else:
64
+ query_group_by_ = None
65
+
66
+ self.value = dict(
67
+ self.runner.dispatch_query_select_first(
68
+ count,
69
+ unique_count.scalar_subquery().label("uniqueCount"),
70
+ query_group_by_=query_group_by_,
71
+ )
72
+ ) # type: ignore
61
73
  res = self.value.get(Metrics.COUNT.name)
62
74
  except Exception as exc:
63
75
  raise SQLAlchemyError(exc)
@@ -1,8 +1,8 @@
1
- # Copyright 2021 Collate
2
- # Licensed under the Apache License, Version 2.0 (the "License");
1
+ # Copyright 2025 Collate
2
+ # Licensed under the Collate Community License, Version 1.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
5
- # http://www.apache.org/licenses/LICENSE-2.0
5
+ # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6
6
  # Unless required by applicable law or agreed to in writing, software
7
7
  # distributed under the License is distributed on an "AS IS" BASIS,
8
8
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -76,7 +76,7 @@ class ColumnValuesToMatchRegexValidator(
76
76
  res = dict(
77
77
  self.runner.dispatch_query_select_first(
78
78
  Metrics.COUNT(column).fn(),
79
- regex_count,
79
+ regex_count_fn,
80
80
  )
81
81
  )
82
82