openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (885) hide show
  1. airflow_provider_openmetadata/lineage/runner.py +44 -5
  2. airflow_provider_openmetadata/lineage/status.py +1 -1
  3. metadata/antlr/split_listener.py +1 -4
  4. metadata/applications/auto_tagger.py +212 -0
  5. metadata/cli/app.py +47 -0
  6. metadata/cli/db_dump.py +7 -3
  7. metadata/clients/aws_client.py +4 -0
  8. metadata/clients/azure_client.py +85 -0
  9. metadata/clients/domo_client.py +0 -2
  10. metadata/cmd.py +24 -13
  11. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +40 -48
  12. metadata/data_insight/processor/reports/data_processor.py +1 -0
  13. metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
  14. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
  15. metadata/data_insight/producer/cost_analysis_producer.py +78 -14
  16. metadata/data_insight/producer/entity_producer.py +1 -1
  17. metadata/data_insight/producer/producer_interface.py +1 -1
  18. metadata/data_insight/producer/web_analytics_producer.py +1 -1
  19. metadata/data_insight/source/metadata.py +21 -5
  20. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
  21. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
  22. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
  23. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
  24. metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
  25. metadata/data_quality/processor/test_case_runner.py +17 -8
  26. metadata/data_quality/source/test_suite.py +10 -4
  27. metadata/data_quality/validations/base_test_handler.py +21 -17
  28. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
  29. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
  30. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
  31. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
  32. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
  33. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
  34. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
  35. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
  36. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
  37. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
  38. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
  39. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
  40. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
  41. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
  42. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
  43. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
  44. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
  45. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
  46. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
  47. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
  48. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
  49. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
  50. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
  51. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
  52. metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +2 -1
  53. metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +2 -1
  54. metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +7 -2
  55. metadata/examples/workflows/bigtable.yaml +32 -0
  56. metadata/examples/workflows/databricks.yaml +0 -1
  57. metadata/examples/workflows/datalake_azure_default.yaml +29 -0
  58. metadata/examples/workflows/mongodb.yaml +4 -2
  59. metadata/examples/workflows/mstr.yaml +24 -0
  60. metadata/examples/workflows/sas.yaml +28 -0
  61. metadata/examples/workflows/unity_catalog.yaml +27 -0
  62. metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
  63. metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
  64. metadata/generated/antlr/EntityLinkLexer.py +425 -292
  65. metadata/generated/antlr/EntityLinkListener.py +12 -12
  66. metadata/generated/antlr/EntityLinkParser.py +228 -103
  67. metadata/generated/schema/analytics/__init__.py +1 -1
  68. metadata/generated/schema/analytics/basic.py +1 -1
  69. metadata/generated/schema/analytics/reportData.py +1 -1
  70. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  71. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  72. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  73. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  74. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  75. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  76. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  77. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  78. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  79. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  80. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  81. metadata/generated/schema/api/__init__.py +1 -1
  82. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  83. metadata/generated/schema/api/analytics/__init__.py +1 -1
  84. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  85. metadata/generated/schema/api/automations/__init__.py +1 -1
  86. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  87. metadata/generated/schema/api/bulkAssets.py +1 -1
  88. metadata/generated/schema/api/classification/__init__.py +1 -1
  89. metadata/generated/schema/api/classification/createClassification.py +1 -1
  90. metadata/generated/schema/api/classification/createTag.py +1 -1
  91. metadata/generated/schema/api/classification/loadTags.py +1 -1
  92. metadata/generated/schema/api/createBot.py +2 -3
  93. metadata/generated/schema/api/createEventPublisherJob.py +2 -2
  94. metadata/generated/schema/api/createType.py +1 -1
  95. metadata/generated/schema/api/data/__init__.py +1 -1
  96. metadata/generated/schema/api/data/createChart.py +1 -1
  97. metadata/generated/schema/api/data/createContainer.py +2 -2
  98. metadata/generated/schema/api/data/createCustomProperty.py +7 -12
  99. metadata/generated/schema/api/data/createDashboard.py +1 -1
  100. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  101. metadata/generated/schema/api/data/createDatabase.py +2 -3
  102. metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
  103. metadata/generated/schema/api/data/createGlossary.py +2 -3
  104. metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
  105. metadata/generated/schema/api/data/createMlModel.py +1 -1
  106. metadata/generated/schema/api/data/createPipeline.py +2 -2
  107. metadata/generated/schema/api/data/createQuery.py +2 -2
  108. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  109. metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
  110. metadata/generated/schema/api/data/createTable.py +2 -2
  111. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  112. metadata/generated/schema/api/data/createTopic.py +1 -1
  113. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  114. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  115. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  116. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  117. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  118. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  119. metadata/generated/schema/api/docStore/__init__.py +1 -1
  120. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  121. metadata/generated/schema/api/domains/__init__.py +1 -1
  122. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  123. metadata/generated/schema/api/domains/createDomain.py +1 -1
  124. metadata/generated/schema/api/feed/__init__.py +1 -1
  125. metadata/generated/schema/api/feed/closeTask.py +8 -1
  126. metadata/generated/schema/api/feed/createPost.py +1 -1
  127. metadata/generated/schema/api/feed/createSuggestion.py +30 -0
  128. metadata/generated/schema/api/feed/createThread.py +5 -1
  129. metadata/generated/schema/api/feed/resolveTask.py +12 -1
  130. metadata/generated/schema/api/feed/threadCount.py +18 -14
  131. metadata/generated/schema/api/lineage/__init__.py +1 -1
  132. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  133. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  134. metadata/generated/schema/api/policies/__init__.py +1 -1
  135. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  136. metadata/generated/schema/api/services/__init__.py +1 -1
  137. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  138. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  139. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  140. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  141. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  142. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  143. metadata/generated/schema/api/services/createSearchService.py +1 -1
  144. metadata/generated/schema/api/services/createStorageService.py +1 -1
  145. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  146. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  147. metadata/generated/schema/api/setOwner.py +1 -1
  148. metadata/generated/schema/api/teams/__init__.py +1 -1
  149. metadata/generated/schema/api/teams/createPersona.py +1 -1
  150. metadata/generated/schema/api/teams/createRole.py +2 -3
  151. metadata/generated/schema/api/teams/createTeam.py +1 -1
  152. metadata/generated/schema/api/teams/createUser.py +2 -2
  153. metadata/generated/schema/api/tests/__init__.py +1 -1
  154. metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
  155. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  156. metadata/generated/schema/api/tests/createTestCase.py +4 -1
  157. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
  158. metadata/generated/schema/api/tests/createTestDefinition.py +2 -1
  159. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  160. metadata/generated/schema/api/voteRequest.py +1 -1
  161. metadata/generated/schema/auth/__init__.py +1 -1
  162. metadata/generated/schema/auth/basicAuth.py +1 -1
  163. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  164. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  165. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  166. metadata/generated/schema/auth/emailRequest.py +1 -1
  167. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  168. metadata/generated/schema/auth/generateToken.py +1 -1
  169. metadata/generated/schema/auth/jwtAuth.py +1 -1
  170. metadata/generated/schema/auth/loginRequest.py +1 -1
  171. metadata/generated/schema/auth/logoutRequest.py +1 -1
  172. metadata/generated/schema/auth/passwordResetRequest.py +3 -3
  173. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  174. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  175. metadata/generated/schema/auth/refreshToken.py +1 -1
  176. metadata/generated/schema/auth/registrationRequest.py +2 -2
  177. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  178. metadata/generated/schema/auth/revokeToken.py +1 -1
  179. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  180. metadata/generated/schema/auth/ssoAuth.py +1 -1
  181. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  182. metadata/generated/schema/configuration/__init__.py +1 -1
  183. metadata/generated/schema/configuration/appsPrivateConfiguration.py +32 -0
  184. metadata/generated/schema/configuration/authConfig.py +1 -1
  185. metadata/generated/schema/configuration/authenticationConfiguration.py +13 -2
  186. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  187. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  188. metadata/generated/schema/configuration/dataQualityConfiguration.py +16 -0
  189. metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
  190. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
  195. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  196. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  197. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  198. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  199. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  200. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  201. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  202. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  203. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  204. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  205. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  206. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  207. metadata/generated/schema/dataInsight/__init__.py +1 -1
  208. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  209. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  210. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  211. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  212. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  213. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  214. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  215. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  216. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  217. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  218. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  219. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  220. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  221. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  222. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
  223. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
  224. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
  225. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
  226. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
  227. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
  228. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  229. metadata/generated/schema/email/__init__.py +1 -1
  230. metadata/generated/schema/email/emailRequest.py +1 -1
  231. metadata/generated/schema/email/smtpSettings.py +1 -1
  232. metadata/generated/schema/entity/__init__.py +1 -1
  233. metadata/generated/schema/entity/applications/__init__.py +1 -1
  234. metadata/generated/schema/entity/applications/app.py +24 -5
  235. metadata/generated/schema/entity/applications/appRunRecord.py +2 -7
  236. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  237. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
  238. metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
  240. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
  241. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
  242. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
  243. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
  244. metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +14 -4
  245. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
  246. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
  247. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  248. metadata/generated/schema/entity/applications/jobStatus.py +3 -3
  249. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  250. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  251. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
  252. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
  253. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  254. metadata/generated/schema/entity/automations/__init__.py +1 -1
  255. metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
  256. metadata/generated/schema/entity/automations/workflow.py +1 -1
  257. metadata/generated/schema/entity/bot.py +2 -3
  258. metadata/generated/schema/entity/classification/__init__.py +1 -1
  259. metadata/generated/schema/entity/classification/classification.py +1 -1
  260. metadata/generated/schema/entity/classification/tag.py +2 -2
  261. metadata/generated/schema/entity/data/__init__.py +1 -1
  262. metadata/generated/schema/entity/data/chart.py +1 -1
  263. metadata/generated/schema/entity/data/container.py +4 -9
  264. metadata/generated/schema/entity/data/dashboard.py +1 -1
  265. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  266. metadata/generated/schema/entity/data/database.py +4 -8
  267. metadata/generated/schema/entity/data/databaseSchema.py +2 -8
  268. metadata/generated/schema/entity/data/glossary.py +1 -1
  269. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  270. metadata/generated/schema/entity/data/metrics.py +1 -1
  271. metadata/generated/schema/entity/data/mlmodel.py +2 -2
  272. metadata/generated/schema/entity/data/pipeline.py +6 -9
  273. metadata/generated/schema/entity/data/query.py +2 -2
  274. metadata/generated/schema/entity/data/report.py +1 -1
  275. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  276. metadata/generated/schema/entity/data/storedProcedure.py +3 -9
  277. metadata/generated/schema/entity/data/table.py +18 -12
  278. metadata/generated/schema/entity/data/topic.py +1 -1
  279. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  280. metadata/generated/schema/entity/docStore/document.py +1 -1
  281. metadata/generated/schema/entity/domains/__init__.py +1 -1
  282. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  283. metadata/generated/schema/entity/domains/domain.py +1 -1
  284. metadata/generated/schema/entity/events/__init__.py +1 -1
  285. metadata/generated/schema/entity/events/webhook.py +6 -3
  286. metadata/generated/schema/entity/feed/__init__.py +1 -1
  287. metadata/generated/schema/entity/feed/suggestion.py +61 -0
  288. metadata/generated/schema/entity/feed/thread.py +17 -1
  289. metadata/generated/schema/entity/policies/__init__.py +1 -1
  290. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  291. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  292. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  293. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  294. metadata/generated/schema/entity/policies/filters.py +2 -5
  295. metadata/generated/schema/entity/policies/policy.py +1 -1
  296. metadata/generated/schema/entity/services/__init__.py +1 -1
  297. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  298. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  299. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  300. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  301. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  302. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  304. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  305. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  306. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  307. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  308. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  309. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  310. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
  311. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  312. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
  313. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  314. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  315. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  316. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
  319. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +35 -2
  320. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
  321. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +41 -0
  322. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
  323. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +20 -0
  325. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  326. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
  329. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  330. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
  331. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  332. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  333. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  334. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  335. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
  336. metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
  337. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
  340. metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
  341. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  342. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  343. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
  344. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
  345. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
  346. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
  347. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
  348. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
  349. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
  350. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
  351. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
  352. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
  353. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
  354. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
  355. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
  356. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
  357. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +8 -4
  358. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
  359. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +9 -7
  360. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +8 -4
  361. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
  362. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
  363. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +3 -0
  365. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +20 -0
  366. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +36 -0
  367. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +7 -41
  368. metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
  369. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
  370. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
  371. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
  372. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
  373. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
  374. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
  375. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  376. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +14 -2
  378. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +14 -2
  381. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  382. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  383. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +2 -2
  387. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
  388. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  389. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  394. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
  396. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
  405. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  407. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  409. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  410. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  411. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  415. metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
  416. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  420. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  421. metadata/generated/schema/entity/services/dashboardService.py +4 -1
  422. metadata/generated/schema/entity/services/databaseService.py +16 -1
  423. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
  425. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  426. metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
  427. metadata/generated/schema/entity/services/messagingService.py +1 -1
  428. metadata/generated/schema/entity/services/metadataService.py +1 -1
  429. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  430. metadata/generated/schema/entity/services/pipelineService.py +4 -1
  431. metadata/generated/schema/entity/services/searchService.py +1 -1
  432. metadata/generated/schema/entity/services/serviceType.py +1 -1
  433. metadata/generated/schema/entity/services/storageService.py +10 -3
  434. metadata/generated/schema/entity/teams/__init__.py +1 -1
  435. metadata/generated/schema/entity/teams/persona.py +1 -1
  436. metadata/generated/schema/entity/teams/role.py +2 -6
  437. metadata/generated/schema/entity/teams/team.py +1 -1
  438. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  439. metadata/generated/schema/entity/teams/user.py +3 -10
  440. metadata/generated/schema/entity/type.py +4 -19
  441. metadata/generated/schema/entity/utils/__init__.py +1 -1
  442. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  443. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  444. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  445. metadata/generated/schema/events/__init__.py +1 -1
  446. metadata/generated/schema/events/alertMetrics.py +27 -0
  447. metadata/generated/schema/events/api/__init__.py +1 -1
  448. metadata/generated/schema/events/api/createEventSubscription.py +14 -12
  449. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  450. metadata/generated/schema/events/eventFilterRule.py +13 -2
  451. metadata/generated/schema/events/eventSubscription.py +91 -49
  452. metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
  453. metadata/generated/schema/events/failedEvent.py +26 -0
  454. metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
  455. metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
  456. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  457. metadata/generated/schema/metadataIngestion/application.py +40 -0
  458. metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
  459. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  460. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  461. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
  462. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +11 -1
  463. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  464. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  465. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  466. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  467. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
  468. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  469. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  470. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
  471. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  472. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  473. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
  474. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  475. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  476. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  477. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  478. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  479. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  480. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  481. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  482. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  483. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
  484. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
  485. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  486. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  487. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  488. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
  489. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  490. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  491. metadata/generated/schema/monitoring/__init__.py +1 -1
  492. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  493. metadata/generated/schema/security/__init__.py +1 -1
  494. metadata/generated/schema/security/client/__init__.py +1 -1
  495. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  496. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  497. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  498. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  499. metadata/generated/schema/security/client/oidcClientConfig.py +46 -0
  500. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  501. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  502. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  503. metadata/generated/schema/security/credentials/__init__.py +1 -1
  504. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  505. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  506. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  507. metadata/generated/schema/security/credentials/azureCredentials.py +21 -9
  508. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  509. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  510. metadata/generated/schema/security/credentials/gcpCredentials.py +7 -3
  511. metadata/generated/schema/security/credentials/gcpExternalAccount.py +37 -0
  512. metadata/generated/schema/security/credentials/gcpValues.py +2 -2
  513. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  514. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  515. metadata/generated/schema/security/secrets/__init__.py +1 -1
  516. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  517. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
  518. metadata/generated/schema/security/secrets/secretsManagerProvider.py +4 -2
  519. metadata/generated/schema/security/securityConfiguration.py +1 -1
  520. metadata/generated/schema/security/ssl/__init__.py +1 -1
  521. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  522. metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
  523. metadata/generated/schema/settings/__init__.py +1 -1
  524. metadata/generated/schema/settings/settings.py +1 -1
  525. metadata/generated/schema/system/__init__.py +1 -1
  526. metadata/generated/schema/system/entityError.py +17 -0
  527. metadata/generated/schema/system/eventPublisherJob.py +5 -19
  528. metadata/generated/schema/system/indexingError.py +34 -0
  529. metadata/generated/schema/system/ui/__init__.py +1 -1
  530. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  531. metadata/generated/schema/system/ui/page.py +1 -1
  532. metadata/generated/schema/system/validationResponse.py +43 -0
  533. metadata/generated/schema/tests/__init__.py +1 -1
  534. metadata/generated/schema/tests/assigned.py +18 -0
  535. metadata/generated/schema/tests/basic.py +17 -39
  536. metadata/generated/schema/tests/customMetric.py +4 -2
  537. metadata/generated/schema/tests/resolved.py +35 -0
  538. metadata/generated/schema/tests/testCase.py +11 -2
  539. metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
  540. metadata/generated/schema/tests/testDefinition.py +6 -1
  541. metadata/generated/schema/tests/testSuite.py +1 -1
  542. metadata/generated/schema/type/__init__.py +1 -1
  543. metadata/generated/schema/type/auditLog.py +1 -1
  544. metadata/generated/schema/type/basic.py +10 -6
  545. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  546. metadata/generated/schema/type/changeEvent.py +5 -75
  547. metadata/generated/schema/type/changeEventType.py +29 -0
  548. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  549. metadata/generated/schema/type/csvDocumentation.py +1 -1
  550. metadata/generated/schema/type/csvErrorType.py +1 -1
  551. metadata/generated/schema/type/csvFile.py +1 -1
  552. metadata/generated/schema/type/csvImportResult.py +1 -1
  553. metadata/generated/schema/type/customProperties/__init__.py +3 -0
  554. metadata/generated/schema/type/customProperties/enumConfig.py +17 -0
  555. metadata/generated/schema/type/customProperty.py +52 -0
  556. metadata/generated/schema/type/dailyCount.py +1 -1
  557. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  558. metadata/generated/schema/type/entityHistory.py +1 -1
  559. metadata/generated/schema/type/entityLineage.py +2 -1
  560. metadata/generated/schema/type/entityReference.py +5 -1
  561. metadata/generated/schema/type/entityReferenceList.py +1 -1
  562. metadata/generated/schema/type/entityRelationship.py +1 -1
  563. metadata/generated/schema/type/entityUsage.py +1 -1
  564. metadata/generated/schema/type/filterPattern.py +1 -1
  565. metadata/generated/schema/type/function.py +7 -6
  566. metadata/generated/schema/type/include.py +1 -1
  567. metadata/generated/schema/type/jdbcConnection.py +1 -1
  568. metadata/generated/schema/type/lifeCycle.py +1 -1
  569. metadata/generated/schema/type/paging.py +1 -1
  570. metadata/generated/schema/type/profile.py +1 -1
  571. metadata/generated/schema/type/queryParserData.py +2 -2
  572. metadata/generated/schema/type/reaction.py +1 -1
  573. metadata/generated/schema/type/schedule.py +1 -1
  574. metadata/generated/schema/type/schema.py +1 -1
  575. metadata/generated/schema/type/tableQuery.py +2 -2
  576. metadata/generated/schema/type/tableUsageCount.py +1 -1
  577. metadata/generated/schema/type/tagLabel.py +1 -1
  578. metadata/generated/schema/type/usageDetails.py +1 -1
  579. metadata/generated/schema/type/usageRequest.py +1 -1
  580. metadata/generated/schema/type/votes.py +1 -1
  581. metadata/great_expectations/action.py +12 -20
  582. metadata/ingestion/api/delete.py +5 -2
  583. metadata/ingestion/api/models.py +4 -10
  584. metadata/ingestion/api/parser.py +3 -37
  585. metadata/ingestion/api/status.py +15 -4
  586. metadata/ingestion/api/step.py +39 -6
  587. metadata/ingestion/api/steps.py +34 -1
  588. metadata/ingestion/api/topology_runner.py +117 -168
  589. metadata/ingestion/bulksink/metadata_usage.py +11 -5
  590. metadata/ingestion/lineage/models.py +4 -0
  591. metadata/ingestion/lineage/parser.py +28 -12
  592. metadata/ingestion/lineage/sql_lineage.py +9 -17
  593. metadata/ingestion/models/custom_properties.py +0 -1
  594. metadata/ingestion/models/custom_pydantic.py +4 -2
  595. metadata/ingestion/models/lf_tags_model.py +33 -0
  596. metadata/ingestion/models/patch_request.py +160 -2
  597. metadata/ingestion/models/tests_data.py +9 -0
  598. metadata/ingestion/models/topology.py +177 -60
  599. metadata/ingestion/ometa/auth_provider.py +0 -349
  600. metadata/ingestion/ometa/client.py +12 -3
  601. metadata/ingestion/ometa/mixins/custom_property_mixin.py +11 -11
  602. metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
  603. metadata/ingestion/ometa/mixins/patch_mixin.py +21 -72
  604. metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
  605. metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
  606. metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
  607. metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
  608. metadata/ingestion/ometa/ometa_api.py +19 -26
  609. metadata/ingestion/ometa/routes.py +7 -0
  610. metadata/ingestion/processor/query_parser.py +9 -2
  611. metadata/ingestion/sink/metadata_rest.py +29 -8
  612. metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
  613. metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
  614. metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
  615. metadata/ingestion/source/dashboard/looker/metadata.py +41 -29
  616. metadata/ingestion/source/dashboard/looker/utils.py +4 -2
  617. metadata/ingestion/source/dashboard/metabase/client.py +4 -0
  618. metadata/ingestion/source/dashboard/metabase/metadata.py +28 -10
  619. metadata/ingestion/source/dashboard/metabase/models.py +2 -2
  620. metadata/ingestion/source/dashboard/mode/client.py +10 -23
  621. metadata/ingestion/source/dashboard/mode/connection.py +6 -1
  622. metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
  623. metadata/ingestion/source/dashboard/mstr/client.py +208 -0
  624. metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
  625. metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
  626. metadata/ingestion/source/dashboard/mstr/models.py +144 -0
  627. metadata/ingestion/source/dashboard/powerbi/client.py +4 -1
  628. metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
  629. metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
  630. metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
  631. metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
  632. metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
  633. metadata/ingestion/source/dashboard/superset/client.py +16 -9
  634. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  635. metadata/ingestion/source/dashboard/superset/db_source.py +14 -11
  636. metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
  637. metadata/ingestion/source/dashboard/superset/queries.py +1 -1
  638. metadata/ingestion/source/dashboard/tableau/client.py +91 -11
  639. metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
  640. metadata/ingestion/source/dashboard/tableau/metadata.py +76 -70
  641. metadata/ingestion/source/dashboard/tableau/models.py +0 -8
  642. metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
  643. metadata/ingestion/source/database/athena/client.py +80 -0
  644. metadata/ingestion/source/database/athena/connection.py +7 -0
  645. metadata/ingestion/source/database/athena/metadata.py +161 -19
  646. metadata/ingestion/source/database/azuresql/connection.py +21 -3
  647. metadata/ingestion/source/database/azuresql/metadata.py +0 -1
  648. metadata/ingestion/source/database/bigquery/connection.py +24 -3
  649. metadata/ingestion/source/database/bigquery/helper.py +68 -1
  650. metadata/ingestion/source/database/bigquery/metadata.py +49 -28
  651. metadata/ingestion/source/database/bigquery/queries.py +33 -4
  652. metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
  653. metadata/ingestion/source/database/bigquery/usage.py +1 -3
  654. metadata/ingestion/source/database/bigtable/client.py +62 -0
  655. metadata/ingestion/source/database/bigtable/connection.py +116 -0
  656. metadata/ingestion/source/database/bigtable/metadata.py +224 -0
  657. metadata/ingestion/source/database/bigtable/models.py +60 -0
  658. metadata/ingestion/source/database/column_helpers.py +0 -10
  659. metadata/ingestion/source/database/column_type_parser.py +11 -5
  660. metadata/ingestion/source/database/common_db_source.py +33 -8
  661. metadata/ingestion/source/database/common_nosql_source.py +27 -6
  662. metadata/ingestion/source/database/database_service.py +89 -7
  663. metadata/ingestion/source/database/databricks/client.py +55 -103
  664. metadata/ingestion/source/database/databricks/connection.py +16 -55
  665. metadata/ingestion/source/database/databricks/lineage.py +29 -26
  666. metadata/ingestion/source/database/databricks/metadata.py +534 -11
  667. metadata/ingestion/source/database/databricks/queries.py +27 -0
  668. metadata/ingestion/source/database/databricks/query_parser.py +5 -1
  669. metadata/ingestion/source/database/databricks/usage.py +3 -3
  670. metadata/ingestion/source/database/datalake/connection.py +33 -18
  671. metadata/ingestion/source/database/datalake/metadata.py +113 -27
  672. metadata/ingestion/source/database/dbt/dbt_config.py +7 -14
  673. metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
  674. metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
  675. metadata/ingestion/source/database/dbt/metadata.py +27 -53
  676. metadata/ingestion/source/database/deltalake/metadata.py +6 -3
  677. metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
  678. metadata/ingestion/source/database/doris/connection.py +72 -0
  679. metadata/ingestion/source/database/doris/metadata.py +315 -0
  680. metadata/ingestion/source/database/doris/queries.py +54 -0
  681. metadata/ingestion/source/database/doris/utils.py +64 -0
  682. metadata/ingestion/source/database/extended_sample_data.py +532 -0
  683. metadata/ingestion/source/database/glue/metadata.py +8 -5
  684. metadata/ingestion/source/database/hive/connection.py +0 -2
  685. metadata/ingestion/source/database/hive/utils.py +3 -0
  686. metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
  687. metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
  688. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
  689. metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
  690. metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
  691. metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
  692. metadata/ingestion/source/database/iceberg/connection.py +68 -0
  693. metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
  694. metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
  695. metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
  696. metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
  697. metadata/ingestion/source/database/iceberg/helper.py +124 -0
  698. metadata/ingestion/source/database/iceberg/metadata.py +311 -0
  699. metadata/ingestion/source/database/iceberg/models.py +66 -0
  700. metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
  701. metadata/ingestion/source/database/mongodb/connection.py +1 -5
  702. metadata/ingestion/source/database/mssql/lineage.py +3 -0
  703. metadata/ingestion/source/database/mssql/metadata.py +108 -4
  704. metadata/ingestion/source/database/mssql/models.py +30 -0
  705. metadata/ingestion/source/database/mssql/queries.py +178 -1
  706. metadata/ingestion/source/database/mssql/usage.py +5 -1
  707. metadata/ingestion/source/database/mssql/utils.py +207 -4
  708. metadata/ingestion/source/database/mysql/connection.py +14 -0
  709. metadata/ingestion/source/database/mysql/metadata.py +0 -2
  710. metadata/ingestion/source/database/oracle/metadata.py +108 -2
  711. metadata/ingestion/source/database/oracle/models.py +30 -0
  712. metadata/ingestion/source/database/oracle/queries.py +99 -18
  713. metadata/ingestion/source/database/oracle/utils.py +0 -1
  714. metadata/ingestion/source/database/postgres/connection.py +15 -0
  715. metadata/ingestion/source/database/postgres/lineage.py +32 -14
  716. metadata/ingestion/source/database/postgres/metadata.py +15 -7
  717. metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
  718. metadata/ingestion/source/database/postgres/queries.py +4 -2
  719. metadata/ingestion/source/database/postgres/query_parser.py +4 -72
  720. metadata/ingestion/source/database/postgres/usage.py +41 -0
  721. metadata/ingestion/source/database/postgres/utils.py +34 -0
  722. metadata/ingestion/source/database/query_parser_source.py +8 -2
  723. metadata/ingestion/source/database/redshift/metadata.py +14 -4
  724. metadata/ingestion/source/database/redshift/queries.py +10 -4
  725. metadata/ingestion/source/database/redshift/query_parser.py +16 -0
  726. metadata/ingestion/source/database/redshift/usage.py +0 -2
  727. metadata/ingestion/source/database/salesforce/metadata.py +32 -3
  728. metadata/ingestion/source/database/sample_data.py +120 -6
  729. metadata/ingestion/source/database/sas/client.py +184 -0
  730. metadata/ingestion/source/database/sas/connection.py +47 -0
  731. metadata/ingestion/source/database/sas/extension_attr.py +103 -0
  732. metadata/ingestion/source/database/sas/metadata.py +914 -0
  733. metadata/ingestion/source/database/snowflake/metadata.py +34 -48
  734. metadata/ingestion/source/database/snowflake/models.py +6 -1
  735. metadata/ingestion/source/database/snowflake/queries.py +6 -4
  736. metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
  737. metadata/ingestion/source/database/snowflake/utils.py +23 -8
  738. metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
  739. metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
  740. metadata/ingestion/source/database/unitycatalog/client.py +87 -0
  741. metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
  742. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
  743. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
  744. metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
  745. metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
  746. metadata/ingestion/source/database/usage_source.py +3 -2
  747. metadata/ingestion/source/messaging/common_broker_source.py +15 -11
  748. metadata/ingestion/source/messaging/kafka/connection.py +45 -4
  749. metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
  750. metadata/ingestion/source/messaging/messaging_service.py +6 -2
  751. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  752. metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
  753. metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
  754. metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
  755. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +20 -8
  756. metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
  757. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
  758. metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
  759. metadata/ingestion/source/pipeline/airflow/models.py +5 -4
  760. metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
  761. metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -5
  762. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +14 -11
  763. metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
  764. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
  765. metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
  766. metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
  767. metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
  768. metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
  769. metadata/ingestion/source/search/search_service.py +6 -2
  770. metadata/ingestion/source/storage/s3/metadata.py +22 -17
  771. metadata/ingestion/source/storage/storage_service.py +53 -11
  772. metadata/ingestion/stage/table_usage.py +9 -2
  773. metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
  774. metadata/parsers/protobuf_parser.py +29 -11
  775. metadata/pii/processor.py +9 -2
  776. metadata/pii/scanners/ner_scanner.py +5 -3
  777. metadata/profiler/api/models.py +19 -1
  778. metadata/profiler/interface/pandas/profiler_interface.py +59 -18
  779. metadata/profiler/interface/profiler_interface.py +14 -4
  780. metadata/profiler/interface/profiler_interface_factory.py +49 -14
  781. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
  782. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
  783. metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
  784. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
  785. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
  786. metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
  787. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  788. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
  789. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  790. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
  791. metadata/profiler/metrics/composed/null_ratio.py +1 -1
  792. metadata/profiler/metrics/hybrid/histogram.py +1 -0
  793. metadata/profiler/metrics/static/max.py +4 -1
  794. metadata/profiler/metrics/static/min.py +4 -1
  795. metadata/profiler/metrics/system/queries/snowflake.py +89 -17
  796. metadata/profiler/metrics/system/system.py +62 -20
  797. metadata/profiler/orm/functions/conn_test.py +1 -0
  798. metadata/profiler/orm/functions/length.py +1 -0
  799. metadata/profiler/orm/functions/median.py +9 -1
  800. metadata/profiler/orm/functions/sum.py +1 -0
  801. metadata/profiler/orm/functions/table_metric_computer.py +462 -0
  802. metadata/profiler/orm/registry.py +2 -0
  803. metadata/profiler/processor/core.py +129 -62
  804. metadata/profiler/processor/default.py +14 -3
  805. metadata/profiler/processor/handle_partition.py +2 -2
  806. metadata/profiler/processor/processor.py +10 -7
  807. metadata/profiler/processor/sample_data_handler.py +6 -2
  808. metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
  809. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
  810. metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
  811. metadata/profiler/source/base/profiler_source.py +5 -2
  812. metadata/profiler/source/bigquery/type_mapper.py +0 -1
  813. metadata/profiler/source/databricks/profiler_source.py +36 -0
  814. metadata/profiler/source/mariadb/functions/median.py +20 -0
  815. metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
  816. metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
  817. metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
  818. metadata/profiler/source/metadata.py +49 -10
  819. metadata/profiler/source/metadata_ext.py +16 -50
  820. metadata/profiler/source/profiler_source_factory.py +8 -0
  821. metadata/py.typed +0 -0
  822. metadata/readers/dataframe/json.py +5 -4
  823. metadata/readers/file/api_reader.py +0 -1
  824. metadata/utils/constants.py +5 -0
  825. metadata/utils/credentials.py +36 -19
  826. metadata/utils/datalake/datalake_utils.py +369 -129
  827. metadata/utils/entity_link.py +26 -6
  828. metadata/utils/execution_time_tracker.py +199 -0
  829. metadata/utils/filters.py +4 -0
  830. metadata/utils/fqn.py +20 -0
  831. metadata/utils/helpers.py +34 -39
  832. metadata/utils/importer.py +2 -3
  833. metadata/utils/life_cycle_utils.py +4 -4
  834. metadata/utils/logger.py +13 -2
  835. metadata/utils/partition.py +10 -5
  836. metadata/utils/secrets/aws_based_secrets_manager.py +67 -4
  837. metadata/utils/secrets/aws_secrets_manager.py +7 -2
  838. metadata/utils/secrets/aws_ssm_secrets_manager.py +7 -2
  839. metadata/utils/secrets/azure_kv_secrets_manager.py +132 -0
  840. metadata/utils/secrets/{noop_secrets_manager.py → db_secrets_manager.py} +4 -3
  841. metadata/utils/secrets/external_secrets_manager.py +25 -3
  842. metadata/utils/secrets/secrets_manager_factory.py +15 -33
  843. metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
  844. metadata/utils/sqlalchemy_utils.py +21 -0
  845. metadata/utils/storage_metadata_config.py +42 -14
  846. metadata/utils/tag_utils.py +5 -2
  847. metadata/workflow/application.py +154 -0
  848. metadata/workflow/application_output_handler.py +34 -0
  849. metadata/workflow/base.py +88 -153
  850. metadata/workflow/data_insight.py +8 -7
  851. metadata/workflow/data_quality.py +3 -2
  852. metadata/workflow/ingestion.py +203 -0
  853. metadata/workflow/metadata.py +2 -3
  854. metadata/workflow/output_handler.py +226 -0
  855. metadata/workflow/profiler.py +2 -2
  856. metadata/workflow/usage.py +3 -4
  857. metadata/workflow/workflow_output_handler.py +15 -255
  858. metadata/workflow/workflow_status_mixin.py +44 -52
  859. openmetadata_ingestion-1.3.2.0rc1.dist-info/METADATA +758 -0
  860. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/RECORD +870 -762
  861. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/WHEEL +1 -1
  862. metadata/generated/schema/entity/applications/appConfig.py +0 -21
  863. metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
  864. metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
  865. metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
  866. metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
  867. metadata/generated/schema/events/entitySpelFilters.py +0 -19
  868. metadata/ingestion/models/es_documents.py +0 -339
  869. metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
  870. metadata/ingestion/ometa/provider_registry.py +0 -144
  871. metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
  872. metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
  873. metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
  874. metadata/profiler/orm/functions/table_metric_construct.py +0 -365
  875. metadata/utils/secrets/client/loader.py +0 -78
  876. openmetadata_ingestion-1.2.4.0.dist-info/METADATA +0 -429
  877. /metadata/examples/workflows/{datalake_azure.yaml → datalake_azure_client_secret.yaml} +0 -0
  878. /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
  879. /metadata/ingestion/source/database/{databricks/unity_catalog → bigtable}/__init__.py +0 -0
  880. /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/doris}/__init__.py +0 -0
  881. /metadata/{utils/secrets/client → ingestion/source/database/sas}/__init__.py +0 -0
  882. /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
  883. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/LICENSE +0 -0
  884. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/entry_points.txt +0 -0
  885. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,41 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+ """
12
+ Mixin class containing Suggestions specific methods
13
+
14
+ To be used by OpenMetadata class
15
+ """
16
+ from metadata.generated.schema.entity.feed.suggestion import Suggestion
17
+ from metadata.ingestion.ometa.client import REST
18
+ from metadata.utils.logger import ometa_logger
19
+
20
+ logger = ometa_logger()
21
+
22
+
23
+ class OMetaSuggestionsMixin:
24
+ """
25
+ OpenMetadata API methods related to the Suggestion Entity
26
+
27
+ To be inherited by OpenMetadata
28
+ """
29
+
30
+ client: REST
31
+
32
+ def update_suggestion(self, suggestion: Suggestion) -> Suggestion:
33
+ """
34
+ Update an existing Suggestion with new fields
35
+ """
36
+ resp = self.client.put(
37
+ f"{self.get_suffix(Suggestion)}/{str(suggestion.id.__root__)}",
38
+ data=suggestion.json(),
39
+ )
40
+
41
+ return Suggestion(**resp)
@@ -22,6 +22,9 @@ from requests.utils import quote
22
22
  from metadata.generated.schema.api.data.createTableProfile import (
23
23
  CreateTableProfileRequest,
24
24
  )
25
+ from metadata.generated.schema.api.tests.createCustomMetric import (
26
+ CreateCustomMetricRequest,
27
+ )
25
28
  from metadata.generated.schema.entity.data.table import (
26
29
  ColumnProfile,
27
30
  DataModel,
@@ -288,3 +291,18 @@ class OMetaTableMixin:
288
291
  Optional[Table]: OM table object
289
292
  """
290
293
  return self._get(Table, f"{quote(model_str(fqn))}/tableProfile/latest")
294
+
295
+ def create_or_update_custom_metric(
296
+ self, custom_metric: CreateCustomMetricRequest, table_id: str
297
+ ) -> Table:
298
+ """Create or update custom metric. If custom metric name matches an existing
299
+ one then it will be updated.
300
+
301
+ Args:
302
+ custom_metric (CreateCustomMetricRequest): custom metric to be create or updated
303
+ """
304
+ resp = self.client.put(
305
+ f"{self.get_suffix(Table)}/{table_id}/customMetric",
306
+ data=custom_metric.json(),
307
+ )
308
+ return Table(**resp)
@@ -23,6 +23,9 @@ from metadata.generated.schema.api.tests.createLogicalTestCases import (
23
23
  CreateLogicalTestCases,
24
24
  )
25
25
  from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
26
+ from metadata.generated.schema.api.tests.createTestCaseResolutionStatus import (
27
+ CreateTestCaseResolutionStatus,
28
+ )
26
29
  from metadata.generated.schema.api.tests.createTestDefinition import (
27
30
  CreateTestDefinitionRequest,
28
31
  )
@@ -30,6 +33,9 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
30
33
  from metadata.generated.schema.entity.data.table import Table
31
34
  from metadata.generated.schema.tests.basic import TestCaseResult
32
35
  from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
36
+ from metadata.generated.schema.tests.testCaseResolutionStatus import (
37
+ TestCaseResolutionStatus,
38
+ )
33
39
  from metadata.generated.schema.tests.testDefinition import (
34
40
  EntityType,
35
41
  TestCaseParameterDefinition,
@@ -242,10 +248,9 @@ class OMetaTestsMixin:
242
248
  end_ts (int): timestamp
243
249
  """
244
250
 
245
- # timestamp should be changed to milliseconds in https://github.com/open-metadata/OpenMetadata/issues/8930
246
251
  params = {
247
- "startTs": start_ts // 1000,
248
- "endTs": end_ts // 1000,
252
+ "startTs": start_ts,
253
+ "endTs": end_ts,
249
254
  }
250
255
 
251
256
  resp = self.client.get(
@@ -302,3 +307,19 @@ class OMetaTestsMixin:
302
307
  """
303
308
  path = self.get_suffix(TestCase) + "/logicalTestCases"
304
309
  self.client.put(path, data=data.json(encoder=show_secrets_encoder))
310
+
311
+ def create_test_case_resolution(
312
+ self, data: CreateTestCaseResolutionStatus
313
+ ) -> TestCaseResolutionStatus:
314
+ """Create a test case resolution
315
+
316
+ Args:
317
+ data (CreateTestCaseResolutionStatus): test case resolution
318
+
319
+ Returns:
320
+ TestCaseResolutionStatus
321
+ """
322
+ path = self.get_suffix(TestCase) + "/testCaseIncidentStatus"
323
+ response = self.client.post(path, data=data.json(encoder=show_secrets_encoder))
324
+
325
+ return TestCaseResolutionStatus(**response)
@@ -13,12 +13,15 @@ Mixin class containing User specific methods
13
13
 
14
14
  To be used by OpenMetadata class
15
15
  """
16
- import traceback
17
16
  from functools import lru_cache
18
- from typing import Optional
17
+ from typing import Optional, Type
19
18
 
19
+ from metadata.generated.schema.entity.teams.team import Team
20
20
  from metadata.generated.schema.entity.teams.user import User
21
+ from metadata.generated.schema.type.entityReference import EntityReference
22
+ from metadata.ingestion.api.common import T
21
23
  from metadata.ingestion.ometa.client import REST
24
+ from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
22
25
  from metadata.utils.elasticsearch import ES_INDEX_MAP
23
26
  from metadata.utils.logger import ometa_logger
24
27
 
@@ -34,42 +37,134 @@ class OMetaUserMixin:
34
37
 
35
38
  client: REST
36
39
 
37
- email_search = (
38
- "/search/query?q=email.keyword:{email}&from={from_}&size={size}&index="
39
- + ES_INDEX_MAP[User.__name__]
40
- )
40
+ @staticmethod
41
+ def email_search_query_es(entity: Type[T]) -> str:
42
+ return (
43
+ "/search/query?q=email.keyword:{email}&from={from_}&size={size}&index="
44
+ + ES_INDEX_MAP[entity.__name__]
45
+ )
41
46
 
42
- @lru_cache(maxsize=None)
43
- def get_user_by_email(
47
+ @staticmethod
48
+ def name_search_query_es(entity: Type[T]) -> str:
49
+ """
50
+ Allow for more flexible lookup following what the UI is doing when searching users.
51
+
52
+ We don't want to stick to `q=name:{name}` since in case a user is named `random.user`
53
+ but looked as `Random User`, we want to find this match.
54
+ """
55
+ return (
56
+ "/search/query?q={name}&from={from_}&size={size}&index="
57
+ + ES_INDEX_MAP[entity.__name__]
58
+ )
59
+
60
+ def _search_by_email(
44
61
  self,
62
+ entity: Type[T],
45
63
  email: Optional[str],
46
64
  from_count: int = 0,
47
65
  size: int = 1,
48
66
  fields: Optional[list] = None,
49
- ) -> Optional[User]:
67
+ ) -> Optional[T]:
50
68
  """
51
- GET user entity by name
69
+ GET user or team entity by mail
52
70
 
53
71
  Args:
54
72
  email: user email to search
55
73
  from_count: records to expect
56
74
  size: number of records
75
+ fields: Optional field list to pass to ES request
57
76
  """
58
77
  if email:
59
- query_string = self.email_search.format(
78
+ query_string = self.email_search_query_es(entity=entity).format(
60
79
  email=email, from_=from_count, size=size
61
80
  )
81
+ return self._get_entity_from_es(
82
+ entity=entity, query_string=query_string, fields=fields
83
+ )
84
+
85
+ return None
86
+
87
+ def _search_by_name(
88
+ self,
89
+ entity: Type[T],
90
+ name: Optional[str],
91
+ from_count: int = 0,
92
+ size: int = 1,
93
+ fields: Optional[list] = None,
94
+ ) -> Optional[T]:
95
+ """
96
+ GET entity by name
97
+
98
+ Args:
99
+ name: user name to search
100
+ from_count: records to expect
101
+ size: number of records
102
+ fields: Optional field list to pass to ES request
103
+ """
104
+ if name:
105
+ query_string = self.name_search_query_es(entity=entity).format(
106
+ name=name, from_=from_count, size=size
107
+ )
108
+ return self._get_entity_from_es(
109
+ entity=entity, query_string=query_string, fields=fields
110
+ )
62
111
 
63
- try:
64
- entity_list = self._search_es_entity(
65
- entity_type=User, query_string=query_string, fields=fields
66
- )
67
- for user in entity_list or []:
68
- return user
69
- except Exception as err:
70
- logger.debug(traceback.format_exc())
71
- logger.warning(
72
- f"Could not get user info from ES for user email {email} due to {err}"
73
- )
112
+ return None
113
+
114
+ @lru_cache(maxsize=None)
115
+ def get_reference_by_email(
116
+ self,
117
+ email: Optional[str],
118
+ from_count: int = 0,
119
+ size: int = 1,
120
+ fields: Optional[list] = None,
121
+ ) -> Optional[EntityReference]:
122
+ """
123
+ Get a User or Team Entity Reference by searching by its mail
124
+ """
125
+ maybe_user = self._search_by_email(
126
+ entity=User, email=email, from_count=from_count, size=size, fields=fields
127
+ )
128
+ if maybe_user:
129
+ return EntityReference(
130
+ id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
131
+ )
132
+
133
+ maybe_team = self._search_by_email(
134
+ entity=Team, email=email, from_count=from_count, size=size, fields=fields
135
+ )
136
+ if maybe_team:
137
+ return EntityReference(
138
+ id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
139
+ )
140
+
141
+ return None
142
+
143
+ @lru_cache(maxsize=None)
144
+ def get_reference_by_name(
145
+ self,
146
+ name: Optional[str],
147
+ from_count: int = 0,
148
+ size: int = 1,
149
+ fields: Optional[list] = None,
150
+ ) -> Optional[EntityReference]:
151
+ """
152
+ Get a User or Team Entity Reference by searching by its name
153
+ """
154
+ maybe_user = self._search_by_name(
155
+ entity=User, name=name, from_count=from_count, size=size, fields=fields
156
+ )
157
+ if maybe_user:
158
+ return EntityReference(
159
+ id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
160
+ )
161
+
162
+ maybe_team = self._search_by_name(
163
+ entity=Team, name=name, from_count=from_count, size=size, fields=fields
164
+ )
165
+ if maybe_team:
166
+ return EntityReference(
167
+ id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
168
+ )
74
169
 
75
170
  return None
@@ -31,7 +31,7 @@ from metadata.generated.schema.type.basic import FullyQualifiedEntityName
31
31
  from metadata.generated.schema.type.entityHistory import EntityVersionHistory
32
32
  from metadata.generated.schema.type.entityReference import EntityReference
33
33
  from metadata.ingestion.models.encoders import show_secrets_encoder
34
- from metadata.ingestion.ometa.auth_provider import AuthenticationProvider
34
+ from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
35
35
  from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
36
36
  from metadata.ingestion.ometa.mixins.custom_property_mixin import (
37
37
  OMetaCustomPropertyMixin,
@@ -39,7 +39,6 @@ from metadata.ingestion.ometa.mixins.custom_property_mixin import (
39
39
  from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
40
40
  from metadata.ingestion.ometa.mixins.data_insight_mixin import DataInsightMixin
41
41
  from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
42
- from metadata.ingestion.ometa.mixins.glossary_mixin import GlossaryMixin
43
42
  from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
44
43
  OMetaIngestionPipelineMixin,
45
44
  )
@@ -51,16 +50,13 @@ from metadata.ingestion.ometa.mixins.role_policy_mixin import OMetaRolePolicyMix
51
50
  from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin
52
51
  from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
53
52
  from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin
53
+ from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin
54
54
  from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin
55
55
  from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin
56
56
  from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin
57
57
  from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
58
58
  from metadata.ingestion.ometa.mixins.version_mixin import OMetaVersionMixin
59
59
  from metadata.ingestion.ometa.models import EntityList
60
- from metadata.ingestion.ometa.provider_registry import (
61
- InvalidAuthProviderException,
62
- auth_provider_registry,
63
- )
64
60
  from metadata.ingestion.ometa.routes import ROUTES
65
61
  from metadata.ingestion.ometa.utils import get_entity_type, model_str
66
62
  from metadata.utils.logger import ometa_logger
@@ -100,7 +96,6 @@ class OpenMetadata(
100
96
  OMetaTableMixin,
101
97
  OMetaTopicMixin,
102
98
  OMetaVersionMixin,
103
- GlossaryMixin,
104
99
  OMetaServiceMixin,
105
100
  ESMixin,
106
101
  OMetaServerMixin,
@@ -114,6 +109,7 @@ class OpenMetadata(
114
109
  OMetaRolePolicyMixin,
115
110
  OMetaSearchIndexMixin,
116
111
  OMetaCustomPropertyMixin,
112
+ OMetaSuggestionsMixin,
117
113
  Generic[T, C],
118
114
  ):
119
115
  """
@@ -125,7 +121,7 @@ class OpenMetadata(
125
121
  """
126
122
 
127
123
  client: REST
128
- _auth_provider: AuthenticationProvider
124
+ _auth_provider: OpenMetadataAuthenticationProvider
129
125
  config: OpenMetadataConnection
130
126
 
131
127
  class_root = ".".join(["metadata", "generated", "schema"])
@@ -146,16 +142,7 @@ class OpenMetadata(
146
142
  config.secretsManagerLoader,
147
143
  ).get_secrets_manager()
148
144
 
149
- # Load the auth provider init from the registry
150
- auth_provider_fn = auth_provider_registry.registry.get(
151
- self.config.authProvider.value
152
- )
153
- if not auth_provider_fn:
154
- raise InvalidAuthProviderException(
155
- f"Cannot find {self.config.authProvider.value} in {auth_provider_registry.registry}"
156
- )
157
-
158
- self._auth_provider = auth_provider_fn(self.config)
145
+ self._auth_provider = OpenMetadataAuthenticationProvider.create(self.config)
159
146
 
160
147
  get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
161
148
 
@@ -261,11 +248,9 @@ class OpenMetadata(
261
248
  )
262
249
  return entity_class
263
250
 
264
- def create_or_update(self, data: C) -> T:
251
+ def _create(self, data: C, method: str) -> T:
265
252
  """
266
- We allow CreateEntity for PUT, so we expect a type C.
267
-
268
- We PUT to the endpoint and return the Entity generated result
253
+ Internal logic to run POST vs. PUT
269
254
  """
270
255
  entity = data.__class__
271
256
  is_create = "create" in data.__class__.__name__.lower()
@@ -277,15 +262,23 @@ class OpenMetadata(
277
262
  raise InvalidEntityException(
278
263
  f"PUT operations need a CreateEntity, not {entity}"
279
264
  )
280
- resp = self.client.put(
281
- self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder)
282
- )
265
+
266
+ fn = getattr(self.client, method)
267
+ resp = fn(self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder))
283
268
  if not resp:
284
269
  raise EmptyPayloadException(
285
270
  f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.json()}"
286
271
  )
287
272
  return entity_class(**resp)
288
273
 
274
+ def create_or_update(self, data: C) -> T:
275
+ """Run a PUT requesting via create request C"""
276
+ return self._create(data=data, method="put")
277
+
278
+ def create(self, data: C) -> T:
279
+ """Run a POST requesting via create request C"""
280
+ return self._create(data=data, method="post")
281
+
289
282
  def get_by_name(
290
283
  self,
291
284
  entity: Type[T],
@@ -425,7 +418,7 @@ class OpenMetadata(
425
418
  self,
426
419
  entity: Type[T],
427
420
  fields: Optional[List[str]] = None,
428
- limit: int = 1000,
421
+ limit: int = 100,
429
422
  params: Optional[Dict[str, str]] = None,
430
423
  skip_on_failure: bool = False,
431
424
  ) -> Iterable[T]:
@@ -50,6 +50,7 @@ from metadata.generated.schema.api.domains.createDataProduct import (
50
50
  CreateDataProductRequest,
51
51
  )
52
52
  from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
53
+ from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest
53
54
  from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
54
55
  from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest
55
56
  from metadata.generated.schema.api.services.createDashboardService import (
@@ -90,6 +91,7 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
90
91
  from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
91
92
  from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
92
93
  from metadata.generated.schema.entity.automations.workflow import Workflow
94
+ from metadata.generated.schema.entity.bot import Bot
93
95
  from metadata.generated.schema.entity.classification.classification import (
94
96
  Classification,
95
97
  )
@@ -113,6 +115,7 @@ from metadata.generated.schema.entity.data.table import Table
113
115
  from metadata.generated.schema.entity.data.topic import Topic
114
116
  from metadata.generated.schema.entity.domains.dataProduct import DataProduct
115
117
  from metadata.generated.schema.entity.domains.domain import Domain
118
+ from metadata.generated.schema.entity.feed.suggestion import Suggestion
116
119
  from metadata.generated.schema.entity.policies.policy import Policy
117
120
  from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
118
121
  TestConnectionDefinition,
@@ -181,6 +184,7 @@ ROUTES = {
181
184
  User.__name__: "/users",
182
185
  CreateUserRequest.__name__: "/users",
183
186
  AuthenticationMechanism.__name__: "/users/auth-mechanism",
187
+ Bot.__name__: "/bots", # We won't allow bot creation from the client
184
188
  # Roles
185
189
  Role.__name__: "/roles",
186
190
  CreateRoleRequest.__name__: "/roles",
@@ -225,4 +229,7 @@ ROUTES = {
225
229
  CreateDomainRequest.__name__: "/domains",
226
230
  DataProduct.__name__: "/dataProducts",
227
231
  CreateDataProductRequest.__name__: "/dataProducts",
232
+ # Suggestions
233
+ Suggestion.__name__: "/suggestions",
234
+ CreateSuggestionRequest.__name__: "/suggestions",
228
235
  }
@@ -17,10 +17,13 @@ import traceback
17
17
  from typing import Optional
18
18
 
19
19
  from metadata.config.common import ConfigModel
20
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
21
+ StackTraceError,
22
+ )
20
23
  from metadata.generated.schema.type.basic import DateTime
21
24
  from metadata.generated.schema.type.queryParserData import ParsedData, QueryParserData
22
25
  from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
23
- from metadata.ingestion.api.models import Either, StackTraceError
26
+ from metadata.ingestion.api.models import Either
24
27
  from metadata.ingestion.api.steps import Processor
25
28
  from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
26
29
  from metadata.ingestion.lineage.parser import LineageParser
@@ -84,6 +87,10 @@ class QueryParserProcessor(Processor):
84
87
  self.metadata = metadata
85
88
  self.connection_type = connection_type
86
89
 
90
+ @property
91
+ def name(self) -> str:
92
+ return "Query Parser"
93
+
87
94
  @classmethod
88
95
  def create(cls, config_dict: dict, metadata: OpenMetadata, **kwargs):
89
96
  config = ConfigModel.parse_obj(config_dict)
@@ -106,7 +113,7 @@ class QueryParserProcessor(Processor):
106
113
  left=StackTraceError(
107
114
  name="Query",
108
115
  error=f"Error processing query [{table_query.query}]: {exc}",
109
- stack_trace=traceback.format_exc(),
116
+ stackTrace=traceback.format_exc(),
110
117
  )
111
118
  )
112
119
 
@@ -47,6 +47,9 @@ from metadata.generated.schema.entity.teams.team import Team
47
47
  from metadata.generated.schema.entity.teams.user import User
48
48
  from metadata.generated.schema.tests.basic import TestCaseResult
49
49
  from metadata.generated.schema.tests.testCase import TestCase
50
+ from metadata.generated.schema.tests.testCaseResolutionStatus import (
51
+ TestCaseResolutionStatus,
52
+ )
50
53
  from metadata.generated.schema.tests.testSuite import TestSuite
51
54
  from metadata.generated.schema.type.schema import Topic
52
55
  from metadata.ingestion.api.models import Either, Entity, StackTraceError
@@ -59,7 +62,9 @@ from metadata.ingestion.models.ometa_classification import OMetaTagAndClassifica
59
62
  from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData
60
63
  from metadata.ingestion.models.patch_request import (
61
64
  ALLOWED_COMMON_PATCH_FIELDS,
65
+ ARRAY_ENTITY_FIELDS,
62
66
  RESTRICT_UPDATE_LIST,
67
+ PatchedEntity,
63
68
  PatchRequest,
64
69
  )
65
70
  from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
@@ -67,6 +72,7 @@ from metadata.ingestion.models.profile_data import OMetaTableProfileSampleData
67
72
  from metadata.ingestion.models.search_index_data import OMetaIndexSampleData
68
73
  from metadata.ingestion.models.tests_data import (
69
74
  OMetaLogicalTestSuiteSample,
75
+ OMetaTestCaseResolutionStatus,
70
76
  OMetaTestCaseResultsSample,
71
77
  OMetaTestCaseSample,
72
78
  OMetaTestSuiteSample,
@@ -77,7 +83,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
77
83
  from metadata.ingestion.source.dashboard.dashboard_service import DashboardUsage
78
84
  from metadata.ingestion.source.database.database_service import DataModelLink
79
85
  from metadata.profiler.api.models import ProfilerResponse
80
- from metadata.utils.helpers import calculate_execution_time
86
+ from metadata.utils.execution_time_tracker import calculate_execution_time
81
87
  from metadata.utils.logger import get_log_name, ingestion_logger
82
88
 
83
89
  logger = ingestion_logger()
@@ -115,12 +121,16 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
115
121
  config = MetadataRestSinkConfig.parse_obj(config_dict)
116
122
  return cls(config, metadata)
117
123
 
124
+ @property
125
+ def name(self) -> str:
126
+ return "OpenMetadata"
127
+
118
128
  @singledispatchmethod
119
129
  def _run_dispatch(self, record: Entity) -> Either[Any]:
120
130
  logger.debug(f"Processing Create request {type(record)}")
121
131
  return self.write_create_request(record)
122
132
 
123
- @calculate_execution_time
133
+ @calculate_execution_time(store=False)
124
134
  def _run(self, record: Entity, *_, **__) -> Either[Any]:
125
135
  """
126
136
  Default implementation for the single dispatch
@@ -132,14 +142,14 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
132
142
  error = f"Failed to ingest {log} due to api request failure: {err}"
133
143
  return Either(
134
144
  left=StackTraceError(
135
- name=log, error=error, stack_trace=traceback.format_exc()
145
+ name=log, error=error, stackTrace=traceback.format_exc()
136
146
  )
137
147
  )
138
148
  except Exception as exc:
139
149
  error = f"Failed to ingest {log}: {exc}"
140
150
  return Either(
141
151
  left=StackTraceError(
142
- name=log, error=error, stack_trace=traceback.format_exc()
152
+ name=log, error=error, stackTrace=traceback.format_exc()
143
153
  )
144
154
  )
145
155
 
@@ -155,7 +165,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
155
165
  error = f"Failed to ingest {type(entity_request).__name__}"
156
166
  return Either(
157
167
  left=StackTraceError(
158
- name=type(entity_request).__name__, error=error, stack_trace=None
168
+ name=type(entity_request).__name__, error=error, stackTrace=None
159
169
  )
160
170
  )
161
171
 
@@ -170,8 +180,10 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
170
180
  destination=record.new_entity,
171
181
  allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
172
182
  restrict_update_fields=RESTRICT_UPDATE_LIST,
183
+ array_entity_fields=ARRAY_ENTITY_FIELDS,
173
184
  )
174
- return Either(right=entity)
185
+ patched_entity = PatchedEntity(new_entity=entity) if entity else None
186
+ return Either(right=patched_entity)
175
187
 
176
188
  @_run_dispatch.register
177
189
  def write_custom_properties(self, record: OMetaCustomProperties) -> Either[Dict]:
@@ -200,7 +212,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
200
212
  left=StackTraceError(
201
213
  name="Data Model",
202
214
  error="Sink did not receive a table. We cannot ingest the data model.",
203
- stack_trace=None,
215
+ stackTrace=None,
204
216
  )
205
217
  )
206
218
 
@@ -275,7 +287,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
275
287
  for role in record.roles:
276
288
  try:
277
289
  role_entity = self.metadata.get_by_name(
278
- entity=Role, fqn=str(role.name.__root__.__root__)
290
+ entity=Role, fqn=str(role.name.__root__)
279
291
  )
280
292
  except APIError:
281
293
  role_entity = self._create_role(role)
@@ -411,6 +423,15 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
411
423
  )
412
424
  return Either(right=res)
413
425
 
426
+ @_run_dispatch.register
427
+ def write_test_case_resolution_status(
428
+ self, record: OMetaTestCaseResolutionStatus
429
+ ) -> TestCaseResolutionStatus:
430
+ """For sample data"""
431
+ res = self.metadata.create_test_case_resolution(record.test_case_resolution)
432
+
433
+ return Either(right=res)
434
+
414
435
  @_run_dispatch.register
415
436
  def write_data_insight_sample(
416
437
  self, record: OMetaDataInsightSample