openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (885) hide show
  1. airflow_provider_openmetadata/lineage/runner.py +44 -5
  2. airflow_provider_openmetadata/lineage/status.py +1 -1
  3. metadata/antlr/split_listener.py +1 -4
  4. metadata/applications/auto_tagger.py +212 -0
  5. metadata/cli/app.py +47 -0
  6. metadata/cli/db_dump.py +7 -3
  7. metadata/clients/aws_client.py +4 -0
  8. metadata/clients/azure_client.py +85 -0
  9. metadata/clients/domo_client.py +0 -2
  10. metadata/cmd.py +24 -13
  11. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +40 -48
  12. metadata/data_insight/processor/reports/data_processor.py +1 -0
  13. metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
  14. metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
  15. metadata/data_insight/producer/cost_analysis_producer.py +78 -14
  16. metadata/data_insight/producer/entity_producer.py +1 -1
  17. metadata/data_insight/producer/producer_interface.py +1 -1
  18. metadata/data_insight/producer/web_analytics_producer.py +1 -1
  19. metadata/data_insight/source/metadata.py +21 -5
  20. metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
  21. metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
  22. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
  23. metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
  24. metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
  25. metadata/data_quality/processor/test_case_runner.py +17 -8
  26. metadata/data_quality/source/test_suite.py +10 -4
  27. metadata/data_quality/validations/base_test_handler.py +21 -17
  28. metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
  29. metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
  30. metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
  31. metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
  32. metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
  33. metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
  34. metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
  35. metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
  36. metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
  37. metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
  38. metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
  39. metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
  40. metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
  41. metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
  42. metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
  43. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
  44. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
  45. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
  46. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
  47. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
  48. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
  49. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
  50. metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
  51. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
  52. metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +2 -1
  53. metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +2 -1
  54. metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +7 -2
  55. metadata/examples/workflows/bigtable.yaml +32 -0
  56. metadata/examples/workflows/databricks.yaml +0 -1
  57. metadata/examples/workflows/datalake_azure_default.yaml +29 -0
  58. metadata/examples/workflows/mongodb.yaml +4 -2
  59. metadata/examples/workflows/mstr.yaml +24 -0
  60. metadata/examples/workflows/sas.yaml +28 -0
  61. metadata/examples/workflows/unity_catalog.yaml +27 -0
  62. metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
  63. metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
  64. metadata/generated/antlr/EntityLinkLexer.py +425 -292
  65. metadata/generated/antlr/EntityLinkListener.py +12 -12
  66. metadata/generated/antlr/EntityLinkParser.py +228 -103
  67. metadata/generated/schema/analytics/__init__.py +1 -1
  68. metadata/generated/schema/analytics/basic.py +1 -1
  69. metadata/generated/schema/analytics/reportData.py +1 -1
  70. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  71. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  72. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  73. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  74. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  75. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  76. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  77. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  78. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  79. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  80. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  81. metadata/generated/schema/api/__init__.py +1 -1
  82. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  83. metadata/generated/schema/api/analytics/__init__.py +1 -1
  84. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  85. metadata/generated/schema/api/automations/__init__.py +1 -1
  86. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  87. metadata/generated/schema/api/bulkAssets.py +1 -1
  88. metadata/generated/schema/api/classification/__init__.py +1 -1
  89. metadata/generated/schema/api/classification/createClassification.py +1 -1
  90. metadata/generated/schema/api/classification/createTag.py +1 -1
  91. metadata/generated/schema/api/classification/loadTags.py +1 -1
  92. metadata/generated/schema/api/createBot.py +2 -3
  93. metadata/generated/schema/api/createEventPublisherJob.py +2 -2
  94. metadata/generated/schema/api/createType.py +1 -1
  95. metadata/generated/schema/api/data/__init__.py +1 -1
  96. metadata/generated/schema/api/data/createChart.py +1 -1
  97. metadata/generated/schema/api/data/createContainer.py +2 -2
  98. metadata/generated/schema/api/data/createCustomProperty.py +7 -12
  99. metadata/generated/schema/api/data/createDashboard.py +1 -1
  100. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  101. metadata/generated/schema/api/data/createDatabase.py +2 -3
  102. metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
  103. metadata/generated/schema/api/data/createGlossary.py +2 -3
  104. metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
  105. metadata/generated/schema/api/data/createMlModel.py +1 -1
  106. metadata/generated/schema/api/data/createPipeline.py +2 -2
  107. metadata/generated/schema/api/data/createQuery.py +2 -2
  108. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  109. metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
  110. metadata/generated/schema/api/data/createTable.py +2 -2
  111. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  112. metadata/generated/schema/api/data/createTopic.py +1 -1
  113. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  114. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  115. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  116. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  117. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  118. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  119. metadata/generated/schema/api/docStore/__init__.py +1 -1
  120. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  121. metadata/generated/schema/api/domains/__init__.py +1 -1
  122. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  123. metadata/generated/schema/api/domains/createDomain.py +1 -1
  124. metadata/generated/schema/api/feed/__init__.py +1 -1
  125. metadata/generated/schema/api/feed/closeTask.py +8 -1
  126. metadata/generated/schema/api/feed/createPost.py +1 -1
  127. metadata/generated/schema/api/feed/createSuggestion.py +30 -0
  128. metadata/generated/schema/api/feed/createThread.py +5 -1
  129. metadata/generated/schema/api/feed/resolveTask.py +12 -1
  130. metadata/generated/schema/api/feed/threadCount.py +18 -14
  131. metadata/generated/schema/api/lineage/__init__.py +1 -1
  132. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  133. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  134. metadata/generated/schema/api/policies/__init__.py +1 -1
  135. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  136. metadata/generated/schema/api/services/__init__.py +1 -1
  137. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  138. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  139. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  140. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  141. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  142. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  143. metadata/generated/schema/api/services/createSearchService.py +1 -1
  144. metadata/generated/schema/api/services/createStorageService.py +1 -1
  145. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  146. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  147. metadata/generated/schema/api/setOwner.py +1 -1
  148. metadata/generated/schema/api/teams/__init__.py +1 -1
  149. metadata/generated/schema/api/teams/createPersona.py +1 -1
  150. metadata/generated/schema/api/teams/createRole.py +2 -3
  151. metadata/generated/schema/api/teams/createTeam.py +1 -1
  152. metadata/generated/schema/api/teams/createUser.py +2 -2
  153. metadata/generated/schema/api/tests/__init__.py +1 -1
  154. metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
  155. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  156. metadata/generated/schema/api/tests/createTestCase.py +4 -1
  157. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
  158. metadata/generated/schema/api/tests/createTestDefinition.py +2 -1
  159. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  160. metadata/generated/schema/api/voteRequest.py +1 -1
  161. metadata/generated/schema/auth/__init__.py +1 -1
  162. metadata/generated/schema/auth/basicAuth.py +1 -1
  163. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  164. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  165. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  166. metadata/generated/schema/auth/emailRequest.py +1 -1
  167. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  168. metadata/generated/schema/auth/generateToken.py +1 -1
  169. metadata/generated/schema/auth/jwtAuth.py +1 -1
  170. metadata/generated/schema/auth/loginRequest.py +1 -1
  171. metadata/generated/schema/auth/logoutRequest.py +1 -1
  172. metadata/generated/schema/auth/passwordResetRequest.py +3 -3
  173. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  174. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  175. metadata/generated/schema/auth/refreshToken.py +1 -1
  176. metadata/generated/schema/auth/registrationRequest.py +2 -2
  177. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  178. metadata/generated/schema/auth/revokeToken.py +1 -1
  179. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  180. metadata/generated/schema/auth/ssoAuth.py +1 -1
  181. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  182. metadata/generated/schema/configuration/__init__.py +1 -1
  183. metadata/generated/schema/configuration/appsPrivateConfiguration.py +32 -0
  184. metadata/generated/schema/configuration/authConfig.py +1 -1
  185. metadata/generated/schema/configuration/authenticationConfiguration.py +13 -2
  186. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  187. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  188. metadata/generated/schema/configuration/dataQualityConfiguration.py +16 -0
  189. metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
  190. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  191. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
  195. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  196. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  197. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  198. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  199. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  200. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  201. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  202. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  203. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  204. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  205. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  206. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  207. metadata/generated/schema/dataInsight/__init__.py +1 -1
  208. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  209. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  210. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  211. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  212. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  213. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  214. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  215. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  216. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  217. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  218. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  219. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  220. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  221. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  222. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
  223. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
  224. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
  225. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
  226. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
  227. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
  228. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  229. metadata/generated/schema/email/__init__.py +1 -1
  230. metadata/generated/schema/email/emailRequest.py +1 -1
  231. metadata/generated/schema/email/smtpSettings.py +1 -1
  232. metadata/generated/schema/entity/__init__.py +1 -1
  233. metadata/generated/schema/entity/applications/__init__.py +1 -1
  234. metadata/generated/schema/entity/applications/app.py +24 -5
  235. metadata/generated/schema/entity/applications/appRunRecord.py +2 -7
  236. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  237. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
  238. metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
  240. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
  241. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
  242. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
  243. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
  244. metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +14 -4
  245. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
  246. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
  247. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  248. metadata/generated/schema/entity/applications/jobStatus.py +3 -3
  249. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  250. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  251. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
  252. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
  253. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  254. metadata/generated/schema/entity/automations/__init__.py +1 -1
  255. metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
  256. metadata/generated/schema/entity/automations/workflow.py +1 -1
  257. metadata/generated/schema/entity/bot.py +2 -3
  258. metadata/generated/schema/entity/classification/__init__.py +1 -1
  259. metadata/generated/schema/entity/classification/classification.py +1 -1
  260. metadata/generated/schema/entity/classification/tag.py +2 -2
  261. metadata/generated/schema/entity/data/__init__.py +1 -1
  262. metadata/generated/schema/entity/data/chart.py +1 -1
  263. metadata/generated/schema/entity/data/container.py +4 -9
  264. metadata/generated/schema/entity/data/dashboard.py +1 -1
  265. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  266. metadata/generated/schema/entity/data/database.py +4 -8
  267. metadata/generated/schema/entity/data/databaseSchema.py +2 -8
  268. metadata/generated/schema/entity/data/glossary.py +1 -1
  269. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  270. metadata/generated/schema/entity/data/metrics.py +1 -1
  271. metadata/generated/schema/entity/data/mlmodel.py +2 -2
  272. metadata/generated/schema/entity/data/pipeline.py +6 -9
  273. metadata/generated/schema/entity/data/query.py +2 -2
  274. metadata/generated/schema/entity/data/report.py +1 -1
  275. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  276. metadata/generated/schema/entity/data/storedProcedure.py +3 -9
  277. metadata/generated/schema/entity/data/table.py +18 -12
  278. metadata/generated/schema/entity/data/topic.py +1 -1
  279. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  280. metadata/generated/schema/entity/docStore/document.py +1 -1
  281. metadata/generated/schema/entity/domains/__init__.py +1 -1
  282. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  283. metadata/generated/schema/entity/domains/domain.py +1 -1
  284. metadata/generated/schema/entity/events/__init__.py +1 -1
  285. metadata/generated/schema/entity/events/webhook.py +6 -3
  286. metadata/generated/schema/entity/feed/__init__.py +1 -1
  287. metadata/generated/schema/entity/feed/suggestion.py +61 -0
  288. metadata/generated/schema/entity/feed/thread.py +17 -1
  289. metadata/generated/schema/entity/policies/__init__.py +1 -1
  290. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  291. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  292. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  293. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  294. metadata/generated/schema/entity/policies/filters.py +2 -5
  295. metadata/generated/schema/entity/policies/policy.py +1 -1
  296. metadata/generated/schema/entity/services/__init__.py +1 -1
  297. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  298. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  299. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  300. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  301. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  302. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  304. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  305. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  306. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  307. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  308. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  309. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  310. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
  311. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  312. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
  313. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  314. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  315. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  316. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
  319. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +35 -2
  320. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
  321. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +41 -0
  322. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
  323. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +20 -0
  325. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  326. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
  329. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  330. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
  331. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  332. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  333. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  334. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  335. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
  336. metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
  337. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
  340. metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
  341. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  342. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  343. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
  344. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
  345. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
  346. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
  347. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
  348. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
  349. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
  350. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
  351. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
  352. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
  353. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
  354. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
  355. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
  356. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
  357. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +8 -4
  358. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
  359. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +9 -7
  360. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +8 -4
  361. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
  362. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
  363. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +3 -0
  365. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +20 -0
  366. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +36 -0
  367. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +7 -41
  368. metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
  369. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
  370. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
  371. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
  372. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
  373. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
  374. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
  375. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  376. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +14 -2
  378. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +14 -2
  381. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  382. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  383. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +2 -2
  387. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
  388. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  389. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  394. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
  396. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
  405. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  407. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  409. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  410. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  411. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  415. metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
  416. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  420. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  421. metadata/generated/schema/entity/services/dashboardService.py +4 -1
  422. metadata/generated/schema/entity/services/databaseService.py +16 -1
  423. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
  425. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  426. metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
  427. metadata/generated/schema/entity/services/messagingService.py +1 -1
  428. metadata/generated/schema/entity/services/metadataService.py +1 -1
  429. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  430. metadata/generated/schema/entity/services/pipelineService.py +4 -1
  431. metadata/generated/schema/entity/services/searchService.py +1 -1
  432. metadata/generated/schema/entity/services/serviceType.py +1 -1
  433. metadata/generated/schema/entity/services/storageService.py +10 -3
  434. metadata/generated/schema/entity/teams/__init__.py +1 -1
  435. metadata/generated/schema/entity/teams/persona.py +1 -1
  436. metadata/generated/schema/entity/teams/role.py +2 -6
  437. metadata/generated/schema/entity/teams/team.py +1 -1
  438. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  439. metadata/generated/schema/entity/teams/user.py +3 -10
  440. metadata/generated/schema/entity/type.py +4 -19
  441. metadata/generated/schema/entity/utils/__init__.py +1 -1
  442. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  443. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  444. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  445. metadata/generated/schema/events/__init__.py +1 -1
  446. metadata/generated/schema/events/alertMetrics.py +27 -0
  447. metadata/generated/schema/events/api/__init__.py +1 -1
  448. metadata/generated/schema/events/api/createEventSubscription.py +14 -12
  449. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  450. metadata/generated/schema/events/eventFilterRule.py +13 -2
  451. metadata/generated/schema/events/eventSubscription.py +91 -49
  452. metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
  453. metadata/generated/schema/events/failedEvent.py +26 -0
  454. metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
  455. metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
  456. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  457. metadata/generated/schema/metadataIngestion/application.py +40 -0
  458. metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
  459. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  460. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  461. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
  462. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +11 -1
  463. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  464. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  465. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  466. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  467. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
  468. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  469. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  470. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
  471. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  472. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  473. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
  474. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  475. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  476. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  477. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  478. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  479. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  480. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  481. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  482. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  483. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
  484. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
  485. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  486. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  487. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  488. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
  489. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  490. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  491. metadata/generated/schema/monitoring/__init__.py +1 -1
  492. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  493. metadata/generated/schema/security/__init__.py +1 -1
  494. metadata/generated/schema/security/client/__init__.py +1 -1
  495. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  496. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  497. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  498. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  499. metadata/generated/schema/security/client/oidcClientConfig.py +46 -0
  500. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  501. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  502. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  503. metadata/generated/schema/security/credentials/__init__.py +1 -1
  504. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  505. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  506. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  507. metadata/generated/schema/security/credentials/azureCredentials.py +21 -9
  508. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  509. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  510. metadata/generated/schema/security/credentials/gcpCredentials.py +7 -3
  511. metadata/generated/schema/security/credentials/gcpExternalAccount.py +37 -0
  512. metadata/generated/schema/security/credentials/gcpValues.py +2 -2
  513. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  514. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  515. metadata/generated/schema/security/secrets/__init__.py +1 -1
  516. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  517. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
  518. metadata/generated/schema/security/secrets/secretsManagerProvider.py +4 -2
  519. metadata/generated/schema/security/securityConfiguration.py +1 -1
  520. metadata/generated/schema/security/ssl/__init__.py +1 -1
  521. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  522. metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
  523. metadata/generated/schema/settings/__init__.py +1 -1
  524. metadata/generated/schema/settings/settings.py +1 -1
  525. metadata/generated/schema/system/__init__.py +1 -1
  526. metadata/generated/schema/system/entityError.py +17 -0
  527. metadata/generated/schema/system/eventPublisherJob.py +5 -19
  528. metadata/generated/schema/system/indexingError.py +34 -0
  529. metadata/generated/schema/system/ui/__init__.py +1 -1
  530. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  531. metadata/generated/schema/system/ui/page.py +1 -1
  532. metadata/generated/schema/system/validationResponse.py +43 -0
  533. metadata/generated/schema/tests/__init__.py +1 -1
  534. metadata/generated/schema/tests/assigned.py +18 -0
  535. metadata/generated/schema/tests/basic.py +17 -39
  536. metadata/generated/schema/tests/customMetric.py +4 -2
  537. metadata/generated/schema/tests/resolved.py +35 -0
  538. metadata/generated/schema/tests/testCase.py +11 -2
  539. metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
  540. metadata/generated/schema/tests/testDefinition.py +6 -1
  541. metadata/generated/schema/tests/testSuite.py +1 -1
  542. metadata/generated/schema/type/__init__.py +1 -1
  543. metadata/generated/schema/type/auditLog.py +1 -1
  544. metadata/generated/schema/type/basic.py +10 -6
  545. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  546. metadata/generated/schema/type/changeEvent.py +5 -75
  547. metadata/generated/schema/type/changeEventType.py +29 -0
  548. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  549. metadata/generated/schema/type/csvDocumentation.py +1 -1
  550. metadata/generated/schema/type/csvErrorType.py +1 -1
  551. metadata/generated/schema/type/csvFile.py +1 -1
  552. metadata/generated/schema/type/csvImportResult.py +1 -1
  553. metadata/generated/schema/type/customProperties/__init__.py +3 -0
  554. metadata/generated/schema/type/customProperties/enumConfig.py +17 -0
  555. metadata/generated/schema/type/customProperty.py +52 -0
  556. metadata/generated/schema/type/dailyCount.py +1 -1
  557. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  558. metadata/generated/schema/type/entityHistory.py +1 -1
  559. metadata/generated/schema/type/entityLineage.py +2 -1
  560. metadata/generated/schema/type/entityReference.py +5 -1
  561. metadata/generated/schema/type/entityReferenceList.py +1 -1
  562. metadata/generated/schema/type/entityRelationship.py +1 -1
  563. metadata/generated/schema/type/entityUsage.py +1 -1
  564. metadata/generated/schema/type/filterPattern.py +1 -1
  565. metadata/generated/schema/type/function.py +7 -6
  566. metadata/generated/schema/type/include.py +1 -1
  567. metadata/generated/schema/type/jdbcConnection.py +1 -1
  568. metadata/generated/schema/type/lifeCycle.py +1 -1
  569. metadata/generated/schema/type/paging.py +1 -1
  570. metadata/generated/schema/type/profile.py +1 -1
  571. metadata/generated/schema/type/queryParserData.py +2 -2
  572. metadata/generated/schema/type/reaction.py +1 -1
  573. metadata/generated/schema/type/schedule.py +1 -1
  574. metadata/generated/schema/type/schema.py +1 -1
  575. metadata/generated/schema/type/tableQuery.py +2 -2
  576. metadata/generated/schema/type/tableUsageCount.py +1 -1
  577. metadata/generated/schema/type/tagLabel.py +1 -1
  578. metadata/generated/schema/type/usageDetails.py +1 -1
  579. metadata/generated/schema/type/usageRequest.py +1 -1
  580. metadata/generated/schema/type/votes.py +1 -1
  581. metadata/great_expectations/action.py +12 -20
  582. metadata/ingestion/api/delete.py +5 -2
  583. metadata/ingestion/api/models.py +4 -10
  584. metadata/ingestion/api/parser.py +3 -37
  585. metadata/ingestion/api/status.py +15 -4
  586. metadata/ingestion/api/step.py +39 -6
  587. metadata/ingestion/api/steps.py +34 -1
  588. metadata/ingestion/api/topology_runner.py +117 -168
  589. metadata/ingestion/bulksink/metadata_usage.py +11 -5
  590. metadata/ingestion/lineage/models.py +4 -0
  591. metadata/ingestion/lineage/parser.py +28 -12
  592. metadata/ingestion/lineage/sql_lineage.py +9 -17
  593. metadata/ingestion/models/custom_properties.py +0 -1
  594. metadata/ingestion/models/custom_pydantic.py +4 -2
  595. metadata/ingestion/models/lf_tags_model.py +33 -0
  596. metadata/ingestion/models/patch_request.py +160 -2
  597. metadata/ingestion/models/tests_data.py +9 -0
  598. metadata/ingestion/models/topology.py +177 -60
  599. metadata/ingestion/ometa/auth_provider.py +0 -349
  600. metadata/ingestion/ometa/client.py +12 -3
  601. metadata/ingestion/ometa/mixins/custom_property_mixin.py +11 -11
  602. metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
  603. metadata/ingestion/ometa/mixins/patch_mixin.py +21 -72
  604. metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
  605. metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
  606. metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
  607. metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
  608. metadata/ingestion/ometa/ometa_api.py +19 -26
  609. metadata/ingestion/ometa/routes.py +7 -0
  610. metadata/ingestion/processor/query_parser.py +9 -2
  611. metadata/ingestion/sink/metadata_rest.py +29 -8
  612. metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
  613. metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
  614. metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
  615. metadata/ingestion/source/dashboard/looker/metadata.py +41 -29
  616. metadata/ingestion/source/dashboard/looker/utils.py +4 -2
  617. metadata/ingestion/source/dashboard/metabase/client.py +4 -0
  618. metadata/ingestion/source/dashboard/metabase/metadata.py +28 -10
  619. metadata/ingestion/source/dashboard/metabase/models.py +2 -2
  620. metadata/ingestion/source/dashboard/mode/client.py +10 -23
  621. metadata/ingestion/source/dashboard/mode/connection.py +6 -1
  622. metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
  623. metadata/ingestion/source/dashboard/mstr/client.py +208 -0
  624. metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
  625. metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
  626. metadata/ingestion/source/dashboard/mstr/models.py +144 -0
  627. metadata/ingestion/source/dashboard/powerbi/client.py +4 -1
  628. metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
  629. metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
  630. metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
  631. metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
  632. metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
  633. metadata/ingestion/source/dashboard/superset/client.py +16 -9
  634. metadata/ingestion/source/dashboard/superset/connection.py +3 -3
  635. metadata/ingestion/source/dashboard/superset/db_source.py +14 -11
  636. metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
  637. metadata/ingestion/source/dashboard/superset/queries.py +1 -1
  638. metadata/ingestion/source/dashboard/tableau/client.py +91 -11
  639. metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
  640. metadata/ingestion/source/dashboard/tableau/metadata.py +76 -70
  641. metadata/ingestion/source/dashboard/tableau/models.py +0 -8
  642. metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
  643. metadata/ingestion/source/database/athena/client.py +80 -0
  644. metadata/ingestion/source/database/athena/connection.py +7 -0
  645. metadata/ingestion/source/database/athena/metadata.py +161 -19
  646. metadata/ingestion/source/database/azuresql/connection.py +21 -3
  647. metadata/ingestion/source/database/azuresql/metadata.py +0 -1
  648. metadata/ingestion/source/database/bigquery/connection.py +24 -3
  649. metadata/ingestion/source/database/bigquery/helper.py +68 -1
  650. metadata/ingestion/source/database/bigquery/metadata.py +49 -28
  651. metadata/ingestion/source/database/bigquery/queries.py +33 -4
  652. metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
  653. metadata/ingestion/source/database/bigquery/usage.py +1 -3
  654. metadata/ingestion/source/database/bigtable/client.py +62 -0
  655. metadata/ingestion/source/database/bigtable/connection.py +116 -0
  656. metadata/ingestion/source/database/bigtable/metadata.py +224 -0
  657. metadata/ingestion/source/database/bigtable/models.py +60 -0
  658. metadata/ingestion/source/database/column_helpers.py +0 -10
  659. metadata/ingestion/source/database/column_type_parser.py +11 -5
  660. metadata/ingestion/source/database/common_db_source.py +33 -8
  661. metadata/ingestion/source/database/common_nosql_source.py +27 -6
  662. metadata/ingestion/source/database/database_service.py +89 -7
  663. metadata/ingestion/source/database/databricks/client.py +55 -103
  664. metadata/ingestion/source/database/databricks/connection.py +16 -55
  665. metadata/ingestion/source/database/databricks/lineage.py +29 -26
  666. metadata/ingestion/source/database/databricks/metadata.py +534 -11
  667. metadata/ingestion/source/database/databricks/queries.py +27 -0
  668. metadata/ingestion/source/database/databricks/query_parser.py +5 -1
  669. metadata/ingestion/source/database/databricks/usage.py +3 -3
  670. metadata/ingestion/source/database/datalake/connection.py +33 -18
  671. metadata/ingestion/source/database/datalake/metadata.py +113 -27
  672. metadata/ingestion/source/database/dbt/dbt_config.py +7 -14
  673. metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
  674. metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
  675. metadata/ingestion/source/database/dbt/metadata.py +27 -53
  676. metadata/ingestion/source/database/deltalake/metadata.py +6 -3
  677. metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
  678. metadata/ingestion/source/database/doris/connection.py +72 -0
  679. metadata/ingestion/source/database/doris/metadata.py +315 -0
  680. metadata/ingestion/source/database/doris/queries.py +54 -0
  681. metadata/ingestion/source/database/doris/utils.py +64 -0
  682. metadata/ingestion/source/database/extended_sample_data.py +532 -0
  683. metadata/ingestion/source/database/glue/metadata.py +8 -5
  684. metadata/ingestion/source/database/hive/connection.py +0 -2
  685. metadata/ingestion/source/database/hive/utils.py +3 -0
  686. metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
  687. metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
  688. metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
  689. metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
  690. metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
  691. metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
  692. metadata/ingestion/source/database/iceberg/connection.py +68 -0
  693. metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
  694. metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
  695. metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
  696. metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
  697. metadata/ingestion/source/database/iceberg/helper.py +124 -0
  698. metadata/ingestion/source/database/iceberg/metadata.py +311 -0
  699. metadata/ingestion/source/database/iceberg/models.py +66 -0
  700. metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
  701. metadata/ingestion/source/database/mongodb/connection.py +1 -5
  702. metadata/ingestion/source/database/mssql/lineage.py +3 -0
  703. metadata/ingestion/source/database/mssql/metadata.py +108 -4
  704. metadata/ingestion/source/database/mssql/models.py +30 -0
  705. metadata/ingestion/source/database/mssql/queries.py +178 -1
  706. metadata/ingestion/source/database/mssql/usage.py +5 -1
  707. metadata/ingestion/source/database/mssql/utils.py +207 -4
  708. metadata/ingestion/source/database/mysql/connection.py +14 -0
  709. metadata/ingestion/source/database/mysql/metadata.py +0 -2
  710. metadata/ingestion/source/database/oracle/metadata.py +108 -2
  711. metadata/ingestion/source/database/oracle/models.py +30 -0
  712. metadata/ingestion/source/database/oracle/queries.py +99 -18
  713. metadata/ingestion/source/database/oracle/utils.py +0 -1
  714. metadata/ingestion/source/database/postgres/connection.py +15 -0
  715. metadata/ingestion/source/database/postgres/lineage.py +32 -14
  716. metadata/ingestion/source/database/postgres/metadata.py +15 -7
  717. metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
  718. metadata/ingestion/source/database/postgres/queries.py +4 -2
  719. metadata/ingestion/source/database/postgres/query_parser.py +4 -72
  720. metadata/ingestion/source/database/postgres/usage.py +41 -0
  721. metadata/ingestion/source/database/postgres/utils.py +34 -0
  722. metadata/ingestion/source/database/query_parser_source.py +8 -2
  723. metadata/ingestion/source/database/redshift/metadata.py +14 -4
  724. metadata/ingestion/source/database/redshift/queries.py +10 -4
  725. metadata/ingestion/source/database/redshift/query_parser.py +16 -0
  726. metadata/ingestion/source/database/redshift/usage.py +0 -2
  727. metadata/ingestion/source/database/salesforce/metadata.py +32 -3
  728. metadata/ingestion/source/database/sample_data.py +120 -6
  729. metadata/ingestion/source/database/sas/client.py +184 -0
  730. metadata/ingestion/source/database/sas/connection.py +47 -0
  731. metadata/ingestion/source/database/sas/extension_attr.py +103 -0
  732. metadata/ingestion/source/database/sas/metadata.py +914 -0
  733. metadata/ingestion/source/database/snowflake/metadata.py +34 -48
  734. metadata/ingestion/source/database/snowflake/models.py +6 -1
  735. metadata/ingestion/source/database/snowflake/queries.py +6 -4
  736. metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
  737. metadata/ingestion/source/database/snowflake/utils.py +23 -8
  738. metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
  739. metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
  740. metadata/ingestion/source/database/unitycatalog/client.py +87 -0
  741. metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
  742. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
  743. metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
  744. metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
  745. metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
  746. metadata/ingestion/source/database/usage_source.py +3 -2
  747. metadata/ingestion/source/messaging/common_broker_source.py +15 -11
  748. metadata/ingestion/source/messaging/kafka/connection.py +45 -4
  749. metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
  750. metadata/ingestion/source/messaging/messaging_service.py +6 -2
  751. metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
  752. metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
  753. metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
  754. metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
  755. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +20 -8
  756. metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
  757. metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
  758. metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
  759. metadata/ingestion/source/pipeline/airflow/models.py +5 -4
  760. metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
  761. metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -5
  762. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +14 -11
  763. metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
  764. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
  765. metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
  766. metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
  767. metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
  768. metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
  769. metadata/ingestion/source/search/search_service.py +6 -2
  770. metadata/ingestion/source/storage/s3/metadata.py +22 -17
  771. metadata/ingestion/source/storage/storage_service.py +53 -11
  772. metadata/ingestion/stage/table_usage.py +9 -2
  773. metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
  774. metadata/parsers/protobuf_parser.py +29 -11
  775. metadata/pii/processor.py +9 -2
  776. metadata/pii/scanners/ner_scanner.py +5 -3
  777. metadata/profiler/api/models.py +19 -1
  778. metadata/profiler/interface/pandas/profiler_interface.py +59 -18
  779. metadata/profiler/interface/profiler_interface.py +14 -4
  780. metadata/profiler/interface/profiler_interface_factory.py +49 -14
  781. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
  782. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
  783. metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
  784. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
  785. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
  786. metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
  787. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  788. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
  789. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  790. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
  791. metadata/profiler/metrics/composed/null_ratio.py +1 -1
  792. metadata/profiler/metrics/hybrid/histogram.py +1 -0
  793. metadata/profiler/metrics/static/max.py +4 -1
  794. metadata/profiler/metrics/static/min.py +4 -1
  795. metadata/profiler/metrics/system/queries/snowflake.py +89 -17
  796. metadata/profiler/metrics/system/system.py +62 -20
  797. metadata/profiler/orm/functions/conn_test.py +1 -0
  798. metadata/profiler/orm/functions/length.py +1 -0
  799. metadata/profiler/orm/functions/median.py +9 -1
  800. metadata/profiler/orm/functions/sum.py +1 -0
  801. metadata/profiler/orm/functions/table_metric_computer.py +462 -0
  802. metadata/profiler/orm/registry.py +2 -0
  803. metadata/profiler/processor/core.py +129 -62
  804. metadata/profiler/processor/default.py +14 -3
  805. metadata/profiler/processor/handle_partition.py +2 -2
  806. metadata/profiler/processor/processor.py +10 -7
  807. metadata/profiler/processor/sample_data_handler.py +6 -2
  808. metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
  809. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
  810. metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
  811. metadata/profiler/source/base/profiler_source.py +5 -2
  812. metadata/profiler/source/bigquery/type_mapper.py +0 -1
  813. metadata/profiler/source/databricks/profiler_source.py +36 -0
  814. metadata/profiler/source/mariadb/functions/median.py +20 -0
  815. metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
  816. metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
  817. metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
  818. metadata/profiler/source/metadata.py +49 -10
  819. metadata/profiler/source/metadata_ext.py +16 -50
  820. metadata/profiler/source/profiler_source_factory.py +8 -0
  821. metadata/py.typed +0 -0
  822. metadata/readers/dataframe/json.py +5 -4
  823. metadata/readers/file/api_reader.py +0 -1
  824. metadata/utils/constants.py +5 -0
  825. metadata/utils/credentials.py +36 -19
  826. metadata/utils/datalake/datalake_utils.py +369 -129
  827. metadata/utils/entity_link.py +26 -6
  828. metadata/utils/execution_time_tracker.py +199 -0
  829. metadata/utils/filters.py +4 -0
  830. metadata/utils/fqn.py +20 -0
  831. metadata/utils/helpers.py +34 -39
  832. metadata/utils/importer.py +2 -3
  833. metadata/utils/life_cycle_utils.py +4 -4
  834. metadata/utils/logger.py +13 -2
  835. metadata/utils/partition.py +10 -5
  836. metadata/utils/secrets/aws_based_secrets_manager.py +67 -4
  837. metadata/utils/secrets/aws_secrets_manager.py +7 -2
  838. metadata/utils/secrets/aws_ssm_secrets_manager.py +7 -2
  839. metadata/utils/secrets/azure_kv_secrets_manager.py +132 -0
  840. metadata/utils/secrets/{noop_secrets_manager.py → db_secrets_manager.py} +4 -3
  841. metadata/utils/secrets/external_secrets_manager.py +25 -3
  842. metadata/utils/secrets/secrets_manager_factory.py +15 -33
  843. metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
  844. metadata/utils/sqlalchemy_utils.py +21 -0
  845. metadata/utils/storage_metadata_config.py +42 -14
  846. metadata/utils/tag_utils.py +5 -2
  847. metadata/workflow/application.py +154 -0
  848. metadata/workflow/application_output_handler.py +34 -0
  849. metadata/workflow/base.py +88 -153
  850. metadata/workflow/data_insight.py +8 -7
  851. metadata/workflow/data_quality.py +3 -2
  852. metadata/workflow/ingestion.py +203 -0
  853. metadata/workflow/metadata.py +2 -3
  854. metadata/workflow/output_handler.py +226 -0
  855. metadata/workflow/profiler.py +2 -2
  856. metadata/workflow/usage.py +3 -4
  857. metadata/workflow/workflow_output_handler.py +15 -255
  858. metadata/workflow/workflow_status_mixin.py +44 -52
  859. openmetadata_ingestion-1.3.2.0rc1.dist-info/METADATA +758 -0
  860. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/RECORD +870 -762
  861. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/WHEEL +1 -1
  862. metadata/generated/schema/entity/applications/appConfig.py +0 -21
  863. metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
  864. metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
  865. metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
  866. metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
  867. metadata/generated/schema/events/entitySpelFilters.py +0 -19
  868. metadata/ingestion/models/es_documents.py +0 -339
  869. metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
  870. metadata/ingestion/ometa/provider_registry.py +0 -144
  871. metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
  872. metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
  873. metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
  874. metadata/profiler/orm/functions/table_metric_construct.py +0 -365
  875. metadata/utils/secrets/client/loader.py +0 -78
  876. openmetadata_ingestion-1.2.4.0.dist-info/METADATA +0 -429
  877. /metadata/examples/workflows/{datalake_azure.yaml → datalake_azure_client_secret.yaml} +0 -0
  878. /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
  879. /metadata/ingestion/source/database/{databricks/unity_catalog → bigtable}/__init__.py +0 -0
  880. /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/doris}/__init__.py +0 -0
  881. /metadata/{utils/secrets/client → ingestion/source/database/sas}/__init__.py +0 -0
  882. /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
  883. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/LICENSE +0 -0
  884. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/entry_points.txt +0 -0
  885. {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/top_level.txt +0 -0
@@ -18,7 +18,7 @@ from __future__ import annotations
18
18
  import traceback
19
19
  from collections import defaultdict
20
20
  from copy import deepcopy
21
- from typing import Iterable, Optional
21
+ from typing import Dict, Iterable, Optional
22
22
 
23
23
  from metadata.data_insight.processor.reports.data_processor import DataProcessor
24
24
  from metadata.generated.schema.analytics.reportData import ReportData, ReportDataType
@@ -28,9 +28,11 @@ from metadata.generated.schema.analytics.reportDataType.aggregatedCostAnalysisRe
28
28
  from metadata.generated.schema.analytics.reportDataType.rawCostAnalysisReportData import (
29
29
  RawCostAnalysisReportData,
30
30
  )
31
- from metadata.generated.schema.entity.data.table import Table
31
+ from metadata.generated.schema.type.entityReference import EntityReference
32
32
  from metadata.generated.schema.type.lifeCycle import LifeCycle
33
33
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
34
+ from metadata.ingestion.ometa.utils import model_str
35
+ from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
34
36
  from metadata.utils.logger import data_insight_logger
35
37
  from metadata.utils.time_utils import get_end_of_day_timestamp_mill
36
38
 
@@ -98,36 +100,32 @@ class RawCostAnalysisReportDataProcessor(DataProcessor):
98
100
  data=value,
99
101
  ) # type: ignore
100
102
 
101
- def refine(self, entity: Table) -> None:
103
+ def refine(self, entity: Dict) -> None:
102
104
  """Aggregate data
103
105
  Returns:
104
106
  list:
105
107
  """
106
108
 
107
- try:
108
- cost_analysis_data = RawCostAnalysisReportData(
109
- entity=self.metadata.get_entity_reference(
110
- entity=type(entity), fqn=entity.fullyQualifiedName
109
+ for entity_fqn, cost_analysis_report_data in entity.items():
110
+ try:
111
+ cost_analysis_data = RawCostAnalysisReportData(
112
+ entity=EntityReference(
113
+ id=cost_analysis_report_data.entity.id,
114
+ fullyQualifiedName=model_str(
115
+ cost_analysis_report_data.entity.fullyQualifiedName
116
+ ),
117
+ type=ENTITY_REFERENCE_TYPE_MAP[
118
+ type(cost_analysis_report_data.entity).__name__
119
+ ],
120
+ ),
121
+ lifeCycle=cost_analysis_report_data.life_cycle,
122
+ sizeInByte=cost_analysis_report_data.size,
111
123
  )
112
- )
113
- if entity.lifeCycle:
114
- cost_analysis_data.lifeCycle = entity.lifeCycle
115
-
116
- table_profile = self.metadata.get_latest_table_profile(
117
- fqn=entity.fullyQualifiedName
118
- )
119
- if table_profile.profile:
120
- cost_analysis_data.sizeInByte = table_profile.profile.sizeInByte
121
-
122
- if cost_analysis_data.lifeCycle or cost_analysis_data.sizeInByte:
123
- self._refined_data[
124
- entity.fullyQualifiedName.__root__
125
- ] = cost_analysis_data
126
-
127
- self.processor_status.scanned(entity.name.__root__)
128
- except Exception as err:
129
- logger.debug(traceback.format_exc())
130
- logger.error(f"Error trying fetch cost analysis data -- {err}")
124
+ self._refined_data[entity_fqn] = cost_analysis_data
125
+ self.processor_status.scanned(entity_fqn)
126
+ except Exception as err:
127
+ logger.debug(traceback.format_exc())
128
+ logger.error(f"Error trying fetch cost analysis data -- {err}")
131
129
 
132
130
  def get_status(self):
133
131
  return self.processor_status
@@ -142,6 +140,7 @@ class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
142
140
  super().__init__(metadata)
143
141
  self._refined_data = defaultdict(lambda: defaultdict(dict))
144
142
  self.post_hook = self._post_hook_fn
143
+ self.clean_up_cache = True
145
144
 
146
145
  def yield_refined_data(self) -> Iterable[ReportData]:
147
146
  """Yield refined data"""
@@ -152,27 +151,17 @@ class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
152
151
  data=data,
153
152
  ) # type: ignore
154
153
 
155
- def refine(self, entity: Table) -> None:
154
+ def refine(self, entity: Dict) -> None:
156
155
  """Aggregate data
157
156
  Returns:
158
157
  list:
159
158
  """
160
159
  try:
161
- life_cycle = None
162
- if entity.lifeCycle:
163
- life_cycle = entity.lifeCycle
164
-
165
- size = None
166
- table_profile = self.metadata.get_latest_table_profile(
167
- fqn=entity.fullyQualifiedName
168
- )
169
- if table_profile.profile:
170
- size = table_profile.profile.sizeInByte
171
-
172
- if life_cycle or size:
173
- entity_type = str(entity.__class__.__name__)
174
- service_type = str(entity.serviceType.name)
175
- service_name = str(entity.service.name)
160
+
161
+ for entity_fqn, cost_analysis_report_data in entity.items():
162
+ entity_type = str(cost_analysis_report_data.entity.__class__.__name__)
163
+ service_type = str(cost_analysis_report_data.entity.serviceType.name)
164
+ service_name = str(cost_analysis_report_data.entity.service.name)
176
165
  if not self._refined_data[str(entity_type)][service_type].get(
177
166
  service_name
178
167
  ):
@@ -185,18 +174,18 @@ class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
185
174
  else:
186
175
  self._refined_data[entity_type][service_type][service_name][
187
176
  TOTAL_SIZE
188
- ] += (size or 0)
177
+ ] += (cost_analysis_report_data.size or 0)
189
178
  self._refined_data[entity_type][service_type][service_name][
190
179
  TOTAL_COUNT
191
180
  ] += 1
192
181
 
193
182
  self._get_data_assets_dict(
194
- life_cycle=life_cycle,
195
- size=size,
183
+ life_cycle=cost_analysis_report_data.life_cycle,
184
+ size=cost_analysis_report_data.size,
196
185
  data=self._refined_data[entity_type][service_type][service_name],
197
186
  )
198
187
 
199
- self.processor_status.scanned(entity.name.__root__)
188
+ self.processor_status.scanned(entity_fqn)
200
189
  except Exception as err:
201
190
  logger.debug(traceback.format_exc())
202
191
  logger.error(f"Error trying fetch cost analysis data -- {err}")
@@ -241,13 +230,16 @@ class AggregatedCostAnalysisReportDataProcessor(DataProcessor):
241
230
  Helper method to calculate number of data assets within time period
242
231
  """
243
232
  try:
244
- if not life_cycle:
233
+ if not life_cycle or not life_cycle.accessed:
245
234
  return
246
235
 
247
236
  # Iterate over the different time periods and update the data
248
237
  for days, key in DAYS:
249
238
  days_before_timestamp = get_end_of_day_timestamp_mill(days=days)
250
- if life_cycle.accessed.timestamp.__root__ <= days_before_timestamp:
239
+ if (
240
+ life_cycle.accessed
241
+ and life_cycle.accessed.timestamp.__root__ <= days_before_timestamp
242
+ ):
251
243
  data[UNUSED_DATA_ASSETS][COUNT][key] += 1
252
244
  data[UNUSED_DATA_ASSETS][SIZE][key] += size or 0
253
245
  else:
@@ -47,6 +47,7 @@ class DataProcessor(abc.ABC):
47
47
  self._refined_data = {}
48
48
  self.post_hook: Optional[Callable] = None
49
49
  self.pre_hook: Optional[Callable] = None
50
+ self.clean_up_cache: bool = False
50
51
 
51
52
  @classmethod
52
53
  def create(cls, _data_processor_type, metadata: OpenMetadata):
@@ -38,10 +38,12 @@ from metadata.generated.schema.entity.data import (
38
38
  table,
39
39
  topic,
40
40
  )
41
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
42
+ StackTraceError,
43
+ )
41
44
  from metadata.generated.schema.entity.teams.user import User
42
45
  from metadata.generated.schema.type.entityReference import EntityReference
43
46
  from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
44
- from metadata.ingestion.api.models import StackTraceError
45
47
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
46
48
  from metadata.utils.helpers import get_entity_tier_from_tags
47
49
  from metadata.utils.logger import data_insight_logger
@@ -75,6 +77,10 @@ class EntityReportDataProcessor(DataProcessor):
75
77
  self._refined_data = defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
76
78
  self.post_hook = self._post_hook_fn
77
79
 
80
+ @property
81
+ def name(self) -> str:
82
+ return "Entity Report Processor"
83
+
78
84
  def _get_team(self, owner: EntityReference) -> Optional[str]:
79
85
  """Get the team from an entity. We'll use this info as well to
80
86
  add info if an entity has an owner
@@ -191,7 +197,7 @@ class EntityReportDataProcessor(DataProcessor):
191
197
  StackTraceError(
192
198
  name=entity.name.__root__,
193
199
  error="Error retrieving team",
194
- stack_trace=traceback.format_exc(),
200
+ stackTrace=traceback.format_exc(),
195
201
  )
196
202
  )
197
203
  raise
@@ -78,6 +78,10 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
78
78
  super().__init__(metadata)
79
79
  self.pre_hook = self._pre_hook_fn
80
80
 
81
+ @property
82
+ def name(self) -> str:
83
+ return "Web Analytics Processor"
84
+
81
85
  def _pre_hook_fn(self):
82
86
  """Start our generator function"""
83
87
  # pylint: disable=attribute-defined-outside-init
@@ -13,16 +13,31 @@ Producer class for data insight entity reports
13
13
  """
14
14
 
15
15
  import traceback
16
- from typing import Iterable
16
+ from typing import Dict, Iterable, Optional
17
+
18
+ from pydantic import BaseModel
17
19
 
18
20
  from metadata.data_insight.producer.producer_interface import ProducerInterface
19
21
  from metadata.generated.schema.entity.data.table import Table
20
22
  from metadata.generated.schema.entity.services.databaseService import DatabaseService
23
+ from metadata.generated.schema.type.lifeCycle import LifeCycle
24
+ from metadata.ingestion.api.models import Entity
25
+ from metadata.ingestion.ometa.utils import model_str
21
26
  from metadata.utils.logger import data_insight_logger
22
27
 
23
28
  logger = data_insight_logger()
24
29
 
25
30
 
31
+ class CostAnalysisReportData(BaseModel):
32
+ """
33
+ Query executed get life cycle
34
+ """
35
+
36
+ entity: Entity
37
+ life_cycle: Optional[LifeCycle]
38
+ size: Optional[float]
39
+
40
+
26
41
  class CostAnalysisProducer(ProducerInterface):
27
42
  """entity producer class"""
28
43
 
@@ -36,27 +51,76 @@ class CostAnalysisProducer(ProducerInterface):
36
51
  and database_service.connection.config.supportsProfiler.__root__
37
52
  )
38
53
 
54
+ def _check_life_cycle_and_size_data(
55
+ self, table: Table
56
+ ) -> Optional[CostAnalysisReportData]:
57
+ """
58
+ Method to check if the valid life cycle and table size data is present for the table
59
+ """
60
+ cost_analysis_report_data = CostAnalysisReportData(entity=table)
61
+ if table.lifeCycle and table.lifeCycle.accessed:
62
+ cost_analysis_report_data.life_cycle = table.lifeCycle
63
+
64
+ table_profile = self.metadata.get_latest_table_profile(
65
+ fqn=table.fullyQualifiedName
66
+ )
67
+ if table_profile.profile:
68
+ cost_analysis_report_data.size = table_profile.profile.sizeInByte
69
+
70
+ if cost_analysis_report_data.life_cycle or cost_analysis_report_data.size:
71
+ return cost_analysis_report_data
72
+ return None
73
+
74
+ def life_cycle_data_dict(
75
+ self, entities_cache: Optional[Dict], database_service_fqn: str
76
+ ) -> Iterable[Dict]:
77
+ """
78
+ Cache the required lifecycle data to be used by the processors and return the dict
79
+ """
80
+ if entities_cache.get(database_service_fqn):
81
+ yield entities_cache[database_service_fqn]
82
+ else:
83
+ tables = self.metadata.list_all_entities(
84
+ Table,
85
+ limit=100,
86
+ skip_on_failure=True,
87
+ params={"database": database_service_fqn},
88
+ )
89
+ entities_cache[database_service_fqn] = {}
90
+
91
+ for table in tables:
92
+ try:
93
+ cost_analysis_data = self._check_life_cycle_and_size_data(
94
+ table=table
95
+ )
96
+ if cost_analysis_data:
97
+ entities_cache[database_service_fqn][
98
+ model_str(table.fullyQualifiedName)
99
+ ] = cost_analysis_data
100
+ except Exception as err:
101
+ logger.error(
102
+ f"Error trying to fetch cost analysis data for [{model_str(table.fullyQualifiedName)}] -- {err}"
103
+ )
104
+ logger.debug(traceback.format_exc())
105
+
106
+ yield entities_cache[database_service_fqn]
107
+
39
108
  # pylint: disable=dangerous-default-value
40
- def fetch_data(self, limit=100, fields=["*"]) -> Iterable:
109
+ def fetch_data(
110
+ self, limit=100, fields=["*"], entities_cache=None
111
+ ) -> Optional[Iterable[Dict]]:
41
112
  database_services = self.metadata.list_all_entities(
42
113
  DatabaseService, limit=limit, fields=fields, skip_on_failure=True
43
114
  )
44
- entities_list = []
45
115
  for database_service in database_services or []:
46
116
  try:
47
117
  if self._check_profiler_and_usage_support(database_service):
48
- entities_list.extend(
49
- self.metadata.list_all_entities(
50
- Table,
51
- limit=limit,
52
- fields=fields,
53
- skip_on_failure=True,
54
- params={
55
- "database": database_service.fullyQualifiedName.__root__
56
- },
57
- )
118
+ yield from self.life_cycle_data_dict(
119
+ entities_cache=entities_cache,
120
+ database_service_fqn=model_str(
121
+ database_service.fullyQualifiedName
122
+ ),
58
123
  )
59
124
  except Exception as err:
60
125
  logger.error(f"Error trying to fetch entities -- {err}")
61
126
  logger.debug(traceback.format_exc())
62
- return entities_list
@@ -52,7 +52,7 @@ class EntityProducer(ProducerInterface):
52
52
  ]
53
53
 
54
54
  # pylint: disable=dangerous-default-value
55
- def fetch_data(self, limit=100, fields=["*"]) -> Iterable:
55
+ def fetch_data(self, limit=100, fields=["*"], entities_cache=None) -> Iterable:
56
56
  for entity in self.entities:
57
57
  try:
58
58
  yield from self.metadata.list_all_entities(
@@ -24,6 +24,6 @@ class ProducerInterface(ABC):
24
24
  self.metadata = metadata
25
25
 
26
26
  @abstractmethod
27
- def fetch_data(self, limit, fields):
27
+ def fetch_data(self, limit, fields, entities_cache=None):
28
28
  """fetch data from source"""
29
29
  raise NotImplementedError
@@ -75,7 +75,7 @@ class WebAnalyticsProducer(ProducerInterface):
75
75
  CACHED_EVENTS.clear()
76
76
 
77
77
  def fetch_data(
78
- self, limit=100, fields=["*"]
78
+ self, limit=100, fields=["*"], entities_cache=None
79
79
  ): # pylint: disable=dangerous-default-value
80
80
  """fetch data for web analytics event"""
81
81
  events = self._get_events(None, limit, fields)
@@ -33,7 +33,10 @@ from metadata.data_insight.processor.reports.web_analytic_report_data_processor
33
33
  )
34
34
  from metadata.data_insight.producer.producer_factory import producer_factory
35
35
  from metadata.generated.schema.analytics.reportData import ReportData, ReportDataType
36
- from metadata.ingestion.api.models import Either, StackTraceError
36
+ from metadata.generated.schema.entity.services.ingestionPipelines.status import (
37
+ StackTraceError,
38
+ )
39
+ from metadata.ingestion.api.models import Either
37
40
  from metadata.ingestion.api.step import Step
38
41
  from metadata.ingestion.api.steps import Source
39
42
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -71,6 +74,7 @@ class DataInsightSource(Source):
71
74
  super().__init__()
72
75
  self.metadata = metadata
73
76
  self.date = datetime.utcnow().strftime("%Y-%m-%d")
77
+ self.entities_cache = {}
74
78
 
75
79
  _processors = self._instantiate_processors()
76
80
  self._processors: Dict[
@@ -87,9 +91,13 @@ class DataInsightSource(Source):
87
91
  _processors
88
92
  ) # make an immutable copy of the dict
89
93
 
94
+ @property
95
+ def name(self) -> str:
96
+ return "OpenMetadata Insights"
97
+
90
98
  @property
91
99
  def processors(self) -> Dict[str, Optional[DataProcessor]]:
92
- """dictionnaray of processors"""
100
+ """dictionary of processors"""
93
101
  return self._processors
94
102
 
95
103
  def _instantiate_processors(self) -> Dict[str, DataProcessor]:
@@ -123,11 +131,19 @@ class DataInsightSource(Source):
123
131
  processor = cast(DataProcessor, processor)
124
132
  processor.pre_hook() if processor.pre_hook else None # pylint: disable=expression-not-assigned
125
133
 
126
- for data in producer.fetch_data(fields=["owner", "tags"]):
134
+ for data in (
135
+ producer.fetch_data(
136
+ fields=["owner", "tags"], entities_cache=self.entities_cache
137
+ )
138
+ or []
139
+ ):
127
140
  processor.refine(data)
128
141
 
129
142
  processor.post_hook() if processor.post_hook else None # pylint: disable=expression-not-assigned
130
143
 
144
+ if processor.clean_up_cache:
145
+ self.entities_cache.clear()
146
+
131
147
  for data in processor.yield_refined_data():
132
148
  yield Either(left=None, right=DataInsightRecord(data=data))
133
149
  except KeyError as key_error:
@@ -136,7 +152,7 @@ class DataInsightSource(Source):
136
152
  name=report_data_type.value,
137
153
  error=f"Error retrieving processor with exception [{key_error}]."
138
154
  "Available processors are {self.processors}",
139
- stack_trace=traceback.format_exc(),
155
+ stackTrace=traceback.format_exc(),
140
156
  ),
141
157
  right=None,
142
158
  )
@@ -145,7 +161,7 @@ class DataInsightSource(Source):
145
161
  left=StackTraceError(
146
162
  name=report_data_type.value,
147
163
  error=f"Error listing data for report with exception [{exc}]",
148
- stack_trace=traceback.format_exc(),
164
+ stackTrace=traceback.format_exc(),
149
165
  ),
150
166
  right=None,
151
167
  )
@@ -0,0 +1,29 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Interfaces with database for all database engine
14
+ supporting sqlalchemy abstraction layer
15
+ """
16
+
17
+
18
+ from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
19
+ SQATestSuiteInterface,
20
+ )
21
+
22
+
23
+ class DatabricksTestSuiteInterface(SQATestSuiteInterface):
24
+ def __init__(self, *args, **kwargs):
25
+ super().__init__(*args, **kwargs)
26
+
27
+ def create_session(self):
28
+ super().create_session()
29
+ self.set_catalog(self.session)
@@ -0,0 +1,29 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Interfaces with database for all database engine
14
+ supporting sqlalchemy abstraction layer
15
+ """
16
+
17
+
18
+ from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
19
+ SQATestSuiteInterface,
20
+ )
21
+
22
+
23
+ class SnowflakeTestSuiteInterface(SQATestSuiteInterface):
24
+ def __init__(self, *args, **kwargs):
25
+ super().__init__(*args, **kwargs)
26
+
27
+ def create_session(self):
28
+ super().create_session()
29
+ self.set_session_tag(self.session)
@@ -58,12 +58,7 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteInterface):
58
58
  self.ometa_client = ometa_client
59
59
  self.table_entity = table_entity
60
60
  self.service_connection_config = service_connection_config
61
- self.session = create_and_bind_session(
62
- get_connection(self.service_connection_config)
63
- )
64
- self.set_session_tag(self.session)
65
- self.set_catalog(self.session)
66
-
61
+ self.create_session()
67
62
  self._table = self._convert_table_to_orm_object(sqa_metadata)
68
63
 
69
64
  (
@@ -75,6 +70,11 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteInterface):
75
70
  self._sampler = self._create_sampler()
76
71
  self._runner = self._create_runner()
77
72
 
73
+ def create_session(self):
74
+ self.session = create_and_bind_session(
75
+ get_connection(self.service_connection_config)
76
+ )
77
+
78
78
  @property
79
79
  def sample(self) -> Union[DeclarativeMeta, AliasedClass]:
80
80
  """_summary_
@@ -0,0 +1,35 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+
12
+ """
13
+ Interfaces with database for all database engine
14
+ supporting sqlalchemy abstraction layer
15
+ """
16
+
17
+
18
+ from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
19
+ SQATestSuiteInterface,
20
+ )
21
+ from metadata.ingestion.connections.session import create_and_bind_session
22
+ from metadata.ingestion.source.database.databricks.connection import (
23
+ get_connection as databricks_get_connection,
24
+ )
25
+
26
+
27
+ class UnityCatalogTestSuiteInterface(SQATestSuiteInterface):
28
+ def __init__(self, *args, **kwargs):
29
+ super().__init__(*args, **kwargs)
30
+
31
+ def create_session(self):
32
+ self.session = create_and_bind_session(
33
+ databricks_get_connection(self.service_connection_config)
34
+ )
35
+ self.set_catalog(self.session)
@@ -17,14 +17,32 @@ from logging import Logger
17
17
  from metadata.data_quality.interface.pandas.pandas_test_suite_interface import (
18
18
  PandasTestSuiteInterface,
19
19
  )
20
+ from metadata.data_quality.interface.sqlalchemy.databricks.test_suite_interface import (
21
+ DatabricksTestSuiteInterface,
22
+ )
23
+ from metadata.data_quality.interface.sqlalchemy.snowflake.test_suite_interface import (
24
+ SnowflakeTestSuiteInterface,
25
+ )
20
26
  from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
21
27
  SQATestSuiteInterface,
22
28
  )
29
+ from metadata.data_quality.interface.sqlalchemy.unity_catalog.test_suite_interface import (
30
+ UnityCatalogTestSuiteInterface,
31
+ )
23
32
  from metadata.data_quality.interface.test_suite_interface import TestSuiteInterface
24
33
  from metadata.generated.schema.entity.data.table import Table
34
+ from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
35
+ DatabricksConnection,
36
+ )
25
37
  from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
26
38
  DatalakeConnection,
27
39
  )
40
+ from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
41
+ SnowflakeConnection,
42
+ )
43
+ from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import (
44
+ UnityCatalogConnection,
45
+ )
28
46
  from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
29
47
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
30
48
  from metadata.utils.logger import test_suite_logger
@@ -51,6 +69,17 @@ class TestSuiteInterfaceFactory:
51
69
  """
52
70
  self._interface_type[interface_type] = interface
53
71
 
72
+ def register_many(self, interface_dict):
73
+ """
74
+ Registers multiple profiler interfaces at once.
75
+
76
+ Args:
77
+ interface_dict: A dictionary mapping connection class names (strings) to their
78
+ corresponding profiler interface classes.
79
+ """
80
+ for interface_type, interface_class in interface_dict.items():
81
+ self.register(interface_type, interface_class)
82
+
54
83
  def create(
55
84
  self,
56
85
  service_connection_config: DatabaseConnection,
@@ -86,3 +115,14 @@ class TestSuiteInterfaceFactory:
86
115
 
87
116
 
88
117
  test_suite_interface_factory = TestSuiteInterfaceFactory()
118
+
119
+
120
+ test_suite_interface = {
121
+ DatabaseConnection.__name__: SQATestSuiteInterface,
122
+ DatalakeConnection.__name__: PandasTestSuiteInterface,
123
+ SnowflakeConnection.__name__: SnowflakeTestSuiteInterface,
124
+ UnityCatalogConnection.__name__: UnityCatalogTestSuiteInterface,
125
+ DatabricksConnection.__name__: DatabricksTestSuiteInterface,
126
+ }
127
+
128
+ test_suite_interface_factory.register_many(test_suite_interface)