openmetadata-ingestion 1.6.1.0__py3-none-any.whl → 1.6.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (671) hide show
  1. metadata/clients/aws_client.py +70 -23
  2. metadata/data_quality/api/models.py +4 -0
  3. metadata/data_quality/processor/test_case_runner.py +8 -5
  4. metadata/data_quality/runner/base_test_suite_source.py +4 -5
  5. metadata/data_quality/source/test_suite.py +143 -39
  6. metadata/generated/schema/analytics/__init__.py +1 -1
  7. metadata/generated/schema/analytics/basic.py +1 -1
  8. metadata/generated/schema/analytics/reportData.py +1 -1
  9. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  10. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  11. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  12. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  13. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  14. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  15. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  16. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  17. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  18. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  19. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  20. metadata/generated/schema/api/__init__.py +1 -1
  21. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  22. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  23. metadata/generated/schema/api/analytics/__init__.py +1 -1
  24. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  25. metadata/generated/schema/api/automations/__init__.py +1 -1
  26. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  27. metadata/generated/schema/api/bulkAssets.py +1 -1
  28. metadata/generated/schema/api/classification/__init__.py +1 -1
  29. metadata/generated/schema/api/classification/createClassification.py +1 -1
  30. metadata/generated/schema/api/classification/createTag.py +1 -1
  31. metadata/generated/schema/api/classification/loadTags.py +1 -1
  32. metadata/generated/schema/api/createBot.py +1 -1
  33. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  34. metadata/generated/schema/api/createType.py +1 -1
  35. metadata/generated/schema/api/data/__init__.py +1 -1
  36. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  37. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  38. metadata/generated/schema/api/data/createChart.py +1 -1
  39. metadata/generated/schema/api/data/createContainer.py +1 -1
  40. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  41. metadata/generated/schema/api/data/createDashboard.py +1 -1
  42. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  43. metadata/generated/schema/api/data/createDatabase.py +1 -1
  44. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  45. metadata/generated/schema/api/data/createGlossary.py +1 -1
  46. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  47. metadata/generated/schema/api/data/createMetric.py +1 -1
  48. metadata/generated/schema/api/data/createMlModel.py +1 -1
  49. metadata/generated/schema/api/data/createPipeline.py +1 -1
  50. metadata/generated/schema/api/data/createQuery.py +1 -1
  51. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  52. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  53. metadata/generated/schema/api/data/createTable.py +1 -1
  54. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  55. metadata/generated/schema/api/data/createTopic.py +1 -1
  56. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  57. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  58. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  59. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  60. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  61. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  62. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  63. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  64. metadata/generated/schema/api/docStore/__init__.py +1 -1
  65. metadata/generated/schema/api/docStore/createDocument.py +2 -2
  66. metadata/generated/schema/api/domains/__init__.py +1 -1
  67. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  68. metadata/generated/schema/api/domains/createDomain.py +1 -1
  69. metadata/generated/schema/api/feed/__init__.py +1 -1
  70. metadata/generated/schema/api/feed/closeTask.py +1 -1
  71. metadata/generated/schema/api/feed/createPost.py +1 -1
  72. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  73. metadata/generated/schema/api/feed/createThread.py +1 -1
  74. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  75. metadata/generated/schema/api/feed/threadCount.py +1 -1
  76. metadata/generated/schema/api/governance/__init__.py +1 -1
  77. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  78. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  79. metadata/generated/schema/api/lineage/__init__.py +1 -1
  80. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  81. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  82. metadata/generated/schema/api/policies/__init__.py +1 -1
  83. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  84. metadata/generated/schema/api/services/__init__.py +1 -1
  85. metadata/generated/schema/api/services/createApiService.py +1 -1
  86. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  87. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  88. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  89. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  90. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  91. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  92. metadata/generated/schema/api/services/createSearchService.py +1 -1
  93. metadata/generated/schema/api/services/createStorageService.py +1 -1
  94. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  95. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  96. metadata/generated/schema/api/setOwner.py +1 -1
  97. metadata/generated/schema/api/teams/__init__.py +1 -1
  98. metadata/generated/schema/api/teams/createPersona.py +1 -1
  99. metadata/generated/schema/api/teams/createRole.py +1 -1
  100. metadata/generated/schema/api/teams/createTeam.py +1 -1
  101. metadata/generated/schema/api/teams/createUser.py +1 -1
  102. metadata/generated/schema/api/tests/__init__.py +1 -1
  103. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  104. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  105. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  106. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  107. metadata/generated/schema/api/tests/createTestCaseResult.py +8 -1
  108. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  109. metadata/generated/schema/api/tests/createTestSuite.py +7 -3
  110. metadata/generated/schema/api/voteRequest.py +1 -1
  111. metadata/generated/schema/auth/__init__.py +1 -1
  112. metadata/generated/schema/auth/basicAuth.py +1 -1
  113. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  114. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  115. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  116. metadata/generated/schema/auth/emailRequest.py +1 -1
  117. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  118. metadata/generated/schema/auth/generateToken.py +1 -1
  119. metadata/generated/schema/auth/jwtAuth.py +1 -1
  120. metadata/generated/schema/auth/loginRequest.py +1 -1
  121. metadata/generated/schema/auth/logoutRequest.py +1 -1
  122. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  123. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  124. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  125. metadata/generated/schema/auth/refreshToken.py +1 -1
  126. metadata/generated/schema/auth/registrationRequest.py +1 -1
  127. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  128. metadata/generated/schema/auth/revokeToken.py +1 -1
  129. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  130. metadata/generated/schema/auth/ssoAuth.py +1 -1
  131. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  132. metadata/generated/schema/configuration/__init__.py +1 -1
  133. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  134. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  135. metadata/generated/schema/configuration/authConfig.py +1 -1
  136. metadata/generated/schema/configuration/authenticationConfiguration.py +14 -1
  137. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  138. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  139. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  140. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  141. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  142. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  143. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  144. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  145. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  146. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  147. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  148. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  149. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  150. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  151. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  152. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  153. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  154. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  155. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  156. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  157. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  158. metadata/generated/schema/configuration/searchSettings.py +1 -1
  159. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  160. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  161. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  162. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  163. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  164. metadata/generated/schema/configuration/workflowSettings.py +73 -0
  165. metadata/generated/schema/dataInsight/__init__.py +1 -1
  166. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  167. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  168. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  169. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  170. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  171. metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
  172. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  173. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  174. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  175. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  176. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  177. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  178. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  179. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  180. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  181. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  182. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  183. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  184. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  185. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  186. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  187. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  188. metadata/generated/schema/email/__init__.py +1 -1
  189. metadata/generated/schema/email/emailRequest.py +1 -1
  190. metadata/generated/schema/email/emailTemplate.py +1 -1
  191. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  192. metadata/generated/schema/email/smtpSettings.py +1 -1
  193. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  194. metadata/generated/schema/entity/__init__.py +1 -1
  195. metadata/generated/schema/entity/applications/__init__.py +1 -1
  196. metadata/generated/schema/entity/applications/app.py +1 -1
  197. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  198. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  199. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  200. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +1 -1
  201. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  202. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  203. metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +40 -0
  204. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  205. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  206. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  207. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  208. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  209. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  210. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  211. metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +30 -0
  212. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  213. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  214. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  215. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  216. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  217. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +7 -1
  218. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  219. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  220. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  221. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  222. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  223. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  224. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  225. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  226. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  227. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  228. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  229. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  230. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
  231. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
  232. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  233. metadata/generated/schema/entity/automations/__init__.py +1 -1
  234. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  235. metadata/generated/schema/entity/automations/workflow.py +1 -1
  236. metadata/generated/schema/entity/bot.py +1 -1
  237. metadata/generated/schema/entity/classification/__init__.py +1 -1
  238. metadata/generated/schema/entity/classification/classification.py +1 -1
  239. metadata/generated/schema/entity/classification/tag.py +1 -1
  240. metadata/generated/schema/entity/data/__init__.py +1 -1
  241. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  242. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  243. metadata/generated/schema/entity/data/chart.py +1 -1
  244. metadata/generated/schema/entity/data/container.py +1 -1
  245. metadata/generated/schema/entity/data/dashboard.py +1 -1
  246. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  247. metadata/generated/schema/entity/data/database.py +1 -1
  248. metadata/generated/schema/entity/data/databaseSchema.py +1 -1
  249. metadata/generated/schema/entity/data/glossary.py +1 -1
  250. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  251. metadata/generated/schema/entity/data/metric.py +1 -1
  252. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  253. metadata/generated/schema/entity/data/pipeline.py +1 -1
  254. metadata/generated/schema/entity/data/query.py +1 -1
  255. metadata/generated/schema/entity/data/report.py +1 -1
  256. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  257. metadata/generated/schema/entity/data/storedProcedure.py +1 -1
  258. metadata/generated/schema/entity/data/table.py +1 -1
  259. metadata/generated/schema/entity/data/topic.py +1 -1
  260. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  261. metadata/generated/schema/entity/docStore/document.py +1 -1
  262. metadata/generated/schema/entity/domains/__init__.py +1 -1
  263. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  264. metadata/generated/schema/entity/domains/domain.py +1 -1
  265. metadata/generated/schema/entity/events/__init__.py +1 -1
  266. metadata/generated/schema/entity/events/webhook.py +1 -1
  267. metadata/generated/schema/entity/feed/__init__.py +1 -1
  268. metadata/generated/schema/entity/feed/assets.py +1 -1
  269. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  270. metadata/generated/schema/entity/feed/description.py +1 -1
  271. metadata/generated/schema/entity/feed/domain.py +1 -1
  272. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  273. metadata/generated/schema/entity/feed/owner.py +1 -1
  274. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  275. metadata/generated/schema/entity/feed/tag.py +1 -1
  276. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  277. metadata/generated/schema/entity/feed/thread.py +1 -1
  278. metadata/generated/schema/entity/policies/__init__.py +1 -1
  279. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  280. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  281. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  282. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  283. metadata/generated/schema/entity/policies/filters.py +1 -1
  284. metadata/generated/schema/entity/policies/policy.py +1 -1
  285. metadata/generated/schema/entity/services/__init__.py +1 -1
  286. metadata/generated/schema/entity/services/apiService.py +1 -1
  287. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  288. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  289. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  290. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  291. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  292. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  293. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  294. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  295. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  296. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  297. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  298. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  299. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  300. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  301. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +1 -1
  302. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  304. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  305. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  306. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  307. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  308. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  309. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  310. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  311. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  312. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  313. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  314. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
  315. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  316. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  319. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  320. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
  321. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  322. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  323. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  325. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  326. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  329. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  330. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  331. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  332. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  333. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  334. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  335. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  336. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  337. metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  340. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  341. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  342. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  343. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  344. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  345. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  346. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
  347. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  351. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  352. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  353. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  360. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +1 -1
  361. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  367. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  372. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  384. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  387. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  389. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  390. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  391. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  398. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  404. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +5 -1
  405. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  410. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
  411. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  416. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  422. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
  423. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  425. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  426. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  427. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  428. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  429. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  431. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  432. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  435. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  436. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  437. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  438. metadata/generated/schema/entity/services/dashboardService.py +1 -1
  439. metadata/generated/schema/entity/services/databaseService.py +1 -1
  440. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  441. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +1 -1
  442. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  443. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  444. metadata/generated/schema/entity/services/messagingService.py +1 -1
  445. metadata/generated/schema/entity/services/metadataService.py +1 -1
  446. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  447. metadata/generated/schema/entity/services/pipelineService.py +1 -1
  448. metadata/generated/schema/entity/services/searchService.py +1 -1
  449. metadata/generated/schema/entity/services/serviceType.py +1 -1
  450. metadata/generated/schema/entity/services/storageService.py +1 -1
  451. metadata/generated/schema/entity/teams/__init__.py +1 -1
  452. metadata/generated/schema/entity/teams/persona.py +1 -1
  453. metadata/generated/schema/entity/teams/role.py +1 -1
  454. metadata/generated/schema/entity/teams/team.py +1 -1
  455. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  456. metadata/generated/schema/entity/teams/user.py +1 -1
  457. metadata/generated/schema/entity/type.py +1 -1
  458. metadata/generated/schema/entity/utils/__init__.py +1 -1
  459. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  460. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  461. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  462. metadata/generated/schema/events/__init__.py +1 -1
  463. metadata/generated/schema/events/alertMetrics.py +1 -1
  464. metadata/generated/schema/events/api/__init__.py +1 -1
  465. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  466. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  467. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  468. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  469. metadata/generated/schema/events/api/typedEvent.py +1 -1
  470. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  471. metadata/generated/schema/events/eventFilterRule.py +1 -1
  472. metadata/generated/schema/events/eventSubscription.py +1 -1
  473. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  474. metadata/generated/schema/events/failedEvent.py +1 -1
  475. metadata/generated/schema/events/failedEventResponse.py +1 -1
  476. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  477. metadata/generated/schema/events/statusContext.py +1 -1
  478. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  479. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  480. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  481. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  482. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  483. metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
  484. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
  485. metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
  486. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  487. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  488. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
  489. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
  490. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
  491. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  492. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
  493. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  494. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
  495. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  496. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +2 -6
  497. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  498. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
  499. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +1 -1
  500. metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
  501. metadata/generated/schema/governance/workflows/workflowInstance.py +1 -1
  502. metadata/generated/schema/governance/workflows/workflowInstanceState.py +1 -1
  503. metadata/generated/schema/jobs/__init__.py +3 -0
  504. metadata/generated/schema/jobs/backgroundJob.py +66 -0
  505. metadata/generated/schema/jobs/enumCleanupArgs.py +23 -0
  506. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  507. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  508. metadata/generated/schema/metadataIngestion/application.py +1 -1
  509. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  510. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  511. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  512. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -1
  513. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
  514. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -1
  515. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  516. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  517. metadata/generated/schema/metadataIngestion/dbtPipeline.py +8 -1
  518. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  519. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  520. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  521. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  522. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  523. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  524. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  525. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  526. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  527. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  528. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  529. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  530. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  531. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  532. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +5 -1
  533. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  534. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  535. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  536. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  537. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  538. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  539. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  540. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  541. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +27 -3
  542. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  543. metadata/generated/schema/monitoring/__init__.py +1 -1
  544. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  545. metadata/generated/schema/security/__init__.py +1 -1
  546. metadata/generated/schema/security/client/__init__.py +1 -1
  547. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  548. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  549. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  550. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  551. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  552. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  553. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  554. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  555. metadata/generated/schema/security/credentials/__init__.py +1 -1
  556. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  557. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  558. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  559. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  560. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  561. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  562. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  563. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  564. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  565. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  566. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  567. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  568. metadata/generated/schema/security/sasl/__init__.py +1 -1
  569. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  570. metadata/generated/schema/security/secrets/__init__.py +1 -1
  571. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  572. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  573. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  574. metadata/generated/schema/security/securityConfiguration.py +1 -1
  575. metadata/generated/schema/security/ssl/__init__.py +1 -1
  576. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  577. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  578. metadata/generated/schema/settings/__init__.py +1 -1
  579. metadata/generated/schema/settings/settings.py +4 -1
  580. metadata/generated/schema/system/__init__.py +1 -1
  581. metadata/generated/schema/system/entityError.py +1 -1
  582. metadata/generated/schema/system/eventPublisherJob.py +1 -1
  583. metadata/generated/schema/system/indexingError.py +1 -1
  584. metadata/generated/schema/system/limitsResponse.py +1 -1
  585. metadata/generated/schema/system/ui/__init__.py +1 -1
  586. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  587. metadata/generated/schema/system/ui/page.py +1 -1
  588. metadata/generated/schema/system/validationResponse.py +1 -1
  589. metadata/generated/schema/tests/__init__.py +1 -1
  590. metadata/generated/schema/tests/assigned.py +1 -1
  591. metadata/generated/schema/tests/basic.py +20 -20
  592. metadata/generated/schema/tests/customMetric.py +1 -1
  593. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  594. metadata/generated/schema/tests/resolved.py +1 -1
  595. metadata/generated/schema/tests/testCase.py +8 -3
  596. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  597. metadata/generated/schema/tests/testDefinition.py +1 -1
  598. metadata/generated/schema/tests/testSuite.py +13 -6
  599. metadata/generated/schema/type/__init__.py +1 -1
  600. metadata/generated/schema/type/apiSchema.py +1 -1
  601. metadata/generated/schema/type/assetCertification.py +1 -1
  602. metadata/generated/schema/type/auditLog.py +1 -1
  603. metadata/generated/schema/type/basic.py +1 -1
  604. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  605. metadata/generated/schema/type/changeEvent.py +1 -1
  606. metadata/generated/schema/type/changeEventType.py +1 -1
  607. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  608. metadata/generated/schema/type/csvDocumentation.py +1 -1
  609. metadata/generated/schema/type/csvErrorType.py +1 -1
  610. metadata/generated/schema/type/csvFile.py +1 -1
  611. metadata/generated/schema/type/csvImportResult.py +1 -1
  612. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  613. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  614. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  615. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  616. metadata/generated/schema/type/customProperty.py +1 -1
  617. metadata/generated/schema/type/dailyCount.py +1 -1
  618. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  619. metadata/generated/schema/type/entityHierarchy.py +1 -1
  620. metadata/generated/schema/type/entityHistory.py +1 -1
  621. metadata/generated/schema/type/entityLineage.py +1 -1
  622. metadata/generated/schema/type/entityReference.py +1 -1
  623. metadata/generated/schema/type/entityReferenceList.py +1 -1
  624. metadata/generated/schema/type/entityRelationship.py +1 -1
  625. metadata/generated/schema/type/entityUsage.py +1 -1
  626. metadata/generated/schema/type/filterPattern.py +1 -1
  627. metadata/generated/schema/type/function.py +1 -1
  628. metadata/generated/schema/type/include.py +1 -1
  629. metadata/generated/schema/type/jdbcConnection.py +1 -1
  630. metadata/generated/schema/type/lifeCycle.py +1 -1
  631. metadata/generated/schema/type/paging.py +1 -1
  632. metadata/generated/schema/type/profile.py +1 -1
  633. metadata/generated/schema/type/queryParserData.py +1 -1
  634. metadata/generated/schema/type/reaction.py +1 -1
  635. metadata/generated/schema/type/schedule.py +1 -1
  636. metadata/generated/schema/type/schema.py +1 -1
  637. metadata/generated/schema/type/tableQuery.py +1 -1
  638. metadata/generated/schema/type/tableUsageCount.py +1 -1
  639. metadata/generated/schema/type/tagLabel.py +1 -1
  640. metadata/generated/schema/type/usageDetails.py +1 -1
  641. metadata/generated/schema/type/usageRequest.py +1 -1
  642. metadata/generated/schema/type/votes.py +1 -1
  643. metadata/great_expectations/action.py +1 -1
  644. metadata/ingestion/lineage/models.py +8 -0
  645. metadata/ingestion/ometa/mixins/es_mixin.py +10 -1
  646. metadata/ingestion/ometa/mixins/tests_mixin.py +1 -1
  647. metadata/ingestion/ometa/mixins/user_mixin.py +19 -5
  648. metadata/ingestion/ometa/routes.py +4 -0
  649. metadata/ingestion/source/dashboard/powerbi/client.py +31 -4
  650. metadata/ingestion/source/dashboard/powerbi/metadata.py +157 -135
  651. metadata/ingestion/source/database/db2/metadata.py +6 -0
  652. metadata/ingestion/source/database/db2/utils.py +63 -0
  653. metadata/ingestion/source/database/dbt/dbt_service.py +11 -0
  654. metadata/ingestion/source/database/dbt/metadata.py +42 -0
  655. metadata/ingestion/source/database/redshift/queries.py +1 -1
  656. metadata/ingestion/source/database/sample_data.py +1 -3
  657. metadata/ingestion/source/database/snowflake/queries.py +2 -2
  658. metadata/ingestion/source/database/sql_column_handler.py +6 -0
  659. metadata/ingestion/source/pipeline/airbyte/client.py +3 -2
  660. metadata/ingestion/source/pipeline/dbtcloud/metadata.py +22 -30
  661. metadata/ingestion/source/storage/s3/metadata.py +56 -9
  662. metadata/profiler/orm/converter/common.py +3 -1
  663. metadata/utils/constants.py +11 -12
  664. metadata/utils/elasticsearch.py +9 -1
  665. metadata/workflow/data_quality.py +5 -43
  666. {openmetadata_ingestion-1.6.1.0.dist-info → openmetadata_ingestion-1.6.2.1.dist-info}/METADATA +356 -356
  667. {openmetadata_ingestion-1.6.1.0.dist-info → openmetadata_ingestion-1.6.2.1.dist-info}/RECORD +671 -664
  668. {openmetadata_ingestion-1.6.1.0.dist-info → openmetadata_ingestion-1.6.2.1.dist-info}/LICENSE +0 -0
  669. {openmetadata_ingestion-1.6.1.0.dist-info → openmetadata_ingestion-1.6.2.1.dist-info}/WHEEL +0 -0
  670. {openmetadata_ingestion-1.6.1.0.dist-info → openmetadata_ingestion-1.6.2.1.dist-info}/entry_points.txt +0 -0
  671. {openmetadata_ingestion-1.6.1.0.dist-info → openmetadata_ingestion-1.6.2.1.dist-info}/top_level.txt +0 -0
@@ -11,12 +11,16 @@
11
11
  """
12
12
  Module containing AWS Client
13
13
  """
14
+ import datetime
14
15
  from enum import Enum
15
- from typing import Any, Optional
16
+ from functools import partial
17
+ from typing import Any, Callable, Dict, Optional, Type, TypeVar
16
18
 
17
19
  import boto3
18
20
  from boto3 import Session
19
- from pydantic import BaseModel
21
+ from botocore.credentials import RefreshableCredentials
22
+ from botocore.session import get_session
23
+ from pydantic import BaseModel, Field
20
24
 
21
25
  from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials
22
26
  from metadata.ingestion.models.custom_pydantic import CustomSecretStr
@@ -44,10 +48,27 @@ class AWSAssumeRoleException(Exception):
44
48
  """
45
49
 
46
50
 
51
+ class AWSAssumeRoleCredentialResponse(BaseModel):
52
+ AccessKeyId: str = Field()
53
+ SecretAccessKey: str = Field()
54
+ SessionToken: Optional[str] = Field(
55
+ default=None,
56
+ )
57
+ Expiration: Optional[datetime.datetime] = None
58
+
59
+
47
60
  class AWSAssumeRoleCredentialWrapper(BaseModel):
48
- accessKeyId: str
49
- secretAccessKey: CustomSecretStr
50
- sessionToken: Optional[str] = None
61
+ accessKeyId: str = Field()
62
+ secretAccessKey: CustomSecretStr = Field()
63
+ sessionToken: Optional[str] = Field(
64
+ default=None,
65
+ )
66
+ expiryTime: Optional[str] = Field()
67
+
68
+
69
+ AWSAssumeRoleCredentialFormat = TypeVar(
70
+ "AWSAssumeRoleCredentialFormat", AWSAssumeRoleCredentialWrapper, Dict
71
+ )
51
72
 
52
73
 
53
74
  class AWSClient:
@@ -65,7 +86,10 @@ class AWSClient:
65
86
  @staticmethod
66
87
  def get_assume_role_config(
67
88
  config: AWSCredentials,
68
- ) -> Optional[AWSAssumeRoleCredentialWrapper]:
89
+ return_type: Type[
90
+ AWSAssumeRoleCredentialFormat
91
+ ] = AWSAssumeRoleCredentialWrapper,
92
+ ) -> Optional[AWSAssumeRoleCredentialFormat]:
69
93
  """
70
94
  Get temporary credentials from assumed role
71
95
  """
@@ -90,12 +114,19 @@ class AWSClient:
90
114
  )
91
115
 
92
116
  if resp:
93
- credentials = resp.get("Credentials", {})
94
- return AWSAssumeRoleCredentialWrapper(
95
- accessKeyId=credentials.get("AccessKeyId"),
96
- secretAccessKey=credentials.get("SecretAccessKey"),
97
- sessionToken=credentials.get("SessionToken"),
117
+ credentials: AWSAssumeRoleCredentialResponse = (
118
+ AWSAssumeRoleCredentialResponse(**resp.get("Credentials", {}))
119
+ )
120
+ creds_wrapper = AWSAssumeRoleCredentialWrapper(
121
+ accessKeyId=credentials.AccessKeyId,
122
+ secretAccessKey=credentials.SecretAccessKey,
123
+ sessionToken=credentials.SessionToken,
124
+ expiryTime=credentials.Expiration.isoformat(),
98
125
  )
126
+ if return_type == Dict:
127
+ return creds_wrapper.model_dump(by_alias=True)
128
+ return creds_wrapper
129
+
99
130
  return None
100
131
 
101
132
  @staticmethod
@@ -105,17 +136,32 @@ class AWSClient:
105
136
  aws_session_token: Optional[str],
106
137
  aws_region: str,
107
138
  profile=None,
139
+ refresh_using: Optional[Callable] = None,
108
140
  ) -> Session:
109
141
  """
110
142
  The only required param for boto3 is the region.
111
143
  The rest of credentials will have fallback strategies based on
112
144
  https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials
113
145
  """
146
+ if refresh_using:
147
+ refreshable_creds = RefreshableCredentials.create_from_metadata(
148
+ metadata=refresh_using(),
149
+ refresh_using=refresh_using,
150
+ method="sts-assume-role",
151
+ )
152
+ session = get_session()
153
+ session._credentials = refreshable_creds # pylint: disable=protected-access
154
+ return Session(
155
+ botocore_session=session, region_name=aws_region, profile_name=profile
156
+ )
157
+
114
158
  return Session(
115
159
  aws_access_key_id=aws_access_key_id,
116
- aws_secret_access_key=aws_secret_access_key.get_secret_value()
117
- if aws_secret_access_key
118
- else None,
160
+ aws_secret_access_key=(
161
+ aws_secret_access_key.get_secret_value()
162
+ if aws_secret_access_key
163
+ else None
164
+ ),
119
165
  aws_session_token=aws_session_token,
120
166
  region_name=aws_region,
121
167
  profile_name=profile,
@@ -123,15 +169,16 @@ class AWSClient:
123
169
 
124
170
  def create_session(self) -> Session:
125
171
  if self.config.assumeRoleArn:
126
- assume_creds = AWSClient.get_assume_role_config(self.config)
127
- if assume_creds:
128
- return AWSClient._get_session(
129
- assume_creds.accessKeyId,
130
- assume_creds.secretAccessKey,
131
- assume_creds.sessionToken,
132
- self.config.awsRegion,
133
- self.config.profileName,
134
- )
172
+ return AWSClient._get_session(
173
+ None,
174
+ None,
175
+ None,
176
+ self.config.awsRegion,
177
+ self.config.profileName,
178
+ refresh_using=partial(
179
+ AWSClient.get_assume_role_config, self.config, Dict
180
+ ),
181
+ )
135
182
 
136
183
  return AWSClient._get_session(
137
184
  self.config.awsAccessKeyId,
@@ -23,6 +23,7 @@ from pydantic import Field
23
23
  from metadata.config.common import ConfigModel
24
24
  from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
25
25
  from metadata.generated.schema.entity.data.table import Table
26
+ from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
26
27
  from metadata.generated.schema.tests.basic import TestCaseResult
27
28
  from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
28
29
  from metadata.ingestion.models.custom_pydantic import BaseModel
@@ -63,6 +64,9 @@ class TableAndTests(BaseModel):
63
64
  executable_test_suite: Optional[CreateTestSuiteRequest] = Field(
64
65
  None, description="If no executable test suite is found, we'll create one"
65
66
  )
67
+ service_connection: DatabaseConnection = Field(
68
+ ..., description="Service connection for the given table"
69
+ )
66
70
 
67
71
 
68
72
  class TestCaseResults(BaseModel):
@@ -27,6 +27,7 @@ from metadata.data_quality.runner.base_test_suite_source import BaseTestSuiteRun
27
27
  from metadata.data_quality.runner.core import DataTestsRunner
28
28
  from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
29
29
  from metadata.generated.schema.entity.data.table import Table
30
+ from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
30
31
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
31
32
  StackTraceError,
32
33
  )
@@ -93,7 +94,9 @@ class TestCaseRunner(Processor):
93
94
  record.table, openmetadata_test_cases
94
95
  )
95
96
 
96
- test_suite_runner = self.get_test_suite_runner(record.table)
97
+ test_suite_runner = self.get_test_suite_runner(
98
+ record.table, record.service_connection
99
+ )
97
100
 
98
101
  logger.debug(
99
102
  f"Found {len(openmetadata_test_cases)} test cases for table {record.table.fullyQualifiedName.root}"
@@ -351,9 +354,9 @@ class TestCaseRunner(Processor):
351
354
  result.append(tc)
352
355
  return result
353
356
 
354
- def get_test_suite_runner(self, table: Table):
357
+ def get_test_suite_runner(
358
+ self, table: Table, service_connection: DatabaseConnection
359
+ ):
355
360
  return BaseTestSuiteRunner(
356
- self.config,
357
- self.metadata,
358
- table,
361
+ self.config, self.metadata, table, service_connection
359
362
  ).get_data_quality_runner()
@@ -46,11 +46,12 @@ class BaseTestSuiteRunner:
46
46
  config: OpenMetadataWorkflowConfig,
47
47
  ometa_client: OpenMetadata,
48
48
  entity: Table,
49
+ service_connection: DatabaseConnection,
49
50
  ):
50
51
  self.validator_builder_class = ValidatorBuilder
51
52
  self._interface = None
52
53
  self.entity = entity
53
- self.service_conn_config = self._copy_service_config(config, self.entity.database) # type: ignore
54
+ self.service_conn_config = self._copy_service_config(service_connection, self.entity.database) # type: ignore
54
55
  self._interface_type: str = self.service_conn_config.type.value.lower()
55
56
 
56
57
  self.source_config = TestSuitePipeline.model_validate(
@@ -67,7 +68,7 @@ class BaseTestSuiteRunner:
67
68
  self._interface = interface
68
69
 
69
70
  def _copy_service_config(
70
- self, config: OpenMetadataWorkflowConfig, database: EntityReference
71
+ self, service_connection: DatabaseConnection, database: EntityReference
71
72
  ) -> DatabaseConnection:
72
73
  """Make a copy of the service config and update the database name
73
74
 
@@ -77,9 +78,7 @@ class BaseTestSuiteRunner:
77
78
  Returns:
78
79
  DatabaseService.__config__
79
80
  """
80
- config_copy = deepcopy(
81
- config.source.serviceConnection.root.config # type: ignore
82
- )
81
+ config_copy = deepcopy(service_connection.config) # type: ignore
83
82
  if hasattr(
84
83
  config_copy, # type: ignore
85
84
  "supportsDatabase",
@@ -14,11 +14,17 @@ Test Suite Workflow Source
14
14
 
15
15
  The main goal is to get the configured table from the API.
16
16
  """
17
- from typing import Iterable, List, Optional, cast
17
+ import itertools
18
+ import traceback
19
+ from typing import Dict, Iterable, List, Optional, cast
18
20
 
19
21
  from metadata.data_quality.api.models import TableAndTests
20
22
  from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest
21
23
  from metadata.generated.schema.entity.data.table import Table
24
+ from metadata.generated.schema.entity.services.databaseService import (
25
+ DatabaseConnection,
26
+ DatabaseService,
27
+ )
22
28
  from metadata.generated.schema.entity.services.ingestionPipelines.status import (
23
29
  StackTraceError,
24
30
  )
@@ -36,7 +42,7 @@ from metadata.ingestion.api.parser import parse_workflow_config_gracefully
36
42
  from metadata.ingestion.api.step import Step
37
43
  from metadata.ingestion.api.steps import Source
38
44
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
39
- from metadata.utils import fqn
45
+ from metadata.utils import entity_link, fqn
40
46
  from metadata.utils.constants import CUSTOM_CONNECTOR_PREFIX
41
47
  from metadata.utils.logger import test_suite_logger
42
48
  from metadata.utils.service_spec.service_spec import import_source_class
@@ -61,18 +67,36 @@ class TestSuiteSource(Source):
61
67
 
62
68
  self.source_config: TestSuitePipeline = self.config.source.sourceConfig.config
63
69
 
70
+ # Build at runtime - if not informed in the yaml - the service connection map
71
+ self.service_connection_map: Dict[
72
+ str, DatabaseConnection
73
+ ] = self._load_yaml_service_connections()
74
+
64
75
  self.test_connection()
65
76
 
66
77
  @property
67
78
  def name(self) -> str:
68
79
  return "OpenMetadata"
69
80
 
81
+ def _load_yaml_service_connections(self) -> Dict[str, DatabaseConnection]:
82
+ """Load the service connections from the YAML file"""
83
+ service_connections = self.source_config.serviceConnections
84
+ if not service_connections:
85
+ return {}
86
+ return {
87
+ conn.serviceName: cast(DatabaseConnection, conn.serviceConnection.root)
88
+ for conn in service_connections
89
+ }
90
+
70
91
  def _get_table_entity(self) -> Optional[Table]:
71
92
  """given an entity fqn return the table entity
72
93
 
73
94
  Args:
74
95
  entity_fqn: entity fqn for the test case
75
96
  """
97
+ # Logical test suites don't have associated tables
98
+ if self.source_config.entityFullyQualifiedName is None:
99
+ return None
76
100
  table: Table = self.metadata.get_by_name(
77
101
  entity=Table,
78
102
  fqn=self.source_config.entityFullyQualifiedName.root,
@@ -81,23 +105,51 @@ class TestSuiteSource(Source):
81
105
 
82
106
  return table
83
107
 
84
- def _get_test_cases_from_test_suite(
85
- self, test_suite: Optional[TestSuite]
86
- ) -> List[TestCase]:
108
+ def _get_table_service_connection(self, table: Table) -> DatabaseConnection:
109
+ """Get the service connection for the table"""
110
+ service_name = table.service.name
111
+
112
+ if service_name not in self.service_connection_map:
113
+ try:
114
+ service: DatabaseService = self.metadata.get_by_name(
115
+ DatabaseService, service_name
116
+ )
117
+ if not service:
118
+ raise ConnectionError(
119
+ f"Could not retrieve service with name `{service_name}`. "
120
+ "Typically caused by the `entityFullyQualifiedName` does not exists in OpenMetadata "
121
+ "or the JWT Token is invalid."
122
+ )
123
+ if not service.connection:
124
+ raise ConnectionError(
125
+ f"Service with name `{service_name}` does not have a connection. "
126
+ "If the connection is not stored in OpenMetadata, please provide it in the YAML file."
127
+ )
128
+ self.service_connection_map[service_name] = service.connection
129
+
130
+ except Exception as exc:
131
+ logger.debug(traceback.format_exc())
132
+ logger.error(
133
+ f"Error getting service connection for service name [{service_name}]"
134
+ f" using the secrets manager provider [{self.metadata.config.secretsManagerProvider}]: {exc}"
135
+ )
136
+ raise exc
137
+
138
+ return self.service_connection_map[service_name]
139
+
140
+ def _get_test_cases_from_test_suite(self, test_suite: TestSuite) -> List[TestCase]:
87
141
  """Return test cases if the test suite exists and has them"""
88
- if test_suite:
89
- test_cases = self.metadata.list_all_entities(
90
- entity=TestCase,
91
- fields=["testSuite", "entityLink", "testDefinition"],
92
- params={"testSuiteId": test_suite.id.root},
93
- )
94
- test_cases = cast(List[TestCase], test_cases) # satisfy type checker
95
- if self.source_config.testCases is not None:
96
- test_cases = [
97
- t for t in test_cases if t.name in self.source_config.testCases
98
- ]
99
- return test_cases
100
- return []
142
+ test_cases = self.metadata.list_all_entities(
143
+ entity=TestCase,
144
+ fields=["testSuite", "entityLink", "testDefinition"],
145
+ params={"testSuiteId": test_suite.id.root},
146
+ )
147
+ test_cases = cast(List[TestCase], test_cases) # satisfy type checker
148
+ if self.source_config.testCases is not None:
149
+ test_cases = [
150
+ t for t in test_cases if t.name in self.source_config.testCases
151
+ ]
152
+ return test_cases
101
153
 
102
154
  def prepare(self):
103
155
  """Nothing to prepare"""
@@ -106,6 +158,7 @@ class TestSuiteSource(Source):
106
158
  self.metadata.health_check()
107
159
 
108
160
  def _iter(self) -> Iterable[Either[TableAndTests]]:
161
+ # Basic tests suites will have a table informed
109
162
  table: Table = self._get_table_entity()
110
163
  if table:
111
164
  source_type = table.serviceType.value.lower()
@@ -119,20 +172,29 @@ class TestSuiteSource(Source):
119
172
  )
120
173
  yield from self._process_table_suite(table)
121
174
 
175
+ # Logical test suites won't have a table, we'll need to group the execution by tests
122
176
  else:
123
- yield Either(
124
- left=StackTraceError(
125
- name="Missing Table",
126
- error=f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.root}."
127
- " Make sure the table exists in OpenMetadata and/or the JWT Token provided is valid.",
128
- )
129
- )
177
+ yield from self._process_logical_suite()
130
178
 
131
179
  def _process_table_suite(self, table: Table) -> Iterable[Either[TableAndTests]]:
132
180
  """
133
181
  Check that the table has the proper test suite built in
134
182
  """
183
+ try:
184
+ service_connection: DatabaseConnection = self._get_table_service_connection(
185
+ table
186
+ )
187
+ except Exception as exc:
188
+ yield Either(
189
+ left=StackTraceError(
190
+ name="Error getting service connection",
191
+ error=f"Error getting the service connection for table {table.name.root}: {exc}",
192
+ stackTrace=traceback.format_exc(),
193
+ )
194
+ )
195
+ return
135
196
  # If there is no executable test suite yet for the table, we'll need to create one
197
+ # Then, the suite won't have yet any tests
136
198
  if not table.testSuite:
137
199
  executable_test_suite = CreateTestSuiteRequest(
138
200
  name=fqn.build(
@@ -143,38 +205,80 @@ class TestSuiteSource(Source):
143
205
  displayName=f"{self.source_config.entityFullyQualifiedName.root} Test Suite",
144
206
  description="Test Suite created from YAML processor config file",
145
207
  owners=None,
146
- executableEntityReference=self.source_config.entityFullyQualifiedName.root,
208
+ basicEntityReference=self.source_config.entityFullyQualifiedName.root,
147
209
  )
148
210
  yield Either(
149
211
  right=TableAndTests(
150
212
  executable_test_suite=executable_test_suite,
151
- service_type=self.config.source.serviceConnection.root.config.type.value,
213
+ service_type=service_connection.config.type.value,
214
+ service_connection=service_connection,
152
215
  )
153
216
  )
217
+ test_suite_cases = []
154
218
 
155
- test_suite: Optional[TestSuite] = None
156
- if table.testSuite:
157
- test_suite = self.metadata.get_by_id(
219
+ # Otherwise, we pick the tests already registered in the suite
220
+ else:
221
+ test_suite: Optional[TestSuite] = self.metadata.get_by_id(
158
222
  entity=TestSuite, entity_id=table.testSuite.id.root
159
223
  )
224
+ test_suite_cases = self._get_test_cases_from_test_suite(test_suite)
225
+
226
+ yield Either(
227
+ right=TableAndTests(
228
+ table=table,
229
+ test_cases=test_suite_cases,
230
+ service_type=service_connection.config.type.value,
231
+ service_connection=service_connection,
232
+ )
233
+ )
160
234
 
161
- if test_suite and not test_suite.executable:
235
+ def _process_logical_suite(self):
236
+ """Process logical test suite, collect all test cases and yield them in batches by table"""
237
+ test_suite = self.metadata.get_by_name(
238
+ entity=TestSuite, fqn=self.config.source.serviceName
239
+ )
240
+ if test_suite is None:
162
241
  yield Either(
163
242
  left=StackTraceError(
164
- name="Non-executable Test Suite",
165
- error=f"The table {self.source_config.entityFullyQualifiedName.root} "
166
- "has a test suite that is not executable.",
243
+ name="Test Suite not found",
244
+ error=f"Test Suite with name {self.config.source.serviceName} not found",
167
245
  )
168
246
  )
247
+ test_cases: List[TestCase] = self._get_test_cases_from_test_suite(test_suite)
248
+ grouped_by_table = itertools.groupby(
249
+ test_cases, key=lambda t: entity_link.get_table_fqn(t.entityLink.root)
250
+ )
251
+ for table_fqn, group in grouped_by_table:
252
+ table_entity: Table = self.metadata.get_by_name(Table, table_fqn)
253
+ if table_entity is None:
254
+ yield Either(
255
+ left=StackTraceError(
256
+ name="Table not found",
257
+ error=f"Table with fqn {table_fqn} not found for test suite {test_suite.name.root}",
258
+ )
259
+ )
260
+ continue
169
261
 
170
- else:
171
- test_suite_cases = self._get_test_cases_from_test_suite(test_suite)
262
+ try:
263
+ service_connection: DatabaseConnection = (
264
+ self._get_table_service_connection(table_entity)
265
+ )
266
+ except Exception as exc:
267
+ yield Either(
268
+ left=StackTraceError(
269
+ name="Error getting service connection",
270
+ error=f"Error getting the service connection for table {table_entity.name.root}: {exc}",
271
+ stackTrace=traceback.format_exc(),
272
+ )
273
+ )
274
+ continue
172
275
 
173
276
  yield Either(
174
277
  right=TableAndTests(
175
- table=table,
176
- test_cases=test_suite_cases,
177
- service_type=self.config.source.serviceConnection.root.config.type.value,
278
+ table=table_entity,
279
+ test_cases=list(group),
280
+ service_type=service_connection.config.type.value,
281
+ service_connection=service_connection,
178
282
  )
179
283
  )
180
284
 
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/basic.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/aggregatedCostAnalysisReportData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/entityReportData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/rawCostAnalysisReportData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/webAnalyticEntityViewReportData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/reportDataType/webAnalyticUserActivityReportData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/webAnalyticEvent.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/webAnalyticEventData.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/webAnalyticEventType/customEvent.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: analytics/webAnalyticEventType/pageViewEvent.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: api/addGlossaryToAssetsRequest.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,6 +1,6 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: api/addTagToAssetsRequest.json
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00
4
4
 
5
5
  from __future__ import annotations
6
6
 
@@ -1,3 +1,3 @@
1
1
  # generated by datamodel-codegen:
2
2
  # filename: schema
3
- # timestamp: 2024-12-10T17:32:44+00:00
3
+ # timestamp: 2025-01-17T09:15:19+00:00