openmetadata-ingestion 1.6.0.0rc2__py3-none-any.whl → 1.6.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (687) hide show
  1. metadata/clients/aws_client.py +2 -2
  2. metadata/data_quality/validations/base_test_handler.py +4 -4
  3. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +2 -2
  4. metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +7 -1
  5. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +5 -4
  6. metadata/generated/schema/analytics/__init__.py +1 -1
  7. metadata/generated/schema/analytics/basic.py +1 -1
  8. metadata/generated/schema/analytics/reportData.py +1 -1
  9. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  10. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  11. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  12. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  13. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  14. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  15. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  16. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  17. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  18. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  19. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  20. metadata/generated/schema/api/__init__.py +1 -1
  21. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  22. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  23. metadata/generated/schema/api/analytics/__init__.py +1 -1
  24. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  25. metadata/generated/schema/api/automations/__init__.py +1 -1
  26. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  27. metadata/generated/schema/api/bulkAssets.py +1 -1
  28. metadata/generated/schema/api/classification/__init__.py +1 -1
  29. metadata/generated/schema/api/classification/createClassification.py +1 -1
  30. metadata/generated/schema/api/classification/createTag.py +1 -1
  31. metadata/generated/schema/api/classification/loadTags.py +1 -1
  32. metadata/generated/schema/api/createBot.py +1 -1
  33. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  34. metadata/generated/schema/api/createType.py +1 -1
  35. metadata/generated/schema/api/data/__init__.py +1 -1
  36. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  37. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  38. metadata/generated/schema/api/data/createChart.py +1 -1
  39. metadata/generated/schema/api/data/createContainer.py +1 -1
  40. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  41. metadata/generated/schema/api/data/createDashboard.py +1 -1
  42. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  43. metadata/generated/schema/api/data/createDatabase.py +1 -1
  44. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  45. metadata/generated/schema/api/data/createGlossary.py +1 -1
  46. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  47. metadata/generated/schema/api/data/createMetric.py +1 -1
  48. metadata/generated/schema/api/data/createMlModel.py +1 -1
  49. metadata/generated/schema/api/data/createPipeline.py +1 -1
  50. metadata/generated/schema/api/data/createQuery.py +1 -1
  51. metadata/generated/schema/api/data/createSearchIndex.py +8 -8
  52. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  53. metadata/generated/schema/api/data/createTable.py +1 -1
  54. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  55. metadata/generated/schema/api/data/createTopic.py +1 -1
  56. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  57. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  58. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  59. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  60. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  61. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  62. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  63. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  64. metadata/generated/schema/api/docStore/__init__.py +1 -1
  65. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  66. metadata/generated/schema/api/domains/__init__.py +1 -1
  67. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  68. metadata/generated/schema/api/domains/createDomain.py +1 -1
  69. metadata/generated/schema/api/feed/__init__.py +1 -1
  70. metadata/generated/schema/api/feed/closeTask.py +1 -1
  71. metadata/generated/schema/api/feed/createPost.py +1 -1
  72. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  73. metadata/generated/schema/api/feed/createThread.py +1 -1
  74. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  75. metadata/generated/schema/api/feed/threadCount.py +1 -1
  76. metadata/generated/schema/api/governance/__init__.py +1 -1
  77. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  78. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  79. metadata/generated/schema/api/lineage/__init__.py +1 -1
  80. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  81. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  82. metadata/generated/schema/api/policies/__init__.py +1 -1
  83. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  84. metadata/generated/schema/api/services/__init__.py +1 -1
  85. metadata/generated/schema/api/services/createApiService.py +1 -1
  86. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  87. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  88. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  89. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  90. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  91. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  92. metadata/generated/schema/api/services/createSearchService.py +1 -1
  93. metadata/generated/schema/api/services/createStorageService.py +1 -1
  94. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  95. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  96. metadata/generated/schema/api/setOwner.py +1 -1
  97. metadata/generated/schema/api/teams/__init__.py +1 -1
  98. metadata/generated/schema/api/teams/createPersona.py +1 -1
  99. metadata/generated/schema/api/teams/createRole.py +1 -1
  100. metadata/generated/schema/api/teams/createTeam.py +1 -1
  101. metadata/generated/schema/api/teams/createUser.py +1 -1
  102. metadata/generated/schema/api/tests/__init__.py +1 -1
  103. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  104. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  105. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  106. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  107. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  108. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  109. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  110. metadata/generated/schema/api/voteRequest.py +1 -1
  111. metadata/generated/schema/auth/__init__.py +1 -1
  112. metadata/generated/schema/auth/basicAuth.py +1 -1
  113. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  114. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  115. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  116. metadata/generated/schema/auth/emailRequest.py +1 -1
  117. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  118. metadata/generated/schema/auth/generateToken.py +1 -1
  119. metadata/generated/schema/auth/jwtAuth.py +1 -1
  120. metadata/generated/schema/auth/loginRequest.py +1 -1
  121. metadata/generated/schema/auth/logoutRequest.py +1 -1
  122. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  123. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  124. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  125. metadata/generated/schema/auth/refreshToken.py +1 -1
  126. metadata/generated/schema/auth/registrationRequest.py +1 -1
  127. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  128. metadata/generated/schema/auth/revokeToken.py +1 -1
  129. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  130. metadata/generated/schema/auth/ssoAuth.py +1 -1
  131. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  132. metadata/generated/schema/configuration/__init__.py +1 -1
  133. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  134. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  135. metadata/generated/schema/configuration/authConfig.py +1 -1
  136. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  137. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  138. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  139. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  140. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  141. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  142. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  143. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  144. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  145. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  146. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  147. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  148. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  149. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  150. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  151. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  152. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  153. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  154. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  155. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  156. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  157. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  158. metadata/generated/schema/configuration/searchSettings.py +1 -1
  159. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  160. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  161. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  162. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  163. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  164. metadata/generated/schema/dataInsight/__init__.py +1 -1
  165. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  166. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  167. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  168. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  169. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  170. metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
  171. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  172. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  173. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  174. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  175. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  176. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  177. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  178. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  179. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  180. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  181. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  182. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  183. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  184. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  185. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  186. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  187. metadata/generated/schema/email/__init__.py +1 -1
  188. metadata/generated/schema/email/emailRequest.py +1 -1
  189. metadata/generated/schema/email/emailTemplate.py +1 -1
  190. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  191. metadata/generated/schema/email/smtpSettings.py +1 -1
  192. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  193. metadata/generated/schema/entity/__init__.py +1 -1
  194. metadata/generated/schema/entity/applications/__init__.py +1 -1
  195. metadata/generated/schema/entity/applications/app.py +1 -1
  196. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  197. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  198. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  199. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +1 -1
  200. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  201. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  202. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  203. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  204. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  205. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  206. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  207. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  208. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  209. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  210. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  211. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  212. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  213. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  214. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  215. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  216. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
  217. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  218. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  219. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  220. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  221. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  222. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  223. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  224. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  225. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  226. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  227. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
  228. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
  229. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  230. metadata/generated/schema/entity/automations/__init__.py +1 -1
  231. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  232. metadata/generated/schema/entity/automations/workflow.py +1 -1
  233. metadata/generated/schema/entity/bot.py +1 -1
  234. metadata/generated/schema/entity/classification/__init__.py +1 -1
  235. metadata/generated/schema/entity/classification/classification.py +1 -1
  236. metadata/generated/schema/entity/classification/tag.py +1 -1
  237. metadata/generated/schema/entity/data/__init__.py +1 -1
  238. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  239. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  240. metadata/generated/schema/entity/data/chart.py +1 -1
  241. metadata/generated/schema/entity/data/container.py +1 -1
  242. metadata/generated/schema/entity/data/dashboard.py +1 -1
  243. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  244. metadata/generated/schema/entity/data/database.py +1 -1
  245. metadata/generated/schema/entity/data/databaseSchema.py +1 -1
  246. metadata/generated/schema/entity/data/glossary.py +1 -1
  247. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  248. metadata/generated/schema/entity/data/metric.py +1 -1
  249. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  250. metadata/generated/schema/entity/data/pipeline.py +1 -1
  251. metadata/generated/schema/entity/data/query.py +1 -1
  252. metadata/generated/schema/entity/data/report.py +1 -1
  253. metadata/generated/schema/entity/data/searchIndex.py +13 -13
  254. metadata/generated/schema/entity/data/storedProcedure.py +1 -1
  255. metadata/generated/schema/entity/data/table.py +1 -1
  256. metadata/generated/schema/entity/data/topic.py +1 -1
  257. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  258. metadata/generated/schema/entity/docStore/document.py +1 -1
  259. metadata/generated/schema/entity/domains/__init__.py +1 -1
  260. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  261. metadata/generated/schema/entity/domains/domain.py +1 -1
  262. metadata/generated/schema/entity/events/__init__.py +1 -1
  263. metadata/generated/schema/entity/events/webhook.py +1 -1
  264. metadata/generated/schema/entity/feed/__init__.py +1 -1
  265. metadata/generated/schema/entity/feed/assets.py +1 -1
  266. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  267. metadata/generated/schema/entity/feed/description.py +1 -1
  268. metadata/generated/schema/entity/feed/domain.py +1 -1
  269. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  270. metadata/generated/schema/entity/feed/owner.py +1 -1
  271. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  272. metadata/generated/schema/entity/feed/tag.py +1 -1
  273. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  274. metadata/generated/schema/entity/feed/thread.py +1 -1
  275. metadata/generated/schema/entity/policies/__init__.py +1 -1
  276. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  277. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  278. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  279. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  280. metadata/generated/schema/entity/policies/filters.py +1 -1
  281. metadata/generated/schema/entity/policies/policy.py +1 -1
  282. metadata/generated/schema/entity/services/__init__.py +1 -1
  283. metadata/generated/schema/entity/services/apiService.py +1 -1
  284. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  285. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  286. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  287. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  288. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  289. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  290. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  291. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  292. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  293. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  294. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  295. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  296. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  297. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  298. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +1 -1
  299. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  300. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  301. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  302. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  303. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  304. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  305. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  306. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  307. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  308. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  309. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  310. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  311. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
  312. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  313. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  314. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  315. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  316. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  317. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
  318. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  319. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  320. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  321. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  322. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  323. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  324. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  325. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  326. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  327. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  328. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  329. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  330. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  331. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  332. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  333. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  334. metadata/generated/schema/entity/services/connections/database/db2Connection.py +13 -1
  335. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  336. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  337. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  338. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  339. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  340. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  341. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  342. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  343. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
  344. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  345. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
  346. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
  347. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  348. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  352. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  353. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +1 -1
  360. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
  361. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  362. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  367. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  372. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  381. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  385. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  387. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  388. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  389. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  390. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  395. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  401. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  405. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  410. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  411. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  416. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  418. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  421. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  422. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  423. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  424. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  425. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  427. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  428. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  429. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  431. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  432. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  434. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  435. metadata/generated/schema/entity/services/dashboardService.py +1 -1
  436. metadata/generated/schema/entity/services/databaseService.py +1 -1
  437. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  438. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +1 -1
  439. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  440. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  441. metadata/generated/schema/entity/services/messagingService.py +1 -1
  442. metadata/generated/schema/entity/services/metadataService.py +1 -1
  443. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  444. metadata/generated/schema/entity/services/pipelineService.py +1 -1
  445. metadata/generated/schema/entity/services/searchService.py +1 -1
  446. metadata/generated/schema/entity/services/serviceType.py +1 -1
  447. metadata/generated/schema/entity/services/storageService.py +1 -1
  448. metadata/generated/schema/entity/teams/__init__.py +1 -1
  449. metadata/generated/schema/entity/teams/persona.py +1 -1
  450. metadata/generated/schema/entity/teams/role.py +1 -1
  451. metadata/generated/schema/entity/teams/team.py +1 -1
  452. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  453. metadata/generated/schema/entity/teams/user.py +1 -1
  454. metadata/generated/schema/entity/type.py +1 -1
  455. metadata/generated/schema/entity/utils/__init__.py +1 -1
  456. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  457. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  458. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  459. metadata/generated/schema/events/__init__.py +1 -1
  460. metadata/generated/schema/events/alertMetrics.py +1 -1
  461. metadata/generated/schema/events/api/__init__.py +1 -1
  462. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  463. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  464. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  465. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  466. metadata/generated/schema/events/api/typedEvent.py +1 -1
  467. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  468. metadata/generated/schema/events/eventFilterRule.py +1 -1
  469. metadata/generated/schema/events/eventSubscription.py +1 -1
  470. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  471. metadata/generated/schema/events/failedEvent.py +1 -1
  472. metadata/generated/schema/events/failedEventResponse.py +1 -1
  473. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  474. metadata/generated/schema/events/statusContext.py +1 -1
  475. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  476. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  477. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  478. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  479. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  480. metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
  481. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
  482. metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
  483. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  484. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  485. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
  486. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
  487. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
  488. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  489. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
  490. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  491. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
  492. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  493. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
  494. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  495. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
  496. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +1 -1
  497. metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
  498. metadata/generated/schema/governance/workflows/workflowInstance.py +1 -1
  499. metadata/generated/schema/governance/workflows/workflowInstanceState.py +1 -1
  500. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  501. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  502. metadata/generated/schema/metadataIngestion/application.py +1 -1
  503. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  504. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  505. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  506. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -17
  507. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
  508. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -9
  509. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  510. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  511. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  512. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  513. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  514. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  515. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  516. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  517. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  518. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  519. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  520. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  521. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  522. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  523. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  524. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  525. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  526. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  527. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  528. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  529. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  530. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  531. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  532. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  533. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  534. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  535. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  536. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  537. metadata/generated/schema/monitoring/__init__.py +1 -1
  538. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  539. metadata/generated/schema/security/__init__.py +1 -1
  540. metadata/generated/schema/security/client/__init__.py +1 -1
  541. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  542. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  543. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  544. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  545. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  546. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  547. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  548. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  549. metadata/generated/schema/security/credentials/__init__.py +1 -1
  550. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  551. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  552. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  553. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  554. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  555. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  556. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  557. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  558. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  559. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  560. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  561. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  562. metadata/generated/schema/security/sasl/__init__.py +1 -1
  563. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  564. metadata/generated/schema/security/secrets/__init__.py +1 -1
  565. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  566. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  567. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  568. metadata/generated/schema/security/securityConfiguration.py +1 -1
  569. metadata/generated/schema/security/ssl/__init__.py +1 -1
  570. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  571. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  572. metadata/generated/schema/settings/__init__.py +1 -1
  573. metadata/generated/schema/settings/settings.py +1 -1
  574. metadata/generated/schema/system/__init__.py +1 -1
  575. metadata/generated/schema/system/entityError.py +1 -1
  576. metadata/generated/schema/system/eventPublisherJob.py +1 -1
  577. metadata/generated/schema/system/indexingError.py +1 -1
  578. metadata/generated/schema/system/limitsResponse.py +1 -1
  579. metadata/generated/schema/system/ui/__init__.py +1 -1
  580. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  581. metadata/generated/schema/system/ui/page.py +1 -1
  582. metadata/generated/schema/system/validationResponse.py +1 -1
  583. metadata/generated/schema/tests/__init__.py +1 -1
  584. metadata/generated/schema/tests/assigned.py +1 -1
  585. metadata/generated/schema/tests/basic.py +1 -1
  586. metadata/generated/schema/tests/customMetric.py +1 -1
  587. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  588. metadata/generated/schema/tests/resolved.py +1 -1
  589. metadata/generated/schema/tests/testCase.py +1 -1
  590. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  591. metadata/generated/schema/tests/testDefinition.py +1 -1
  592. metadata/generated/schema/tests/testSuite.py +1 -1
  593. metadata/generated/schema/type/__init__.py +1 -1
  594. metadata/generated/schema/type/apiSchema.py +1 -1
  595. metadata/generated/schema/type/assetCertification.py +1 -1
  596. metadata/generated/schema/type/auditLog.py +1 -1
  597. metadata/generated/schema/type/basic.py +1 -1
  598. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  599. metadata/generated/schema/type/changeEvent.py +1 -1
  600. metadata/generated/schema/type/changeEventType.py +1 -1
  601. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  602. metadata/generated/schema/type/csvDocumentation.py +1 -1
  603. metadata/generated/schema/type/csvErrorType.py +1 -1
  604. metadata/generated/schema/type/csvFile.py +1 -1
  605. metadata/generated/schema/type/csvImportResult.py +1 -1
  606. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  607. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  608. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  609. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  610. metadata/generated/schema/type/customProperty.py +1 -1
  611. metadata/generated/schema/type/dailyCount.py +1 -1
  612. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  613. metadata/generated/schema/type/entityHierarchy.py +1 -1
  614. metadata/generated/schema/type/entityHistory.py +1 -1
  615. metadata/generated/schema/type/entityLineage.py +1 -1
  616. metadata/generated/schema/type/entityReference.py +1 -1
  617. metadata/generated/schema/type/entityReferenceList.py +1 -1
  618. metadata/generated/schema/type/entityRelationship.py +2 -1
  619. metadata/generated/schema/type/entityUsage.py +1 -1
  620. metadata/generated/schema/type/filterPattern.py +1 -1
  621. metadata/generated/schema/type/function.py +1 -1
  622. metadata/generated/schema/type/include.py +1 -1
  623. metadata/generated/schema/type/jdbcConnection.py +1 -1
  624. metadata/generated/schema/type/lifeCycle.py +1 -1
  625. metadata/generated/schema/type/paging.py +1 -1
  626. metadata/generated/schema/type/profile.py +1 -1
  627. metadata/generated/schema/type/queryParserData.py +1 -1
  628. metadata/generated/schema/type/reaction.py +1 -1
  629. metadata/generated/schema/type/schedule.py +1 -1
  630. metadata/generated/schema/type/schema.py +1 -1
  631. metadata/generated/schema/type/tableQuery.py +1 -1
  632. metadata/generated/schema/type/tableUsageCount.py +1 -1
  633. metadata/generated/schema/type/tagLabel.py +1 -1
  634. metadata/generated/schema/type/usageDetails.py +1 -1
  635. metadata/generated/schema/type/usageRequest.py +1 -1
  636. metadata/generated/schema/type/votes.py +1 -1
  637. metadata/ingestion/lineage/parser.py +6 -4
  638. metadata/ingestion/lineage/sql_lineage.py +258 -29
  639. metadata/ingestion/models/patch_request.py +1 -0
  640. metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -2
  641. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +5 -3
  642. metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
  643. metadata/ingestion/ometa/mixins/table_mixin.py +1 -1
  644. metadata/ingestion/source/dashboard/dashboard_service.py +2 -0
  645. metadata/ingestion/source/dashboard/microstrategy/client.py +39 -23
  646. metadata/ingestion/source/dashboard/microstrategy/connection.py +4 -1
  647. metadata/ingestion/source/dashboard/tableau/metadata.py +29 -12
  648. metadata/ingestion/source/dashboard/tableau/models.py +10 -0
  649. metadata/ingestion/source/database/bigquery/profiler/system.py +5 -4
  650. metadata/ingestion/source/database/db2/connection.py +14 -1
  651. metadata/ingestion/source/database/dbt/constants.py +8 -0
  652. metadata/ingestion/source/database/dbt/dbt_config.py +6 -2
  653. metadata/ingestion/source/database/dbt/dbt_service.py +18 -1
  654. metadata/ingestion/source/database/mssql/service_spec.py +2 -0
  655. metadata/ingestion/source/database/redshift/profiler/system.py +5 -4
  656. metadata/ingestion/source/database/snowflake/connection.py +7 -2
  657. metadata/ingestion/source/database/snowflake/lineage.py +35 -1
  658. metadata/ingestion/source/database/snowflake/metadata.py +38 -19
  659. metadata/ingestion/source/database/snowflake/models.py +1 -0
  660. metadata/ingestion/source/database/snowflake/profiler/system.py +5 -4
  661. metadata/ingestion/source/database/snowflake/queries.py +22 -1
  662. metadata/ingestion/source/database/stored_procedures_mixin.py +20 -8
  663. metadata/ingestion/source/pipeline/fivetran/metadata.py +1 -1
  664. metadata/ingestion/source/pipeline/kafkaconnect/client.py +12 -13
  665. metadata/ingestion/source/pipeline/kafkaconnect/connection.py +1 -1
  666. metadata/profiler/interface/sqlalchemy/profiler_interface.py +1 -1
  667. metadata/profiler/metrics/static/max.py +4 -2
  668. metadata/profiler/metrics/static/min.py +4 -2
  669. metadata/profiler/orm/converter/converter_registry.py +2 -0
  670. metadata/profiler/orm/converter/mssql/converter.py +15 -1
  671. metadata/profiler/orm/functions/md5.py +8 -5
  672. metadata/profiler/orm/functions/table_metric_computer.py +6 -6
  673. metadata/profiler/orm/types/custom_timestamp.py +4 -2
  674. metadata/profiler/processor/runner.py +5 -0
  675. metadata/profiler/source/database/base/profiler_source.py +0 -1
  676. metadata/sampler/processor.py +1 -5
  677. metadata/sampler/sqlalchemy/mssql/sampler.py +50 -0
  678. metadata/sampler/sqlalchemy/sampler.py +5 -2
  679. metadata/utils/class_helper.py +7 -0
  680. metadata/utils/importer.py +1 -1
  681. metadata/workflow/data_quality.py +5 -1
  682. {openmetadata_ingestion-1.6.0.0rc2.dist-info → openmetadata_ingestion-1.6.1.0.dist-info}/METADATA +364 -364
  683. {openmetadata_ingestion-1.6.0.0rc2.dist-info → openmetadata_ingestion-1.6.1.0.dist-info}/RECORD +687 -686
  684. {openmetadata_ingestion-1.6.0.0rc2.dist-info → openmetadata_ingestion-1.6.1.0.dist-info}/LICENSE +0 -0
  685. {openmetadata_ingestion-1.6.0.0rc2.dist-info → openmetadata_ingestion-1.6.1.0.dist-info}/WHEEL +0 -0
  686. {openmetadata_ingestion-1.6.0.0rc2.dist-info → openmetadata_ingestion-1.6.1.0.dist-info}/entry_points.txt +0 -0
  687. {openmetadata_ingestion-1.6.0.0rc2.dist-info → openmetadata_ingestion-1.6.1.0.dist-info}/top_level.txt +0 -0
@@ -27,7 +27,10 @@ from metadata.generated.schema.api.data.createStoredProcedure import (
27
27
  )
28
28
  from metadata.generated.schema.entity.data.database import Database
29
29
  from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
30
- from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
30
+ from metadata.generated.schema.entity.data.storedProcedure import (
31
+ StoredProcedureCode,
32
+ StoredProcedureType,
33
+ )
31
34
  from metadata.generated.schema.entity.data.table import (
32
35
  PartitionColumnDetails,
33
36
  PartitionIntervalTypes,
@@ -71,6 +74,7 @@ from metadata.ingestion.source.database.snowflake.models import (
71
74
  SnowflakeStoredProcedure,
72
75
  )
73
76
  from metadata.ingestion.source.database.snowflake.queries import (
77
+ SNOWFLAKE_DESC_FUNCTION,
74
78
  SNOWFLAKE_DESC_STORED_PROCEDURE,
75
79
  SNOWFLAKE_FETCH_ALL_TAGS,
76
80
  SNOWFLAKE_GET_CLUSTER_KEY,
@@ -78,6 +82,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
78
82
  SNOWFLAKE_GET_DATABASE_COMMENTS,
79
83
  SNOWFLAKE_GET_DATABASES,
80
84
  SNOWFLAKE_GET_EXTERNAL_LOCATIONS,
85
+ SNOWFLAKE_GET_FUNCTIONS,
81
86
  SNOWFLAKE_GET_ORGANIZATION_NAME,
82
87
  SNOWFLAKE_GET_SCHEMA_COMMENTS,
83
88
  SNOWFLAKE_GET_STORED_PROCEDURES,
@@ -623,26 +628,34 @@ class SnowflakeSource(
623
628
 
624
629
  return views
625
630
 
631
+ def _get_stored_procedures_internal(
632
+ self, query: str
633
+ ) -> Iterable[SnowflakeStoredProcedure]:
634
+ results = self.engine.execute(
635
+ query.format(
636
+ database_name=self.context.get().database,
637
+ schema_name=self.context.get().database_schema,
638
+ )
639
+ ).all()
640
+ for row in results:
641
+ stored_procedure = SnowflakeStoredProcedure.model_validate(dict(row))
642
+ if stored_procedure.definition is None:
643
+ logger.debug(
644
+ f"Missing ownership permissions on procedure {stored_procedure.name}."
645
+ " Trying to fetch description via DESCRIBE."
646
+ )
647
+ stored_procedure.definition = self.describe_procedure_definition(
648
+ stored_procedure
649
+ )
650
+ yield stored_procedure
651
+
626
652
  def get_stored_procedures(self) -> Iterable[SnowflakeStoredProcedure]:
627
653
  """List Snowflake stored procedures"""
628
654
  if self.source_config.includeStoredProcedures:
629
- results = self.engine.execute(
630
- SNOWFLAKE_GET_STORED_PROCEDURES.format(
631
- database_name=self.context.get().database,
632
- schema_name=self.context.get().database_schema,
633
- )
634
- ).all()
635
- for row in results:
636
- stored_procedure = SnowflakeStoredProcedure.model_validate(dict(row))
637
- if stored_procedure.definition is None:
638
- logger.debug(
639
- f"Missing ownership permissions on procedure {stored_procedure.name}."
640
- " Trying to fetch description via DESCRIBE."
641
- )
642
- stored_procedure.definition = self.describe_procedure_definition(
643
- stored_procedure
644
- )
645
- yield stored_procedure
655
+ yield from self._get_stored_procedures_internal(
656
+ SNOWFLAKE_GET_STORED_PROCEDURES
657
+ )
658
+ yield from self._get_stored_procedures_internal(SNOWFLAKE_GET_FUNCTIONS)
646
659
 
647
660
  def describe_procedure_definition(
648
661
  self, stored_procedure: SnowflakeStoredProcedure
@@ -654,8 +667,12 @@ class SnowflakeSource(
654
667
  Then, if the procedure is created with `EXECUTE AS CALLER`, we can still try to
655
668
  get the definition with a DESCRIBE.
656
669
  """
670
+ if stored_procedure.procedure_type == StoredProcedureType.StoredProcedure.value:
671
+ query = SNOWFLAKE_DESC_STORED_PROCEDURE
672
+ else:
673
+ query = SNOWFLAKE_DESC_FUNCTION
657
674
  res = self.engine.execute(
658
- SNOWFLAKE_DESC_STORED_PROCEDURE.format(
675
+ query.format(
659
676
  database_name=self.context.get().database,
660
677
  schema_name=self.context.get().database_schema,
661
678
  procedure_name=stored_procedure.name,
@@ -677,6 +694,8 @@ class SnowflakeSource(
677
694
  language=STORED_PROC_LANGUAGE_MAP.get(stored_procedure.language),
678
695
  code=stored_procedure.definition,
679
696
  ),
697
+ storedProcedureType=stored_procedure.procedure_type
698
+ or StoredProcedureType.StoredProcedure.value,
680
699
  databaseSchema=fqn.build(
681
700
  metadata=self.metadata,
682
701
  entity_type=DatabaseSchema,
@@ -50,6 +50,7 @@ class SnowflakeStoredProcedure(BaseModel):
50
50
  None, alias="SIGNATURE", description="Used to build the source URL"
51
51
  )
52
52
  comment: Optional[str] = Field(None, alias="COMMENT")
53
+ procedure_type: Optional[str] = Field(None, alias="PROCEDURE_TYPE")
53
54
 
54
55
  # Update the signature to clean it up on read
55
56
  @field_validator("signature")
@@ -20,6 +20,7 @@ from metadata.profiler.metrics.system.system import (
20
20
  SQASessionProvider,
21
21
  SystemMetricsComputer,
22
22
  )
23
+ from metadata.profiler.processor.runner import QueryRunner
23
24
  from metadata.utils.collections import CaseInsensitiveString
24
25
  from metadata.utils.logger import profiler_logger
25
26
  from metadata.utils.lru_cache import LRU_CACHE_SIZE, LRUCache
@@ -294,11 +295,11 @@ class SnowflakeSystemMetricsSource(
294
295
  )
295
296
 
296
297
  def get_kwargs(self, **kwargs):
297
- table = kwargs.get("table")
298
+ runner: QueryRunner = kwargs.get("runner")
298
299
  return {
299
- "table": table.__table__.name,
300
- "database": self.get_session().get_bind().url.database,
301
- "schema": table.__table__.schema,
300
+ "table": runner.table_name,
301
+ "database": runner.session.get_bind().url.database,
302
+ "schema": runner.schema_name,
302
303
  }
303
304
 
304
305
  def get_inserts(self, **kwargs) -> List[SystemProfile]:
@@ -294,17 +294,38 @@ SELECT
294
294
  PROCEDURE_LANGUAGE AS language,
295
295
  PROCEDURE_DEFINITION AS definition,
296
296
  ARGUMENT_SIGNATURE AS signature,
297
- COMMENT as comment
297
+ COMMENT as comment,
298
+ 'StoredProcedure' as procedure_type
298
299
  FROM INFORMATION_SCHEMA.PROCEDURES
299
300
  WHERE PROCEDURE_CATALOG = '{database_name}'
300
301
  AND PROCEDURE_SCHEMA = '{schema_name}'
301
302
  """
302
303
  )
303
304
 
305
+ SNOWFLAKE_GET_FUNCTIONS = textwrap.dedent(
306
+ """
307
+ SELECT
308
+ FUNCTION_NAME AS name,
309
+ FUNCTION_OWNER AS owner,
310
+ FUNCTION_LANGUAGE AS language,
311
+ FUNCTION_DEFINITION AS definition,
312
+ ARGUMENT_SIGNATURE AS signature,
313
+ COMMENT as comment,
314
+ 'UDF' as procedure_type
315
+ FROM INFORMATION_SCHEMA.FUNCTIONS
316
+ WHERE FUNCTION_CATALOG = '{database_name}'
317
+ AND FUNCTION_SCHEMA = '{schema_name}'
318
+ """
319
+ )
320
+
304
321
  SNOWFLAKE_DESC_STORED_PROCEDURE = (
305
322
  "DESC PROCEDURE {database_name}.{schema_name}.{procedure_name}{procedure_signature}"
306
323
  )
307
324
 
325
+ SNOWFLAKE_DESC_FUNCTION = (
326
+ "DESC FUNCTION {database_name}.{schema_name}.{procedure_name}{procedure_signature}"
327
+ )
328
+
308
329
  SNOWFLAKE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
309
330
  """
310
331
  WITH SP_HISTORY AS (
@@ -217,14 +217,26 @@ class StoredProcedureLineageMixin(ABC):
217
217
  self, procedure_and_query: ProcedureAndQuery
218
218
  ) -> Iterable[Either[Union[AddLineageRequest, CreateQueryRequest]]]:
219
219
 
220
- yield from self._yield_procedure_lineage(
221
- query_by_procedure=procedure_and_query.query_by_procedure,
222
- procedure=procedure_and_query.procedure,
223
- )
224
- yield from self.yield_procedure_query(
225
- query_by_procedure=procedure_and_query.query_by_procedure,
226
- procedure=procedure_and_query.procedure,
227
- )
220
+ try:
221
+ yield from self._yield_procedure_lineage(
222
+ query_by_procedure=procedure_and_query.query_by_procedure,
223
+ procedure=procedure_and_query.procedure,
224
+ )
225
+ except Exception as exc:
226
+ logger.debug(traceback.format_exc())
227
+ logger.warning(
228
+ f"Could not get lineage for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
229
+ )
230
+ try:
231
+ yield from self.yield_procedure_query(
232
+ query_by_procedure=procedure_and_query.query_by_procedure,
233
+ procedure=procedure_and_query.procedure,
234
+ )
235
+ except Exception as exc:
236
+ logger.debug(traceback.format_exc())
237
+ logger.warning(
238
+ f"Could not get query for store procedure '{procedure_and_query.procedure.fullyQualifiedName}' due to [{exc}]."
239
+ )
228
240
 
229
241
  def procedure_lineage_generator(self) -> Iterable[ProcedureAndQuery]:
230
242
  query = {
@@ -275,7 +275,7 @@ class FivetranSource(PipelineServiceSource):
275
275
  )
276
276
 
277
277
  def get_pipeline_name(self, pipeline_details: FivetranPipelineDetails) -> str:
278
- return pipeline_details.pipeline_name
278
+ return pipeline_details.pipeline_display_name or pipeline_details.pipeline_name
279
279
 
280
280
  def get_source_url(
281
281
  self,
@@ -61,15 +61,18 @@ class KafkaConnectClient:
61
61
  """
62
62
  Get the version and other details of the Kafka Connect cluster.
63
63
  """
64
- try:
65
- result = self.client.get_cluster_info()
66
- return result
67
-
68
- except Exception as exc:
69
- logger.debug(traceback.format_exc())
70
- logger.error(f"Unable to get cluster info :{exc}")
64
+ return self.client.get_cluster_info()
71
65
 
72
- return None
66
+ def get_connectors_list(
67
+ self,
68
+ expand: str = None,
69
+ pattern: str = None,
70
+ state: str = None,
71
+ ) -> dict:
72
+ """
73
+ Get the list of connectors from Kafka Connect cluster.
74
+ """
75
+ return self.client.list_connectors(expand=expand, pattern=pattern, state=state)
73
76
 
74
77
  def get_connectors(
75
78
  self,
@@ -85,12 +88,8 @@ class KafkaConnectClient:
85
88
  pattern (str): Only list connectors that match the regex pattern.
86
89
  state (str): Only list connectors that match the state.
87
90
  """
88
-
89
91
  try:
90
- result = self.client.list_connectors(
91
- expand=expand, pattern=pattern, state=state
92
- )
93
- return result
92
+ return self.get_connectors_list(expand=expand, pattern=pattern, state=state)
94
93
  except Exception as exc:
95
94
  logger.debug(traceback.format_exc())
96
95
  logger.error(f"Unable to get connectors list {exc}")
@@ -51,7 +51,7 @@ def test_connection(
51
51
 
52
52
  test_fn = {
53
53
  "GetClusterInfo": client.get_cluster_info,
54
- "GetPipelines": client.get_connectors,
54
+ "GetPipelines": client.get_connectors_list,
55
55
  "GetPlugins": client.get_connector_plugins,
56
56
  }
57
57
 
@@ -372,7 +372,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
372
372
  dictionnary of results
373
373
  """
374
374
  logger.debug(f"Computing system metrics for {runner.table_name}")
375
- return self.system_metrics_computer.get_system_metrics(table=runner.dataset)
375
+ return self.system_metrics_computer.get_system_metrics(runner=runner)
376
376
 
377
377
  def _create_thread_safe_runner(self, session, column=None):
378
378
  """Create thread safe runner"""
@@ -88,10 +88,12 @@ class Max(StaticMetric):
88
88
  def fn(self):
89
89
  """sqlalchemy function"""
90
90
  if is_concatenable(self.col.type):
91
- return MaxFn(LenFn(column(self.col.name, self.col.type)))
91
+ return MaxFn(
92
+ LenFn(column(self.col.name, self.col.type)), type_=self.col.type
93
+ )
92
94
  if (not is_quantifiable(self.col.type)) and (not is_date_time(self.col.type)):
93
95
  return None
94
- return MaxFn(column(self.col.name, self.col.type))
96
+ return MaxFn(column(self.col.name, self.col.type), type_=self.col.type)
95
97
 
96
98
  def df_fn(self, dfs=None):
97
99
  """pandas function"""
@@ -88,11 +88,13 @@ class Min(StaticMetric):
88
88
  def fn(self):
89
89
  """sqlalchemy function"""
90
90
  if is_concatenable(self.col.type):
91
- return MinFn(LenFn(column(self.col.name, self.col.type)))
91
+ return MinFn(
92
+ LenFn(column(self.col.name, self.col.type)), type_=self.col.type
93
+ )
92
94
 
93
95
  if (not is_quantifiable(self.col.type)) and (not is_date_time(self.col.type)):
94
96
  return None
95
- return MinFn(column(self.col.name, self.col.type))
97
+ return MinFn(column(self.col.name, self.col.type), type_=self.col.type)
96
98
 
97
99
  def df_fn(self, dfs=None):
98
100
  """pandas function"""
@@ -19,6 +19,7 @@ from metadata.generated.schema.entity.services.databaseService import (
19
19
  )
20
20
  from metadata.profiler.orm.converter.bigquery.converter import BigqueryMapTypes
21
21
  from metadata.profiler.orm.converter.common import CommonMapTypes
22
+ from metadata.profiler.orm.converter.mssql.converter import MssqlMapTypes
22
23
  from metadata.profiler.orm.converter.redshift.converter import RedshiftMapTypes
23
24
  from metadata.profiler.orm.converter.snowflake.converter import SnowflakeMapTypes
24
25
 
@@ -26,3 +27,4 @@ converter_registry = defaultdict(lambda: CommonMapTypes)
26
27
  converter_registry[DatabaseServiceType.BigQuery] = BigqueryMapTypes
27
28
  converter_registry[DatabaseServiceType.Snowflake] = SnowflakeMapTypes
28
29
  converter_registry[DatabaseServiceType.Redshift] = RedshiftMapTypes
30
+ converter_registry[DatabaseServiceType.Mssql] = MssqlMapTypes
@@ -16,10 +16,24 @@ Map Types to convert/cast mssql related data types to relevant data types
16
16
 
17
17
  from sqlalchemy import NVARCHAR, TEXT
18
18
 
19
- from metadata.profiler.orm.registry import CustomImage
19
+ from metadata.profiler.orm.converter.common import CommonMapTypes
20
+ from metadata.profiler.orm.registry import CustomImage, CustomTypes, DataType
20
21
 
21
22
  cast_dict = {
22
23
  CustomImage: "VARBINARY(max)",
23
24
  TEXT: "VARCHAR(max)",
24
25
  NVARCHAR: "NVARCHAR(max)",
25
26
  }
27
+
28
+
29
+ class MssqlMapTypes(CommonMapTypes):
30
+ """
31
+ Mssql type mapper
32
+ """
33
+
34
+ def __init__(self) -> None:
35
+ self._TYPE_MAP.update(
36
+ {
37
+ DataType.TIMESTAMP: CustomTypes.TIMESTAMP.value,
38
+ }
39
+ )
@@ -10,15 +10,13 @@
10
10
  # limitations under the License.
11
11
 
12
12
  """
13
- Define Concat function
13
+ Define MD5 hashing function
14
14
  """
15
- # Keep SQA docs style defining custom constructs
16
-
17
15
  from sqlalchemy.ext.compiler import compiles
18
16
  from sqlalchemy.sql.functions import FunctionElement
19
17
 
20
18
  from metadata.profiler.metrics.core import CACHE
21
- from metadata.profiler.orm.registry import Dialects
19
+ from metadata.profiler.orm.registry import PythonDialects
22
20
  from metadata.utils.logger import profiler_logger
23
21
 
24
22
  logger = profiler_logger()
@@ -33,7 +31,12 @@ def _(element, compiler, **kw):
33
31
  return f"MD5({compiler.process(element.clauses, **kw)})"
34
32
 
35
33
 
36
- @compiles(MD5, Dialects.MSSQL)
34
+ @compiles(MD5, PythonDialects.MSSQL.value)
37
35
  def _(element, compiler, **kw):
38
36
  # TODO requires separate where clauses for each table
39
37
  return f"CONVERT(VARCHAR(8), HashBytes('MD5', {compiler.process(element.clauses, **kw)}), 2)"
38
+
39
+
40
+ @compiles(MD5, PythonDialects.BigQuery.value)
41
+ def _(element, compiler, **kw):
42
+ return f"TO_HEX(MD5(CAST({compiler.process(element.clauses, **kw)} AS STRING)))"
@@ -91,8 +91,8 @@ class AbstractTableMetricComputer(ABC):
91
91
  table (DeclarativeMeta): _description_
92
92
  """
93
93
  try:
94
- self._schema_name = self.table.schema
95
- self._table_name = self.table.name
94
+ self._schema_name = self.runner.schema_name
95
+ self._table_name = self.runner.table_name
96
96
  except AttributeError:
97
97
  raise AttributeError(ERROR_MSG)
98
98
 
@@ -119,10 +119,10 @@ class AbstractTableMetricComputer(ABC):
119
119
  Returns:
120
120
  Tuple[str, int]
121
121
  """
122
- col_names = literal(",".join(inspect(self.table).c.keys()), type_=String).label(
123
- COLUMN_NAMES
124
- )
125
- col_count = literal(len(inspect(self.table).c)).label(COLUMN_COUNT)
122
+ col_names = literal(
123
+ ",".join(inspect(self.runner.raw_dataset).c.keys()), type_=String
124
+ ).label(COLUMN_NAMES)
125
+ col_count = literal(len(inspect(self.runner.raw_dataset).c)).label(COLUMN_COUNT)
126
126
  return col_names, col_count
127
127
 
128
128
  def _build_query(
@@ -45,6 +45,8 @@ class CustomTimestamp(TypeDecorator):
45
45
  import struct # pylint: disable=import-outside-toplevel
46
46
 
47
47
  if dialect.name == "mssql" and isinstance(value, bytes):
48
- bytes_to_int = struct.unpack(">Q", value)[0]
49
- return bytes_to_int
48
+ unpacked_value = struct.unpack("@Q", value)
49
+ if unpacked_value:
50
+ return unpacked_value[0]
51
+ return None
50
52
  return value
@@ -83,6 +83,11 @@ class QueryRunner:
83
83
  """Table name attribute access"""
84
84
  return self.raw_dataset.__table__.schema
85
85
 
86
+ @property
87
+ def session(self):
88
+ """Table name attribute access"""
89
+ return self._session
90
+
86
91
  def _build_query(self, *entities, **kwargs) -> Query:
87
92
  """Build query object
88
93
 
@@ -154,7 +154,6 @@ class ProfilerSource(ProfilerSourceInterface):
154
154
  profile_sample_type=self.source_config.profileSampleType,
155
155
  sampling_method_type=self.source_config.samplingMethodType,
156
156
  ),
157
- default_sample_data_count=self.source_config.sampleDataCount,
158
157
  )
159
158
 
160
159
  profiler_interface: ProfilerInterface = profiler_class.create(
@@ -88,11 +88,7 @@ class SamplerProcessor(Processor):
88
88
  schema_entity=schema_entity,
89
89
  database_entity=database_entity,
90
90
  table_config=get_config_for_table(entity, self.profiler_config),
91
- default_sample_config=SampleConfig(
92
- profile_sample=self.source_config.profileSample,
93
- profile_sample_type=self.source_config.profileSampleType,
94
- sampling_method_type=self.source_config.samplingMethodType,
95
- ),
91
+ default_sample_config=SampleConfig(),
96
92
  default_sample_data_count=self.source_config.sampleDataCount,
97
93
  )
98
94
  sample_data = SampleData(
@@ -0,0 +1,50 @@
1
+ # Copyright 2021 Collate
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ # http://www.apache.org/licenses/LICENSE-2.0
6
+ # Unless required by applicable law or agreed to in writing, software
7
+ # distributed under the License is distributed on an "AS IS" BASIS,
8
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9
+ # See the License for the specific language governing permissions and
10
+ # limitations under the License.
11
+ """
12
+ Helper module to handle data sampling
13
+ for the profiler
14
+ """
15
+
16
+
17
+ from sqlalchemy import Table, text
18
+ from sqlalchemy.sql.selectable import CTE
19
+
20
+ from metadata.generated.schema.entity.data.table import ProfileSampleType
21
+ from metadata.sampler.sqlalchemy.sampler import SQASampler
22
+
23
+
24
+ class MssqlSampler(SQASampler):
25
+ """
26
+ Generates a sample of the data to not
27
+ run the query in the whole table.
28
+ """
29
+
30
+ def set_tablesample(self, selectable: Table):
31
+ """Set the TABLESAMPLE clause for MSSQL
32
+ Args:
33
+ selectable (Table): _description_
34
+ """
35
+ if self.sample_config.profile_sample_type == ProfileSampleType.PERCENTAGE:
36
+ return selectable.tablesample(
37
+ text(f"{self.sample_config.profile_sample or 100} PERCENT")
38
+ )
39
+
40
+ return selectable.tablesample(
41
+ text(f"{int(self.sample_config.profile_sample or 100)} ROWS")
42
+ )
43
+
44
+ def get_sample_query(self, *, column=None) -> CTE:
45
+ """get query for sample data"""
46
+ rnd = self._base_sample_query(column).cte(
47
+ f"{self.raw_dataset.__tablename__}_rnd"
48
+ )
49
+ query = self.client.query(rnd)
50
+ return query.cte(f"{self.raw_dataset.__tablename__}_sample")
@@ -223,8 +223,11 @@ class SQASampler(SamplerInterface, SQAInterfaceMixin):
223
223
  if not is_safe_sql_query(self.sample_query):
224
224
  raise RuntimeError(f"SQL expression is not safe\n\n{self.sample_query}")
225
225
 
226
- return self.client.query(self.raw_dataset).from_statement(
227
- text(f"{self.sample_query}")
226
+ stmt = text(f"{self.sample_query}")
227
+ stmt = stmt.columns(*list(inspect(self.raw_dataset).c))
228
+
229
+ return self.client.query(stmt.subquery()).cte(
230
+ f"{self.raw_dataset.__tablename__}_user_sampled"
228
231
  )
229
232
 
230
233
  def _partitioned_table(self) -> Query:
@@ -66,6 +66,9 @@ from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
66
66
  TestSuitePipeline,
67
67
  )
68
68
  from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
69
+ from metadata.generated.schema.tests.testSuite import (
70
+ ServiceType as TestSuiteServiceType,
71
+ )
69
72
 
70
73
  SERVICE_TYPE_REF = {
71
74
  ServiceType.Api.value: "apiService",
@@ -77,6 +80,8 @@ SERVICE_TYPE_REF = {
77
80
  ServiceType.Metadata.value: "metadataService",
78
81
  ServiceType.Search.value: "searchService",
79
82
  ServiceType.Storage.value: "storageService",
83
+ # We use test suites as "services" for DQ Ingestion Pipelines
84
+ TestSuiteServiceType.TestSuite.value: "testSuite",
80
85
  }
81
86
 
82
87
  SOURCE_CONFIG_TYPE_INGESTION = {
@@ -123,6 +128,8 @@ def get_pipeline_type_from_source_config(source_config: SourceConfig) -> Pipelin
123
128
  def _get_service_type_from( # pylint: disable=inconsistent-return-statements
124
129
  service_subtype: str,
125
130
  ) -> ServiceType:
131
+ if service_subtype.lower() == "testsuite":
132
+ return TestSuiteServiceType.TestSuite
126
133
  for service_type in ServiceType:
127
134
  if service_subtype.lower() in [
128
135
  subtype.value.lower()
@@ -131,7 +131,7 @@ def import_from_module(key: str) -> Type[Any]:
131
131
  try:
132
132
  obj = getattr(importlib.import_module(module_name), obj_name)
133
133
  return obj
134
- except ModuleNotFoundError as err:
134
+ except (ModuleNotFoundError, ImportError) as err:
135
135
  logger.debug(traceback.format_exc())
136
136
  raise DynamicImportException(module=module_name, key=obj_name, cause=err)
137
137
 
@@ -22,6 +22,7 @@ from metadata.generated.schema.entity.services.connections.serviceConnection imp
22
22
  from metadata.generated.schema.entity.services.databaseService import DatabaseService
23
23
  from metadata.generated.schema.tests.testSuite import ServiceType, TestSuite
24
24
  from metadata.ingestion.api.steps import Processor, Sink
25
+ from metadata.ingestion.ometa.utils import model_str
25
26
  from metadata.utils import fqn
26
27
  from metadata.utils.importer import import_sink_class
27
28
  from metadata.utils.logger import test_suite_logger
@@ -40,6 +41,7 @@ class TestSuiteWorkflow(IngestionWorkflow):
40
41
  """
41
42
 
42
43
  __test__ = False
44
+ service_type = ServiceType.TestSuite
43
45
 
44
46
  def set_steps(self):
45
47
  self.source = TestSuiteSource.create(self.config.model_dump(), self.metadata)
@@ -113,6 +115,8 @@ class TestSuiteWorkflow(IngestionWorkflow):
113
115
  fqn=fqn.build(
114
116
  metadata=None,
115
117
  entity_type=TestSuite,
116
- table_fqn=self.config.source.sourceConfig.config.entityFullyQualifiedName,
118
+ table_fqn=model_str(
119
+ self.config.source.sourceConfig.config.entityFullyQualifiedName
120
+ ),
117
121
  ),
118
122
  )