openmetadata-ingestion 1.6.0.0rc1__py3-none-any.whl → 1.6.0.0rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (815) hide show
  1. metadata/cli/classify.py +52 -0
  2. metadata/clients/aws_client.py +2 -2
  3. metadata/cmd.py +9 -0
  4. metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +40 -29
  5. metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -25
  6. metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -50
  7. metadata/data_quality/interface/test_suite_interface.py +45 -56
  8. metadata/data_quality/processor/test_case_runner.py +9 -9
  9. metadata/data_quality/runner/base_test_suite_source.py +52 -26
  10. metadata/data_quality/validations/base_test_handler.py +14 -9
  11. metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
  12. metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
  13. metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
  14. metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
  15. metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
  16. metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
  17. metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
  18. metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
  19. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
  20. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
  21. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
  22. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
  23. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
  24. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
  25. metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
  26. metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
  27. metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
  28. metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
  29. metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +21 -16
  30. metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +7 -1
  31. metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +21 -18
  32. metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
  33. metadata/examples/workflows/bigquery_classifier.yaml +56 -0
  34. metadata/examples/workflows/bigquery_profiler.yaml +1 -2
  35. metadata/examples/workflows/db2_profiler.yaml +1 -2
  36. metadata/examples/workflows/dbtcloud.yaml +2 -1
  37. metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
  38. metadata/examples/workflows/mongodb.yaml +4 -0
  39. metadata/examples/workflows/mysql_profiler.yaml +0 -1
  40. metadata/examples/workflows/redshift_classifier.yaml +38 -0
  41. metadata/examples/workflows/redshift_profiler.yaml +2 -3
  42. metadata/generated/antlr/FqnLexer.py +15 -15
  43. metadata/generated/schema/analytics/__init__.py +1 -1
  44. metadata/generated/schema/analytics/basic.py +1 -1
  45. metadata/generated/schema/analytics/reportData.py +1 -1
  46. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  47. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  48. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  49. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  50. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  51. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  52. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  53. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  54. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  55. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  56. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  57. metadata/generated/schema/api/__init__.py +1 -1
  58. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  59. metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
  60. metadata/generated/schema/api/analytics/__init__.py +1 -1
  61. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  62. metadata/generated/schema/api/automations/__init__.py +1 -1
  63. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  64. metadata/generated/schema/api/bulkAssets.py +1 -1
  65. metadata/generated/schema/api/classification/__init__.py +1 -1
  66. metadata/generated/schema/api/classification/createClassification.py +1 -1
  67. metadata/generated/schema/api/classification/createTag.py +1 -1
  68. metadata/generated/schema/api/classification/loadTags.py +1 -1
  69. metadata/generated/schema/api/createBot.py +1 -1
  70. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  71. metadata/generated/schema/api/createType.py +1 -1
  72. metadata/generated/schema/api/data/__init__.py +1 -1
  73. metadata/generated/schema/api/data/createAPICollection.py +1 -1
  74. metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
  75. metadata/generated/schema/api/data/createChart.py +1 -1
  76. metadata/generated/schema/api/data/createContainer.py +1 -1
  77. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  78. metadata/generated/schema/api/data/createDashboard.py +1 -1
  79. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  80. metadata/generated/schema/api/data/createDatabase.py +1 -1
  81. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  82. metadata/generated/schema/api/data/createGlossary.py +1 -1
  83. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  84. metadata/generated/schema/api/data/createMetric.py +1 -1
  85. metadata/generated/schema/api/data/createMlModel.py +1 -1
  86. metadata/generated/schema/api/data/createPipeline.py +1 -1
  87. metadata/generated/schema/api/data/createQuery.py +1 -1
  88. metadata/generated/schema/api/data/createSearchIndex.py +8 -1
  89. metadata/generated/schema/api/data/createStoredProcedure.py +8 -1
  90. metadata/generated/schema/api/data/createTable.py +1 -1
  91. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  92. metadata/generated/schema/api/data/createTopic.py +1 -1
  93. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  94. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  95. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  96. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  97. metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
  98. metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
  99. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  100. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  101. metadata/generated/schema/api/docStore/__init__.py +1 -1
  102. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  103. metadata/generated/schema/api/domains/__init__.py +1 -1
  104. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  105. metadata/generated/schema/api/domains/createDomain.py +1 -1
  106. metadata/generated/schema/api/feed/__init__.py +1 -1
  107. metadata/generated/schema/api/feed/closeTask.py +1 -1
  108. metadata/generated/schema/api/feed/createPost.py +1 -1
  109. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  110. metadata/generated/schema/api/feed/createThread.py +1 -1
  111. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  112. metadata/generated/schema/api/feed/threadCount.py +1 -1
  113. metadata/generated/schema/api/governance/__init__.py +1 -1
  114. metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
  115. metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
  116. metadata/generated/schema/api/lineage/__init__.py +1 -1
  117. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  118. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  119. metadata/generated/schema/api/policies/__init__.py +1 -1
  120. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  121. metadata/generated/schema/api/services/__init__.py +1 -1
  122. metadata/generated/schema/api/services/createApiService.py +1 -1
  123. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  124. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  125. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  126. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  127. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  128. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  129. metadata/generated/schema/api/services/createSearchService.py +1 -1
  130. metadata/generated/schema/api/services/createStorageService.py +1 -1
  131. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  132. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  133. metadata/generated/schema/api/setOwner.py +1 -1
  134. metadata/generated/schema/api/teams/__init__.py +1 -1
  135. metadata/generated/schema/api/teams/createPersona.py +1 -1
  136. metadata/generated/schema/api/teams/createRole.py +1 -1
  137. metadata/generated/schema/api/teams/createTeam.py +1 -1
  138. metadata/generated/schema/api/teams/createUser.py +1 -1
  139. metadata/generated/schema/api/tests/__init__.py +1 -1
  140. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  141. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  142. metadata/generated/schema/api/tests/createTestCase.py +1 -1
  143. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  144. metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
  145. metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
  146. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  147. metadata/generated/schema/api/voteRequest.py +1 -1
  148. metadata/generated/schema/auth/__init__.py +1 -1
  149. metadata/generated/schema/auth/basicAuth.py +1 -1
  150. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  151. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  152. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  153. metadata/generated/schema/auth/emailRequest.py +1 -1
  154. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  155. metadata/generated/schema/auth/generateToken.py +1 -1
  156. metadata/generated/schema/auth/jwtAuth.py +1 -1
  157. metadata/generated/schema/auth/loginRequest.py +1 -1
  158. metadata/generated/schema/auth/logoutRequest.py +1 -1
  159. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  160. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  161. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  162. metadata/generated/schema/auth/refreshToken.py +1 -1
  163. metadata/generated/schema/auth/registrationRequest.py +1 -1
  164. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  165. metadata/generated/schema/auth/revokeToken.py +1 -1
  166. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  167. metadata/generated/schema/auth/ssoAuth.py +1 -1
  168. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  169. metadata/generated/schema/configuration/__init__.py +1 -1
  170. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  171. metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
  172. metadata/generated/schema/configuration/authConfig.py +1 -1
  173. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  174. metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
  175. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  176. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  177. metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
  178. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  179. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  180. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  181. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  182. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  183. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  184. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  185. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  186. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  187. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  188. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  189. metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
  190. metadata/generated/schema/configuration/lineageSettings.py +1 -1
  191. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  192. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  193. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  194. metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
  195. metadata/generated/schema/configuration/searchSettings.py +1 -1
  196. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
  197. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  198. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  199. metadata/generated/schema/configuration/themeConfiguration.py +1 -1
  200. metadata/generated/schema/configuration/uiThemePreference.py +1 -1
  201. metadata/generated/schema/dataInsight/__init__.py +1 -1
  202. metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
  203. metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
  204. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
  205. metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
  206. metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
  207. metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
  208. metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
  209. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  210. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  211. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  212. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  213. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  214. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  215. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  216. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  217. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  218. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  219. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  220. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  221. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  222. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  223. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  224. metadata/generated/schema/email/__init__.py +1 -1
  225. metadata/generated/schema/email/emailRequest.py +1 -1
  226. metadata/generated/schema/email/emailTemplate.py +1 -1
  227. metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
  228. metadata/generated/schema/email/smtpSettings.py +1 -1
  229. metadata/generated/schema/email/templateValidationReponse.py +1 -1
  230. metadata/generated/schema/entity/__init__.py +1 -1
  231. metadata/generated/schema/entity/applications/__init__.py +1 -1
  232. metadata/generated/schema/entity/applications/app.py +2 -1
  233. metadata/generated/schema/entity/applications/appExtension.py +1 -1
  234. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  235. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  236. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
  237. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  238. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
  239. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
  240. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
  241. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
  242. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
  243. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
  244. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
  245. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
  246. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
  247. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
  248. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
  249. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
  250. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
  251. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
  252. metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
  253. metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
  254. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  255. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  256. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  257. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  258. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  259. metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
  260. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  261. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  262. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  263. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  264. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
  265. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
  266. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  267. metadata/generated/schema/entity/automations/__init__.py +1 -1
  268. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  269. metadata/generated/schema/entity/automations/workflow.py +1 -1
  270. metadata/generated/schema/entity/bot.py +1 -1
  271. metadata/generated/schema/entity/classification/__init__.py +1 -1
  272. metadata/generated/schema/entity/classification/classification.py +1 -1
  273. metadata/generated/schema/entity/classification/tag.py +1 -1
  274. metadata/generated/schema/entity/data/__init__.py +1 -1
  275. metadata/generated/schema/entity/data/apiCollection.py +1 -1
  276. metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
  277. metadata/generated/schema/entity/data/chart.py +1 -1
  278. metadata/generated/schema/entity/data/container.py +1 -1
  279. metadata/generated/schema/entity/data/dashboard.py +1 -1
  280. metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
  281. metadata/generated/schema/entity/data/database.py +1 -1
  282. metadata/generated/schema/entity/data/databaseSchema.py +1 -1
  283. metadata/generated/schema/entity/data/glossary.py +1 -1
  284. metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
  285. metadata/generated/schema/entity/data/metric.py +1 -1
  286. metadata/generated/schema/entity/data/mlmodel.py +1 -1
  287. metadata/generated/schema/entity/data/pipeline.py +1 -1
  288. metadata/generated/schema/entity/data/query.py +1 -1
  289. metadata/generated/schema/entity/data/report.py +1 -1
  290. metadata/generated/schema/entity/data/searchIndex.py +13 -1
  291. metadata/generated/schema/entity/data/storedProcedure.py +13 -1
  292. metadata/generated/schema/entity/data/table.py +1 -1
  293. metadata/generated/schema/entity/data/topic.py +1 -1
  294. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  295. metadata/generated/schema/entity/docStore/document.py +1 -1
  296. metadata/generated/schema/entity/domains/__init__.py +1 -1
  297. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  298. metadata/generated/schema/entity/domains/domain.py +1 -1
  299. metadata/generated/schema/entity/events/__init__.py +1 -1
  300. metadata/generated/schema/entity/events/webhook.py +1 -1
  301. metadata/generated/schema/entity/feed/__init__.py +1 -1
  302. metadata/generated/schema/entity/feed/assets.py +1 -1
  303. metadata/generated/schema/entity/feed/customProperty.py +1 -1
  304. metadata/generated/schema/entity/feed/description.py +1 -1
  305. metadata/generated/schema/entity/feed/domain.py +1 -1
  306. metadata/generated/schema/entity/feed/entityInfo.py +1 -1
  307. metadata/generated/schema/entity/feed/owner.py +1 -1
  308. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  309. metadata/generated/schema/entity/feed/tag.py +1 -1
  310. metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
  311. metadata/generated/schema/entity/feed/thread.py +1 -1
  312. metadata/generated/schema/entity/policies/__init__.py +1 -1
  313. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  314. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
  315. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  316. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  317. metadata/generated/schema/entity/policies/filters.py +1 -1
  318. metadata/generated/schema/entity/policies/policy.py +1 -1
  319. metadata/generated/schema/entity/services/__init__.py +1 -1
  320. metadata/generated/schema/entity/services/apiService.py +1 -1
  321. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  322. metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
  323. metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
  324. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  325. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  326. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  327. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  328. metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
  329. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  330. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  331. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  332. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  333. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  334. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  335. metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
  336. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  337. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
  338. metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
  339. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
  340. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
  341. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
  342. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
  343. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
  344. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
  345. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
  346. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  347. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  348. metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
  349. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  350. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  351. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  352. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  353. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  354. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
  355. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  356. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
  357. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  358. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  359. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  360. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  361. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  362. metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
  363. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  364. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  365. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
  366. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  367. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  368. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  369. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  370. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  371. metadata/generated/schema/entity/services/connections/database/db2Connection.py +13 -1
  372. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
  373. metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
  374. metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
  375. metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
  376. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
  384. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  385. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  386. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  387. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  388. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  389. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  390. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  391. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +6 -2
  395. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
  396. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +6 -3
  397. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  399. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
  401. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  409. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  410. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
  411. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  412. metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
  413. metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  416. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  418. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
  420. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  422. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  423. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  424. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  425. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
  427. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  428. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  429. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  431. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  432. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  434. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  435. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  436. metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
  437. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  438. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  439. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  440. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  441. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  442. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  443. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  444. metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
  446. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
  449. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
  451. metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
  454. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  455. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  456. metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  458. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  459. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  460. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  461. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  462. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  463. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  464. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  466. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  467. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  469. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
  470. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  471. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  472. metadata/generated/schema/entity/services/dashboardService.py +4 -4
  473. metadata/generated/schema/entity/services/databaseService.py +1 -1
  474. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  475. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
  476. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  477. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  478. metadata/generated/schema/entity/services/messagingService.py +1 -1
  479. metadata/generated/schema/entity/services/metadataService.py +1 -1
  480. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  481. metadata/generated/schema/entity/services/pipelineService.py +1 -1
  482. metadata/generated/schema/entity/services/searchService.py +1 -1
  483. metadata/generated/schema/entity/services/serviceType.py +1 -1
  484. metadata/generated/schema/entity/services/storageService.py +1 -1
  485. metadata/generated/schema/entity/teams/__init__.py +1 -1
  486. metadata/generated/schema/entity/teams/persona.py +1 -1
  487. metadata/generated/schema/entity/teams/role.py +1 -1
  488. metadata/generated/schema/entity/teams/team.py +1 -1
  489. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  490. metadata/generated/schema/entity/teams/user.py +1 -1
  491. metadata/generated/schema/entity/type.py +1 -1
  492. metadata/generated/schema/entity/utils/__init__.py +1 -1
  493. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  494. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  495. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  496. metadata/generated/schema/events/__init__.py +1 -1
  497. metadata/generated/schema/events/alertMetrics.py +1 -1
  498. metadata/generated/schema/events/api/__init__.py +1 -1
  499. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  500. metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
  501. metadata/generated/schema/events/api/eventsRecord.py +1 -1
  502. metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
  503. metadata/generated/schema/events/api/typedEvent.py +1 -1
  504. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  505. metadata/generated/schema/events/eventFilterRule.py +1 -1
  506. metadata/generated/schema/events/eventSubscription.py +1 -1
  507. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  508. metadata/generated/schema/events/failedEvent.py +1 -1
  509. metadata/generated/schema/events/failedEventResponse.py +1 -1
  510. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  511. metadata/generated/schema/events/statusContext.py +1 -1
  512. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  513. metadata/generated/schema/events/subscriptionStatus.py +1 -1
  514. metadata/generated/schema/events/testDestinationStatus.py +1 -1
  515. metadata/generated/schema/governance/workflows/__init__.py +1 -1
  516. metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
  517. metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
  518. metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
  519. metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
  520. metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
  521. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
  522. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
  523. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
  524. metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
  525. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
  526. metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
  527. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
  528. metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
  529. metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
  530. metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
  531. metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
  532. metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
  533. metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +2 -2
  534. metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
  535. metadata/generated/schema/governance/workflows/workflowInstance.py +8 -1
  536. metadata/generated/schema/governance/workflows/workflowInstanceState.py +8 -1
  537. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  538. metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
  539. metadata/generated/schema/metadataIngestion/application.py +1 -1
  540. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  541. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
  542. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  543. metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +111 -0
  544. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
  545. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -33
  546. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  547. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  548. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  549. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  550. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  551. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  552. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  553. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  554. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
  555. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
  556. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  557. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  558. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  560. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
  561. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
  562. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  563. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  564. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  565. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  566. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  567. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  568. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  569. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  570. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  571. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  572. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  573. metadata/generated/schema/metadataIngestion/workflow.py +3 -1
  574. metadata/generated/schema/monitoring/__init__.py +1 -1
  575. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  576. metadata/generated/schema/security/__init__.py +1 -1
  577. metadata/generated/schema/security/client/__init__.py +1 -1
  578. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  579. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  580. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  581. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  582. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  583. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  584. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  585. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  586. metadata/generated/schema/security/credentials/__init__.py +1 -1
  587. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  588. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  589. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  590. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  591. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  592. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  593. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  594. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  595. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  596. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  597. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  598. metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
  599. metadata/generated/schema/security/sasl/__init__.py +1 -1
  600. metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
  601. metadata/generated/schema/security/secrets/__init__.py +1 -1
  602. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  603. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  604. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  605. metadata/generated/schema/security/securityConfiguration.py +1 -1
  606. metadata/generated/schema/security/ssl/__init__.py +1 -1
  607. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
  608. metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
  609. metadata/generated/schema/settings/__init__.py +1 -1
  610. metadata/generated/schema/settings/settings.py +1 -1
  611. metadata/generated/schema/system/__init__.py +1 -1
  612. metadata/generated/schema/system/entityError.py +1 -1
  613. metadata/generated/schema/system/eventPublisherJob.py +2 -1
  614. metadata/generated/schema/system/indexingError.py +1 -1
  615. metadata/generated/schema/system/limitsResponse.py +1 -1
  616. metadata/generated/schema/system/ui/__init__.py +1 -1
  617. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  618. metadata/generated/schema/system/ui/page.py +1 -1
  619. metadata/generated/schema/system/validationResponse.py +1 -1
  620. metadata/generated/schema/tests/__init__.py +1 -1
  621. metadata/generated/schema/tests/assigned.py +1 -1
  622. metadata/generated/schema/tests/basic.py +1 -1
  623. metadata/generated/schema/tests/customMetric.py +1 -1
  624. metadata/generated/schema/tests/dataQualityReport.py +1 -1
  625. metadata/generated/schema/tests/resolved.py +1 -1
  626. metadata/generated/schema/tests/testCase.py +1 -1
  627. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  628. metadata/generated/schema/tests/testDefinition.py +1 -1
  629. metadata/generated/schema/tests/testSuite.py +1 -1
  630. metadata/generated/schema/type/__init__.py +1 -1
  631. metadata/generated/schema/type/apiSchema.py +1 -1
  632. metadata/generated/schema/type/assetCertification.py +1 -1
  633. metadata/generated/schema/type/auditLog.py +1 -1
  634. metadata/generated/schema/type/basic.py +1 -1
  635. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  636. metadata/generated/schema/type/changeEvent.py +1 -1
  637. metadata/generated/schema/type/changeEventType.py +1 -1
  638. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  639. metadata/generated/schema/type/csvDocumentation.py +1 -1
  640. metadata/generated/schema/type/csvErrorType.py +1 -1
  641. metadata/generated/schema/type/csvFile.py +1 -1
  642. metadata/generated/schema/type/csvImportResult.py +1 -1
  643. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  644. metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
  645. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  646. metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
  647. metadata/generated/schema/type/customProperty.py +1 -1
  648. metadata/generated/schema/type/dailyCount.py +1 -1
  649. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  650. metadata/generated/schema/type/entityHierarchy.py +1 -1
  651. metadata/generated/schema/type/entityHistory.py +1 -1
  652. metadata/generated/schema/type/entityLineage.py +1 -1
  653. metadata/generated/schema/type/entityReference.py +1 -1
  654. metadata/generated/schema/type/entityReferenceList.py +1 -1
  655. metadata/generated/schema/type/entityRelationship.py +2 -1
  656. metadata/generated/schema/type/entityUsage.py +1 -1
  657. metadata/generated/schema/type/filterPattern.py +1 -1
  658. metadata/generated/schema/type/function.py +1 -1
  659. metadata/generated/schema/type/include.py +1 -1
  660. metadata/generated/schema/type/jdbcConnection.py +1 -1
  661. metadata/generated/schema/type/lifeCycle.py +1 -1
  662. metadata/generated/schema/type/paging.py +1 -1
  663. metadata/generated/schema/type/profile.py +1 -1
  664. metadata/generated/schema/type/queryParserData.py +1 -1
  665. metadata/generated/schema/type/reaction.py +1 -1
  666. metadata/generated/schema/type/schedule.py +1 -1
  667. metadata/generated/schema/type/schema.py +1 -1
  668. metadata/generated/schema/type/tableQuery.py +1 -1
  669. metadata/generated/schema/type/tableUsageCount.py +1 -1
  670. metadata/generated/schema/type/tagLabel.py +1 -1
  671. metadata/generated/schema/type/usageDetails.py +1 -1
  672. metadata/generated/schema/type/usageRequest.py +1 -1
  673. metadata/generated/schema/type/votes.py +1 -1
  674. metadata/ingestion/api/models.py +9 -5
  675. metadata/ingestion/lineage/parser.py +6 -4
  676. metadata/ingestion/lineage/sql_lineage.py +258 -29
  677. metadata/ingestion/models/patch_request.py +2 -0
  678. metadata/ingestion/ometa/client.py +13 -8
  679. metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -2
  680. metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +5 -3
  681. metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
  682. metadata/ingestion/ometa/mixins/table_mixin.py +1 -1
  683. metadata/ingestion/ometa/models.py +1 -0
  684. metadata/ingestion/ometa/ometa_api.py +6 -3
  685. metadata/ingestion/sink/metadata_rest.py +23 -18
  686. metadata/ingestion/source/dashboard/dashboard_service.py +2 -0
  687. metadata/ingestion/source/dashboard/microstrategy/client.py +268 -0
  688. metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +11 -11
  689. metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +74 -51
  690. metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +9 -3
  691. metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
  692. metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
  693. metadata/ingestion/source/dashboard/superset/db_source.py +1 -1
  694. metadata/ingestion/source/dashboard/tableau/metadata.py +29 -12
  695. metadata/ingestion/source/dashboard/tableau/models.py +10 -0
  696. metadata/ingestion/source/database/athena/metadata.py +8 -0
  697. metadata/ingestion/source/database/azuresql/service_spec.py +2 -0
  698. metadata/ingestion/source/database/bigquery/profiler/profiler.py +1 -1
  699. metadata/ingestion/source/database/bigquery/profiler/system.py +5 -4
  700. metadata/ingestion/source/database/bigquery/service_spec.py +2 -0
  701. metadata/ingestion/source/database/common_nosql_source.py +7 -0
  702. metadata/ingestion/source/database/databricks/metadata.py +1 -3
  703. metadata/ingestion/source/database/databricks/service_spec.py +4 -0
  704. metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
  705. metadata/ingestion/source/database/datalake/clients/base.py +10 -1
  706. metadata/ingestion/source/database/datalake/clients/gcs.py +4 -2
  707. metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
  708. metadata/ingestion/source/database/datalake/service_spec.py +8 -1
  709. metadata/ingestion/source/database/db2/connection.py +14 -1
  710. metadata/ingestion/source/database/dbt/constants.py +12 -0
  711. metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
  712. metadata/ingestion/source/database/dbt/dbt_service.py +25 -1
  713. metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
  714. metadata/ingestion/source/database/dbt/metadata.py +70 -1
  715. metadata/ingestion/source/database/dbt/models.py +2 -0
  716. metadata/ingestion/source/database/dynamodb/service_spec.py +4 -1
  717. metadata/ingestion/source/database/mongodb/service_spec.py +4 -1
  718. metadata/ingestion/source/database/mysql/connection.py +5 -0
  719. metadata/ingestion/source/database/mysql/lineage.py +12 -30
  720. metadata/ingestion/source/database/mysql/queries.py +43 -0
  721. metadata/ingestion/source/database/mysql/query_parser.py +46 -0
  722. metadata/ingestion/source/database/mysql/service_spec.py +7 -1
  723. metadata/{data_quality/builders/pandas_validator_builder.py → ingestion/source/database/mysql/usage.py} +9 -14
  724. metadata/ingestion/source/database/redshift/profiler/system.py +5 -4
  725. metadata/ingestion/source/database/sample_data.py +4 -12
  726. metadata/ingestion/source/database/snowflake/connection.py +8 -3
  727. metadata/ingestion/source/database/snowflake/lineage.py +35 -1
  728. metadata/ingestion/source/database/snowflake/metadata.py +38 -19
  729. metadata/ingestion/source/database/snowflake/models.py +1 -0
  730. metadata/ingestion/source/database/snowflake/profiler/system.py +5 -4
  731. metadata/ingestion/source/database/snowflake/queries.py +22 -1
  732. metadata/ingestion/source/database/snowflake/service_spec.py +6 -0
  733. metadata/ingestion/source/database/stored_procedures_mixin.py +20 -8
  734. metadata/ingestion/source/database/trino/service_spec.py +2 -0
  735. metadata/ingestion/source/database/unitycatalog/service_spec.py +4 -0
  736. metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
  737. metadata/ingestion/source/pipeline/airflow/metadata.py +35 -25
  738. metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
  739. metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
  740. metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
  741. metadata/ingestion/source/pipeline/fivetran/metadata.py +1 -1
  742. metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
  743. metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
  744. metadata/ingestion/source/search/search_service.py +44 -1
  745. metadata/mixins/pandas/pandas_mixin.py +2 -31
  746. metadata/mixins/sqalchemy/sqa_mixin.py +16 -16
  747. metadata/pii/processor.py +10 -9
  748. metadata/profiler/api/models.py +3 -79
  749. metadata/profiler/config.py +39 -0
  750. metadata/profiler/interface/nosql/profiler_interface.py +1 -26
  751. metadata/profiler/interface/pandas/profiler_interface.py +37 -77
  752. metadata/profiler/interface/profiler_interface.py +10 -282
  753. metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
  754. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
  755. metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
  756. metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +2 -2
  757. metadata/profiler/interface/sqlalchemy/profiler_interface.py +46 -109
  758. metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
  759. metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
  760. metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +6 -8
  761. metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
  762. metadata/profiler/orm/converter/base.py +21 -12
  763. metadata/profiler/orm/functions/table_metric_computer.py +9 -8
  764. metadata/profiler/processor/core.py +5 -58
  765. metadata/profiler/processor/handle_partition.py +0 -48
  766. metadata/profiler/processor/runner.py +116 -35
  767. metadata/profiler/processor/sample_data_handler.py +7 -4
  768. metadata/profiler/source/database/base/profiler_source.py +56 -138
  769. metadata/profiler/source/database/bigquery/profiler_source.py +3 -3
  770. metadata/profiler/source/database/databricks/profiler_source.py +2 -3
  771. metadata/profiler/source/fetcher/config.py +44 -0
  772. metadata/profiler/source/fetcher/fetcher_strategy.py +2 -4
  773. metadata/sampler/config.py +237 -0
  774. metadata/sampler/models.py +106 -0
  775. metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
  776. metadata/sampler/pandas/sampler.py +239 -0
  777. metadata/{utils → sampler}/partition.py +66 -51
  778. metadata/sampler/processor.py +154 -0
  779. metadata/sampler/sampler_interface.py +251 -0
  780. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
  781. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
  782. metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
  783. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +93 -98
  784. metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
  785. metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
  786. metadata/utils/class_helper.py +7 -0
  787. metadata/utils/constants.py +63 -0
  788. metadata/utils/helpers.py +1 -2
  789. metadata/utils/importer.py +1 -1
  790. metadata/utils/logger.py +9 -0
  791. metadata/utils/profiler_utils.py +42 -0
  792. metadata/utils/service_spec/default.py +6 -0
  793. metadata/utils/service_spec/service_spec.py +47 -0
  794. metadata/utils/sqlalchemy_utils.py +1 -1
  795. metadata/utils/ssl_manager.py +39 -1
  796. metadata/workflow/classification.py +53 -0
  797. metadata/workflow/data_quality.py +5 -1
  798. metadata/workflow/profiler.py +1 -17
  799. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/METADATA +373 -373
  800. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/RECORD +805 -795
  801. metadata/data_quality/builders/sqa_validator_builder.py +0 -25
  802. metadata/data_quality/interface/test_suite_interface_factory.py +0 -158
  803. metadata/data_quality/runner/test_suite_source_factory.py +0 -38
  804. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
  805. metadata/ingestion/source/dashboard/mstr/client.py +0 -209
  806. metadata/ingestion/source/dashboard/mstr/service_spec.py +0 -4
  807. metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
  808. metadata/profiler/processor/sampler/sampler_factory.py +0 -100
  809. metadata/profiler/processor/sampler/sampler_interface.py +0 -74
  810. metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -83
  811. /metadata/ingestion/source/dashboard/{mstr → microstrategy}/__init__.py +0 -0
  812. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/LICENSE +0 -0
  813. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/WHEEL +0 -0
  814. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/entry_points.txt +0 -0
  815. {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/top_level.txt +0 -0
@@ -77,11 +77,11 @@ class MariaDBProfilerInterface(SQAProfilerInterface):
77
77
  return dict(row)
78
78
  except ProgrammingError:
79
79
  logger.info(
80
- f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to overflow"
80
+ f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
81
81
  )
82
82
  return None
83
83
 
84
84
  except Exception as exc:
85
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
85
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
86
86
  handle_query_exception(msg, exc, session)
87
87
  return None
@@ -21,7 +21,7 @@ import threading
21
21
  import traceback
22
22
  from collections import defaultdict
23
23
  from datetime import datetime
24
- from typing import Any, Dict, List, Optional, Type
24
+ from typing import Any, Dict, List, Optional, Type, Union
25
25
 
26
26
  from sqlalchemy import Column, inspect, text
27
27
  from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
@@ -30,10 +30,18 @@ from sqlalchemy.orm import scoped_session
30
30
  from metadata.generated.schema.entity.data.table import (
31
31
  CustomMetricProfile,
32
32
  SystemProfile,
33
- TableData,
33
+ Table,
34
+ )
35
+ from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
36
+ DatalakeConnection,
37
+ )
38
+ from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
39
+ from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
40
+ DatabaseServiceProfilerPipeline,
34
41
  )
35
42
  from metadata.generated.schema.tests.customMetric import CustomMetric
36
43
  from metadata.ingestion.connections.session import create_and_bind_thread_safe_session
44
+ from metadata.ingestion.ometa.ometa_api import OpenMetadata
37
45
  from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
38
46
  from metadata.profiler.api.models import ThreadPoolMetrics
39
47
  from metadata.profiler.interface.profiler_interface import ProfilerInterface
@@ -47,7 +55,7 @@ from metadata.profiler.orm.functions.table_metric_computer import TableMetricCom
47
55
  from metadata.profiler.orm.registry import Dialects
48
56
  from metadata.profiler.processor.metric_filter import MetricFilter
49
57
  from metadata.profiler.processor.runner import QueryRunner
50
- from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
58
+ from metadata.sampler.sampler_interface import SamplerInterface
51
59
  from metadata.utils.custom_thread_pool import CustomThreadPoolExecutor
52
60
  from metadata.utils.helpers import is_safe_sql_query
53
61
  from metadata.utils.logger import profiler_interface_registry_logger
@@ -75,40 +83,32 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
75
83
  """
76
84
 
77
85
  # pylint: disable=too-many-arguments
78
-
79
86
  def __init__(
80
87
  self,
81
- service_connection_config,
82
- ometa_client,
83
- entity,
84
- storage_config,
85
- profile_sample_config,
86
- source_config,
87
- sample_query,
88
- table_partition_config,
88
+ service_connection_config: Union[DatabaseConnection, DatalakeConnection],
89
+ ometa_client: OpenMetadata,
90
+ entity: Table,
91
+ source_config: DatabaseServiceProfilerPipeline,
92
+ sampler: SamplerInterface,
89
93
  thread_count: int = 5,
90
94
  timeout_seconds: int = 43200,
91
- sqa_metadata=None,
92
- sample_data_count: Optional[int] = SAMPLE_DATA_DEFAULT_COUNT,
93
95
  **kwargs,
94
96
  ):
95
97
  """Instantiate SQA Interface object"""
98
+ self.session_factory = None
99
+ self.session = None
96
100
 
97
101
  super().__init__(
98
- service_connection_config,
99
- ometa_client,
100
- entity,
101
- storage_config,
102
- profile_sample_config,
103
- source_config,
104
- sample_query,
105
- table_partition_config,
106
- thread_count,
107
- timeout_seconds,
108
- sample_data_count,
102
+ service_connection_config=service_connection_config,
103
+ ometa_client=ometa_client,
104
+ entity=entity,
105
+ source_config=source_config,
106
+ sampler=sampler,
107
+ thread_count=thread_count,
108
+ timeout_seconds=timeout_seconds,
109
109
  )
110
110
 
111
- self._table = self._convert_table_to_orm_object(sqa_metadata)
111
+ self._table = self.sampler.raw_dataset
112
112
  self.create_session()
113
113
  self.system_metrics_computer = self.initialize_system_metrics_computer()
114
114
 
@@ -126,25 +126,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
126
126
  def table(self):
127
127
  return self._table
128
128
 
129
- def _get_sampler(self, **kwargs):
130
- """get sampler object"""
131
- from metadata.profiler.processor.sampler.sampler_factory import ( # pylint: disable=import-outside-toplevel
132
- sampler_factory_,
133
- )
134
-
135
- session = kwargs.get("session")
136
- table = kwargs["table"]
137
-
138
- return sampler_factory_.create(
139
- self.service_connection_config.__class__.__name__,
140
- client=session or self.session,
141
- table=table,
142
- profile_sample_config=self.profile_sample_config,
143
- partition_details=self.partition_details,
144
- profile_sample_query=self.profile_query,
145
- sample_data_count=self.sample_data_count,
146
- )
147
-
148
129
  def _session_factory(self) -> scoped_session:
149
130
  """Create thread safe session that will be automatically
150
131
  garbage collected once the application thread ends
@@ -175,7 +156,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
175
156
  )
176
157
  return dict(row)
177
158
  except Exception as exc:
178
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
159
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
179
160
  handle_query_exception(msg, exc, session)
180
161
  return None
181
162
 
@@ -213,7 +194,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
213
194
  except Exception as exc:
214
195
  logger.debug(traceback.format_exc())
215
196
  logger.warning(
216
- f"Error trying to compute profile for {runner.table.__tablename__}: {exc}" # type: ignore
197
+ f"Error trying to compute profile for {runner.table_name}: {exc}" # type: ignore
217
198
  )
218
199
  session.rollback()
219
200
  raise RuntimeError(exc)
@@ -250,7 +231,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
250
231
  runner, column, exc, session, metrics
251
232
  )
252
233
  except Exception as exc:
253
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
234
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
254
235
  handle_query_exception(msg, exc, session)
255
236
  return None
256
237
 
@@ -293,10 +274,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
293
274
  runner._session.get_bind().dialect.name
294
275
  != Dialects.Druid
295
276
  ):
296
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
277
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
297
278
  handle_query_exception(msg, exc, session)
298
279
  except Exception as exc:
299
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
280
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
300
281
  handle_query_exception(msg, exc, session)
301
282
  return None
302
283
 
@@ -329,10 +310,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
329
310
  return dict(row)
330
311
  except ProgrammingError as exc:
331
312
  logger.info(
332
- f"Skipping metrics for {runner.table.__tablename__}.{column.name} due to {exc}"
313
+ f"Skipping metrics for {runner.table_name}.{column.name} due to {exc}"
333
314
  )
334
315
  except Exception as exc:
335
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
316
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
336
317
  handle_query_exception(msg, exc, session)
337
318
  return None
338
319
 
@@ -366,7 +347,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
366
347
  )
367
348
 
368
349
  except Exception as exc:
369
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{metric.columnName}: {exc}"
350
+ msg = f"Error trying to compute profile for {runner.table_name}.{metric.columnName}: {exc}"
370
351
  logger.debug(traceback.format_exc())
371
352
  logger.warning(msg)
372
353
  if custom_metrics:
@@ -390,39 +371,21 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
390
371
  Returns:
391
372
  dictionnary of results
392
373
  """
393
- logger.debug(f"Computing system metrics for {runner.table.__tablename__}")
394
- return self.system_metrics_computer.get_system_metrics(table=runner.table)
374
+ logger.debug(f"Computing system metrics for {runner.table_name}")
375
+ return self.system_metrics_computer.get_system_metrics(runner=runner)
395
376
 
396
- def _create_thread_safe_sampler(
397
- self,
398
- session,
399
- table,
400
- ):
401
- """Create thread safe runner"""
402
- if not hasattr(thread_local, "sampler"):
403
- thread_local.sampler = self._get_sampler(
404
- table=table,
405
- session=session,
406
- )
407
- return thread_local.sampler
408
-
409
- def _create_thread_safe_runner(
410
- self,
411
- session,
412
- table,
413
- sample,
414
- ):
377
+ def _create_thread_safe_runner(self, session, column=None):
415
378
  """Create thread safe runner"""
416
379
  if not hasattr(thread_local, "runner"):
417
380
  thread_local.runner = QueryRunner(
418
381
  session=session,
419
- table=table,
420
- sample=sample,
421
- partition_details=self.partition_details,
422
- profile_sample_query=self.profile_query,
382
+ dataset=self.sampler.get_dataset(column=column),
383
+ raw_dataset=self.sampler.raw_dataset,
384
+ partition_details=self.sampler.partition_details,
385
+ profile_sample_query=self.sampler.sample_query,
423
386
  )
424
387
  return thread_local.runner
425
- thread_local.runner._sample = sample # pylint: disable=protected-access
388
+ thread_local.runner.dataset = self.sampler.get_dataset(column=column)
426
389
  return thread_local.runner
427
390
 
428
391
  def compute_metrics_in_thread(
@@ -437,16 +400,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
437
400
  with Session() as session:
438
401
  self.set_session_tag(session)
439
402
  self.set_catalog(session)
440
- sampler = self._create_thread_safe_sampler(
441
- session,
442
- metric_func.table,
443
- )
444
- sample = sampler.random_sample(metric_func.column)
445
- runner = self._create_thread_safe_runner(
446
- session,
447
- metric_func.table,
448
- sample,
449
- )
403
+ runner = self._create_thread_safe_runner(session, metric_func.column)
450
404
  row = None
451
405
  try:
452
406
  row = self._get_metric_fn[metric_func.metric_type.value](
@@ -454,7 +408,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
454
408
  runner=runner,
455
409
  session=session,
456
410
  column=metric_func.column,
457
- sample=sample,
411
+ sample=runner.dataset,
458
412
  )
459
413
  if isinstance(row, dict):
460
414
  row = self._validate_nulls(row)
@@ -546,21 +500,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
546
500
 
547
501
  return profile_results
548
502
 
549
- def fetch_sample_data(self, table, columns) -> TableData:
550
- """Fetch sample data from database
551
-
552
- Args:
553
- table: ORM declarative table
554
-
555
- Returns:
556
- TableData: sample table data
557
- """
558
- sampler = self._get_sampler(
559
- table=table,
560
- )
561
-
562
- return sampler.fetch_sample_data(columns)
563
-
564
503
  def get_composed_metrics(
565
504
  self, column: Column, metric: Metrics, column_results: Dict
566
505
  ):
@@ -586,7 +525,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
586
525
  column: Column,
587
526
  metric: Type[HybridMetric],
588
527
  column_results: Dict[str, Any],
589
- **kwargs,
590
528
  ):
591
529
  """Given a list of metrics, compute the given results
592
530
  and returns the values
@@ -598,10 +536,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
598
536
  Returns:
599
537
  dictionnary of results
600
538
  """
601
- sampler = self._get_sampler(table=kwargs.get("table"))
602
- sample = sampler.random_sample(column)
539
+ dataset = self.sampler.get_dataset(column=column)
603
540
  try:
604
- return metric(column).fn(sample, column_results, self.session)
541
+ return metric(column).fn(dataset, column_results, self.session)
605
542
  except Exception as exc:
606
543
  logger.debug(traceback.format_exc())
607
544
  logger.warning(f"Unexpected exception computing metrics: {exc}")
@@ -76,11 +76,11 @@ class SingleStoreProfilerInterface(SQAProfilerInterface):
76
76
  return dict(row)
77
77
  except ProgrammingError:
78
78
  logger.info(
79
- f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to overflow"
79
+ f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
80
80
  )
81
81
  return None
82
82
 
83
83
  except Exception as exc:
84
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
84
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
85
85
  handle_query_exception(msg, exc, session)
86
86
  return None
@@ -41,7 +41,7 @@ class SnowflakeProfilerInterface(SQAProfilerInterface):
41
41
  session.bind.dialect.name
42
42
  ):
43
43
  logger.info(
44
- f"Computing metrics without sum for {runner.table.__tablename__}.{column.name}"
44
+ f"Computing metrics without sum for {runner.table_name}.{column.name}"
45
45
  )
46
46
  return self._compute_static_metrics_wo_sum(metrics, runner, session, column)
47
47
  return None
@@ -79,8 +79,8 @@ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
79
79
  list,
80
80
  partition(self.is_statistic_metric, metrics),
81
81
  )
82
- schema = runner.table.__table_args__["schema"]
83
- table_name = runner.table.__tablename__
82
+ schema = runner.schema_name
83
+ table_name = runner.table_name
84
84
  logger.debug(
85
85
  "Getting statistics for column: %s.%s.%s",
86
86
  schema,
@@ -118,8 +118,8 @@ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
118
118
  list,
119
119
  partition(self.is_statistic_metric, metrics),
120
120
  )
121
- schema = runner.table.__table_args__["schema"]
122
- table_name = runner.table.__tablename__
121
+ schema = runner.schema_name
122
+ table_name = runner.table_name
123
123
  logger.debug("Geting statistics for table: %s.%s", schema, table_name)
124
124
  result.update(
125
125
  super().get_table_statistics(stat_metrics, schema, table_name)
@@ -135,12 +135,10 @@ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
135
135
  result.update(super_table_metrics)
136
136
  return result
137
137
 
138
- def get_hybrid_metrics(
139
- self, column: Column, metric: Metric, column_results: Dict, **kwargs
140
- ):
138
+ def get_hybrid_metrics(self, column: Column, metric: Metric, column_results: Dict):
141
139
  # this metrics might have been computed in a previous step
142
140
  return column_results.get(metric.name()) or super().get_hybrid_metrics(
143
- column, metric, column_results, **kwargs
141
+ column, metric, column_results
144
142
  )
145
143
 
146
144
  def is_statistic_metric(self, metric: Metric) -> bool:
@@ -76,11 +76,11 @@ class TrinoProfilerInterface(ProfilerWithStatistics, TrinoStoredStatisticsSource
76
76
  return dict(row)
77
77
  except ProgrammingError as err:
78
78
  logger.info(
79
- f"Skipping window metrics for {runner.table.__tablename__}.{column.name} due to {err}"
79
+ f"Skipping window metrics for {runner.table_name}.{column.name} due to {err}"
80
80
  )
81
81
  return None
82
82
 
83
83
  except Exception as exc:
84
- msg = f"Error trying to compute profile for {runner.table.__tablename__}.{column.name}: {exc}"
84
+ msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
85
85
  handle_query_exception(msg, exc, session)
86
86
  return None
@@ -109,10 +109,29 @@ def ometa_to_sqa_orm(
109
109
  We are building the class dynamically using
110
110
  `type` and passing SQLAlchemy `Base` class
111
111
  as the bases tuple for inheritance.
112
+
113
+ Args:
114
+ table (Table): OpenMetadata Table instance
115
+ metadata (OpenMetadata): OpenMetadata connection
116
+ sqa_metadata_obj (MetaData): For advanced use cases, you can pass a custom MetaData object. For most cases, this
117
+ can be left as None so that the global_metadata object is used.
112
118
  """
119
+ _metadata = sqa_metadata_obj or Base.metadata
113
120
  table.serviceType = cast(
114
121
  databaseService.DatabaseServiceType, table.serviceType
115
122
  ) # satisfy mypy
123
+
124
+ orm_database_name = get_orm_database(table, metadata)
125
+ # SQLite does not support schemas
126
+ orm_schema_name = (
127
+ get_orm_schema(table, metadata)
128
+ if table.serviceType != databaseService.DatabaseServiceType.SQLite
129
+ else None
130
+ )
131
+ orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.root}".replace(
132
+ ".", "_"
133
+ )
134
+
116
135
  cols = {
117
136
  (
118
137
  col.name.root + "_"
@@ -122,12 +141,6 @@ def ometa_to_sqa_orm(
122
141
  for idx, col in enumerate(table.columns)
123
142
  }
124
143
 
125
- orm_database_name = get_orm_database(table, metadata)
126
- orm_schema_name = get_orm_schema(table, metadata)
127
- orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.root}".replace(
128
- ".", "_"
129
- )
130
-
131
144
  # Type takes positional arguments in the form of (name, bases, dict)
132
145
  orm = type(
133
146
  orm_name, # Output class name
@@ -135,10 +148,7 @@ def ometa_to_sqa_orm(
135
148
  {
136
149
  "__tablename__": str(table.name.root),
137
150
  "__table_args__": {
138
- # SQLite does not support schemas
139
- "schema": orm_schema_name
140
- if table.serviceType != databaseService.DatabaseServiceType.SQLite
141
- else None,
151
+ "schema": orm_schema_name,
142
152
  "extend_existing": True, # Recreates the table ORM object if it already exists. Useful for testing
143
153
  "quote": check_snowflake_case_sensitive(
144
154
  table.serviceType, table.name.root
@@ -146,13 +156,12 @@ def ometa_to_sqa_orm(
146
156
  or None,
147
157
  },
148
158
  **cols,
149
- "metadata": sqa_metadata_obj or Base.metadata,
159
+ "metadata": _metadata,
150
160
  },
151
161
  )
152
162
 
153
163
  if not isinstance(orm, DeclarativeMeta):
154
164
  raise ValueError("OMeta to ORM did not create a DeclarativeMeta")
155
-
156
165
  return orm
157
166
 
158
167
 
@@ -53,7 +53,7 @@ class AbstractTableMetricComputer(ABC):
53
53
  self._metrics = metrics
54
54
  self._conn_config = conn_config
55
55
  self._database = self._runner._session.get_bind().url.database
56
- self._table = self._runner.table
56
+ self._table = self._runner.dataset
57
57
  self._entity = entity
58
58
 
59
59
  @property
@@ -91,8 +91,8 @@ class AbstractTableMetricComputer(ABC):
91
91
  table (DeclarativeMeta): _description_
92
92
  """
93
93
  try:
94
- self._schema_name = self.table.__table_args__.get("schema")
95
- self._table_name = self.table.__tablename__
94
+ self._schema_name = self.runner.schema_name
95
+ self._table_name = self.runner.table_name
96
96
  except AttributeError:
97
97
  raise AttributeError(ERROR_MSG)
98
98
 
@@ -119,10 +119,10 @@ class AbstractTableMetricComputer(ABC):
119
119
  Returns:
120
120
  Tuple[str, int]
121
121
  """
122
- col_names = literal(",".join(inspect(self.table).c.keys()), type_=String).label(
123
- COLUMN_NAMES
124
- )
125
- col_count = literal(len(inspect(self.table).c)).label(COLUMN_COUNT)
122
+ col_names = literal(
123
+ ",".join(inspect(self.runner.raw_dataset).c.keys()), type_=String
124
+ ).label(COLUMN_NAMES)
125
+ col_count = literal(len(inspect(self.runner.raw_dataset).c)).label(COLUMN_COUNT)
126
126
  return col_names, col_count
127
127
 
128
128
  def _build_query(
@@ -147,7 +147,8 @@ class BaseTableMetricComputer(AbstractTableMetricComputer):
147
147
  """Base table computer"""
148
148
 
149
149
  def compute(self):
150
- """Default compute behavior for table metrics"""
150
+ """Default compute behavior for table metrics. This method will use the raw table
151
+ to compute metrics and omit any sampling or partitioning logic."""
151
152
  return self.runner.select_first_from_table(
152
153
  *[metric().fn() for metric in self.metrics]
153
154
  )
@@ -40,7 +40,7 @@ from metadata.generated.schema.tests.customMetric import (
40
40
  CustomMetric as CustomMetricEntity,
41
41
  )
42
42
  from metadata.generated.schema.type.basic import Timestamp
43
- from metadata.profiler.api.models import ProfilerResponse, SampleData, ThreadPoolMetrics
43
+ from metadata.profiler.api.models import ProfilerResponse, ThreadPoolMetrics
44
44
  from metadata.profiler.interface.profiler_interface import ProfilerInterface
45
45
  from metadata.profiler.metrics.core import (
46
46
  ComposedMetric,
@@ -54,9 +54,6 @@ from metadata.profiler.metrics.static.row_count import RowCount
54
54
  from metadata.profiler.orm.functions.table_metric_computer import CREATE_DATETIME
55
55
  from metadata.profiler.orm.registry import NOT_COMPUTE
56
56
  from metadata.profiler.processor.metric_filter import MetricFilter
57
- from metadata.profiler.processor.sample_data_handler import upload_sample_data
58
- from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
59
- from metadata.utils.execution_time_tracker import calculate_execution_time
60
57
  from metadata.utils.logger import profiler_logger
61
58
 
62
59
  logger = profiler_logger()
@@ -107,7 +104,6 @@ class Profiler(Generic[TMetric]):
107
104
  self.exclude_columns = exclude_columns
108
105
  self._metrics = metrics
109
106
  self._profile_ts = Timestamp(int(datetime.now().timestamp() * 1000))
110
- self.profile_sample_config = self.profiler_interface.profile_sample_config
111
107
 
112
108
  self.metric_filter = MetricFilter(
113
109
  metrics=self.metrics,
@@ -249,11 +245,6 @@ class Profiler(Generic[TMetric]):
249
245
  return column.customMetrics or None
250
246
  return None
251
247
 
252
- @property
253
- def sample(self):
254
- """Return the sample used for the profiler"""
255
- return self.profiler_interface.sample
256
-
257
248
  def validate_composed_metric(self) -> None:
258
249
  """
259
250
  Make sure that all composed metrics have
@@ -317,7 +308,6 @@ class Profiler(Generic[TMetric]):
317
308
  col,
318
309
  metric,
319
310
  current_col_results,
320
- table=self.table,
321
311
  )
322
312
 
323
313
  def _prepare_table_metrics(self) -> List:
@@ -495,16 +485,6 @@ class Profiler(Generic[TMetric]):
495
485
  )
496
486
  self.compute_metrics()
497
487
 
498
- # We need the sample data for Sample Data or PII Sensitive processing.
499
- # We'll nullify the Sample Data after the PII processing so that it's not stored.
500
- if (
501
- self.source_config.generateSampleData
502
- or self.source_config.processPiiSensitive
503
- ):
504
- sample_data = self.generate_sample_data()
505
- else:
506
- sample_data = None
507
-
508
488
  profile = self.get_profile()
509
489
  if self.source_config.computeMetrics:
510
490
  self._check_profile_and_handle(profile)
@@ -512,43 +492,10 @@ class Profiler(Generic[TMetric]):
512
492
  table_profile = ProfilerResponse(
513
493
  table=self.profiler_interface.table_entity,
514
494
  profile=profile,
515
- sample_data=sample_data,
516
495
  )
517
496
 
518
497
  return table_profile
519
498
 
520
- @calculate_execution_time(store=False)
521
- def generate_sample_data(self) -> Optional[SampleData]:
522
- """Fetch and ingest sample data
523
-
524
- Returns:
525
- TableData: sample data
526
- """
527
- try:
528
- logger.debug(
529
- "Fetching sample data for "
530
- f"{self.profiler_interface.table_entity.fullyQualifiedName.root}..." # type: ignore
531
- )
532
- table_data = self.profiler_interface.fetch_sample_data(
533
- self.table, self.columns
534
- )
535
- upload_sample_data(
536
- data=table_data, profiler_interface=self.profiler_interface
537
- )
538
- table_data.rows = table_data.rows[
539
- : min(
540
- SAMPLE_DATA_DEFAULT_COUNT, self.profiler_interface.sample_data_count
541
- )
542
- ]
543
- return SampleData(
544
- data=table_data, store=self.source_config.generateSampleData
545
- )
546
-
547
- except Exception as err:
548
- logger.debug(traceback.format_exc())
549
- logger.warning(f"Error fetching sample data: {err}")
550
- return None
551
-
552
499
  def get_profile(self) -> CreateTableProfileRequest:
553
500
  """
554
501
  After executing the profiler, get all results
@@ -603,13 +550,13 @@ class Profiler(Generic[TMetric]):
603
550
  createDateTime=raw_create_date,
604
551
  sizeInByte=self._table_results.get("sizeInBytes"),
605
552
  profileSample=(
606
- self.profile_sample_config.profile_sample
607
- if self.profile_sample_config
553
+ self.profiler_interface.sampler.sample_config.profile_sample
554
+ if self.profiler_interface.sampler.sample_config
608
555
  else None
609
556
  ),
610
557
  profileSampleType=(
611
- self.profile_sample_config.profile_sample_type
612
- if self.profile_sample_config
558
+ self.profiler_interface.sampler.sample_config.profile_sample_type
559
+ if self.profiler_interface.sampler.sample_config
613
560
  else None
614
561
  ),
615
562
  customMetrics=self._table_results.get("customMetrics"),