openmetadata-ingestion 1.3.4.0__py3-none-any.whl → 1.4.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openmetadata-ingestion might be problematic. Click here for more details.

Files changed (982) hide show
  1. metadata/__version__.py +13 -0
  2. metadata/automations/runner.py +9 -0
  3. metadata/cli/app.py +0 -1
  4. metadata/cli/lineage.py +5 -1
  5. metadata/cli/restore.py +7 -0
  6. metadata/cmd.py +2 -345
  7. metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +0 -1
  8. metadata/data_quality/api/models.py +1 -0
  9. metadata/data_quality/processor/test_case_runner.py +9 -2
  10. metadata/data_quality/source/test_suite.py +14 -3
  11. metadata/data_quality/validations/base_test_handler.py +1 -1
  12. metadata/data_quality/validations/column/base/columnValuesMissingCount.py +4 -1
  13. metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +28 -17
  14. metadata/data_quality/validations/mixins/sqa_validator_mixin.py +4 -1
  15. metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +1 -1
  16. metadata/examples/workflows/airbyte.yaml +1 -0
  17. metadata/examples/workflows/airflow.yaml +1 -1
  18. metadata/examples/workflows/airflow_backend.yaml +1 -1
  19. metadata/examples/workflows/airflow_postgres.yaml +1 -1
  20. metadata/examples/workflows/amundsen.yaml +1 -0
  21. metadata/examples/workflows/athena.yaml +1 -0
  22. metadata/examples/workflows/athena_lineage.yaml +1 -1
  23. metadata/examples/workflows/athena_usage.yaml +1 -0
  24. metadata/examples/workflows/atlas.yaml +1 -0
  25. metadata/examples/workflows/azuresql.yaml +1 -0
  26. metadata/examples/workflows/azuresql_lineage.yaml +1 -0
  27. metadata/examples/workflows/azuresql_usage.yaml +1 -0
  28. metadata/examples/workflows/bigquery.yaml +1 -0
  29. metadata/examples/workflows/bigquery_lineage.yaml +1 -0
  30. metadata/examples/workflows/bigquery_profiler.yaml +1 -0
  31. metadata/examples/workflows/bigquery_usage.yaml +1 -0
  32. metadata/examples/workflows/bigtable.yaml +1 -1
  33. metadata/examples/workflows/clickhouse.yaml +1 -0
  34. metadata/examples/workflows/clickhouse_lineage.yaml +1 -0
  35. metadata/examples/workflows/clickhouse_usage.yaml +1 -0
  36. metadata/examples/workflows/couchbase.yaml +1 -1
  37. metadata/examples/workflows/dagster.yaml +1 -0
  38. metadata/examples/workflows/data_insight.yaml +1 -1
  39. metadata/examples/workflows/databricks.yaml +1 -0
  40. metadata/examples/workflows/databricks_lineage.yaml +1 -0
  41. metadata/examples/workflows/databricks_pipeline.yaml +1 -0
  42. metadata/examples/workflows/databricks_usage.yaml +1 -0
  43. metadata/examples/workflows/datalake_azure_client_secret.yaml +1 -0
  44. metadata/examples/workflows/datalake_gcs.yaml +1 -0
  45. metadata/examples/workflows/datalake_profiler.yaml +1 -0
  46. metadata/examples/workflows/datalake_s3.yaml +1 -0
  47. metadata/examples/workflows/db2.yaml +1 -0
  48. metadata/examples/workflows/db2_profiler.yaml +1 -0
  49. metadata/examples/workflows/dbt.yaml +2 -0
  50. metadata/examples/workflows/deltalake.yaml +1 -0
  51. metadata/examples/workflows/domodashboard.yaml +1 -0
  52. metadata/examples/workflows/dynamodb.yaml +1 -0
  53. metadata/examples/workflows/elasticsearch.yaml +1 -1
  54. metadata/examples/workflows/fivetran.yaml +1 -0
  55. metadata/examples/workflows/glue.yaml +1 -0
  56. metadata/examples/workflows/gluepipeline.yaml +1 -0
  57. metadata/examples/workflows/hive.yaml +1 -0
  58. metadata/examples/workflows/impala.yaml +1 -0
  59. metadata/examples/workflows/kafka.yaml +1 -1
  60. metadata/examples/workflows/kafka_connect.yaml +30 -0
  61. metadata/examples/workflows/kinesis.yaml +1 -1
  62. metadata/examples/workflows/lightdash.yaml +1 -0
  63. metadata/examples/workflows/looker.yaml +1 -0
  64. metadata/examples/workflows/mariadb.yaml +1 -0
  65. metadata/examples/workflows/metabase.yaml +1 -0
  66. metadata/examples/workflows/mlflow.yaml +1 -0
  67. metadata/examples/workflows/mode.yaml +3 -1
  68. metadata/examples/workflows/mongodb.yaml +1 -0
  69. metadata/examples/workflows/mssql.yaml +1 -0
  70. metadata/examples/workflows/mssql_lineage.yaml +1 -0
  71. metadata/examples/workflows/mssql_usage.yaml +1 -0
  72. metadata/examples/workflows/mstr.yaml +1 -0
  73. metadata/examples/workflows/mysql.yaml +1 -0
  74. metadata/examples/workflows/mysql_profiler.yaml +1 -0
  75. metadata/examples/workflows/openmetadata.yaml +1 -0
  76. metadata/examples/workflows/oracle.yaml +1 -0
  77. metadata/examples/workflows/oracle_lineage.yaml +1 -1
  78. metadata/examples/workflows/oracle_usage.yaml +1 -0
  79. metadata/examples/workflows/pinotdb.yaml +1 -0
  80. metadata/examples/workflows/postgres.yaml +1 -0
  81. metadata/examples/workflows/postgres_lineage.yaml +1 -1
  82. metadata/examples/workflows/postgres_usage.yaml +1 -1
  83. metadata/examples/workflows/powerbi.yaml +52 -0
  84. metadata/examples/workflows/presto.yaml +1 -0
  85. metadata/examples/workflows/qlik_sense.yaml +6 -4
  86. metadata/examples/workflows/qlikcloud.yaml +31 -0
  87. metadata/examples/workflows/query_log_usage.yaml +1 -0
  88. metadata/examples/workflows/quicksight.yaml +1 -0
  89. metadata/examples/workflows/redash.yaml +1 -0
  90. metadata/examples/workflows/redpanda.yaml +1 -0
  91. metadata/examples/workflows/redshift.yaml +1 -0
  92. metadata/examples/workflows/redshift_lineage.yaml +1 -0
  93. metadata/examples/workflows/redshift_profiler.yaml +1 -0
  94. metadata/examples/workflows/redshift_usage.yaml +1 -0
  95. metadata/examples/workflows/sagemaker.yaml +1 -1
  96. metadata/examples/workflows/salesforce.yaml +1 -0
  97. metadata/examples/workflows/sas.yaml +1 -1
  98. metadata/examples/workflows/singlestore.yaml +1 -0
  99. metadata/examples/workflows/snowflake.yaml +1 -0
  100. metadata/examples/workflows/snowflake_lineage.yaml +1 -0
  101. metadata/examples/workflows/snowflake_usage.yaml +1 -0
  102. metadata/examples/workflows/spline.yaml +5 -4
  103. metadata/examples/workflows/sqlite.yaml +1 -0
  104. metadata/examples/workflows/superset.yaml +1 -0
  105. metadata/examples/workflows/tableau.yaml +1 -0
  106. metadata/examples/workflows/test_suite.yaml +13 -16
  107. metadata/examples/workflows/trino.yaml +1 -0
  108. metadata/examples/workflows/trino_lineage.yaml +1 -0
  109. metadata/examples/workflows/trino_usage.yaml +1 -0
  110. metadata/examples/workflows/unity_catalog.yaml +1 -1
  111. metadata/examples/workflows/unity_catalog_lineage.yaml +1 -1
  112. metadata/examples/workflows/unity_catalog_usage.yaml +1 -1
  113. metadata/examples/workflows/vertica.yaml +1 -0
  114. metadata/generated/antlr/EntityLinkLexer.py +354 -349
  115. metadata/generated/schema/analytics/__init__.py +1 -1
  116. metadata/generated/schema/analytics/basic.py +1 -1
  117. metadata/generated/schema/analytics/reportData.py +1 -1
  118. metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
  119. metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
  120. metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
  121. metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
  122. metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
  123. metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
  124. metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
  125. metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
  126. metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
  127. metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
  128. metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
  129. metadata/generated/schema/api/__init__.py +1 -1
  130. metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
  131. metadata/generated/schema/api/analytics/__init__.py +1 -1
  132. metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
  133. metadata/generated/schema/api/automations/__init__.py +1 -1
  134. metadata/generated/schema/api/automations/createWorkflow.py +1 -1
  135. metadata/generated/schema/api/bulkAssets.py +1 -1
  136. metadata/generated/schema/api/classification/__init__.py +1 -1
  137. metadata/generated/schema/api/classification/createClassification.py +2 -3
  138. metadata/generated/schema/api/classification/createTag.py +2 -3
  139. metadata/generated/schema/api/classification/loadTags.py +1 -1
  140. metadata/generated/schema/api/createBot.py +1 -1
  141. metadata/generated/schema/api/createEventPublisherJob.py +1 -1
  142. metadata/generated/schema/api/createType.py +1 -1
  143. metadata/generated/schema/api/data/__init__.py +1 -1
  144. metadata/generated/schema/api/data/createChart.py +1 -1
  145. metadata/generated/schema/api/data/createContainer.py +4 -1
  146. metadata/generated/schema/api/data/createCustomProperty.py +1 -1
  147. metadata/generated/schema/api/data/createDashboard.py +1 -1
  148. metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
  149. metadata/generated/schema/api/data/createDatabase.py +1 -1
  150. metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
  151. metadata/generated/schema/api/data/createGlossary.py +1 -1
  152. metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
  153. metadata/generated/schema/api/data/createMlModel.py +1 -1
  154. metadata/generated/schema/api/data/createPipeline.py +1 -1
  155. metadata/generated/schema/api/data/createQuery.py +1 -1
  156. metadata/generated/schema/api/data/createSearchIndex.py +1 -1
  157. metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
  158. metadata/generated/schema/api/data/createTable.py +4 -3
  159. metadata/generated/schema/api/data/createTableProfile.py +1 -1
  160. metadata/generated/schema/api/data/createTopic.py +1 -1
  161. metadata/generated/schema/api/data/loadGlossary.py +1 -1
  162. metadata/generated/schema/api/data/restoreEntity.py +1 -1
  163. metadata/generated/schema/api/dataInsight/__init__.py +1 -1
  164. metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
  165. metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
  166. metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
  167. metadata/generated/schema/api/docStore/__init__.py +1 -1
  168. metadata/generated/schema/api/docStore/createDocument.py +1 -1
  169. metadata/generated/schema/api/domains/__init__.py +1 -1
  170. metadata/generated/schema/api/domains/createDataProduct.py +1 -1
  171. metadata/generated/schema/api/domains/createDomain.py +1 -1
  172. metadata/generated/schema/api/feed/__init__.py +1 -1
  173. metadata/generated/schema/api/feed/closeTask.py +1 -1
  174. metadata/generated/schema/api/feed/createPost.py +1 -1
  175. metadata/generated/schema/api/feed/createSuggestion.py +1 -1
  176. metadata/generated/schema/api/feed/createThread.py +1 -1
  177. metadata/generated/schema/api/feed/resolveTask.py +1 -1
  178. metadata/generated/schema/api/feed/threadCount.py +1 -1
  179. metadata/generated/schema/api/lineage/__init__.py +1 -1
  180. metadata/generated/schema/api/lineage/addLineage.py +1 -1
  181. metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
  182. metadata/generated/schema/api/policies/__init__.py +1 -1
  183. metadata/generated/schema/api/policies/createPolicy.py +1 -1
  184. metadata/generated/schema/api/services/__init__.py +1 -1
  185. metadata/generated/schema/api/services/createDashboardService.py +1 -1
  186. metadata/generated/schema/api/services/createDatabaseService.py +1 -1
  187. metadata/generated/schema/api/services/createMessagingService.py +1 -1
  188. metadata/generated/schema/api/services/createMetadataService.py +1 -1
  189. metadata/generated/schema/api/services/createMlModelService.py +1 -1
  190. metadata/generated/schema/api/services/createPipelineService.py +1 -1
  191. metadata/generated/schema/api/services/createSearchService.py +1 -1
  192. metadata/generated/schema/api/services/createStorageService.py +1 -1
  193. metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
  194. metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
  195. metadata/generated/schema/api/setOwner.py +1 -1
  196. metadata/generated/schema/api/teams/__init__.py +1 -1
  197. metadata/generated/schema/api/teams/createPersona.py +1 -1
  198. metadata/generated/schema/api/teams/createRole.py +1 -1
  199. metadata/generated/schema/api/teams/createTeam.py +1 -1
  200. metadata/generated/schema/api/teams/createUser.py +2 -2
  201. metadata/generated/schema/api/tests/__init__.py +1 -1
  202. metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
  203. metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
  204. metadata/generated/schema/api/tests/createTestCase.py +2 -2
  205. metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
  206. metadata/generated/schema/api/tests/createTestDefinition.py +2 -2
  207. metadata/generated/schema/api/tests/createTestSuite.py +1 -1
  208. metadata/generated/schema/api/voteRequest.py +1 -1
  209. metadata/generated/schema/auth/__init__.py +1 -1
  210. metadata/generated/schema/auth/basicAuth.py +1 -1
  211. metadata/generated/schema/auth/basicLoginRequest.py +1 -1
  212. metadata/generated/schema/auth/changePasswordRequest.py +1 -1
  213. metadata/generated/schema/auth/createPersonalToken.py +1 -1
  214. metadata/generated/schema/auth/emailRequest.py +1 -1
  215. metadata/generated/schema/auth/emailVerificationToken.py +1 -1
  216. metadata/generated/schema/auth/generateToken.py +1 -1
  217. metadata/generated/schema/auth/jwtAuth.py +1 -1
  218. metadata/generated/schema/auth/loginRequest.py +1 -1
  219. metadata/generated/schema/auth/logoutRequest.py +1 -1
  220. metadata/generated/schema/auth/passwordResetRequest.py +1 -1
  221. metadata/generated/schema/auth/passwordResetToken.py +1 -1
  222. metadata/generated/schema/auth/personalAccessToken.py +1 -1
  223. metadata/generated/schema/auth/refreshToken.py +1 -1
  224. metadata/generated/schema/auth/registrationRequest.py +1 -1
  225. metadata/generated/schema/auth/revokePersonalToken.py +1 -1
  226. metadata/generated/schema/auth/revokeToken.py +1 -1
  227. metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
  228. metadata/generated/schema/auth/ssoAuth.py +1 -1
  229. metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
  230. metadata/generated/schema/configuration/__init__.py +1 -1
  231. metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
  232. metadata/generated/schema/configuration/authConfig.py +1 -1
  233. metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
  234. metadata/generated/schema/configuration/authorizerConfiguration.py +4 -1
  235. metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
  236. metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
  237. metadata/generated/schema/configuration/elasticSearchConfiguration.py +4 -1
  238. metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
  239. metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
  240. metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
  241. metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
  242. metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
  243. metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
  244. metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
  245. metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
  246. metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
  247. metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
  248. metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
  249. metadata/generated/schema/configuration/loginConfiguration.py +1 -1
  250. metadata/generated/schema/configuration/logoConfiguration.py +1 -1
  251. metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
  252. metadata/generated/schema/configuration/profilerConfiguration.py +66 -0
  253. metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
  254. metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
  255. metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
  256. metadata/generated/schema/configuration/themeConfiguration.py +32 -0
  257. metadata/generated/schema/configuration/uiThemePreference.py +23 -0
  258. metadata/generated/schema/dataInsight/__init__.py +1 -1
  259. metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
  260. metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
  261. metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
  262. metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
  263. metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
  264. metadata/generated/schema/dataInsight/type/__init__.py +1 -1
  265. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
  266. metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
  267. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
  268. metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
  269. metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
  270. metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
  271. metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
  272. metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
  273. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
  274. metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
  275. metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
  276. metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
  277. metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
  278. metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
  279. metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
  280. metadata/generated/schema/email/__init__.py +1 -1
  281. metadata/generated/schema/email/emailRequest.py +1 -1
  282. metadata/generated/schema/email/smtpSettings.py +3 -3
  283. metadata/generated/schema/entity/__init__.py +1 -1
  284. metadata/generated/schema/entity/applications/__init__.py +1 -1
  285. metadata/generated/schema/entity/applications/app.py +1 -1
  286. metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
  287. metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
  288. metadata/generated/schema/entity/applications/configuration/applicationConfig.py +3 -3
  289. metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
  290. metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +5 -0
  291. metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +36 -0
  292. metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +31 -0
  293. metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +31 -0
  294. metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +36 -0
  295. metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +31 -0
  296. metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +28 -0
  297. metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +22 -0
  298. metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +30 -0
  299. metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +22 -0
  300. metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +22 -0
  301. metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +31 -0
  302. metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +22 -0
  303. metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +75 -0
  304. metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +1 -1
  305. metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
  306. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
  307. metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
  308. metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
  309. metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
  310. metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +1 -1
  311. metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
  312. metadata/generated/schema/entity/applications/jobStatus.py +1 -1
  313. metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
  314. metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
  315. metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
  316. metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
  317. metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
  318. metadata/generated/schema/entity/automations/__init__.py +1 -1
  319. metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
  320. metadata/generated/schema/entity/automations/workflow.py +1 -1
  321. metadata/generated/schema/entity/bot.py +1 -1
  322. metadata/generated/schema/entity/classification/__init__.py +1 -1
  323. metadata/generated/schema/entity/classification/classification.py +2 -3
  324. metadata/generated/schema/entity/classification/tag.py +3 -9
  325. metadata/generated/schema/entity/data/__init__.py +1 -1
  326. metadata/generated/schema/entity/data/chart.py +1 -1
  327. metadata/generated/schema/entity/data/container.py +4 -1
  328. metadata/generated/schema/entity/data/dashboard.py +1 -1
  329. metadata/generated/schema/entity/data/dashboardDataModel.py +2 -2
  330. metadata/generated/schema/entity/data/database.py +1 -1
  331. metadata/generated/schema/entity/data/databaseSchema.py +1 -1
  332. metadata/generated/schema/entity/data/glossary.py +1 -1
  333. metadata/generated/schema/entity/data/glossaryTerm.py +4 -1
  334. metadata/generated/schema/entity/data/metrics.py +1 -1
  335. metadata/generated/schema/entity/data/mlmodel.py +2 -2
  336. metadata/generated/schema/entity/data/pipeline.py +1 -1
  337. metadata/generated/schema/entity/data/query.py +1 -1
  338. metadata/generated/schema/entity/data/report.py +1 -1
  339. metadata/generated/schema/entity/data/searchIndex.py +1 -1
  340. metadata/generated/schema/entity/data/storedProcedure.py +1 -1
  341. metadata/generated/schema/entity/data/table.py +26 -24
  342. metadata/generated/schema/entity/data/topic.py +1 -1
  343. metadata/generated/schema/entity/docStore/__init__.py +1 -1
  344. metadata/generated/schema/entity/docStore/document.py +1 -1
  345. metadata/generated/schema/entity/domains/__init__.py +1 -1
  346. metadata/generated/schema/entity/domains/dataProduct.py +1 -1
  347. metadata/generated/schema/entity/domains/domain.py +1 -1
  348. metadata/generated/schema/entity/events/__init__.py +1 -1
  349. metadata/generated/schema/entity/events/webhook.py +1 -1
  350. metadata/generated/schema/entity/feed/__init__.py +1 -1
  351. metadata/generated/schema/entity/feed/assets.py +18 -0
  352. metadata/generated/schema/entity/feed/customProperty.py +17 -0
  353. metadata/generated/schema/entity/feed/description.py +24 -0
  354. metadata/generated/schema/entity/feed/domain.py +23 -0
  355. metadata/generated/schema/entity/feed/entityInfo.py +19 -0
  356. metadata/generated/schema/entity/feed/owner.py +23 -0
  357. metadata/generated/schema/entity/feed/suggestion.py +1 -1
  358. metadata/generated/schema/entity/feed/tag.py +23 -0
  359. metadata/generated/schema/entity/feed/testCaseResult.py +23 -0
  360. metadata/generated/schema/entity/feed/thread.py +81 -2
  361. metadata/generated/schema/entity/policies/__init__.py +1 -1
  362. metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
  363. metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +3 -1
  364. metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
  365. metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
  366. metadata/generated/schema/entity/policies/filters.py +3 -3
  367. metadata/generated/schema/entity/policies/policy.py +1 -1
  368. metadata/generated/schema/entity/services/__init__.py +1 -1
  369. metadata/generated/schema/entity/services/connections/__init__.py +1 -1
  370. metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
  371. metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
  372. metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
  373. metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
  374. metadata/generated/schema/entity/services/connections/connectionBasicType.py +5 -1
  375. metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
  376. metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
  377. metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
  378. metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
  379. metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
  380. metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
  381. metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
  382. metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +1 -1
  383. metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +33 -2
  384. metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +3 -0
  385. metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +37 -0
  386. metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +25 -0
  387. metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +37 -0
  388. metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +37 -0
  389. metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +38 -0
  390. metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +6 -15
  391. metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
  392. metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
  393. metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
  394. metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
  395. metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
  396. metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
  397. metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
  398. metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +4 -1
  399. metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
  400. metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +4 -1
  401. metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
  402. metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
  403. metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
  404. metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
  405. metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
  406. metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
  407. metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
  408. metadata/generated/schema/entity/services/connections/database/databricksConnection.py +4 -1
  409. metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
  410. metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
  411. metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
  412. metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
  413. metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
  414. metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
  415. metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +2 -2
  416. metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
  417. metadata/generated/schema/entity/services/connections/database/dorisConnection.py +4 -13
  418. metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
  419. metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +4 -1
  420. metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
  421. metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -15
  422. metadata/generated/schema/entity/services/connections/database/hiveConnection.py +4 -1
  423. metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
  424. metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
  425. metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
  426. metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
  427. metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
  428. metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
  429. metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
  430. metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
  431. metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
  432. metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
  433. metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +4 -1
  434. metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +4 -1
  435. metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +7 -13
  436. metadata/generated/schema/entity/services/connections/database/oracleConnection.py +4 -1
  437. metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
  438. metadata/generated/schema/entity/services/connections/database/postgresConnection.py +5 -15
  439. metadata/generated/schema/entity/services/connections/database/prestoConnection.py +11 -1
  440. metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +5 -15
  441. metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
  442. metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
  443. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
  444. metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
  445. metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
  446. metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
  447. metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
  448. metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +4 -1
  449. metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
  450. metadata/generated/schema/entity/services/connections/database/trinoConnection.py +12 -3
  451. metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
  452. metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
  453. metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
  454. metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
  455. metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +7 -1
  456. metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
  457. metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
  458. metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
  459. metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
  460. metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
  461. metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +7 -3
  462. metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
  463. metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
  464. metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
  465. metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
  466. metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
  467. metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
  468. metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
  469. metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
  470. metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
  471. metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
  472. metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
  473. metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
  474. metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
  475. metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
  476. metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
  477. metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
  478. metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
  479. metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
  480. metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
  481. metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +61 -0
  482. metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
  483. metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +73 -0
  484. metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
  485. metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
  486. metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
  487. metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
  488. metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
  489. metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
  490. metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
  491. metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
  492. metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
  493. metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
  494. metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
  495. metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
  496. metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
  497. metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
  498. metadata/generated/schema/entity/services/connections/storage/s3Connection.py +5 -2
  499. metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
  500. metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
  501. metadata/generated/schema/entity/services/dashboardService.py +4 -1
  502. metadata/generated/schema/entity/services/databaseService.py +1 -1
  503. metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
  504. metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +1 -1
  505. metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
  506. metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
  507. metadata/generated/schema/entity/services/messagingService.py +1 -1
  508. metadata/generated/schema/entity/services/metadataService.py +1 -1
  509. metadata/generated/schema/entity/services/mlmodelService.py +1 -1
  510. metadata/generated/schema/entity/services/pipelineService.py +7 -1
  511. metadata/generated/schema/entity/services/searchService.py +1 -1
  512. metadata/generated/schema/entity/services/serviceType.py +1 -1
  513. metadata/generated/schema/entity/services/storageService.py +1 -1
  514. metadata/generated/schema/entity/teams/__init__.py +1 -1
  515. metadata/generated/schema/entity/teams/persona.py +1 -1
  516. metadata/generated/schema/entity/teams/role.py +1 -1
  517. metadata/generated/schema/entity/teams/team.py +1 -1
  518. metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
  519. metadata/generated/schema/entity/teams/user.py +2 -2
  520. metadata/generated/schema/entity/type.py +1 -1
  521. metadata/generated/schema/entity/utils/__init__.py +1 -1
  522. metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
  523. metadata/generated/schema/entity/utils/servicesCount.py +1 -1
  524. metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
  525. metadata/generated/schema/events/__init__.py +1 -1
  526. metadata/generated/schema/events/alertMetrics.py +1 -1
  527. metadata/generated/schema/events/api/__init__.py +1 -1
  528. metadata/generated/schema/events/api/createEventSubscription.py +1 -1
  529. metadata/generated/schema/events/emailAlertConfig.py +1 -1
  530. metadata/generated/schema/events/eventFilterRule.py +10 -1
  531. metadata/generated/schema/events/eventSubscription.py +6 -2
  532. metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
  533. metadata/generated/schema/events/failedEvent.py +1 -1
  534. metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
  535. metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
  536. metadata/generated/schema/metadataIngestion/__init__.py +1 -1
  537. metadata/generated/schema/metadataIngestion/application.py +1 -1
  538. metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
  539. metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +19 -6
  540. metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
  541. metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +37 -1
  542. metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +23 -23
  543. metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
  544. metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
  545. metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
  546. metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
  547. metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
  548. metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
  549. metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
  550. metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
  551. metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
  552. metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
  553. metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
  554. metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
  555. metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
  556. metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
  557. metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +22 -4
  558. metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
  559. metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
  560. metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
  561. metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
  562. metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
  563. metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
  564. metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
  565. metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
  566. metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
  567. metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
  568. metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
  569. metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
  570. metadata/generated/schema/metadataIngestion/workflow.py +1 -1
  571. metadata/generated/schema/monitoring/__init__.py +1 -1
  572. metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
  573. metadata/generated/schema/security/__init__.py +1 -1
  574. metadata/generated/schema/security/client/__init__.py +1 -1
  575. metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
  576. metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
  577. metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
  578. metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
  579. metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
  580. metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
  581. metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
  582. metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
  583. metadata/generated/schema/security/credentials/__init__.py +1 -1
  584. metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
  585. metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
  586. metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
  587. metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
  588. metadata/generated/schema/security/credentials/basicAuth.py +1 -1
  589. metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
  590. metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
  591. metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
  592. metadata/generated/schema/security/credentials/gcpValues.py +1 -1
  593. metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
  594. metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
  595. metadata/generated/schema/security/secrets/__init__.py +1 -1
  596. metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
  597. metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
  598. metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
  599. metadata/generated/schema/security/securityConfiguration.py +1 -1
  600. metadata/generated/schema/security/ssl/__init__.py +1 -1
  601. metadata/generated/schema/security/ssl/validateSSLClientConfig.py +16 -4
  602. metadata/generated/schema/security/ssl/verifySSLConfig.py +10 -1
  603. metadata/generated/schema/settings/__init__.py +1 -1
  604. metadata/generated/schema/settings/settings.py +5 -2
  605. metadata/generated/schema/system/__init__.py +1 -1
  606. metadata/generated/schema/system/entityError.py +1 -1
  607. metadata/generated/schema/system/eventPublisherJob.py +1 -1
  608. metadata/generated/schema/system/indexingError.py +1 -1
  609. metadata/generated/schema/system/ui/__init__.py +1 -1
  610. metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
  611. metadata/generated/schema/system/ui/page.py +1 -1
  612. metadata/generated/schema/system/validationResponse.py +1 -1
  613. metadata/generated/schema/tests/__init__.py +1 -1
  614. metadata/generated/schema/tests/assigned.py +1 -1
  615. metadata/generated/schema/tests/basic.py +31 -15
  616. metadata/generated/schema/tests/customMetric.py +1 -1
  617. metadata/generated/schema/tests/resolved.py +1 -1
  618. metadata/generated/schema/tests/testCase.py +13 -2
  619. metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
  620. metadata/generated/schema/tests/testDefinition.py +22 -2
  621. metadata/generated/schema/tests/testSuite.py +1 -1
  622. metadata/generated/schema/type/__init__.py +1 -1
  623. metadata/generated/schema/type/auditLog.py +1 -1
  624. metadata/generated/schema/type/basic.py +7 -1
  625. metadata/generated/schema/type/bulkOperationResult.py +1 -1
  626. metadata/generated/schema/type/changeEvent.py +1 -1
  627. metadata/generated/schema/type/changeEventType.py +1 -1
  628. metadata/generated/schema/type/collectionDescriptor.py +1 -1
  629. metadata/generated/schema/type/csvDocumentation.py +1 -1
  630. metadata/generated/schema/type/csvErrorType.py +1 -1
  631. metadata/generated/schema/type/csvFile.py +1 -1
  632. metadata/generated/schema/type/csvImportResult.py +1 -1
  633. metadata/generated/schema/type/customProperties/__init__.py +1 -1
  634. metadata/generated/schema/type/customProperties/complexTypes.py +86 -0
  635. metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
  636. metadata/generated/schema/type/customProperty.py +3 -3
  637. metadata/generated/schema/type/dailyCount.py +1 -1
  638. metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
  639. metadata/generated/schema/type/entityHierarchy.py +39 -0
  640. metadata/generated/schema/type/entityHistory.py +1 -1
  641. metadata/generated/schema/type/entityLineage.py +3 -1
  642. metadata/generated/schema/type/entityReference.py +1 -1
  643. metadata/generated/schema/type/entityReferenceList.py +1 -1
  644. metadata/generated/schema/type/entityRelationship.py +1 -1
  645. metadata/generated/schema/type/entityUsage.py +1 -1
  646. metadata/generated/schema/type/filterPattern.py +1 -1
  647. metadata/generated/schema/type/function.py +1 -1
  648. metadata/generated/schema/type/include.py +1 -1
  649. metadata/generated/schema/type/jdbcConnection.py +1 -1
  650. metadata/generated/schema/type/lifeCycle.py +1 -1
  651. metadata/generated/schema/type/paging.py +4 -1
  652. metadata/generated/schema/type/profile.py +1 -1
  653. metadata/generated/schema/type/queryParserData.py +1 -1
  654. metadata/generated/schema/type/reaction.py +1 -1
  655. metadata/generated/schema/type/schedule.py +1 -1
  656. metadata/generated/schema/type/schema.py +1 -1
  657. metadata/generated/schema/type/tableQuery.py +1 -1
  658. metadata/generated/schema/type/tableUsageCount.py +1 -1
  659. metadata/generated/schema/type/tagLabel.py +1 -1
  660. metadata/generated/schema/type/usageDetails.py +1 -1
  661. metadata/generated/schema/type/usageRequest.py +1 -1
  662. metadata/generated/schema/type/votes.py +1 -1
  663. metadata/great_expectations/action.py +0 -1
  664. metadata/ingestion/api/delete.py +33 -1
  665. metadata/ingestion/api/step.py +6 -1
  666. metadata/ingestion/api/topology_runner.py +122 -25
  667. metadata/ingestion/bulksink/metadata_usage.py +6 -1
  668. metadata/ingestion/lineage/models.py +34 -1
  669. metadata/ingestion/lineage/parser.py +15 -8
  670. metadata/ingestion/lineage/sql_lineage.py +21 -3
  671. metadata/ingestion/models/custom_properties.py +3 -0
  672. metadata/ingestion/models/life_cycle.py +4 -1
  673. metadata/ingestion/models/patch_request.py +177 -35
  674. metadata/ingestion/models/topology.py +81 -1
  675. metadata/ingestion/ometa/client.py +3 -0
  676. metadata/ingestion/ometa/mixins/es_mixin.py +95 -42
  677. metadata/ingestion/ometa/mixins/lineage_mixin.py +169 -4
  678. metadata/ingestion/ometa/mixins/patch_mixin.py +9 -4
  679. metadata/ingestion/ometa/mixins/server_mixin.py +50 -5
  680. metadata/ingestion/ometa/mixins/tests_mixin.py +55 -1
  681. metadata/ingestion/ometa/mixins/user_mixin.py +18 -6
  682. metadata/ingestion/ometa/ometa_api.py +6 -1
  683. metadata/ingestion/ometa/routes.py +3 -0
  684. metadata/ingestion/ometa/utils.py +2 -0
  685. metadata/ingestion/processor/query_parser.py +36 -26
  686. metadata/ingestion/sink/file.py +4 -1
  687. metadata/ingestion/sink/metadata_rest.py +19 -5
  688. metadata/ingestion/source/dashboard/dashboard_service.py +75 -16
  689. metadata/ingestion/source/dashboard/domodashboard/metadata.py +7 -5
  690. metadata/ingestion/source/dashboard/lightdash/metadata.py +7 -5
  691. metadata/ingestion/source/dashboard/looker/metadata.py +25 -18
  692. metadata/ingestion/source/dashboard/metabase/client.py +39 -8
  693. metadata/ingestion/source/dashboard/metabase/connection.py +2 -1
  694. metadata/ingestion/source/dashboard/metabase/metadata.py +9 -7
  695. metadata/ingestion/source/dashboard/metabase/models.py +1 -1
  696. metadata/ingestion/source/dashboard/mode/metadata.py +7 -5
  697. metadata/ingestion/source/dashboard/mstr/metadata.py +10 -5
  698. metadata/ingestion/source/dashboard/powerbi/client.py +13 -3
  699. metadata/ingestion/source/dashboard/powerbi/connection.py +13 -5
  700. metadata/ingestion/source/dashboard/powerbi/file_client.py +327 -0
  701. metadata/ingestion/source/dashboard/powerbi/metadata.py +156 -53
  702. metadata/ingestion/source/dashboard/powerbi/models.py +26 -0
  703. metadata/ingestion/source/dashboard/qlikcloud/__init__.py +0 -0
  704. metadata/ingestion/source/dashboard/qlikcloud/client.py +167 -0
  705. metadata/ingestion/source/dashboard/qlikcloud/connection.py +56 -0
  706. metadata/ingestion/source/dashboard/qlikcloud/constants.py +66 -0
  707. metadata/ingestion/source/dashboard/qlikcloud/metadata.py +249 -0
  708. metadata/ingestion/source/dashboard/qlikcloud/models.py +33 -0
  709. metadata/ingestion/source/dashboard/qliksense/client.py +9 -26
  710. metadata/ingestion/source/dashboard/qliksense/metadata.py +30 -17
  711. metadata/ingestion/source/dashboard/qliksense/models.py +1 -0
  712. metadata/ingestion/source/dashboard/quicksight/metadata.py +7 -5
  713. metadata/ingestion/source/dashboard/redash/metadata.py +10 -5
  714. metadata/ingestion/source/dashboard/superset/api_source.py +10 -5
  715. metadata/ingestion/source/dashboard/superset/db_source.py +12 -6
  716. metadata/ingestion/source/dashboard/superset/metadata.py +8 -1
  717. metadata/ingestion/source/dashboard/superset/mixin.py +20 -3
  718. metadata/ingestion/source/dashboard/superset/models.py +2 -0
  719. metadata/ingestion/source/dashboard/superset/queries.py +23 -6
  720. metadata/ingestion/source/dashboard/tableau/metadata.py +71 -10
  721. metadata/ingestion/source/database/athena/client.py +1 -1
  722. metadata/ingestion/source/database/athena/lineage.py +2 -0
  723. metadata/ingestion/source/database/athena/metadata.py +83 -15
  724. metadata/ingestion/source/database/athena/models.py +1 -1
  725. metadata/ingestion/source/database/athena/query_parser.py +6 -1
  726. metadata/ingestion/source/database/athena/usage.py +2 -0
  727. metadata/ingestion/source/database/azuresql/metadata.py +4 -2
  728. metadata/ingestion/source/database/azuresql/query_parser.py +4 -1
  729. metadata/ingestion/source/database/bigquery/helper.py +23 -12
  730. metadata/ingestion/source/database/bigquery/incremental_table_processor.py +86 -0
  731. metadata/ingestion/source/database/bigquery/metadata.py +224 -52
  732. metadata/ingestion/source/database/bigquery/models.py +35 -1
  733. metadata/ingestion/source/database/bigquery/queries.py +25 -12
  734. metadata/ingestion/source/database/bigquery/query_parser.py +3 -1
  735. metadata/ingestion/source/database/bigtable/metadata.py +6 -4
  736. metadata/ingestion/source/database/clickhouse/metadata.py +27 -17
  737. metadata/ingestion/source/database/clickhouse/query_parser.py +4 -2
  738. metadata/ingestion/source/database/common_db_source.py +117 -65
  739. metadata/ingestion/source/database/common_nosql_source.py +15 -15
  740. metadata/ingestion/source/database/couchbase/metadata.py +8 -6
  741. metadata/ingestion/source/database/database_service.py +43 -27
  742. metadata/ingestion/source/database/databricks/metadata.py +80 -34
  743. metadata/ingestion/source/database/databricks/queries.py +0 -1
  744. metadata/ingestion/source/database/databricks/query_parser.py +9 -2
  745. metadata/ingestion/source/database/datalake/connection.py +1 -2
  746. metadata/ingestion/source/database/datalake/metadata.py +21 -19
  747. metadata/ingestion/source/database/db2/metadata.py +4 -1
  748. metadata/ingestion/source/database/dbt/dbt_service.py +13 -11
  749. metadata/ingestion/source/database/dbt/metadata.py +97 -33
  750. metadata/ingestion/source/database/dbt/models.py +10 -1
  751. metadata/ingestion/source/database/deltalake/connection.py +2 -2
  752. metadata/ingestion/source/database/deltalake/metadata.py +19 -17
  753. metadata/ingestion/source/database/domodatabase/metadata.py +16 -11
  754. metadata/ingestion/source/database/doris/connection.py +0 -10
  755. metadata/ingestion/source/database/doris/metadata.py +23 -4
  756. metadata/ingestion/source/database/druid/metadata.py +5 -1
  757. metadata/ingestion/source/database/dynamodb/metadata.py +3 -1
  758. metadata/ingestion/source/database/extended_sample_data.py +4 -2
  759. metadata/ingestion/source/database/external_table_lineage_mixin.py +82 -0
  760. metadata/ingestion/source/database/glue/metadata.py +23 -21
  761. metadata/ingestion/source/database/greenplum/connection.py +0 -10
  762. metadata/ingestion/source/database/greenplum/metadata.py +26 -11
  763. metadata/ingestion/source/database/hive/metadata.py +29 -4
  764. metadata/ingestion/source/database/iceberg/helper.py +42 -5
  765. metadata/ingestion/source/database/iceberg/metadata.py +18 -16
  766. metadata/ingestion/source/database/iceberg/models.py +12 -4
  767. metadata/ingestion/source/database/impala/metadata.py +4 -1
  768. metadata/ingestion/source/database/incremental_metadata_extraction.py +144 -0
  769. metadata/ingestion/source/database/life_cycle_query_mixin.py +39 -36
  770. metadata/ingestion/source/database/mariadb/metadata.py +5 -1
  771. metadata/ingestion/source/database/mongodb/metadata.py +4 -2
  772. metadata/ingestion/source/database/mssql/metadata.py +15 -7
  773. metadata/ingestion/source/database/mssql/queries.py +18 -16
  774. metadata/ingestion/source/database/mssql/query_parser.py +4 -1
  775. metadata/ingestion/source/database/mssql/utils.py +3 -0
  776. metadata/ingestion/source/database/mysql/connection.py +0 -11
  777. metadata/ingestion/source/database/mysql/metadata.py +9 -2
  778. metadata/ingestion/source/database/oracle/connection.py +2 -2
  779. metadata/ingestion/source/database/oracle/metadata.py +33 -17
  780. metadata/ingestion/source/database/oracle/queries.py +13 -13
  781. metadata/ingestion/source/database/oracle/query_parser.py +4 -1
  782. metadata/ingestion/source/database/pinotdb/metadata.py +4 -1
  783. metadata/ingestion/source/database/postgres/connection.py +0 -10
  784. metadata/ingestion/source/database/postgres/metadata.py +38 -15
  785. metadata/ingestion/source/database/postgres/queries.py +17 -1
  786. metadata/ingestion/source/database/postgres/query_parser.py +4 -2
  787. metadata/ingestion/source/database/postgres/utils.py +98 -0
  788. metadata/ingestion/source/database/presto/connection.py +14 -0
  789. metadata/ingestion/source/database/presto/metadata.py +8 -5
  790. metadata/ingestion/source/database/query/lineage.py +5 -1
  791. metadata/ingestion/source/database/query/usage.py +4 -1
  792. metadata/ingestion/source/database/redshift/connection.py +0 -10
  793. metadata/ingestion/source/database/redshift/incremental_table_processor.py +192 -0
  794. metadata/ingestion/source/database/redshift/metadata.py +175 -26
  795. metadata/ingestion/source/database/redshift/models.py +74 -1
  796. metadata/ingestion/source/database/redshift/queries.py +17 -2
  797. metadata/ingestion/source/database/redshift/query_parser.py +3 -1
  798. metadata/ingestion/source/database/redshift/utils.py +13 -7
  799. metadata/ingestion/source/database/salesforce/metadata.py +13 -11
  800. metadata/ingestion/source/database/sample_data.py +56 -13
  801. metadata/ingestion/source/database/sample_usage.py +3 -1
  802. metadata/ingestion/source/database/saphana/metadata.py +6 -5
  803. metadata/ingestion/source/database/sas/metadata.py +8 -3
  804. metadata/ingestion/source/database/singlestore/metadata.py +5 -1
  805. metadata/ingestion/source/database/snowflake/metadata.py +193 -59
  806. metadata/ingestion/source/database/snowflake/models.py +24 -1
  807. metadata/ingestion/source/database/snowflake/queries.py +147 -25
  808. metadata/ingestion/source/database/snowflake/query_parser.py +3 -1
  809. metadata/ingestion/source/database/snowflake/utils.py +187 -11
  810. metadata/ingestion/source/database/sql_column_handler.py +2 -0
  811. metadata/ingestion/source/database/sqlalchemy_source.py +4 -4
  812. metadata/ingestion/source/database/sqlite/metadata.py +5 -1
  813. metadata/ingestion/source/database/stored_procedures_mixin.py +11 -9
  814. metadata/ingestion/source/database/trino/connection.py +20 -0
  815. metadata/ingestion/source/database/trino/metadata.py +7 -4
  816. metadata/ingestion/source/database/trino/query_parser.py +4 -1
  817. metadata/ingestion/source/database/unitycatalog/lineage.py +3 -1
  818. metadata/ingestion/source/database/unitycatalog/metadata.py +43 -32
  819. metadata/ingestion/source/database/unitycatalog/query_parser.py +4 -1
  820. metadata/ingestion/source/database/vertica/metadata.py +4 -2
  821. metadata/ingestion/source/database/vertica/query_parser.py +4 -2
  822. metadata/ingestion/source/messaging/common_broker_source.py +45 -6
  823. metadata/ingestion/source/messaging/kafka/metadata.py +21 -1
  824. metadata/ingestion/source/messaging/kinesis/metadata.py +11 -5
  825. metadata/ingestion/source/messaging/messaging_service.py +3 -3
  826. metadata/ingestion/source/messaging/redpanda/metadata.py +5 -1
  827. metadata/ingestion/source/metadata/amundsen/metadata.py +3 -1
  828. metadata/ingestion/source/metadata/atlas/metadata.py +3 -1
  829. metadata/ingestion/source/mlmodel/mlflow/metadata.py +4 -2
  830. metadata/ingestion/source/mlmodel/mlmodel_service.py +3 -3
  831. metadata/ingestion/source/mlmodel/sagemaker/metadata.py +8 -3
  832. metadata/ingestion/source/pipeline/airbyte/metadata.py +9 -7
  833. metadata/ingestion/source/pipeline/airflow/metadata.py +30 -14
  834. metadata/ingestion/source/pipeline/dagster/metadata.py +17 -15
  835. metadata/ingestion/source/pipeline/dagster/queries.py +53 -82
  836. metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +10 -8
  837. metadata/ingestion/source/pipeline/domopipeline/metadata.py +6 -4
  838. metadata/ingestion/source/pipeline/fivetran/metadata.py +6 -4
  839. metadata/ingestion/source/pipeline/gluepipeline/metadata.py +7 -5
  840. metadata/ingestion/source/pipeline/kafkaconnect/__init__.py +0 -0
  841. metadata/ingestion/source/pipeline/kafkaconnect/client.py +278 -0
  842. metadata/ingestion/source/pipeline/kafkaconnect/connection.py +58 -0
  843. metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +319 -0
  844. metadata/ingestion/source/pipeline/kafkaconnect/models.py +62 -0
  845. metadata/ingestion/source/pipeline/nifi/metadata.py +4 -2
  846. metadata/ingestion/source/pipeline/openlineage/__init__.py +0 -0
  847. metadata/ingestion/source/pipeline/openlineage/connection.py +88 -0
  848. metadata/ingestion/source/pipeline/openlineage/metadata.py +520 -0
  849. metadata/ingestion/source/pipeline/openlineage/models.py +88 -0
  850. metadata/ingestion/source/pipeline/openlineage/utils.py +59 -0
  851. metadata/ingestion/source/pipeline/pipeline_service.py +13 -3
  852. metadata/ingestion/source/pipeline/spline/metadata.py +8 -6
  853. metadata/ingestion/source/search/elasticsearch/metadata.py +8 -6
  854. metadata/ingestion/source/search/search_service.py +3 -3
  855. metadata/ingestion/source/sqa_types.py +20 -1
  856. metadata/ingestion/source/storage/s3/connection.py +10 -1
  857. metadata/ingestion/source/storage/s3/metadata.py +45 -30
  858. metadata/ingestion/source/storage/s3/models.py +8 -4
  859. metadata/ingestion/source/storage/storage_service.py +4 -4
  860. metadata/ingestion/stage/table_usage.py +6 -1
  861. metadata/mixins/pandas/pandas_mixin.py +3 -3
  862. metadata/pii/processor.py +6 -1
  863. metadata/profiler/__init__.py +0 -0
  864. metadata/profiler/adaptors/__init__.py +0 -0
  865. metadata/profiler/adaptors/adaptor_factory.py +43 -0
  866. metadata/profiler/adaptors/dynamodb.py +40 -0
  867. metadata/profiler/adaptors/factory.py +76 -0
  868. metadata/profiler/adaptors/mongodb.py +178 -0
  869. metadata/profiler/adaptors/nosql_adaptor.py +65 -0
  870. metadata/profiler/api/models.py +1 -1
  871. metadata/profiler/factory.py +41 -0
  872. metadata/profiler/interface/nosql/profiler_interface.py +236 -0
  873. metadata/profiler/interface/pandas/profiler_interface.py +51 -30
  874. metadata/profiler/interface/profiler_interface.py +6 -2
  875. metadata/profiler/interface/profiler_interface_factory.py +11 -22
  876. metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +77 -0
  877. metadata/profiler/interface/sqlalchemy/profiler_interface.py +2 -1
  878. metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +3 -6
  879. metadata/profiler/metrics/composed/distinct_ratio.py +2 -1
  880. metadata/profiler/metrics/composed/duplicate_count.py +2 -1
  881. metadata/profiler/metrics/composed/ilike_ratio.py +2 -1
  882. metadata/profiler/metrics/composed/iqr.py +2 -1
  883. metadata/profiler/metrics/composed/like_ratio.py +2 -1
  884. metadata/profiler/metrics/composed/non_parametric_skew.py +2 -1
  885. metadata/profiler/metrics/composed/null_ratio.py +7 -4
  886. metadata/profiler/metrics/composed/unique_ratio.py +2 -1
  887. metadata/profiler/metrics/core.py +14 -1
  888. metadata/profiler/metrics/hybrid/histogram.py +2 -1
  889. metadata/profiler/metrics/registry.py +4 -0
  890. metadata/profiler/metrics/static/column_count.py +2 -1
  891. metadata/profiler/metrics/static/column_names.py +2 -1
  892. metadata/profiler/metrics/static/count.py +2 -1
  893. metadata/profiler/metrics/static/count_in_set.py +2 -1
  894. metadata/profiler/metrics/static/distinct_count.py +11 -2
  895. metadata/profiler/metrics/static/ilike_count.py +2 -1
  896. metadata/profiler/metrics/static/like_count.py +2 -1
  897. metadata/profiler/metrics/static/max.py +26 -7
  898. metadata/profiler/metrics/static/max_length.py +3 -2
  899. metadata/profiler/metrics/static/mean.py +30 -22
  900. metadata/profiler/metrics/static/min.py +25 -6
  901. metadata/profiler/metrics/static/min_length.py +3 -2
  902. metadata/profiler/metrics/static/not_like_count.py +2 -1
  903. metadata/profiler/metrics/static/not_regexp_match_count.py +2 -1
  904. metadata/profiler/metrics/static/null_count.py +2 -1
  905. metadata/profiler/metrics/static/null_missing_count.py +73 -0
  906. metadata/profiler/metrics/static/regexp_match_count.py +2 -1
  907. metadata/profiler/metrics/static/row_count.py +9 -3
  908. metadata/profiler/metrics/static/stddev.py +2 -1
  909. metadata/profiler/metrics/static/sum.py +22 -4
  910. metadata/profiler/metrics/static/unique_count.py +13 -4
  911. metadata/profiler/metrics/system/system.py +2 -1
  912. metadata/profiler/metrics/window/first_quartile.py +2 -1
  913. metadata/profiler/metrics/window/median.py +8 -2
  914. metadata/profiler/metrics/window/third_quartile.py +2 -1
  915. metadata/profiler/orm/converter/base.py +4 -1
  916. metadata/profiler/orm/converter/bigquery/converter.py +11 -0
  917. metadata/profiler/orm/converter/common.py +47 -0
  918. metadata/profiler/orm/converter/converter_registry.py +2 -0
  919. metadata/profiler/orm/converter/redshift/converter.py +48 -0
  920. metadata/profiler/orm/converter/snowflake/converter.py +13 -0
  921. metadata/profiler/orm/functions/conn_test.py +1 -1
  922. metadata/profiler/orm/functions/sum.py +4 -2
  923. metadata/profiler/orm/functions/table_metric_computer.py +1 -1
  924. metadata/profiler/orm/registry.py +6 -3
  925. metadata/profiler/processor/core.py +58 -97
  926. metadata/profiler/processor/default.py +5 -0
  927. metadata/profiler/processor/handle_partition.py +3 -3
  928. metadata/profiler/processor/metric_filter.py +267 -0
  929. metadata/profiler/processor/processor.py +4 -2
  930. metadata/profiler/processor/sampler/nosql/sampler.py +72 -0
  931. metadata/profiler/processor/sampler/pandas/sampler.py +5 -6
  932. metadata/profiler/processor/sampler/sampler_factory.py +9 -0
  933. metadata/profiler/processor/sampler/sampler_interface.py +2 -2
  934. metadata/profiler/processor/sampler/sqlalchemy/sampler.py +3 -3
  935. metadata/profiler/source/base/profiler_source.py +11 -3
  936. metadata/profiler/source/databricks/profiler_source.py +5 -1
  937. metadata/profiler/source/metadata.py +8 -1
  938. metadata/profiler/source/metadata_ext.py +2 -0
  939. metadata/readers/dataframe/avro.py +5 -5
  940. metadata/readers/dataframe/json.py +1 -1
  941. metadata/readers/file/adls.py +20 -0
  942. metadata/readers/file/base.py +6 -0
  943. metadata/readers/file/config_source_factory.py +12 -0
  944. metadata/readers/file/gcs.py +18 -0
  945. metadata/readers/file/local.py +12 -0
  946. metadata/readers/file/s3.py +16 -0
  947. metadata/utils/constants.py +32 -24
  948. metadata/utils/datalake/datalake_utils.py +40 -4
  949. metadata/utils/elasticsearch.py +8 -0
  950. metadata/utils/execution_time_tracker.py +69 -28
  951. metadata/utils/fqn.py +61 -7
  952. metadata/utils/helpers.py +2 -0
  953. metadata/utils/importer.py +15 -6
  954. metadata/utils/logger.py +1 -1
  955. metadata/utils/lru_cache.py +2 -0
  956. metadata/utils/messaging_utils.py +43 -0
  957. metadata/utils/partition.py +93 -28
  958. metadata/utils/sqlalchemy_utils.py +60 -1
  959. metadata/utils/ssl_manager.py +181 -0
  960. metadata/utils/tag_utils.py +30 -13
  961. metadata/utils/test_utils.py +65 -0
  962. metadata/workflow/application.py +6 -1
  963. metadata/workflow/base.py +1 -1
  964. metadata/workflow/ingestion.py +19 -0
  965. metadata/workflow/metadata.py +9 -1
  966. metadata/workflow/output_handler.py +24 -2
  967. metadata/workflow/workflow_output_handler.py +2 -0
  968. {openmetadata_ingestion-1.3.4.0.dist-info → openmetadata_ingestion-1.4.0.0rc2.dist-info}/METADATA +325 -326
  969. openmetadata_ingestion-1.4.0.0rc2.dist-info/RECORD +1501 -0
  970. metadata/applications/auto_tagger.py +0 -212
  971. metadata/cli/backup.py +0 -202
  972. metadata/cli/db_dump.py +0 -239
  973. metadata/cli/docker.py +0 -382
  974. metadata/cli/openmetadata_dag_config_migration.py +0 -109
  975. metadata/cli/openmetadata_imports_migration.py +0 -71
  976. metadata/cli/utils.py +0 -56
  977. metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +0 -28
  978. openmetadata_ingestion-1.3.4.0.dist-info/RECORD +0 -1435
  979. {openmetadata_ingestion-1.3.4.0.dist-info → openmetadata_ingestion-1.4.0.0rc2.dist-info}/LICENSE +0 -0
  980. {openmetadata_ingestion-1.3.4.0.dist-info → openmetadata_ingestion-1.4.0.0rc2.dist-info}/WHEEL +0 -0
  981. {openmetadata_ingestion-1.3.4.0.dist-info → openmetadata_ingestion-1.4.0.0rc2.dist-info}/entry_points.txt +0 -0
  982. {openmetadata_ingestion-1.3.4.0.dist-info → openmetadata_ingestion-1.4.0.0rc2.dist-info}/top_level.txt +0 -0
@@ -40,7 +40,11 @@ from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipelin
40
40
  from metadata.generated.schema.metadataIngestion.workflow import (
41
41
  Source as WorkflowSource,
42
42
  )
43
- from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
43
+ from metadata.generated.schema.type.entityLineage import (
44
+ ColumnLineage,
45
+ EntitiesEdge,
46
+ LineageDetails,
47
+ )
44
48
  from metadata.generated.schema.type.entityLineage import Source as LineageSource
45
49
  from metadata.generated.schema.type.entityReference import EntityReference
46
50
  from metadata.generated.schema.type.usageRequest import UsageRequest
@@ -48,13 +52,14 @@ from metadata.ingestion.api.delete import delete_entity_from_source
48
52
  from metadata.ingestion.api.models import Either, Entity
49
53
  from metadata.ingestion.api.steps import Source
50
54
  from metadata.ingestion.api.topology_runner import C, TopologyRunnerMixin
55
+ from metadata.ingestion.lineage.sql_lineage import get_column_fqn
51
56
  from metadata.ingestion.models.delete_entity import DeleteEntity
52
57
  from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
53
58
  from metadata.ingestion.models.patch_request import PatchRequest
54
59
  from metadata.ingestion.models.topology import (
55
60
  NodeStage,
56
61
  ServiceTopology,
57
- TopologyContext,
62
+ TopologyContextManager,
58
63
  TopologyNode,
59
64
  )
60
65
  from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -194,7 +199,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
194
199
  service_connection: DashboardConnection.__fields__["config"].type_
195
200
 
196
201
  topology = DashboardServiceTopology()
197
- context = TopologyContext.create(topology)
202
+ context = TopologyContextManager(topology)
198
203
  dashboard_source_state: Set = set()
199
204
  datamodel_source_state: Set = set()
200
205
 
@@ -288,13 +293,13 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
288
293
  Returns:
289
294
  Lineage request between Data Models and Dashboards
290
295
  """
291
- if hasattr(self.context, "dataModels") and self.context.dataModels:
292
- for datamodel in self.context.dataModels:
296
+ if hasattr(self.context.get(), "dataModels") and self.context.get().dataModels:
297
+ for datamodel in self.context.get().dataModels:
293
298
  try:
294
299
  datamodel_fqn = fqn.build(
295
300
  metadata=self.metadata,
296
301
  entity_type=DashboardDataModel,
297
- service_name=self.context.dashboard_service,
302
+ service_name=self.context.get().dashboard_service,
298
303
  data_model_name=datamodel,
299
304
  )
300
305
  datamodel_entity = self.metadata.get_by_name(
@@ -304,8 +309,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
304
309
  dashboard_fqn = fqn.build(
305
310
  self.metadata,
306
311
  entity_type=Dashboard,
307
- service_name=self.context.dashboard_service,
308
- dashboard_name=self.context.dashboard,
312
+ service_name=self.context.get().dashboard_service,
313
+ dashboard_name=self.context.get().dashboard,
309
314
  )
310
315
  dashboard_entity = self.metadata.get_by_name(
311
316
  entity=Dashboard, fqn=dashboard_fqn
@@ -319,6 +324,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
319
324
  f"Error to yield dashboard lineage details for data model name [{datamodel.name}]: {err}"
320
325
  )
321
326
 
327
+ def get_db_service_names(self) -> List[str]:
328
+ """
329
+ Get the list of db service names
330
+ """
331
+ return (
332
+ self.source_config.lineageInformation.dbServiceNames or []
333
+ if self.source_config.lineageInformation
334
+ else []
335
+ )
336
+
322
337
  def yield_dashboard_lineage(
323
338
  self, dashboard_details: Any
324
339
  ) -> Iterable[Either[AddLineageRequest]]:
@@ -330,7 +345,9 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
330
345
  """
331
346
  yield from self.yield_datamodel_dashboard_lineage() or []
332
347
 
333
- for db_service_name in self.source_config.dbServiceNames or []:
348
+ db_service_names = self.get_db_service_names()
349
+
350
+ for db_service_name in db_service_names or []:
334
351
  yield from self.yield_dashboard_lineage_details(
335
352
  dashboard_details, db_service_name
336
353
  ) or []
@@ -380,7 +397,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
380
397
  entity_type=Dashboard,
381
398
  entity_source_state=self.dashboard_source_state,
382
399
  mark_deleted_entity=self.source_config.markDeletedDashboards,
383
- params={"service": self.context.dashboard_service},
400
+ params={"service": self.context.get().dashboard_service},
384
401
  )
385
402
 
386
403
  def mark_datamodels_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
@@ -394,7 +411,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
394
411
  entity_type=DashboardDataModel,
395
412
  entity_source_state=self.datamodel_source_state,
396
413
  mark_deleted_entity=self.source_config.markDeletedDataModels,
397
- params={"service": self.context.dashboard_service},
414
+ params={"service": self.context.get().dashboard_service},
398
415
  )
399
416
 
400
417
  def get_owner_ref( # pylint: disable=unused-argument, useless-return
@@ -440,6 +457,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
440
457
  def _get_add_lineage_request(
441
458
  to_entity: Union[Dashboard, DashboardDataModel],
442
459
  from_entity: Union[Table, DashboardDataModel, Dashboard],
460
+ column_lineage: List[ColumnLineage] = None,
443
461
  ) -> Optional[Either[AddLineageRequest]]:
444
462
  if from_entity and to_entity:
445
463
  return Either(
@@ -454,7 +472,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
454
472
  type=LINEAGE_MAP[type(to_entity)],
455
473
  ),
456
474
  lineageDetails=LineageDetails(
457
- source=LineageSource.DashboardLineage
475
+ source=LineageSource.DashboardLineage,
476
+ columnsLineage=column_lineage,
458
477
  ),
459
478
  )
460
479
  )
@@ -462,6 +481,20 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
462
481
 
463
482
  return None
464
483
 
484
+ @staticmethod
485
+ def _get_data_model_column_fqn(
486
+ data_model_entity: DashboardDataModel, column: str
487
+ ) -> Optional[str]:
488
+ """
489
+ Get fqn of column if exist in table entity
490
+ """
491
+ if not data_model_entity:
492
+ return None
493
+ for tbl_column in data_model_entity.columns:
494
+ if tbl_column.displayName.lower() == column.lower():
495
+ return tbl_column.fullyQualifiedName.__root__
496
+ return None
497
+
465
498
  def get_dashboard(self) -> Any:
466
499
  """
467
500
  Method to iterate through dashboard lists filter dashboards & yield dashboard details
@@ -480,15 +513,15 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
480
513
 
481
514
  try:
482
515
  dashboard_details = self.get_dashboard_details(dashboard)
483
- self.context.project_name = ( # pylint: disable=assignment-from-none
516
+ self.context.get().project_name = ( # pylint: disable=E1128
484
517
  self.get_project_name(dashboard_details=dashboard_details)
485
518
  )
486
519
  if filter_by_project(
487
520
  self.source_config.projectFilterPattern,
488
- self.context.project_name,
521
+ self.context.get().project_name,
489
522
  ):
490
523
  self.status.filter(
491
- self.context.project_name,
524
+ self.context.get().project_name,
492
525
  "Project / Workspace Filtered Out",
493
526
  )
494
527
  continue
@@ -519,7 +552,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
519
552
  :return: Entity FQN derived from context
520
553
  """
521
554
  context_names = [
522
- self.context.__dict__[dependency]
555
+ self.context.get().__dict__[dependency]
523
556
  for dependency in stage.consumer or [] # root nodes do not have consumers
524
557
  ]
525
558
 
@@ -597,3 +630,29 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
597
630
  )
598
631
  patch_request.new_entity.dataModels = datamodel_entity_ref_list
599
632
  return patch_request
633
+
634
+ def _get_column_lineage(
635
+ self,
636
+ om_table: Table,
637
+ data_model_entity: DashboardDataModel,
638
+ columns_list: List[str],
639
+ ) -> List[ColumnLineage]:
640
+ """
641
+ Get the column lineage from the fields
642
+ """
643
+ try:
644
+ column_lineage = []
645
+ for field in columns_list or []:
646
+ from_column = get_column_fqn(table_entity=om_table, column=field)
647
+ to_column = self._get_data_model_column_fqn(
648
+ data_model_entity=data_model_entity,
649
+ column=field,
650
+ )
651
+ if from_column and to_column:
652
+ column_lineage.append(
653
+ ColumnLineage(fromColumns=[from_column], toColumn=to_column)
654
+ )
655
+ return column_lineage
656
+ except Exception as exc:
657
+ logger.debug(f"Error to get column lineage: {exc}")
658
+ logger.debug(traceback.format_exc())
@@ -61,7 +61,9 @@ class DomodashboardSource(DashboardServiceSource):
61
61
  metadata_config: OpenMetadataConnection
62
62
 
63
63
  @classmethod
64
- def create(cls, config_dict, metadata: OpenMetadata):
64
+ def create(
65
+ cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
66
+ ):
65
67
  config = WorkflowSource.parse_obj(config_dict)
66
68
  connection: DomoDashboardConnection = config.serviceConnection.__root__.config
67
69
  if not isinstance(connection, DomoDashboardConnection):
@@ -124,12 +126,12 @@ class DomodashboardSource(DashboardServiceSource):
124
126
  fqn.build(
125
127
  self.metadata,
126
128
  entity_type=Chart,
127
- service_name=self.context.dashboard_service,
129
+ service_name=self.context.get().dashboard_service,
128
130
  chart_name=chart,
129
131
  )
130
- for chart in self.context.charts or []
132
+ for chart in self.context.get().charts or []
131
133
  ],
132
- service=self.context.dashboard_service,
134
+ service=self.context.get().dashboard_service,
133
135
  owner=self.get_owner_ref(dashboard_details=dashboard_details),
134
136
  )
135
137
  yield Either(right=dashboard_request)
@@ -219,7 +221,7 @@ class DomodashboardSource(DashboardServiceSource):
219
221
  description=chart.description,
220
222
  displayName=chart.name,
221
223
  sourceUrl=chart_url,
222
- service=self.context.dashboard_service,
224
+ service=self.context.get().dashboard_service,
223
225
  chartType=get_standard_chart_type(chart.metadata.chartType),
224
226
  )
225
227
  )
@@ -50,7 +50,9 @@ class LightdashSource(DashboardServiceSource):
50
50
  metadata_config: OpenMetadataConnection
51
51
 
52
52
  @classmethod
53
- def create(cls, config_dict, metadata: OpenMetadata):
53
+ def create(
54
+ cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
55
+ ):
54
56
  config = WorkflowSource.parse_obj(config_dict)
55
57
  connection: LightdashConnection = config.serviceConnection.__root__.config
56
58
  if not isinstance(connection, LightdashConnection):
@@ -111,12 +113,12 @@ class LightdashSource(DashboardServiceSource):
111
113
  fqn.build(
112
114
  self.metadata,
113
115
  entity_type=Chart,
114
- service_name=self.context.dashboard_service,
116
+ service_name=self.context.get().dashboard_service,
115
117
  chart_name=chart,
116
118
  )
117
- for chart in self.context.charts or []
119
+ for chart in self.context.get().charts or []
118
120
  ],
119
- service=self.context.dashboard_service,
121
+ service=self.context.get().dashboard_service,
120
122
  owner=self.get_owner_ref(dashboard_details=dashboard_details),
121
123
  )
122
124
  yield dashboard_request
@@ -152,7 +154,7 @@ class LightdashSource(DashboardServiceSource):
152
154
  displayName=chart.name,
153
155
  description=chart.description,
154
156
  sourceUrl=chart_url,
155
- service=self.context.dashboard_service,
157
+ service=self.context.get().dashboard_service,
156
158
  )
157
159
  self.status.scanned(chart.name)
158
160
  except Exception as exc: # pylint: disable=broad-except
@@ -174,7 +174,12 @@ class LookerSource(DashboardServiceSource):
174
174
  self._added_lineage: Optional[Dict] = {}
175
175
 
176
176
  @classmethod
177
- def create(cls, config_dict: dict, metadata: OpenMetadata) -> "LookerSource":
177
+ def create(
178
+ cls,
179
+ config_dict: dict,
180
+ metadata: OpenMetadata,
181
+ pipeline_name: Optional[str] = None,
182
+ ) -> "LookerSource":
178
183
  config = WorkflowSource.parse_obj(config_dict)
179
184
  connection: LookerConnection = config.serviceConnection.__root__.config
180
185
  if not isinstance(connection, LookerConnection):
@@ -367,7 +372,7 @@ class LookerSource(DashboardServiceSource):
367
372
  fqn_datamodel = fqn.build(
368
373
  self.metadata,
369
374
  DashboardDataModel,
370
- service_name=self.context.dashboard_service,
375
+ service_name=self.context.get().dashboard_service,
371
376
  data_model_name=data_model_name,
372
377
  )
373
378
 
@@ -396,7 +401,7 @@ class LookerSource(DashboardServiceSource):
396
401
  name=datamodel_name,
397
402
  displayName=model.name,
398
403
  description=model.description,
399
- service=self.context.dashboard_service,
404
+ service=self.context.get().dashboard_service,
400
405
  dataModelType=DataModelType.LookMlExplore.value,
401
406
  serviceType=DashboardServiceType.Looker.value,
402
407
  columns=get_columns_from_model(model),
@@ -408,14 +413,14 @@ class LookerSource(DashboardServiceSource):
408
413
  self.register_record_datamodel(datamodel_request=explore_datamodel)
409
414
 
410
415
  # build datamodel by our hand since ack_sink=False
411
- self.context.dataModel = self._build_data_model(datamodel_name)
412
- self._view_data_model = copy.deepcopy(self.context.dataModel)
416
+ self.context.get().dataModel = self._build_data_model(datamodel_name)
417
+ self._view_data_model = copy.deepcopy(self.context.get().dataModel)
413
418
 
414
419
  # Maybe use the project_name as key too?
415
420
  # Save the explores for when we create the lineage with the dashboards and views
416
421
  self._explores_cache[
417
422
  explore_datamodel.name.__root__
418
- ] = self.context.dataModel # This is the newly created explore
423
+ ] = self.context.get().dataModel # This is the newly created explore
419
424
 
420
425
  # We can get VIEWs from the JOINs to know the dependencies
421
426
  # We will only try and fetch if we have the credentials
@@ -494,7 +499,7 @@ class LookerSource(DashboardServiceSource):
494
499
  name=build_datamodel_name(explore.model_name, view.name),
495
500
  displayName=view.name,
496
501
  description=view.description,
497
- service=self.context.dashboard_service,
502
+ service=self.context.get().dashboard_service,
498
503
  dataModelType=DataModelType.LookMlView.value,
499
504
  serviceType=DashboardServiceType.Looker.value,
500
505
  columns=get_columns_from_model(view),
@@ -541,11 +546,13 @@ class LookerSource(DashboardServiceSource):
541
546
  " while processing view lineage."
542
547
  )
543
548
 
549
+ db_service_names = self.get_db_service_names()
550
+
544
551
  if view.sql_table_name:
545
552
  source_table_name = self._clean_table_name(view.sql_table_name)
546
553
 
547
554
  # View to the source is only there if we are informing the dbServiceNames
548
- for db_service_name in self.source_config.dbServiceNames or []:
555
+ for db_service_name in db_service_names or []:
549
556
  yield self.build_lineage_request(
550
557
  source=source_table_name,
551
558
  db_service_name=db_service_name,
@@ -556,7 +563,7 @@ class LookerSource(DashboardServiceSource):
556
563
  sql_query = view.derived_table.sql
557
564
  if not sql_query:
558
565
  return
559
- for db_service_name in self.source_config.dbServiceNames or []:
566
+ for db_service_name in db_service_names or []:
560
567
  db_service = self.metadata.get_by_name(
561
568
  DatabaseService, db_service_name
562
569
  )
@@ -654,16 +661,16 @@ class LookerSource(DashboardServiceSource):
654
661
  fqn.build(
655
662
  self.metadata,
656
663
  entity_type=Chart,
657
- service_name=self.context.dashboard_service,
664
+ service_name=self.context.get().dashboard_service,
658
665
  chart_name=chart,
659
666
  )
660
- for chart in self.context.charts or []
667
+ for chart in self.context.get().charts or []
661
668
  ],
662
669
  # Dashboards are created from the UI directly. They are not linked to a project
663
670
  # like LookML assets, but rather just organised in folders.
664
671
  project=self.get_project_name(dashboard_details),
665
672
  sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/dashboards/{dashboard_details.id}",
666
- service=self.context.dashboard_service,
673
+ service=self.context.get().dashboard_service,
667
674
  owner=self.get_owner_ref(dashboard_details=dashboard_details),
668
675
  )
669
676
  yield Either(right=dashboard_request)
@@ -732,7 +739,7 @@ class LookerSource(DashboardServiceSource):
732
739
  fqn=fqn.build(
733
740
  self.metadata,
734
741
  entity_type=DashboardDataModel,
735
- service_name=self.context.dashboard_service,
742
+ service_name=self.context.get().dashboard_service,
736
743
  data_model_name=explore_name,
737
744
  ),
738
745
  )
@@ -757,8 +764,8 @@ class LookerSource(DashboardServiceSource):
757
764
  dashboard_fqn = fqn.build(
758
765
  self.metadata,
759
766
  entity_type=Dashboard,
760
- service_name=self.context.dashboard_service,
761
- dashboard_name=self.context.dashboard,
767
+ service_name=self.context.get().dashboard_service,
768
+ dashboard_name=self.context.get().dashboard,
762
769
  )
763
770
  dashboard_entity = self.metadata.get_by_name(
764
771
  entity=Dashboard, fqn=dashboard_fqn
@@ -869,7 +876,7 @@ class LookerSource(DashboardServiceSource):
869
876
  sourceUrl=chart.query.share_url
870
877
  if chart.query is not None
871
878
  else f"{clean_uri(self.service_connection.hostPort)}/merge?mid={chart.merge_result_id}",
872
- service=self.context.dashboard_service,
879
+ service=self.context.get().dashboard_service,
873
880
  )
874
881
  )
875
882
 
@@ -938,13 +945,13 @@ class LookerSource(DashboardServiceSource):
938
945
  :return: UsageRequest, if not computed
939
946
  """
940
947
 
941
- dashboard_name = self.context.dashboard
948
+ dashboard_name = self.context.get().dashboard
942
949
 
943
950
  try:
944
951
  dashboard_fqn = fqn.build(
945
952
  metadata=self.metadata,
946
953
  entity_type=Dashboard,
947
- service_name=self.context.dashboard_service,
954
+ service_name=self.context.get().dashboard_service,
948
955
  dashboard_name=dashboard_name,
949
956
  )
950
957
 
@@ -85,18 +85,49 @@ class MetabaseClient:
85
85
  )
86
86
  self.client = REST(client_config)
87
87
 
88
- def get_dashboards_list(self) -> List[MetabaseDashboard]:
88
+ def get_dashboards_list(
89
+ self, collections: List[MetabaseCollection]
90
+ ) -> List[MetabaseDashboard]:
89
91
  """
90
92
  Get List of all dashboards
91
93
  """
92
- try:
93
- resp_dashboards = self.client.get("/dashboard")
94
+ dashboards = []
95
+ for collection in collections or []:
96
+ try:
97
+ resp_dashboards = self.client.get(
98
+ f"/collection/{collection.id}/items?models=dashboard"
99
+ )
100
+ if resp_dashboards:
101
+ dashboard_list = MetabaseDashboardList(**resp_dashboards)
102
+ dashboards.extend(dashboard_list.data)
103
+ except Exception:
104
+ logger.debug(traceback.format_exc())
105
+ logger.warning("Failed to fetch the dashboard list")
106
+ return dashboards
107
+
108
+ def get_dashboards_list_test_conn(
109
+ self, collections: List[MetabaseCollection]
110
+ ) -> List[MetabaseDashboard]:
111
+ """
112
+ Get List of all dashboards
113
+ """
114
+ for collection in collections or []:
115
+ resp_dashboards = self.client.get(
116
+ f"/collection/{collection.id}/items?models=dashboard"
117
+ )
94
118
  if resp_dashboards:
95
- dashboard_list = MetabaseDashboardList(dashboards=resp_dashboards)
96
- return dashboard_list.dashboards
97
- except Exception:
98
- logger.debug(traceback.format_exc())
99
- logger.warning("Failed to fetch the dashboard list")
119
+ dashboard_list = MetabaseDashboardList(**resp_dashboards)
120
+ return dashboard_list.data
121
+ return []
122
+
123
+ def get_collections_list_test_conn(self) -> List[MetabaseCollection]:
124
+ """
125
+ Get List of all collections
126
+ """
127
+ resp_collections = self.client.get("/collection")
128
+ if resp_collections:
129
+ collection_list = MetabaseCollectionList(collections=resp_collections)
130
+ return collection_list.collections
100
131
  return []
101
132
 
102
133
  def get_collections_list(self) -> List[MetabaseCollection]:
@@ -44,7 +44,8 @@ def test_connection(
44
44
  """
45
45
 
46
46
  def custom_executor():
47
- return client.get_dashboards_list()
47
+ collections = client.get_collections_list_test_conn()
48
+ return client.get_dashboards_list_test_conn(collections)
48
49
 
49
50
  test_fn = {"GetDashboards": custom_executor}
50
51
 
@@ -67,7 +67,9 @@ class MetabaseSource(DashboardServiceSource):
67
67
  metadata_config: OpenMetadataConnection
68
68
 
69
69
  @classmethod
70
- def create(cls, config_dict, metadata: OpenMetadata):
70
+ def create(
71
+ cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
72
+ ):
71
73
  config = WorkflowSource.parse_obj(config_dict)
72
74
  connection: MetabaseConnection = config.serviceConnection.__root__.config
73
75
  if not isinstance(connection, MetabaseConnection):
@@ -92,7 +94,7 @@ class MetabaseSource(DashboardServiceSource):
92
94
  """
93
95
  Get List of all dashboards
94
96
  """
95
- return self.client.get_dashboards_list()
97
+ return self.client.get_dashboards_list(self.collections)
96
98
 
97
99
  def get_dashboard_name(self, dashboard: MetabaseDashboard) -> str:
98
100
  """
@@ -144,17 +146,17 @@ class MetabaseSource(DashboardServiceSource):
144
146
  sourceUrl=dashboard_url,
145
147
  displayName=dashboard_details.name,
146
148
  description=dashboard_details.description,
147
- project=self.context.project_name,
149
+ project=self.context.get().project_name,
148
150
  charts=[
149
151
  fqn.build(
150
152
  self.metadata,
151
153
  entity_type=Chart,
152
- service_name=self.context.dashboard_service,
154
+ service_name=self.context.get().dashboard_service,
153
155
  chart_name=chart,
154
156
  )
155
- for chart in self.context.charts or []
157
+ for chart in self.context.get().charts or []
156
158
  ],
157
- service=self.context.dashboard_service,
159
+ service=self.context.get().dashboard_service,
158
160
  owner=self.get_owner_ref(dashboard_details=dashboard_details),
159
161
  )
160
162
  yield Either(right=dashboard_request)
@@ -200,7 +202,7 @@ class MetabaseSource(DashboardServiceSource):
200
202
  description=chart_details.description,
201
203
  chartType=get_standard_chart_type(chart_details.display).value,
202
204
  sourceUrl=chart_url,
203
- service=self.context.dashboard_service,
205
+ service=self.context.get().dashboard_service,
204
206
  )
205
207
  )
206
208
  except Exception as exc: # pylint: disable=broad-except
@@ -37,7 +37,7 @@ class MetabaseCollection(BaseModel):
37
37
 
38
38
 
39
39
  class MetabaseDashboardList(BaseModel):
40
- dashboards: Optional[List[MetabaseDashboard]]
40
+ data: Optional[List[MetabaseDashboard]]
41
41
 
42
42
 
43
43
  class MetabaseCollectionList(BaseModel):
@@ -59,7 +59,9 @@ class ModeSource(DashboardServiceSource):
59
59
  self.data_sources = self.client.get_all_data_sources(self.workspace_name)
60
60
 
61
61
  @classmethod
62
- def create(cls, config_dict, metadata: OpenMetadata):
62
+ def create(
63
+ cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
64
+ ):
63
65
  config = WorkflowSource.parse_obj(config_dict)
64
66
  connection: ModeConnection = config.serviceConnection.__root__.config
65
67
  if not isinstance(connection, ModeConnection):
@@ -103,12 +105,12 @@ class ModeSource(DashboardServiceSource):
103
105
  fqn.build(
104
106
  self.metadata,
105
107
  entity_type=Chart,
106
- service_name=self.context.dashboard_service,
108
+ service_name=self.context.get().dashboard_service,
107
109
  chart_name=chart,
108
110
  )
109
- for chart in self.context.charts or []
111
+ for chart in self.context.get().charts or []
110
112
  ],
111
- service=self.context.dashboard_service,
113
+ service=self.context.get().dashboard_service,
112
114
  owner=self.get_owner_ref(dashboard_details=dashboard_details),
113
115
  )
114
116
  yield Either(right=dashboard_request)
@@ -203,7 +205,7 @@ class ModeSource(DashboardServiceSource):
203
205
  displayName=chart_name,
204
206
  chartType=ChartType.Other,
205
207
  sourceUrl=chart_url,
206
- service=self.context.dashboard_service,
208
+ service=self.context.get().dashboard_service,
207
209
  )
208
210
  )
209
211
  except Exception as exc:
@@ -48,7 +48,12 @@ class MstrSource(DashboardServiceSource):
48
48
  """
49
49
 
50
50
  @classmethod
51
- def create(cls, config_dict: dict, metadata: OpenMetadata):
51
+ def create(
52
+ cls,
53
+ config_dict: dict,
54
+ metadata: OpenMetadata,
55
+ pipeline_name: Optional[str] = None,
56
+ ):
52
57
  config = WorkflowSource.parse_obj(config_dict)
53
58
  connection: MstrConnection = config.serviceConnection.__root__.config
54
59
  if not isinstance(connection, MstrConnection):
@@ -110,12 +115,12 @@ class MstrSource(DashboardServiceSource):
110
115
  fqn.build(
111
116
  self.metadata,
112
117
  entity_type=Chart,
113
- service_name=self.context.dashboard_service,
118
+ service_name=self.context.get().dashboard_service,
114
119
  chart_name=chart,
115
120
  )
116
- for chart in self.context.charts or []
121
+ for chart in self.context.get().charts or []
117
122
  ],
118
- service=self.context.dashboard_service,
123
+ service=self.context.get().dashboard_service,
119
124
  owner=self.get_owner_ref(dashboard_details=dashboard_details),
120
125
  )
121
126
  yield Either(right=dashboard_request)
@@ -169,7 +174,7 @@ class MstrSource(DashboardServiceSource):
169
174
  chartType=get_standard_chart_type(
170
175
  chart.visualizationType
171
176
  ).value,
172
- service=self.context.dashboard_service,
177
+ service=self.context.get().dashboard_service,
173
178
  )
174
179
  )
175
180
  except Exception as exc: