acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (414) hide show
  1. {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/METADATA +2615 -2547
  2. {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/RECORD +412 -338
  3. {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/entry_points.txt +5 -0
  4. datahub/_version.py +1 -1
  5. datahub/api/entities/assertion/assertion.py +1 -1
  6. datahub/api/entities/common/serialized_value.py +1 -1
  7. datahub/api/entities/corpgroup/corpgroup.py +1 -1
  8. datahub/api/entities/dataproduct/dataproduct.py +32 -3
  9. datahub/api/entities/dataset/dataset.py +26 -23
  10. datahub/api/entities/external/__init__.py +0 -0
  11. datahub/api/entities/external/external_entities.py +724 -0
  12. datahub/api/entities/external/external_tag.py +147 -0
  13. datahub/api/entities/external/lake_formation_external_entites.py +162 -0
  14. datahub/api/entities/external/restricted_text.py +172 -0
  15. datahub/api/entities/external/unity_catalog_external_entites.py +172 -0
  16. datahub/api/entities/forms/forms.py +3 -3
  17. datahub/api/entities/structuredproperties/structuredproperties.py +4 -4
  18. datahub/api/graphql/operation.py +10 -6
  19. datahub/cli/check_cli.py +88 -7
  20. datahub/cli/cli_utils.py +63 -0
  21. datahub/cli/config_utils.py +18 -10
  22. datahub/cli/container_cli.py +5 -0
  23. datahub/cli/delete_cli.py +125 -27
  24. datahub/cli/docker_check.py +110 -14
  25. datahub/cli/docker_cli.py +153 -229
  26. datahub/cli/exists_cli.py +0 -2
  27. datahub/cli/get_cli.py +0 -2
  28. datahub/cli/graphql_cli.py +1422 -0
  29. datahub/cli/iceberg_cli.py +5 -0
  30. datahub/cli/ingest_cli.py +3 -15
  31. datahub/cli/migrate.py +2 -0
  32. datahub/cli/put_cli.py +1 -4
  33. datahub/cli/quickstart_versioning.py +53 -10
  34. datahub/cli/specific/assertions_cli.py +37 -6
  35. datahub/cli/specific/datacontract_cli.py +54 -7
  36. datahub/cli/specific/dataproduct_cli.py +2 -15
  37. datahub/cli/specific/dataset_cli.py +1 -8
  38. datahub/cli/specific/forms_cli.py +0 -4
  39. datahub/cli/specific/group_cli.py +0 -2
  40. datahub/cli/specific/structuredproperties_cli.py +1 -4
  41. datahub/cli/specific/user_cli.py +172 -3
  42. datahub/cli/state_cli.py +0 -2
  43. datahub/cli/timeline_cli.py +0 -2
  44. datahub/configuration/common.py +40 -1
  45. datahub/configuration/connection_resolver.py +5 -2
  46. datahub/configuration/env_vars.py +331 -0
  47. datahub/configuration/import_resolver.py +7 -4
  48. datahub/configuration/kafka.py +21 -1
  49. datahub/configuration/pydantic_migration_helpers.py +6 -13
  50. datahub/configuration/source_common.py +3 -2
  51. datahub/configuration/validate_field_deprecation.py +5 -2
  52. datahub/configuration/validate_field_removal.py +8 -2
  53. datahub/configuration/validate_field_rename.py +6 -5
  54. datahub/configuration/validate_multiline_string.py +5 -2
  55. datahub/emitter/mce_builder.py +8 -4
  56. datahub/emitter/rest_emitter.py +103 -30
  57. datahub/entrypoints.py +6 -3
  58. datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py +297 -1
  59. datahub/ingestion/api/auto_work_units/auto_validate_input_fields.py +87 -0
  60. datahub/ingestion/api/decorators.py +15 -3
  61. datahub/ingestion/api/report.py +381 -3
  62. datahub/ingestion/api/sink.py +27 -2
  63. datahub/ingestion/api/source.py +165 -58
  64. datahub/ingestion/api/source_protocols.py +23 -0
  65. datahub/ingestion/autogenerated/__init__.py +0 -0
  66. datahub/ingestion/autogenerated/capability_summary.json +3652 -0
  67. datahub/ingestion/autogenerated/lineage.json +402 -0
  68. datahub/ingestion/autogenerated/lineage_helper.py +177 -0
  69. datahub/ingestion/extractor/schema_util.py +13 -4
  70. datahub/ingestion/glossary/classification_mixin.py +5 -0
  71. datahub/ingestion/graph/client.py +330 -25
  72. datahub/ingestion/graph/config.py +3 -2
  73. datahub/ingestion/graph/filters.py +30 -11
  74. datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +21 -11
  75. datahub/ingestion/run/pipeline.py +81 -11
  76. datahub/ingestion/run/pipeline_config.py +2 -2
  77. datahub/ingestion/sink/datahub_kafka.py +1 -0
  78. datahub/ingestion/sink/datahub_rest.py +13 -5
  79. datahub/ingestion/sink/file.py +1 -0
  80. datahub/ingestion/source/abs/config.py +1 -1
  81. datahub/ingestion/source/abs/datalake_profiler_config.py +1 -1
  82. datahub/ingestion/source/abs/source.py +15 -30
  83. datahub/ingestion/source/aws/aws_common.py +185 -13
  84. datahub/ingestion/source/aws/glue.py +517 -244
  85. datahub/ingestion/source/aws/platform_resource_repository.py +30 -0
  86. datahub/ingestion/source/aws/s3_boto_utils.py +100 -5
  87. datahub/ingestion/source/aws/tag_entities.py +270 -0
  88. datahub/ingestion/source/azure/azure_common.py +3 -3
  89. datahub/ingestion/source/bigquery_v2/bigquery.py +67 -24
  90. datahub/ingestion/source/bigquery_v2/bigquery_config.py +47 -19
  91. datahub/ingestion/source/bigquery_v2/bigquery_connection.py +12 -1
  92. datahub/ingestion/source/bigquery_v2/bigquery_queries.py +3 -0
  93. datahub/ingestion/source/bigquery_v2/bigquery_report.py +0 -2
  94. datahub/ingestion/source/bigquery_v2/bigquery_schema.py +23 -16
  95. datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +20 -5
  96. datahub/ingestion/source/bigquery_v2/common.py +1 -1
  97. datahub/ingestion/source/bigquery_v2/profiler.py +4 -2
  98. datahub/ingestion/source/bigquery_v2/queries.py +3 -3
  99. datahub/ingestion/source/bigquery_v2/queries_extractor.py +45 -9
  100. datahub/ingestion/source/cassandra/cassandra.py +6 -8
  101. datahub/ingestion/source/cassandra/cassandra_api.py +17 -1
  102. datahub/ingestion/source/cassandra/cassandra_config.py +5 -0
  103. datahub/ingestion/source/cassandra/cassandra_profiling.py +7 -6
  104. datahub/ingestion/source/cassandra/cassandra_utils.py +1 -2
  105. datahub/ingestion/source/common/gcp_credentials_config.py +3 -1
  106. datahub/ingestion/source/common/subtypes.py +53 -0
  107. datahub/ingestion/source/data_lake_common/data_lake_utils.py +37 -0
  108. datahub/ingestion/source/data_lake_common/object_store.py +115 -27
  109. datahub/ingestion/source/data_lake_common/path_spec.py +72 -43
  110. datahub/ingestion/source/datahub/config.py +12 -9
  111. datahub/ingestion/source/datahub/datahub_database_reader.py +26 -11
  112. datahub/ingestion/source/datahub/datahub_source.py +10 -0
  113. datahub/ingestion/source/dbt/dbt_cloud.py +16 -5
  114. datahub/ingestion/source/dbt/dbt_common.py +224 -9
  115. datahub/ingestion/source/dbt/dbt_core.py +3 -0
  116. datahub/ingestion/source/debug/__init__.py +0 -0
  117. datahub/ingestion/source/debug/datahub_debug.py +300 -0
  118. datahub/ingestion/source/delta_lake/config.py +9 -5
  119. datahub/ingestion/source/delta_lake/source.py +8 -0
  120. datahub/ingestion/source/dremio/dremio_api.py +114 -73
  121. datahub/ingestion/source/dremio/dremio_aspects.py +3 -2
  122. datahub/ingestion/source/dremio/dremio_config.py +5 -4
  123. datahub/ingestion/source/dremio/dremio_reporting.py +22 -3
  124. datahub/ingestion/source/dremio/dremio_source.py +132 -98
  125. datahub/ingestion/source/dremio/dremio_sql_queries.py +82 -21
  126. datahub/ingestion/source/dynamodb/dynamodb.py +11 -8
  127. datahub/ingestion/source/excel/__init__.py +0 -0
  128. datahub/ingestion/source/excel/config.py +92 -0
  129. datahub/ingestion/source/excel/excel_file.py +539 -0
  130. datahub/ingestion/source/excel/profiling.py +308 -0
  131. datahub/ingestion/source/excel/report.py +49 -0
  132. datahub/ingestion/source/excel/source.py +662 -0
  133. datahub/ingestion/source/excel/util.py +18 -0
  134. datahub/ingestion/source/feast.py +8 -10
  135. datahub/ingestion/source/file.py +3 -0
  136. datahub/ingestion/source/fivetran/config.py +66 -7
  137. datahub/ingestion/source/fivetran/fivetran.py +227 -43
  138. datahub/ingestion/source/fivetran/fivetran_log_api.py +37 -8
  139. datahub/ingestion/source/fivetran/fivetran_query.py +51 -29
  140. datahub/ingestion/source/fivetran/fivetran_rest_api.py +65 -0
  141. datahub/ingestion/source/fivetran/response_models.py +97 -0
  142. datahub/ingestion/source/gc/datahub_gc.py +0 -2
  143. datahub/ingestion/source/gcs/gcs_source.py +32 -4
  144. datahub/ingestion/source/ge_data_profiler.py +108 -31
  145. datahub/ingestion/source/ge_profiling_config.py +26 -11
  146. datahub/ingestion/source/grafana/entity_mcp_builder.py +272 -0
  147. datahub/ingestion/source/grafana/field_utils.py +307 -0
  148. datahub/ingestion/source/grafana/grafana_api.py +142 -0
  149. datahub/ingestion/source/grafana/grafana_config.py +104 -0
  150. datahub/ingestion/source/grafana/grafana_source.py +522 -84
  151. datahub/ingestion/source/grafana/lineage.py +202 -0
  152. datahub/ingestion/source/grafana/models.py +137 -0
  153. datahub/ingestion/source/grafana/report.py +90 -0
  154. datahub/ingestion/source/grafana/types.py +16 -0
  155. datahub/ingestion/source/hex/api.py +28 -1
  156. datahub/ingestion/source/hex/hex.py +16 -5
  157. datahub/ingestion/source/hex/mapper.py +16 -2
  158. datahub/ingestion/source/hex/model.py +2 -0
  159. datahub/ingestion/source/hex/query_fetcher.py +1 -1
  160. datahub/ingestion/source/iceberg/iceberg.py +123 -59
  161. datahub/ingestion/source/iceberg/iceberg_profiler.py +4 -2
  162. datahub/ingestion/source/identity/azure_ad.py +1 -1
  163. datahub/ingestion/source/identity/okta.py +1 -14
  164. datahub/ingestion/source/kafka/kafka.py +16 -0
  165. datahub/ingestion/source/kafka_connect/common.py +2 -2
  166. datahub/ingestion/source/kafka_connect/sink_connectors.py +156 -47
  167. datahub/ingestion/source/kafka_connect/source_connectors.py +62 -4
  168. datahub/ingestion/source/looker/looker_common.py +148 -79
  169. datahub/ingestion/source/looker/looker_config.py +15 -4
  170. datahub/ingestion/source/looker/looker_constant.py +4 -0
  171. datahub/ingestion/source/looker/looker_lib_wrapper.py +36 -3
  172. datahub/ingestion/source/looker/looker_liquid_tag.py +56 -5
  173. datahub/ingestion/source/looker/looker_source.py +503 -547
  174. datahub/ingestion/source/looker/looker_view_id_cache.py +1 -1
  175. datahub/ingestion/source/looker/lookml_concept_context.py +1 -1
  176. datahub/ingestion/source/looker/lookml_config.py +31 -3
  177. datahub/ingestion/source/looker/lookml_refinement.py +1 -1
  178. datahub/ingestion/source/looker/lookml_source.py +96 -117
  179. datahub/ingestion/source/looker/view_upstream.py +494 -1
  180. datahub/ingestion/source/metabase.py +32 -6
  181. datahub/ingestion/source/metadata/business_glossary.py +7 -7
  182. datahub/ingestion/source/metadata/lineage.py +9 -9
  183. datahub/ingestion/source/mlflow.py +12 -2
  184. datahub/ingestion/source/mock_data/__init__.py +0 -0
  185. datahub/ingestion/source/mock_data/datahub_mock_data.py +533 -0
  186. datahub/ingestion/source/mock_data/datahub_mock_data_report.py +12 -0
  187. datahub/ingestion/source/mock_data/table_naming_helper.py +97 -0
  188. datahub/ingestion/source/mode.py +26 -5
  189. datahub/ingestion/source/mongodb.py +11 -1
  190. datahub/ingestion/source/neo4j/neo4j_source.py +83 -144
  191. datahub/ingestion/source/nifi.py +2 -2
  192. datahub/ingestion/source/openapi.py +1 -1
  193. datahub/ingestion/source/powerbi/config.py +47 -21
  194. datahub/ingestion/source/powerbi/m_query/data_classes.py +1 -0
  195. datahub/ingestion/source/powerbi/m_query/parser.py +2 -2
  196. datahub/ingestion/source/powerbi/m_query/pattern_handler.py +100 -10
  197. datahub/ingestion/source/powerbi/powerbi.py +10 -6
  198. datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +0 -1
  199. datahub/ingestion/source/powerbi_report_server/report_server.py +0 -23
  200. datahub/ingestion/source/powerbi_report_server/report_server_domain.py +2 -4
  201. datahub/ingestion/source/preset.py +3 -3
  202. datahub/ingestion/source/qlik_sense/data_classes.py +28 -8
  203. datahub/ingestion/source/qlik_sense/qlik_sense.py +2 -1
  204. datahub/ingestion/source/redash.py +1 -1
  205. datahub/ingestion/source/redshift/config.py +15 -9
  206. datahub/ingestion/source/redshift/datashares.py +1 -1
  207. datahub/ingestion/source/redshift/lineage.py +386 -687
  208. datahub/ingestion/source/redshift/query.py +23 -19
  209. datahub/ingestion/source/redshift/redshift.py +52 -111
  210. datahub/ingestion/source/redshift/redshift_schema.py +17 -12
  211. datahub/ingestion/source/redshift/report.py +0 -2
  212. datahub/ingestion/source/redshift/usage.py +6 -5
  213. datahub/ingestion/source/s3/report.py +4 -2
  214. datahub/ingestion/source/s3/source.py +449 -248
  215. datahub/ingestion/source/sac/sac.py +3 -1
  216. datahub/ingestion/source/salesforce.py +28 -13
  217. datahub/ingestion/source/schema/json_schema.py +14 -14
  218. datahub/ingestion/source/schema_inference/object.py +22 -6
  219. datahub/ingestion/source/sigma/data_classes.py +3 -0
  220. datahub/ingestion/source/sigma/sigma.py +7 -1
  221. datahub/ingestion/source/slack/slack.py +10 -16
  222. datahub/ingestion/source/snaplogic/__init__.py +0 -0
  223. datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
  224. datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
  225. datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
  226. datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
  227. datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
  228. datahub/ingestion/source/snowflake/constants.py +3 -0
  229. datahub/ingestion/source/snowflake/snowflake_config.py +76 -23
  230. datahub/ingestion/source/snowflake/snowflake_connection.py +24 -8
  231. datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +19 -6
  232. datahub/ingestion/source/snowflake/snowflake_queries.py +464 -97
  233. datahub/ingestion/source/snowflake/snowflake_query.py +77 -5
  234. datahub/ingestion/source/snowflake/snowflake_report.py +1 -2
  235. datahub/ingestion/source/snowflake/snowflake_schema.py +352 -16
  236. datahub/ingestion/source/snowflake/snowflake_schema_gen.py +51 -10
  237. datahub/ingestion/source/snowflake/snowflake_summary.py +7 -1
  238. datahub/ingestion/source/snowflake/snowflake_usage_v2.py +8 -2
  239. datahub/ingestion/source/snowflake/snowflake_utils.py +36 -15
  240. datahub/ingestion/source/snowflake/snowflake_v2.py +39 -4
  241. datahub/ingestion/source/snowflake/stored_proc_lineage.py +143 -0
  242. datahub/ingestion/source/sql/athena.py +217 -25
  243. datahub/ingestion/source/sql/athena_properties_extractor.py +795 -0
  244. datahub/ingestion/source/sql/clickhouse.py +24 -8
  245. datahub/ingestion/source/sql/cockroachdb.py +5 -4
  246. datahub/ingestion/source/sql/druid.py +2 -2
  247. datahub/ingestion/source/sql/hana.py +3 -1
  248. datahub/ingestion/source/sql/hive.py +4 -3
  249. datahub/ingestion/source/sql/hive_metastore.py +19 -20
  250. datahub/ingestion/source/sql/mariadb.py +0 -1
  251. datahub/ingestion/source/sql/mssql/job_models.py +3 -1
  252. datahub/ingestion/source/sql/mssql/source.py +336 -57
  253. datahub/ingestion/source/sql/mysql.py +154 -4
  254. datahub/ingestion/source/sql/oracle.py +5 -5
  255. datahub/ingestion/source/sql/postgres.py +142 -6
  256. datahub/ingestion/source/sql/presto.py +2 -1
  257. datahub/ingestion/source/sql/sql_common.py +281 -49
  258. datahub/ingestion/source/sql/sql_generic_profiler.py +2 -1
  259. datahub/ingestion/source/sql/sql_types.py +22 -0
  260. datahub/ingestion/source/sql/sqlalchemy_uri.py +39 -7
  261. datahub/ingestion/source/sql/teradata.py +1028 -245
  262. datahub/ingestion/source/sql/trino.py +11 -1
  263. datahub/ingestion/source/sql/two_tier_sql_source.py +2 -3
  264. datahub/ingestion/source/sql/vertica.py +14 -7
  265. datahub/ingestion/source/sql_queries.py +219 -121
  266. datahub/ingestion/source/state/checkpoint.py +8 -29
  267. datahub/ingestion/source/state/entity_removal_state.py +5 -2
  268. datahub/ingestion/source/state/redundant_run_skip_handler.py +21 -0
  269. datahub/ingestion/source/state/stateful_ingestion_base.py +36 -11
  270. datahub/ingestion/source/superset.py +314 -67
  271. datahub/ingestion/source/tableau/tableau.py +135 -59
  272. datahub/ingestion/source/tableau/tableau_common.py +9 -2
  273. datahub/ingestion/source/tableau/tableau_constant.py +1 -4
  274. datahub/ingestion/source/tableau/tableau_server_wrapper.py +3 -0
  275. datahub/ingestion/source/unity/config.py +160 -40
  276. datahub/ingestion/source/unity/connection.py +61 -0
  277. datahub/ingestion/source/unity/connection_test.py +1 -0
  278. datahub/ingestion/source/unity/platform_resource_repository.py +19 -0
  279. datahub/ingestion/source/unity/proxy.py +794 -51
  280. datahub/ingestion/source/unity/proxy_patch.py +321 -0
  281. datahub/ingestion/source/unity/proxy_types.py +36 -2
  282. datahub/ingestion/source/unity/report.py +15 -3
  283. datahub/ingestion/source/unity/source.py +465 -131
  284. datahub/ingestion/source/unity/tag_entities.py +197 -0
  285. datahub/ingestion/source/unity/usage.py +46 -4
  286. datahub/ingestion/source/usage/clickhouse_usage.py +4 -1
  287. datahub/ingestion/source/usage/starburst_trino_usage.py +5 -2
  288. datahub/ingestion/source/usage/usage_common.py +4 -3
  289. datahub/ingestion/source/vertexai/vertexai.py +1 -1
  290. datahub/ingestion/source_config/pulsar.py +3 -1
  291. datahub/ingestion/source_report/ingestion_stage.py +50 -11
  292. datahub/ingestion/transformer/add_dataset_ownership.py +18 -2
  293. datahub/ingestion/transformer/base_transformer.py +8 -5
  294. datahub/ingestion/transformer/set_browse_path.py +112 -0
  295. datahub/integrations/assertion/snowflake/compiler.py +4 -3
  296. datahub/metadata/_internal_schema_classes.py +6806 -4871
  297. datahub/metadata/_urns/urn_defs.py +1767 -1539
  298. datahub/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
  299. datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +2 -0
  300. datahub/metadata/com/linkedin/pegasus2avro/file/__init__.py +19 -0
  301. datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py +2 -0
  302. datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
  303. datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +6 -0
  304. datahub/metadata/com/linkedin/pegasus2avro/module/__init__.py +31 -0
  305. datahub/metadata/com/linkedin/pegasus2avro/platform/event/v1/__init__.py +4 -0
  306. datahub/metadata/com/linkedin/pegasus2avro/role/__init__.py +2 -0
  307. datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py +19 -0
  308. datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +8 -0
  309. datahub/metadata/com/linkedin/pegasus2avro/template/__init__.py +31 -0
  310. datahub/metadata/schema.avsc +18395 -16979
  311. datahub/metadata/schemas/Actors.avsc +38 -1
  312. datahub/metadata/schemas/ApplicationKey.avsc +31 -0
  313. datahub/metadata/schemas/ApplicationProperties.avsc +72 -0
  314. datahub/metadata/schemas/Applications.avsc +38 -0
  315. datahub/metadata/schemas/AssetSettings.avsc +63 -0
  316. datahub/metadata/schemas/ChartInfo.avsc +2 -1
  317. datahub/metadata/schemas/ChartKey.avsc +1 -0
  318. datahub/metadata/schemas/ContainerKey.avsc +1 -0
  319. datahub/metadata/schemas/ContainerProperties.avsc +8 -0
  320. datahub/metadata/schemas/CorpUserEditableInfo.avsc +1 -1
  321. datahub/metadata/schemas/CorpUserSettings.avsc +50 -0
  322. datahub/metadata/schemas/DashboardKey.avsc +1 -0
  323. datahub/metadata/schemas/DataFlowInfo.avsc +8 -0
  324. datahub/metadata/schemas/DataFlowKey.avsc +1 -0
  325. datahub/metadata/schemas/DataHubFileInfo.avsc +230 -0
  326. datahub/metadata/schemas/DataHubFileKey.avsc +21 -0
  327. datahub/metadata/schemas/DataHubPageModuleKey.avsc +21 -0
  328. datahub/metadata/schemas/DataHubPageModuleProperties.avsc +298 -0
  329. datahub/metadata/schemas/DataHubPageTemplateKey.avsc +21 -0
  330. datahub/metadata/schemas/DataHubPageTemplateProperties.avsc +251 -0
  331. datahub/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
  332. datahub/metadata/schemas/DataJobInfo.avsc +8 -0
  333. datahub/metadata/schemas/DataJobInputOutput.avsc +8 -0
  334. datahub/metadata/schemas/DataJobKey.avsc +1 -0
  335. datahub/metadata/schemas/DataProcessKey.avsc +8 -0
  336. datahub/metadata/schemas/DataProductKey.avsc +3 -1
  337. datahub/metadata/schemas/DataProductProperties.avsc +1 -1
  338. datahub/metadata/schemas/DatasetKey.avsc +11 -1
  339. datahub/metadata/schemas/DatasetUsageStatistics.avsc +8 -0
  340. datahub/metadata/schemas/DomainKey.avsc +2 -1
  341. datahub/metadata/schemas/GlobalSettingsInfo.avsc +134 -0
  342. datahub/metadata/schemas/GlossaryNodeKey.avsc +2 -1
  343. datahub/metadata/schemas/GlossaryTermKey.avsc +3 -1
  344. datahub/metadata/schemas/IcebergWarehouseInfo.avsc +8 -0
  345. datahub/metadata/schemas/IncidentInfo.avsc +3 -3
  346. datahub/metadata/schemas/InstitutionalMemory.avsc +31 -0
  347. datahub/metadata/schemas/LogicalParent.avsc +145 -0
  348. datahub/metadata/schemas/MLFeatureKey.avsc +1 -0
  349. datahub/metadata/schemas/MLFeatureTableKey.avsc +1 -0
  350. datahub/metadata/schemas/MLModelDeploymentKey.avsc +8 -0
  351. datahub/metadata/schemas/MLModelGroupKey.avsc +11 -1
  352. datahub/metadata/schemas/MLModelKey.avsc +9 -0
  353. datahub/metadata/schemas/MLPrimaryKeyKey.avsc +1 -0
  354. datahub/metadata/schemas/MetadataChangeEvent.avsc +151 -47
  355. datahub/metadata/schemas/MetadataChangeLog.avsc +62 -44
  356. datahub/metadata/schemas/MetadataChangeProposal.avsc +61 -0
  357. datahub/metadata/schemas/NotebookKey.avsc +1 -0
  358. datahub/metadata/schemas/Operation.avsc +4 -2
  359. datahub/metadata/schemas/Ownership.avsc +69 -0
  360. datahub/metadata/schemas/QuerySubjects.avsc +1 -12
  361. datahub/metadata/schemas/RelationshipChangeEvent.avsc +215 -0
  362. datahub/metadata/schemas/SchemaFieldKey.avsc +4 -1
  363. datahub/metadata/schemas/StructuredProperties.avsc +69 -0
  364. datahub/metadata/schemas/StructuredPropertySettings.avsc +9 -0
  365. datahub/metadata/schemas/SystemMetadata.avsc +61 -0
  366. datahub/metadata/schemas/UpstreamLineage.avsc +9 -0
  367. datahub/sdk/__init__.py +2 -0
  368. datahub/sdk/_all_entities.py +7 -0
  369. datahub/sdk/_shared.py +249 -5
  370. datahub/sdk/chart.py +386 -0
  371. datahub/sdk/container.py +7 -0
  372. datahub/sdk/dashboard.py +453 -0
  373. datahub/sdk/dataflow.py +7 -0
  374. datahub/sdk/datajob.py +45 -13
  375. datahub/sdk/dataset.py +56 -2
  376. datahub/sdk/entity_client.py +111 -9
  377. datahub/sdk/lineage_client.py +663 -82
  378. datahub/sdk/main_client.py +50 -16
  379. datahub/sdk/mlmodel.py +120 -38
  380. datahub/sdk/mlmodelgroup.py +7 -0
  381. datahub/sdk/search_client.py +7 -3
  382. datahub/sdk/search_filters.py +304 -36
  383. datahub/secret/datahub_secret_store.py +3 -0
  384. datahub/secret/environment_secret_store.py +29 -0
  385. datahub/secret/file_secret_store.py +49 -0
  386. datahub/specific/aspect_helpers/fine_grained_lineage.py +76 -0
  387. datahub/specific/aspect_helpers/siblings.py +73 -0
  388. datahub/specific/aspect_helpers/structured_properties.py +27 -0
  389. datahub/specific/chart.py +1 -1
  390. datahub/specific/datajob.py +15 -1
  391. datahub/specific/dataproduct.py +4 -0
  392. datahub/specific/dataset.py +39 -59
  393. datahub/sql_parsing/split_statements.py +13 -0
  394. datahub/sql_parsing/sql_parsing_aggregator.py +70 -26
  395. datahub/sql_parsing/sqlglot_lineage.py +196 -42
  396. datahub/sql_parsing/sqlglot_utils.py +12 -4
  397. datahub/sql_parsing/tool_meta_extractor.py +1 -3
  398. datahub/telemetry/telemetry.py +28 -14
  399. datahub/testing/sdk_v2_helpers.py +7 -1
  400. datahub/upgrade/upgrade.py +73 -17
  401. datahub/utilities/file_backed_collections.py +8 -9
  402. datahub/utilities/is_pytest.py +3 -2
  403. datahub/utilities/logging_manager.py +22 -6
  404. datahub/utilities/mapping.py +29 -2
  405. datahub/utilities/sample_data.py +5 -4
  406. datahub/utilities/server_config_util.py +10 -1
  407. datahub/utilities/sqlalchemy_query_combiner.py +5 -2
  408. datahub/utilities/stats_collections.py +4 -0
  409. datahub/utilities/urns/urn.py +41 -2
  410. datahub/emitter/sql_parsing_builder.py +0 -306
  411. datahub/ingestion/source/redshift/lineage_v2.py +0 -466
  412. {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/WHEEL +0 -0
  413. {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/licenses/LICENSE +0 -0
  414. {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/top_level.txt +0 -0
@@ -16,6 +16,7 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
16
16
  from datahub.ingestion.graph.config import ClientMode
17
17
  from datahub.metadata.schema_classes import SystemMetadataClass
18
18
  from datahub.telemetry import telemetry
19
+ from datahub.upgrade import upgrade
19
20
 
20
21
  logger = logging.getLogger(__name__)
21
22
 
@@ -164,6 +165,7 @@ def validate_warehouse(data_root: str) -> None:
164
165
  help=f"Expiration duration for temporary credentials used for role. Defaults to {DEFAULT_CREDS_EXPIRY_DURATION_SECONDS} seconds if unspecified",
165
166
  )
166
167
  @telemetry.with_telemetry(capture_kwargs=["duration_seconds"])
168
+ @upgrade.check_upgrade
167
169
  def create(
168
170
  warehouse: str,
169
171
  description: Optional[str],
@@ -317,6 +319,7 @@ def create(
317
319
  help=f"Expiration duration for temporary credentials used for role. Defaults to {DEFAULT_CREDS_EXPIRY_DURATION_SECONDS} seconds if unspecified",
318
320
  )
319
321
  @telemetry.with_telemetry(capture_kwargs=["duration_seconds"])
322
+ @upgrade.check_upgrade
320
323
  def update(
321
324
  warehouse: str,
322
325
  data_root: str,
@@ -403,6 +406,7 @@ def update(
403
406
 
404
407
  @iceberg.command()
405
408
  @telemetry.with_telemetry()
409
+ @upgrade.check_upgrade
406
410
  def list() -> None:
407
411
  """
408
412
  List iceberg warehouses
@@ -419,6 +423,7 @@ def list() -> None:
419
423
  "-w", "--warehouse", required=True, type=str, help="The name of the warehouse"
420
424
  )
421
425
  @telemetry.with_telemetry()
426
+ @upgrade.check_upgrade
422
427
  def get(warehouse: str) -> None:
423
428
  """Fetches the details of the specified iceberg warehouse"""
424
429
  client = get_default_graph(ClientMode.CLI)
datahub/cli/ingest_cli.py CHANGED
@@ -24,7 +24,6 @@ from datahub.ingestion.run.pipeline import Pipeline
24
24
  from datahub.telemetry import telemetry
25
25
  from datahub.upgrade import upgrade
26
26
  from datahub.utilities.ingest_utils import deploy_source_vars
27
- from datahub.utilities.perf_timer import PerfTimer
28
27
 
29
28
  logger = logging.getLogger(__name__)
30
29
 
@@ -114,6 +113,7 @@ def ingest() -> None:
114
113
  "no_progress",
115
114
  ]
116
115
  )
116
+ @upgrade.check_upgrade
117
117
  def run(
118
118
  config: str,
119
119
  dry_run: bool,
@@ -178,14 +178,7 @@ def run(
178
178
  no_progress=no_progress,
179
179
  raw_config=raw_pipeline_config,
180
180
  )
181
- with PerfTimer() as timer:
182
- ret = run_pipeline_to_completion(pipeline)
183
-
184
- # The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
185
- if ret == 0:
186
- upgrade.check_upgrade_post(
187
- main_method_runtime=timer.elapsed_seconds(), graph=pipeline.ctx.graph
188
- )
181
+ ret = run_pipeline_to_completion(pipeline)
189
182
 
190
183
  if ret:
191
184
  sys.exit(ret)
@@ -193,8 +186,6 @@ def run(
193
186
 
194
187
 
195
188
  @ingest.command()
196
- @upgrade.check_upgrade
197
- @telemetry.with_telemetry()
198
189
  @click.option(
199
190
  "-n",
200
191
  "--name",
@@ -252,6 +243,7 @@ def run(
252
243
  required=False,
253
244
  default=None,
254
245
  )
246
+ @upgrade.check_upgrade
255
247
  def deploy(
256
248
  name: Optional[str],
257
249
  config: str,
@@ -386,7 +378,6 @@ def mcps(path: str) -> None:
386
378
  "--source", type=str, default=None, help="Filter by ingestion source name."
387
379
  )
388
380
  @upgrade.check_upgrade
389
- @telemetry.with_telemetry()
390
381
  def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) -> None:
391
382
  """
392
383
  List ingestion source runs with their details, optionally filtered by URN or source.
@@ -515,7 +506,6 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
515
506
  help="If enabled, will list ingestion runs which have been soft deleted",
516
507
  )
517
508
  @upgrade.check_upgrade
518
- @telemetry.with_telemetry()
519
509
  def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> None:
520
510
  """List recent ingestion runs to datahub"""
521
511
 
@@ -565,7 +555,6 @@ def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> N
565
555
  )
566
556
  @click.option("-a", "--show-aspect", required=False, is_flag=True)
567
557
  @upgrade.check_upgrade
568
- @telemetry.with_telemetry()
569
558
  def show(
570
559
  run_id: str, start: int, count: int, include_soft_deletes: bool, show_aspect: bool
571
560
  ) -> None:
@@ -615,7 +604,6 @@ def show(
615
604
  help="Path to directory where rollback reports will be saved to",
616
605
  )
617
606
  @upgrade.check_upgrade
618
- @telemetry.with_telemetry()
619
607
  def rollback(
620
608
  run_id: str, force: bool, dry_run: bool, safe: bool, report_dir: str
621
609
  ) -> None:
datahub/cli/migrate.py CHANGED
@@ -37,6 +37,7 @@ from datahub.metadata.schema_classes import (
37
37
  SystemMetadataClass,
38
38
  )
39
39
  from datahub.telemetry import telemetry
40
+ from datahub.upgrade import upgrade
40
41
  from datahub.utilities.urns.urn import Urn
41
42
 
42
43
  log = logging.getLogger(__name__)
@@ -119,6 +120,7 @@ def _get_type_from_urn(urn: str) -> str:
119
120
  help="When enabled, will not delete (hard/soft) the previous entities.",
120
121
  )
121
122
  @telemetry.with_telemetry()
123
+ @upgrade.check_upgrade
122
124
  def dataplatform2instance(
123
125
  instance: str,
124
126
  platform: str,
datahub/cli/put_cli.py CHANGED
@@ -14,7 +14,6 @@ from datahub.metadata.schema_classes import (
14
14
  PlatformTypeClass,
15
15
  SystemMetadataClass,
16
16
  )
17
- from datahub.telemetry import telemetry
18
17
  from datahub.upgrade import upgrade
19
18
  from datahub.utilities.urns.data_platform_urn import DataPlatformUrn
20
19
  from datahub.utilities.urns.urn import guess_entity_type
@@ -45,7 +44,6 @@ def put() -> None:
45
44
  help="Run ID into which we should log the aspect.",
46
45
  )
47
46
  @upgrade.check_upgrade
48
- @telemetry.with_telemetry()
49
47
  def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> None:
50
48
  """Update a single aspect of an entity"""
51
49
 
@@ -75,8 +73,6 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
75
73
 
76
74
  @put.command()
77
75
  @click.pass_context
78
- @upgrade.check_upgrade
79
- @telemetry.with_telemetry()
80
76
  @click.option(
81
77
  "--name",
82
78
  type=str,
@@ -98,6 +94,7 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
98
94
  @click.option(
99
95
  "--run-id", type=str, help="Run ID into which we should log the platform."
100
96
  )
97
+ @upgrade.check_upgrade
101
98
  def platform(
102
99
  ctx: click.Context, name: str, display_name: Optional[str], logo: str, run_id: str
103
100
  ) -> None:
@@ -1,27 +1,50 @@
1
1
  import json
2
2
  import logging
3
- import os
4
3
  import os.path
5
4
  import re
6
5
  from typing import Dict, Optional
7
6
 
8
7
  import click
8
+ import packaging
9
9
  import requests
10
10
  import yaml
11
11
  from packaging.version import parse
12
12
  from pydantic import BaseModel
13
13
 
14
+ from datahub._version import nice_version_name
15
+ from datahub.configuration.env_vars import get_force_local_quickstart_mapping
16
+
14
17
  logger = logging.getLogger(__name__)
15
18
 
16
- LOCAL_QUICKSTART_MAPPING_FILE = os.environ.get("FORCE_LOCAL_QUICKSTART_MAPPING", "")
19
+ LOCAL_QUICKSTART_MAPPING_FILE = get_force_local_quickstart_mapping()
17
20
  DEFAULT_LOCAL_CONFIG_PATH = "~/.datahub/quickstart/quickstart_version_mapping.yaml"
18
21
  DEFAULT_REMOTE_CONFIG_PATH = "https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/quickstart_version_mapping.yaml"
19
22
 
23
+ MINIMUM_SUPPORTED_VERSION = "v1.1.0"
24
+
25
+
26
+ def get_minimum_supported_version_message(version: str) -> str:
27
+ MINIMUM_SUPPORTED_VERSION_MESSAGE = f"""
28
+ DataHub CLI Version Compatibility Issue
29
+
30
+ You're trying to install DataHub server version {version} which is not supported by this CLI version.
31
+
32
+ This CLI (version {nice_version_name()}) only supports installing DataHub server versions {MINIMUM_SUPPORTED_VERSION} and above.
33
+
34
+ To install older server versions:
35
+ 1. Uninstall current CLI: pip uninstall acryl-datahub
36
+ 2. Install older CLI: pip install acryl-datahub==1.1
37
+ 3. Run quickstart with your desired version: datahub docker quickstart --version <version>
38
+
39
+ For more information: https://docs.datahub.com/docs/quickstart#install-datahub-server
40
+ """
41
+ return MINIMUM_SUPPORTED_VERSION_MESSAGE
42
+
20
43
 
21
44
  class QuickstartExecutionPlan(BaseModel):
22
45
  composefile_git_ref: str
23
46
  docker_tag: str
24
- mysql_tag: Optional[str]
47
+ mysql_tag: Optional[str] = None
25
48
 
26
49
 
27
50
  def _is_it_a_version(version: str) -> bool:
@@ -126,15 +149,23 @@ class QuickstartVersionMappingConfig(BaseModel):
126
149
  mysql_tag=str(mysql_tag),
127
150
  ),
128
151
  )
152
+
153
+ if not is_minimum_supported_version(requested_version):
154
+ click.secho(
155
+ get_minimum_supported_version_message(version=requested_version),
156
+ fg="red",
157
+ )
158
+ raise click.ClickException("Minimum supported version not met")
159
+
129
160
  # new CLI version is downloading the composefile corresponding to the requested version
130
- # if the version is older than v0.10.1, it doesn't contain the setup job labels and the
131
- # the checks will fail, so in those cases we pick the composefile from v0.10.1 which contains
132
- # the setup job labels
161
+ # if the version is older than <MINIMUM_SUPPORTED_VERSION>, it doesn't contain the
162
+ # docker compose based resolved compose file. In those cases, we pick up the composefile from
163
+ # MINIMUM_SUPPORTED_VERSION which contains the compose file.
133
164
  if _is_it_a_version(result.composefile_git_ref):
134
- if parse("v0.10.1") > parse(result.composefile_git_ref):
135
- # The merge commit where the labels were added
136
- # https://github.com/datahub-project/datahub/pull/7473
137
- result.composefile_git_ref = "1d3339276129a7cb8385c07a958fcc93acda3b4e"
165
+ if parse("v1.2.0") > parse(result.composefile_git_ref):
166
+ # The merge commit where profiles based resolved compose file was added.
167
+ # https://github.com/datahub-project/datahub/pull/13566
168
+ result.composefile_git_ref = "21726bc3341490f4182b904626c793091ac95edd"
138
169
 
139
170
  return result
140
171
 
@@ -148,3 +179,15 @@ def save_quickstart_config(
148
179
  with open(path, "w") as f:
149
180
  yaml.dump(config.dict(), f)
150
181
  logger.info(f"Saved quickstart config to {path}.")
182
+
183
+
184
+ def is_minimum_supported_version(version: str) -> bool:
185
+ if not _is_it_a_version(version):
186
+ return True
187
+
188
+ requested_version = packaging.version.parse(version)
189
+ minimum_supported_version = packaging.version.parse(MINIMUM_SUPPORTED_VERSION)
190
+ if requested_version < minimum_supported_version:
191
+ return False
192
+
193
+ return True
@@ -1,3 +1,8 @@
1
+ """
2
+ DEPRECATED: This assertions CLI is no longer supported and will be removed in a future version.
3
+ Please use alternative methods for managing assertions in DataHub.
4
+ """
5
+
1
6
  import logging
2
7
  import os
3
8
  from pathlib import Path
@@ -17,7 +22,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
17
22
  from datahub.ingestion.graph.client import get_default_graph
18
23
  from datahub.ingestion.graph.config import ClientMode
19
24
  from datahub.integrations.assertion.registry import ASSERTION_PLATFORMS
20
- from datahub.telemetry import telemetry
21
25
  from datahub.upgrade import upgrade
22
26
 
23
27
  logger = logging.getLogger(__name__)
@@ -27,16 +31,35 @@ REPORT_FILE_NAME = "compile_report.json"
27
31
 
28
32
  @click.group(cls=DefaultGroup, default="upsert")
29
33
  def assertions() -> None:
30
- """A group of commands to interact with the Assertion entity in DataHub."""
34
+ """A group of commands to interact with the Assertion entity in DataHub.
35
+
36
+ ⚠️ DEPRECATED: This assertions CLI is no longer supported and will be removed
37
+ in a future version. Please use alternative methods for managing assertions in DataHub.
38
+ """
39
+ click.secho(
40
+ "⚠️ WARNING: The assertions CLI is deprecated and no longer supported. "
41
+ "It may be removed in a future version. Please use alternative methods for managing assertions in DataHub.",
42
+ fg="yellow",
43
+ bold=True,
44
+ err=True,
45
+ )
31
46
  pass
32
47
 
33
48
 
34
49
  @assertions.command()
35
50
  @click.option("-f", "--file", required=True, type=click.Path(exists=True))
36
51
  @upgrade.check_upgrade
37
- @telemetry.with_telemetry()
38
52
  def upsert(file: str) -> None:
39
- """Upsert (create or update) a set of assertions in DataHub."""
53
+ """Upsert (create or update) a set of assertions in DataHub.
54
+
55
+ ⚠️ DEPRECATED: This command is deprecated and no longer supported.
56
+ """
57
+ click.secho(
58
+ "⚠️ WARNING: The 'upsert' command is deprecated and no longer supported.",
59
+ fg="yellow",
60
+ bold=True,
61
+ err=True,
62
+ )
40
63
 
41
64
  assertions_spec: AssertionsConfigSpec = AssertionsConfigSpec.from_yaml(file)
42
65
 
@@ -71,8 +94,6 @@ def upsert(file: str) -> None:
71
94
  default=[],
72
95
  help="Platform-specific extra key-value inputs in form key=value",
73
96
  )
74
- @upgrade.check_upgrade
75
- @telemetry.with_telemetry()
76
97
  def compile(
77
98
  file: str, platform: str, output_to: Optional[str], extras: List[str]
78
99
  ) -> None:
@@ -82,7 +103,15 @@ def compile(
82
103
  In future, we may introduce separate command to automatically apply these compiled changes
83
104
  in assertion platform. Currently, generated result artifacts are stored in target folder
84
105
  unless another folder is specified using option `--output-to <folder>`.
106
+
107
+ ⚠️ DEPRECATED: This command is deprecated and no longer supported.
85
108
  """
109
+ click.secho(
110
+ "⚠️ WARNING: The 'compile' command is deprecated and no longer supported.",
111
+ fg="yellow",
112
+ bold=True,
113
+ err=True,
114
+ )
86
115
 
87
116
  if platform not in ASSERTION_PLATFORMS:
88
117
  click.secho(
@@ -150,3 +179,5 @@ def extras_list_to_dict(extras: List[str]) -> Dict[str, str]:
150
179
  # Later:
151
180
  # 3. execute compiled assertions on assertion platform (Later, requires connection details to platform),
152
181
  # 4. cleanup assertions from assertion platform (generate artifacts. optionally execute)
182
+ #
183
+ # NOTE: This entire assertions CLI is deprecated and these TODOs will not be implemented.
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ import warnings
2
3
  from typing import Optional
3
4
 
4
5
  import click
@@ -7,7 +8,6 @@ from click_default_group import DefaultGroup
7
8
  from datahub.api.entities.datacontract.datacontract import DataContract
8
9
  from datahub.ingestion.graph.client import get_default_graph
9
10
  from datahub.ingestion.graph.config import ClientMode
10
- from datahub.telemetry import telemetry
11
11
  from datahub.upgrade import upgrade
12
12
 
13
13
  logger = logging.getLogger(__name__)
@@ -15,16 +15,52 @@ logger = logging.getLogger(__name__)
15
15
 
16
16
  @click.group(cls=DefaultGroup, default="upsert")
17
17
  def datacontract() -> None:
18
- """A group of commands to interact with the DataContract entity in DataHub."""
19
- pass
18
+ """
19
+ A group of commands to interact with the DataContract entity in DataHub.
20
+
21
+ WARNING: This CLI is DEPRECATED and no longer supported.
22
+ Please migrate to alternative data contract solutions.
23
+ """
24
+ # Issue deprecation warning
25
+ warnings.warn(
26
+ "The datacontract CLI is deprecated and no longer supported. "
27
+ "Please migrate to alternative data contract solutions.",
28
+ DeprecationWarning,
29
+ stacklevel=2,
30
+ )
31
+
32
+ # Log deprecation message for runtime visibility
33
+ logger.warning(
34
+ "DEPRECATED: The datacontract CLI is no longer supported and will be removed in a future version. "
35
+ "Please migrate to alternative data contract solutions."
36
+ )
37
+
38
+ # Display deprecation message to user
39
+ click.secho(
40
+ "⚠️ WARNING: This datacontract CLI is DEPRECATED and no longer supported.",
41
+ fg="yellow",
42
+ bold=True,
43
+ )
44
+ click.secho("Please migrate to alternative data contract solutions.", fg="yellow")
20
45
 
21
46
 
22
47
  @datacontract.command()
23
48
  @click.option("-f", "--file", required=True, type=click.Path(exists=True))
24
49
  @upgrade.check_upgrade
25
- @telemetry.with_telemetry()
26
50
  def upsert(file: str) -> None:
27
- """Upsert (create or update) a Data Contract in DataHub."""
51
+ """
52
+ Upsert (create or update) a Data Contract in DataHub.
53
+
54
+ WARNING: This command is DEPRECATED and no longer supported.
55
+ """
56
+
57
+ click.secho(
58
+ "⚠️ WARNING: The 'upsert' command is deprecated and no longer supported.",
59
+ fg="yellow",
60
+ bold=True,
61
+ )
62
+
63
+ logger.warning("DEPRECATED: datacontract upsert command is no longer supported")
28
64
 
29
65
  data_contract: DataContract = DataContract.from_yaml(file)
30
66
  urn = data_contract.urn
@@ -60,9 +96,20 @@ def upsert(file: str) -> None:
60
96
  )
61
97
  @click.option("--hard/--soft", required=False, is_flag=True, default=False)
62
98
  @upgrade.check_upgrade
63
- @telemetry.with_telemetry()
64
99
  def delete(urn: Optional[str], file: Optional[str], hard: bool) -> None:
65
- """Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
100
+ """
101
+ Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata.
102
+
103
+ WARNING: This command is DEPRECATED and no longer supported.
104
+ """
105
+
106
+ click.secho(
107
+ "⚠️ WARNING: The 'delete' command is deprecated and no longer supported.",
108
+ fg="yellow",
109
+ bold=True,
110
+ )
111
+
112
+ logger.warning("DEPRECATED: datacontract delete command is no longer supported")
66
113
 
67
114
  if not urn:
68
115
  if not file:
@@ -1,7 +1,6 @@
1
1
  import difflib
2
2
  import json
3
3
  import logging
4
- import os
5
4
  import pathlib
6
5
  import sys
7
6
  from pathlib import Path
@@ -14,6 +13,7 @@ from click_default_group import DefaultGroup
14
13
 
15
14
  from datahub.api.entities.dataproduct.dataproduct import DataProduct
16
15
  from datahub.cli.specific.file_loader import load_file
16
+ from datahub.configuration.env_vars import get_dataproduct_external_url
17
17
  from datahub.emitter.mce_builder import (
18
18
  make_group_urn,
19
19
  make_user_urn,
@@ -23,7 +23,6 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
23
23
  from datahub.ingestion.graph.config import ClientMode
24
24
  from datahub.metadata.schema_classes import OwnerClass, OwnershipTypeClass
25
25
  from datahub.specific.dataproduct import DataProductPatchBuilder
26
- from datahub.telemetry import telemetry
27
26
  from datahub.upgrade import upgrade
28
27
  from datahub.utilities.urns.urn import Urn
29
28
 
@@ -85,9 +84,7 @@ def mutate(file: Path, validate_assets: bool, external_url: str, upsert: bool) -
85
84
  with get_default_graph(ClientMode.CLI) as graph:
86
85
  data_product: DataProduct = DataProduct.from_yaml(file, graph)
87
86
  external_url_override = (
88
- external_url
89
- or os.getenv("DATAHUB_DATAPRODUCT_EXTERNAL_URL")
90
- or data_product.external_url
87
+ external_url or get_dataproduct_external_url() or data_product.external_url
91
88
  )
92
89
  data_product.external_url = external_url_override
93
90
  if upsert and not graph.exists(data_product.urn):
@@ -130,7 +127,6 @@ def mutate(file: Path, validate_assets: bool, external_url: str, upsert: bool) -
130
127
  )
131
128
  @click.option("--external-url", required=False, type=str)
132
129
  @upgrade.check_upgrade
133
- @telemetry.with_telemetry()
134
130
  def update(file: Path, validate_assets: bool, external_url: str) -> None:
135
131
  """Create or Update a Data Product in DataHub. Use upsert if you want to apply partial updates."""
136
132
 
@@ -146,7 +142,6 @@ def update(file: Path, validate_assets: bool, external_url: str) -> None:
146
142
  )
147
143
  @click.option("--external-url", required=False, type=str)
148
144
  @upgrade.check_upgrade
149
- @telemetry.with_telemetry()
150
145
  def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
151
146
  """Upsert attributes to a Data Product in DataHub."""
152
147
 
@@ -159,7 +154,6 @@ def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
159
154
  @click.option("-f", "--file", required=True, type=click.Path(exists=True))
160
155
  @click.option("--update", required=False, is_flag=True, default=False)
161
156
  @upgrade.check_upgrade
162
- @telemetry.with_telemetry()
163
157
  def diff(file: Path, update: bool) -> None:
164
158
  """Diff a Data Product file with its twin in DataHub"""
165
159
 
@@ -206,7 +200,6 @@ def diff(file: Path, update: bool) -> None:
206
200
  )
207
201
  @click.option("--hard/--soft", required=False, is_flag=True, default=False)
208
202
  @upgrade.check_upgrade
209
- @telemetry.with_telemetry()
210
203
  def delete(urn: str, file: Path, hard: bool) -> None:
211
204
  """Delete a Data Product in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
212
205
 
@@ -242,7 +235,6 @@ def delete(urn: str, file: Path, hard: bool) -> None:
242
235
  @click.option("--urn", required=True, type=str)
243
236
  @click.option("--to-file", required=False, type=str)
244
237
  @upgrade.check_upgrade
245
- @telemetry.with_telemetry()
246
238
  def get(urn: str, to_file: str) -> None:
247
239
  """Get a Data Product from DataHub"""
248
240
 
@@ -279,7 +271,6 @@ def get(urn: str, to_file: str) -> None:
279
271
  help="A markdown file that contains documentation for this data product",
280
272
  )
281
273
  @upgrade.check_upgrade
282
- @telemetry.with_telemetry()
283
274
  def set_description(urn: str, description: str, md_file: Path) -> None:
284
275
  """Set description for a Data Product in DataHub"""
285
276
 
@@ -330,7 +321,6 @@ def set_description(urn: str, description: str, md_file: Path) -> None:
330
321
  default=OwnershipTypeClass.TECHNICAL_OWNER,
331
322
  )
332
323
  @upgrade.check_upgrade
333
- @telemetry.with_telemetry()
334
324
  def add_owner(urn: str, owner: str, owner_type: str) -> None:
335
325
  """Add owner for a Data Product in DataHub"""
336
326
 
@@ -353,7 +343,6 @@ def add_owner(urn: str, owner: str, owner_type: str) -> None:
353
343
  @click.option("--urn", required=True, type=str)
354
344
  @click.argument("owner_urn", required=True, type=str)
355
345
  @upgrade.check_upgrade
356
- @telemetry.with_telemetry()
357
346
  def remove_owner(urn: str, owner_urn: str) -> None:
358
347
  """Remove owner for a Data Product in DataHub"""
359
348
 
@@ -375,7 +364,6 @@ def remove_owner(urn: str, owner_urn: str) -> None:
375
364
  "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
376
365
  )
377
366
  @upgrade.check_upgrade
378
- @telemetry.with_telemetry()
379
367
  def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
380
368
  """Add asset for a Data Product in DataHub"""
381
369
 
@@ -402,7 +390,6 @@ def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
402
390
  "--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
403
391
  )
404
392
  @upgrade.check_upgrade
405
- @telemetry.with_telemetry()
406
393
  def remove_asset(urn: str, asset: str, validate_assets: bool) -> None:
407
394
  """Remove asset for a Data Product in DataHub"""
408
395
 
@@ -14,7 +14,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
14
14
  from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
15
15
  from datahub.ingestion.graph.config import ClientMode
16
16
  from datahub.metadata.com.linkedin.pegasus2avro.common import Siblings
17
- from datahub.telemetry import telemetry
18
17
  from datahub.upgrade import upgrade
19
18
 
20
19
  logger = logging.getLogger(__name__)
@@ -33,8 +32,6 @@ def dataset() -> None:
33
32
  @click.option(
34
33
  "-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
35
34
  )
36
- @upgrade.check_upgrade
37
- @telemetry.with_telemetry()
38
35
  def upsert(file: Path, dry_run: bool) -> None:
39
36
  """Upsert attributes to a Dataset in DataHub."""
40
37
  # Call the sync command with to_datahub=True to perform the upsert operation
@@ -48,7 +45,6 @@ def upsert(file: Path, dry_run: bool) -> None:
48
45
  @click.option("--urn", required=True, type=str)
49
46
  @click.option("--to-file", required=False, type=str)
50
47
  @upgrade.check_upgrade
51
- @telemetry.with_telemetry()
52
48
  def get(urn: str, to_file: str) -> None:
53
49
  """Get a Dataset from DataHub"""
54
50
 
@@ -77,7 +73,7 @@ def get(urn: str, to_file: str) -> None:
77
73
  help="URN of secondary sibling(s)",
78
74
  multiple=True,
79
75
  )
80
- @telemetry.with_telemetry()
76
+ @upgrade.check_upgrade
81
77
  def add_sibling(urn: str, sibling_urns: Tuple[str]) -> None:
82
78
  all_urns = set()
83
79
  all_urns.add(urn)
@@ -117,8 +113,6 @@ def _get_existing_siblings(graph: DataHubGraph, urn: str) -> Set[str]:
117
113
  @click.option("--lintCheck", required=False, is_flag=True)
118
114
  @click.option("--lintFix", required=False, is_flag=True)
119
115
  @click.argument("file", type=click.Path(exists=True))
120
- @upgrade.check_upgrade
121
- @telemetry.with_telemetry()
122
116
  def file(lintcheck: bool, lintfix: bool, file: str) -> None:
123
117
  """Operate on a Dataset file"""
124
118
 
@@ -175,7 +169,6 @@ def file(lintcheck: bool, lintfix: bool, file: str) -> None:
175
169
  "-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
176
170
  )
177
171
  @upgrade.check_upgrade
178
- @telemetry.with_telemetry()
179
172
  def sync(file: str, to_datahub: bool, dry_run: bool) -> None:
180
173
  """Sync a Dataset file to/from DataHub"""
181
174
 
@@ -8,7 +8,6 @@ from click_default_group import DefaultGroup
8
8
  from datahub.api.entities.forms.forms import Forms
9
9
  from datahub.ingestion.graph.client import get_default_graph
10
10
  from datahub.ingestion.graph.config import ClientMode
11
- from datahub.telemetry import telemetry
12
11
  from datahub.upgrade import upgrade
13
12
 
14
13
  logger = logging.getLogger(__name__)
@@ -24,8 +23,6 @@ def forms() -> None:
24
23
  name="upsert",
25
24
  )
26
25
  @click.option("-f", "--file", required=True, type=click.Path(exists=True))
27
- @upgrade.check_upgrade
28
- @telemetry.with_telemetry()
29
26
  def upsert(file: Path) -> None:
30
27
  """Upsert forms in DataHub."""
31
28
 
@@ -38,7 +35,6 @@ def upsert(file: Path) -> None:
38
35
  @click.option("--urn", required=True, type=str)
39
36
  @click.option("--to-file", required=False, type=str)
40
37
  @upgrade.check_upgrade
41
- @telemetry.with_telemetry()
42
38
  def get(urn: str, to_file: str) -> None:
43
39
  """Get form from DataHub"""
44
40
  with get_default_graph(ClientMode.CLI) as graph:
@@ -11,7 +11,6 @@ from datahub.api.entities.corpgroup.corpgroup import (
11
11
  from datahub.cli.specific.file_loader import load_file
12
12
  from datahub.ingestion.graph.client import get_default_graph
13
13
  from datahub.ingestion.graph.config import ClientMode
14
- from datahub.telemetry import telemetry
15
14
  from datahub.upgrade import upgrade
16
15
 
17
16
  logger = logging.getLogger(__name__)
@@ -35,7 +34,6 @@ def group() -> None:
35
34
  help="When set, writes to the editable section of the metadata graph, overwriting writes from the UI",
36
35
  )
37
36
  @upgrade.check_upgrade
38
- @telemetry.with_telemetry()
39
37
  def upsert(file: Path, override_editable: bool) -> None:
40
38
  """Create or Update a Group with embedded Users"""
41
39
 
@@ -12,7 +12,6 @@ from datahub.api.entities.structuredproperties.structuredproperties import (
12
12
  )
13
13
  from datahub.ingestion.graph.client import get_default_graph
14
14
  from datahub.ingestion.graph.config import ClientMode
15
- from datahub.telemetry import telemetry
16
15
  from datahub.upgrade import upgrade
17
16
  from datahub.utilities.urns.urn import Urn
18
17
 
@@ -30,7 +29,6 @@ def properties() -> None:
30
29
  )
31
30
  @click.option("-f", "--file", required=True, type=click.Path(exists=True))
32
31
  @upgrade.check_upgrade
33
- @telemetry.with_telemetry()
34
32
  def upsert(file: Path) -> None:
35
33
  """Upsert structured properties in DataHub."""
36
34
 
@@ -44,7 +42,6 @@ def upsert(file: Path) -> None:
44
42
  @click.option("--urn", required=True, type=str)
45
43
  @click.option("--to-file", required=False, type=str)
46
44
  @upgrade.check_upgrade
47
- @telemetry.with_telemetry()
48
45
  def get(urn: str, to_file: str) -> None:
49
46
  """Get structured properties from DataHub"""
50
47
  urn = Urn.make_structured_property_urn(urn)
@@ -71,7 +68,7 @@ def get(urn: str, to_file: str) -> None:
71
68
  )
72
69
  @click.option("--details/--no-details", is_flag=True, default=True)
73
70
  @click.option("--to-file", required=False, type=str)
74
- @telemetry.with_telemetry()
71
+ @upgrade.check_upgrade
75
72
  def list(details: bool, to_file: str) -> None:
76
73
  """List structured properties in DataHub"""
77
74