acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/METADATA +2615 -2547
- {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/RECORD +412 -338
- {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/entry_points.txt +5 -0
- datahub/_version.py +1 -1
- datahub/api/entities/assertion/assertion.py +1 -1
- datahub/api/entities/common/serialized_value.py +1 -1
- datahub/api/entities/corpgroup/corpgroup.py +1 -1
- datahub/api/entities/dataproduct/dataproduct.py +32 -3
- datahub/api/entities/dataset/dataset.py +26 -23
- datahub/api/entities/external/__init__.py +0 -0
- datahub/api/entities/external/external_entities.py +724 -0
- datahub/api/entities/external/external_tag.py +147 -0
- datahub/api/entities/external/lake_formation_external_entites.py +162 -0
- datahub/api/entities/external/restricted_text.py +172 -0
- datahub/api/entities/external/unity_catalog_external_entites.py +172 -0
- datahub/api/entities/forms/forms.py +3 -3
- datahub/api/entities/structuredproperties/structuredproperties.py +4 -4
- datahub/api/graphql/operation.py +10 -6
- datahub/cli/check_cli.py +88 -7
- datahub/cli/cli_utils.py +63 -0
- datahub/cli/config_utils.py +18 -10
- datahub/cli/container_cli.py +5 -0
- datahub/cli/delete_cli.py +125 -27
- datahub/cli/docker_check.py +110 -14
- datahub/cli/docker_cli.py +153 -229
- datahub/cli/exists_cli.py +0 -2
- datahub/cli/get_cli.py +0 -2
- datahub/cli/graphql_cli.py +1422 -0
- datahub/cli/iceberg_cli.py +5 -0
- datahub/cli/ingest_cli.py +3 -15
- datahub/cli/migrate.py +2 -0
- datahub/cli/put_cli.py +1 -4
- datahub/cli/quickstart_versioning.py +53 -10
- datahub/cli/specific/assertions_cli.py +37 -6
- datahub/cli/specific/datacontract_cli.py +54 -7
- datahub/cli/specific/dataproduct_cli.py +2 -15
- datahub/cli/specific/dataset_cli.py +1 -8
- datahub/cli/specific/forms_cli.py +0 -4
- datahub/cli/specific/group_cli.py +0 -2
- datahub/cli/specific/structuredproperties_cli.py +1 -4
- datahub/cli/specific/user_cli.py +172 -3
- datahub/cli/state_cli.py +0 -2
- datahub/cli/timeline_cli.py +0 -2
- datahub/configuration/common.py +40 -1
- datahub/configuration/connection_resolver.py +5 -2
- datahub/configuration/env_vars.py +331 -0
- datahub/configuration/import_resolver.py +7 -4
- datahub/configuration/kafka.py +21 -1
- datahub/configuration/pydantic_migration_helpers.py +6 -13
- datahub/configuration/source_common.py +3 -2
- datahub/configuration/validate_field_deprecation.py +5 -2
- datahub/configuration/validate_field_removal.py +8 -2
- datahub/configuration/validate_field_rename.py +6 -5
- datahub/configuration/validate_multiline_string.py +5 -2
- datahub/emitter/mce_builder.py +8 -4
- datahub/emitter/rest_emitter.py +103 -30
- datahub/entrypoints.py +6 -3
- datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py +297 -1
- datahub/ingestion/api/auto_work_units/auto_validate_input_fields.py +87 -0
- datahub/ingestion/api/decorators.py +15 -3
- datahub/ingestion/api/report.py +381 -3
- datahub/ingestion/api/sink.py +27 -2
- datahub/ingestion/api/source.py +165 -58
- datahub/ingestion/api/source_protocols.py +23 -0
- datahub/ingestion/autogenerated/__init__.py +0 -0
- datahub/ingestion/autogenerated/capability_summary.json +3652 -0
- datahub/ingestion/autogenerated/lineage.json +402 -0
- datahub/ingestion/autogenerated/lineage_helper.py +177 -0
- datahub/ingestion/extractor/schema_util.py +13 -4
- datahub/ingestion/glossary/classification_mixin.py +5 -0
- datahub/ingestion/graph/client.py +330 -25
- datahub/ingestion/graph/config.py +3 -2
- datahub/ingestion/graph/filters.py +30 -11
- datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +21 -11
- datahub/ingestion/run/pipeline.py +81 -11
- datahub/ingestion/run/pipeline_config.py +2 -2
- datahub/ingestion/sink/datahub_kafka.py +1 -0
- datahub/ingestion/sink/datahub_rest.py +13 -5
- datahub/ingestion/sink/file.py +1 -0
- datahub/ingestion/source/abs/config.py +1 -1
- datahub/ingestion/source/abs/datalake_profiler_config.py +1 -1
- datahub/ingestion/source/abs/source.py +15 -30
- datahub/ingestion/source/aws/aws_common.py +185 -13
- datahub/ingestion/source/aws/glue.py +517 -244
- datahub/ingestion/source/aws/platform_resource_repository.py +30 -0
- datahub/ingestion/source/aws/s3_boto_utils.py +100 -5
- datahub/ingestion/source/aws/tag_entities.py +270 -0
- datahub/ingestion/source/azure/azure_common.py +3 -3
- datahub/ingestion/source/bigquery_v2/bigquery.py +67 -24
- datahub/ingestion/source/bigquery_v2/bigquery_config.py +47 -19
- datahub/ingestion/source/bigquery_v2/bigquery_connection.py +12 -1
- datahub/ingestion/source/bigquery_v2/bigquery_queries.py +3 -0
- datahub/ingestion/source/bigquery_v2/bigquery_report.py +0 -2
- datahub/ingestion/source/bigquery_v2/bigquery_schema.py +23 -16
- datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +20 -5
- datahub/ingestion/source/bigquery_v2/common.py +1 -1
- datahub/ingestion/source/bigquery_v2/profiler.py +4 -2
- datahub/ingestion/source/bigquery_v2/queries.py +3 -3
- datahub/ingestion/source/bigquery_v2/queries_extractor.py +45 -9
- datahub/ingestion/source/cassandra/cassandra.py +6 -8
- datahub/ingestion/source/cassandra/cassandra_api.py +17 -1
- datahub/ingestion/source/cassandra/cassandra_config.py +5 -0
- datahub/ingestion/source/cassandra/cassandra_profiling.py +7 -6
- datahub/ingestion/source/cassandra/cassandra_utils.py +1 -2
- datahub/ingestion/source/common/gcp_credentials_config.py +3 -1
- datahub/ingestion/source/common/subtypes.py +53 -0
- datahub/ingestion/source/data_lake_common/data_lake_utils.py +37 -0
- datahub/ingestion/source/data_lake_common/object_store.py +115 -27
- datahub/ingestion/source/data_lake_common/path_spec.py +72 -43
- datahub/ingestion/source/datahub/config.py +12 -9
- datahub/ingestion/source/datahub/datahub_database_reader.py +26 -11
- datahub/ingestion/source/datahub/datahub_source.py +10 -0
- datahub/ingestion/source/dbt/dbt_cloud.py +16 -5
- datahub/ingestion/source/dbt/dbt_common.py +224 -9
- datahub/ingestion/source/dbt/dbt_core.py +3 -0
- datahub/ingestion/source/debug/__init__.py +0 -0
- datahub/ingestion/source/debug/datahub_debug.py +300 -0
- datahub/ingestion/source/delta_lake/config.py +9 -5
- datahub/ingestion/source/delta_lake/source.py +8 -0
- datahub/ingestion/source/dremio/dremio_api.py +114 -73
- datahub/ingestion/source/dremio/dremio_aspects.py +3 -2
- datahub/ingestion/source/dremio/dremio_config.py +5 -4
- datahub/ingestion/source/dremio/dremio_reporting.py +22 -3
- datahub/ingestion/source/dremio/dremio_source.py +132 -98
- datahub/ingestion/source/dremio/dremio_sql_queries.py +82 -21
- datahub/ingestion/source/dynamodb/dynamodb.py +11 -8
- datahub/ingestion/source/excel/__init__.py +0 -0
- datahub/ingestion/source/excel/config.py +92 -0
- datahub/ingestion/source/excel/excel_file.py +539 -0
- datahub/ingestion/source/excel/profiling.py +308 -0
- datahub/ingestion/source/excel/report.py +49 -0
- datahub/ingestion/source/excel/source.py +662 -0
- datahub/ingestion/source/excel/util.py +18 -0
- datahub/ingestion/source/feast.py +8 -10
- datahub/ingestion/source/file.py +3 -0
- datahub/ingestion/source/fivetran/config.py +66 -7
- datahub/ingestion/source/fivetran/fivetran.py +227 -43
- datahub/ingestion/source/fivetran/fivetran_log_api.py +37 -8
- datahub/ingestion/source/fivetran/fivetran_query.py +51 -29
- datahub/ingestion/source/fivetran/fivetran_rest_api.py +65 -0
- datahub/ingestion/source/fivetran/response_models.py +97 -0
- datahub/ingestion/source/gc/datahub_gc.py +0 -2
- datahub/ingestion/source/gcs/gcs_source.py +32 -4
- datahub/ingestion/source/ge_data_profiler.py +108 -31
- datahub/ingestion/source/ge_profiling_config.py +26 -11
- datahub/ingestion/source/grafana/entity_mcp_builder.py +272 -0
- datahub/ingestion/source/grafana/field_utils.py +307 -0
- datahub/ingestion/source/grafana/grafana_api.py +142 -0
- datahub/ingestion/source/grafana/grafana_config.py +104 -0
- datahub/ingestion/source/grafana/grafana_source.py +522 -84
- datahub/ingestion/source/grafana/lineage.py +202 -0
- datahub/ingestion/source/grafana/models.py +137 -0
- datahub/ingestion/source/grafana/report.py +90 -0
- datahub/ingestion/source/grafana/types.py +16 -0
- datahub/ingestion/source/hex/api.py +28 -1
- datahub/ingestion/source/hex/hex.py +16 -5
- datahub/ingestion/source/hex/mapper.py +16 -2
- datahub/ingestion/source/hex/model.py +2 -0
- datahub/ingestion/source/hex/query_fetcher.py +1 -1
- datahub/ingestion/source/iceberg/iceberg.py +123 -59
- datahub/ingestion/source/iceberg/iceberg_profiler.py +4 -2
- datahub/ingestion/source/identity/azure_ad.py +1 -1
- datahub/ingestion/source/identity/okta.py +1 -14
- datahub/ingestion/source/kafka/kafka.py +16 -0
- datahub/ingestion/source/kafka_connect/common.py +2 -2
- datahub/ingestion/source/kafka_connect/sink_connectors.py +156 -47
- datahub/ingestion/source/kafka_connect/source_connectors.py +62 -4
- datahub/ingestion/source/looker/looker_common.py +148 -79
- datahub/ingestion/source/looker/looker_config.py +15 -4
- datahub/ingestion/source/looker/looker_constant.py +4 -0
- datahub/ingestion/source/looker/looker_lib_wrapper.py +36 -3
- datahub/ingestion/source/looker/looker_liquid_tag.py +56 -5
- datahub/ingestion/source/looker/looker_source.py +503 -547
- datahub/ingestion/source/looker/looker_view_id_cache.py +1 -1
- datahub/ingestion/source/looker/lookml_concept_context.py +1 -1
- datahub/ingestion/source/looker/lookml_config.py +31 -3
- datahub/ingestion/source/looker/lookml_refinement.py +1 -1
- datahub/ingestion/source/looker/lookml_source.py +96 -117
- datahub/ingestion/source/looker/view_upstream.py +494 -1
- datahub/ingestion/source/metabase.py +32 -6
- datahub/ingestion/source/metadata/business_glossary.py +7 -7
- datahub/ingestion/source/metadata/lineage.py +9 -9
- datahub/ingestion/source/mlflow.py +12 -2
- datahub/ingestion/source/mock_data/__init__.py +0 -0
- datahub/ingestion/source/mock_data/datahub_mock_data.py +533 -0
- datahub/ingestion/source/mock_data/datahub_mock_data_report.py +12 -0
- datahub/ingestion/source/mock_data/table_naming_helper.py +97 -0
- datahub/ingestion/source/mode.py +26 -5
- datahub/ingestion/source/mongodb.py +11 -1
- datahub/ingestion/source/neo4j/neo4j_source.py +83 -144
- datahub/ingestion/source/nifi.py +2 -2
- datahub/ingestion/source/openapi.py +1 -1
- datahub/ingestion/source/powerbi/config.py +47 -21
- datahub/ingestion/source/powerbi/m_query/data_classes.py +1 -0
- datahub/ingestion/source/powerbi/m_query/parser.py +2 -2
- datahub/ingestion/source/powerbi/m_query/pattern_handler.py +100 -10
- datahub/ingestion/source/powerbi/powerbi.py +10 -6
- datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +0 -1
- datahub/ingestion/source/powerbi_report_server/report_server.py +0 -23
- datahub/ingestion/source/powerbi_report_server/report_server_domain.py +2 -4
- datahub/ingestion/source/preset.py +3 -3
- datahub/ingestion/source/qlik_sense/data_classes.py +28 -8
- datahub/ingestion/source/qlik_sense/qlik_sense.py +2 -1
- datahub/ingestion/source/redash.py +1 -1
- datahub/ingestion/source/redshift/config.py +15 -9
- datahub/ingestion/source/redshift/datashares.py +1 -1
- datahub/ingestion/source/redshift/lineage.py +386 -687
- datahub/ingestion/source/redshift/query.py +23 -19
- datahub/ingestion/source/redshift/redshift.py +52 -111
- datahub/ingestion/source/redshift/redshift_schema.py +17 -12
- datahub/ingestion/source/redshift/report.py +0 -2
- datahub/ingestion/source/redshift/usage.py +6 -5
- datahub/ingestion/source/s3/report.py +4 -2
- datahub/ingestion/source/s3/source.py +449 -248
- datahub/ingestion/source/sac/sac.py +3 -1
- datahub/ingestion/source/salesforce.py +28 -13
- datahub/ingestion/source/schema/json_schema.py +14 -14
- datahub/ingestion/source/schema_inference/object.py +22 -6
- datahub/ingestion/source/sigma/data_classes.py +3 -0
- datahub/ingestion/source/sigma/sigma.py +7 -1
- datahub/ingestion/source/slack/slack.py +10 -16
- datahub/ingestion/source/snaplogic/__init__.py +0 -0
- datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
- datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
- datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
- datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
- datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
- datahub/ingestion/source/snowflake/constants.py +3 -0
- datahub/ingestion/source/snowflake/snowflake_config.py +76 -23
- datahub/ingestion/source/snowflake/snowflake_connection.py +24 -8
- datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +19 -6
- datahub/ingestion/source/snowflake/snowflake_queries.py +464 -97
- datahub/ingestion/source/snowflake/snowflake_query.py +77 -5
- datahub/ingestion/source/snowflake/snowflake_report.py +1 -2
- datahub/ingestion/source/snowflake/snowflake_schema.py +352 -16
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py +51 -10
- datahub/ingestion/source/snowflake/snowflake_summary.py +7 -1
- datahub/ingestion/source/snowflake/snowflake_usage_v2.py +8 -2
- datahub/ingestion/source/snowflake/snowflake_utils.py +36 -15
- datahub/ingestion/source/snowflake/snowflake_v2.py +39 -4
- datahub/ingestion/source/snowflake/stored_proc_lineage.py +143 -0
- datahub/ingestion/source/sql/athena.py +217 -25
- datahub/ingestion/source/sql/athena_properties_extractor.py +795 -0
- datahub/ingestion/source/sql/clickhouse.py +24 -8
- datahub/ingestion/source/sql/cockroachdb.py +5 -4
- datahub/ingestion/source/sql/druid.py +2 -2
- datahub/ingestion/source/sql/hana.py +3 -1
- datahub/ingestion/source/sql/hive.py +4 -3
- datahub/ingestion/source/sql/hive_metastore.py +19 -20
- datahub/ingestion/source/sql/mariadb.py +0 -1
- datahub/ingestion/source/sql/mssql/job_models.py +3 -1
- datahub/ingestion/source/sql/mssql/source.py +336 -57
- datahub/ingestion/source/sql/mysql.py +154 -4
- datahub/ingestion/source/sql/oracle.py +5 -5
- datahub/ingestion/source/sql/postgres.py +142 -6
- datahub/ingestion/source/sql/presto.py +2 -1
- datahub/ingestion/source/sql/sql_common.py +281 -49
- datahub/ingestion/source/sql/sql_generic_profiler.py +2 -1
- datahub/ingestion/source/sql/sql_types.py +22 -0
- datahub/ingestion/source/sql/sqlalchemy_uri.py +39 -7
- datahub/ingestion/source/sql/teradata.py +1028 -245
- datahub/ingestion/source/sql/trino.py +11 -1
- datahub/ingestion/source/sql/two_tier_sql_source.py +2 -3
- datahub/ingestion/source/sql/vertica.py +14 -7
- datahub/ingestion/source/sql_queries.py +219 -121
- datahub/ingestion/source/state/checkpoint.py +8 -29
- datahub/ingestion/source/state/entity_removal_state.py +5 -2
- datahub/ingestion/source/state/redundant_run_skip_handler.py +21 -0
- datahub/ingestion/source/state/stateful_ingestion_base.py +36 -11
- datahub/ingestion/source/superset.py +314 -67
- datahub/ingestion/source/tableau/tableau.py +135 -59
- datahub/ingestion/source/tableau/tableau_common.py +9 -2
- datahub/ingestion/source/tableau/tableau_constant.py +1 -4
- datahub/ingestion/source/tableau/tableau_server_wrapper.py +3 -0
- datahub/ingestion/source/unity/config.py +160 -40
- datahub/ingestion/source/unity/connection.py +61 -0
- datahub/ingestion/source/unity/connection_test.py +1 -0
- datahub/ingestion/source/unity/platform_resource_repository.py +19 -0
- datahub/ingestion/source/unity/proxy.py +794 -51
- datahub/ingestion/source/unity/proxy_patch.py +321 -0
- datahub/ingestion/source/unity/proxy_types.py +36 -2
- datahub/ingestion/source/unity/report.py +15 -3
- datahub/ingestion/source/unity/source.py +465 -131
- datahub/ingestion/source/unity/tag_entities.py +197 -0
- datahub/ingestion/source/unity/usage.py +46 -4
- datahub/ingestion/source/usage/clickhouse_usage.py +4 -1
- datahub/ingestion/source/usage/starburst_trino_usage.py +5 -2
- datahub/ingestion/source/usage/usage_common.py +4 -3
- datahub/ingestion/source/vertexai/vertexai.py +1 -1
- datahub/ingestion/source_config/pulsar.py +3 -1
- datahub/ingestion/source_report/ingestion_stage.py +50 -11
- datahub/ingestion/transformer/add_dataset_ownership.py +18 -2
- datahub/ingestion/transformer/base_transformer.py +8 -5
- datahub/ingestion/transformer/set_browse_path.py +112 -0
- datahub/integrations/assertion/snowflake/compiler.py +4 -3
- datahub/metadata/_internal_schema_classes.py +6806 -4871
- datahub/metadata/_urns/urn_defs.py +1767 -1539
- datahub/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/file/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
- datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +6 -0
- datahub/metadata/com/linkedin/pegasus2avro/module/__init__.py +31 -0
- datahub/metadata/com/linkedin/pegasus2avro/platform/event/v1/__init__.py +4 -0
- datahub/metadata/com/linkedin/pegasus2avro/role/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +8 -0
- datahub/metadata/com/linkedin/pegasus2avro/template/__init__.py +31 -0
- datahub/metadata/schema.avsc +18395 -16979
- datahub/metadata/schemas/Actors.avsc +38 -1
- datahub/metadata/schemas/ApplicationKey.avsc +31 -0
- datahub/metadata/schemas/ApplicationProperties.avsc +72 -0
- datahub/metadata/schemas/Applications.avsc +38 -0
- datahub/metadata/schemas/AssetSettings.avsc +63 -0
- datahub/metadata/schemas/ChartInfo.avsc +2 -1
- datahub/metadata/schemas/ChartKey.avsc +1 -0
- datahub/metadata/schemas/ContainerKey.avsc +1 -0
- datahub/metadata/schemas/ContainerProperties.avsc +8 -0
- datahub/metadata/schemas/CorpUserEditableInfo.avsc +1 -1
- datahub/metadata/schemas/CorpUserSettings.avsc +50 -0
- datahub/metadata/schemas/DashboardKey.avsc +1 -0
- datahub/metadata/schemas/DataFlowInfo.avsc +8 -0
- datahub/metadata/schemas/DataFlowKey.avsc +1 -0
- datahub/metadata/schemas/DataHubFileInfo.avsc +230 -0
- datahub/metadata/schemas/DataHubFileKey.avsc +21 -0
- datahub/metadata/schemas/DataHubPageModuleKey.avsc +21 -0
- datahub/metadata/schemas/DataHubPageModuleProperties.avsc +298 -0
- datahub/metadata/schemas/DataHubPageTemplateKey.avsc +21 -0
- datahub/metadata/schemas/DataHubPageTemplateProperties.avsc +251 -0
- datahub/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
- datahub/metadata/schemas/DataJobInfo.avsc +8 -0
- datahub/metadata/schemas/DataJobInputOutput.avsc +8 -0
- datahub/metadata/schemas/DataJobKey.avsc +1 -0
- datahub/metadata/schemas/DataProcessKey.avsc +8 -0
- datahub/metadata/schemas/DataProductKey.avsc +3 -1
- datahub/metadata/schemas/DataProductProperties.avsc +1 -1
- datahub/metadata/schemas/DatasetKey.avsc +11 -1
- datahub/metadata/schemas/DatasetUsageStatistics.avsc +8 -0
- datahub/metadata/schemas/DomainKey.avsc +2 -1
- datahub/metadata/schemas/GlobalSettingsInfo.avsc +134 -0
- datahub/metadata/schemas/GlossaryNodeKey.avsc +2 -1
- datahub/metadata/schemas/GlossaryTermKey.avsc +3 -1
- datahub/metadata/schemas/IcebergWarehouseInfo.avsc +8 -0
- datahub/metadata/schemas/IncidentInfo.avsc +3 -3
- datahub/metadata/schemas/InstitutionalMemory.avsc +31 -0
- datahub/metadata/schemas/LogicalParent.avsc +145 -0
- datahub/metadata/schemas/MLFeatureKey.avsc +1 -0
- datahub/metadata/schemas/MLFeatureTableKey.avsc +1 -0
- datahub/metadata/schemas/MLModelDeploymentKey.avsc +8 -0
- datahub/metadata/schemas/MLModelGroupKey.avsc +11 -1
- datahub/metadata/schemas/MLModelKey.avsc +9 -0
- datahub/metadata/schemas/MLPrimaryKeyKey.avsc +1 -0
- datahub/metadata/schemas/MetadataChangeEvent.avsc +151 -47
- datahub/metadata/schemas/MetadataChangeLog.avsc +62 -44
- datahub/metadata/schemas/MetadataChangeProposal.avsc +61 -0
- datahub/metadata/schemas/NotebookKey.avsc +1 -0
- datahub/metadata/schemas/Operation.avsc +4 -2
- datahub/metadata/schemas/Ownership.avsc +69 -0
- datahub/metadata/schemas/QuerySubjects.avsc +1 -12
- datahub/metadata/schemas/RelationshipChangeEvent.avsc +215 -0
- datahub/metadata/schemas/SchemaFieldKey.avsc +4 -1
- datahub/metadata/schemas/StructuredProperties.avsc +69 -0
- datahub/metadata/schemas/StructuredPropertySettings.avsc +9 -0
- datahub/metadata/schemas/SystemMetadata.avsc +61 -0
- datahub/metadata/schemas/UpstreamLineage.avsc +9 -0
- datahub/sdk/__init__.py +2 -0
- datahub/sdk/_all_entities.py +7 -0
- datahub/sdk/_shared.py +249 -5
- datahub/sdk/chart.py +386 -0
- datahub/sdk/container.py +7 -0
- datahub/sdk/dashboard.py +453 -0
- datahub/sdk/dataflow.py +7 -0
- datahub/sdk/datajob.py +45 -13
- datahub/sdk/dataset.py +56 -2
- datahub/sdk/entity_client.py +111 -9
- datahub/sdk/lineage_client.py +663 -82
- datahub/sdk/main_client.py +50 -16
- datahub/sdk/mlmodel.py +120 -38
- datahub/sdk/mlmodelgroup.py +7 -0
- datahub/sdk/search_client.py +7 -3
- datahub/sdk/search_filters.py +304 -36
- datahub/secret/datahub_secret_store.py +3 -0
- datahub/secret/environment_secret_store.py +29 -0
- datahub/secret/file_secret_store.py +49 -0
- datahub/specific/aspect_helpers/fine_grained_lineage.py +76 -0
- datahub/specific/aspect_helpers/siblings.py +73 -0
- datahub/specific/aspect_helpers/structured_properties.py +27 -0
- datahub/specific/chart.py +1 -1
- datahub/specific/datajob.py +15 -1
- datahub/specific/dataproduct.py +4 -0
- datahub/specific/dataset.py +39 -59
- datahub/sql_parsing/split_statements.py +13 -0
- datahub/sql_parsing/sql_parsing_aggregator.py +70 -26
- datahub/sql_parsing/sqlglot_lineage.py +196 -42
- datahub/sql_parsing/sqlglot_utils.py +12 -4
- datahub/sql_parsing/tool_meta_extractor.py +1 -3
- datahub/telemetry/telemetry.py +28 -14
- datahub/testing/sdk_v2_helpers.py +7 -1
- datahub/upgrade/upgrade.py +73 -17
- datahub/utilities/file_backed_collections.py +8 -9
- datahub/utilities/is_pytest.py +3 -2
- datahub/utilities/logging_manager.py +22 -6
- datahub/utilities/mapping.py +29 -2
- datahub/utilities/sample_data.py +5 -4
- datahub/utilities/server_config_util.py +10 -1
- datahub/utilities/sqlalchemy_query_combiner.py +5 -2
- datahub/utilities/stats_collections.py +4 -0
- datahub/utilities/urns/urn.py +41 -2
- datahub/emitter/sql_parsing_builder.py +0 -306
- datahub/ingestion/source/redshift/lineage_v2.py +0 -466
- {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.1.1rc4.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/top_level.txt +0 -0
datahub/cli/iceberg_cli.py
CHANGED
|
@@ -16,6 +16,7 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
|
|
|
16
16
|
from datahub.ingestion.graph.config import ClientMode
|
|
17
17
|
from datahub.metadata.schema_classes import SystemMetadataClass
|
|
18
18
|
from datahub.telemetry import telemetry
|
|
19
|
+
from datahub.upgrade import upgrade
|
|
19
20
|
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
21
22
|
|
|
@@ -164,6 +165,7 @@ def validate_warehouse(data_root: str) -> None:
|
|
|
164
165
|
help=f"Expiration duration for temporary credentials used for role. Defaults to {DEFAULT_CREDS_EXPIRY_DURATION_SECONDS} seconds if unspecified",
|
|
165
166
|
)
|
|
166
167
|
@telemetry.with_telemetry(capture_kwargs=["duration_seconds"])
|
|
168
|
+
@upgrade.check_upgrade
|
|
167
169
|
def create(
|
|
168
170
|
warehouse: str,
|
|
169
171
|
description: Optional[str],
|
|
@@ -317,6 +319,7 @@ def create(
|
|
|
317
319
|
help=f"Expiration duration for temporary credentials used for role. Defaults to {DEFAULT_CREDS_EXPIRY_DURATION_SECONDS} seconds if unspecified",
|
|
318
320
|
)
|
|
319
321
|
@telemetry.with_telemetry(capture_kwargs=["duration_seconds"])
|
|
322
|
+
@upgrade.check_upgrade
|
|
320
323
|
def update(
|
|
321
324
|
warehouse: str,
|
|
322
325
|
data_root: str,
|
|
@@ -403,6 +406,7 @@ def update(
|
|
|
403
406
|
|
|
404
407
|
@iceberg.command()
|
|
405
408
|
@telemetry.with_telemetry()
|
|
409
|
+
@upgrade.check_upgrade
|
|
406
410
|
def list() -> None:
|
|
407
411
|
"""
|
|
408
412
|
List iceberg warehouses
|
|
@@ -419,6 +423,7 @@ def list() -> None:
|
|
|
419
423
|
"-w", "--warehouse", required=True, type=str, help="The name of the warehouse"
|
|
420
424
|
)
|
|
421
425
|
@telemetry.with_telemetry()
|
|
426
|
+
@upgrade.check_upgrade
|
|
422
427
|
def get(warehouse: str) -> None:
|
|
423
428
|
"""Fetches the details of the specified iceberg warehouse"""
|
|
424
429
|
client = get_default_graph(ClientMode.CLI)
|
datahub/cli/ingest_cli.py
CHANGED
|
@@ -24,7 +24,6 @@ from datahub.ingestion.run.pipeline import Pipeline
|
|
|
24
24
|
from datahub.telemetry import telemetry
|
|
25
25
|
from datahub.upgrade import upgrade
|
|
26
26
|
from datahub.utilities.ingest_utils import deploy_source_vars
|
|
27
|
-
from datahub.utilities.perf_timer import PerfTimer
|
|
28
27
|
|
|
29
28
|
logger = logging.getLogger(__name__)
|
|
30
29
|
|
|
@@ -114,6 +113,7 @@ def ingest() -> None:
|
|
|
114
113
|
"no_progress",
|
|
115
114
|
]
|
|
116
115
|
)
|
|
116
|
+
@upgrade.check_upgrade
|
|
117
117
|
def run(
|
|
118
118
|
config: str,
|
|
119
119
|
dry_run: bool,
|
|
@@ -178,14 +178,7 @@ def run(
|
|
|
178
178
|
no_progress=no_progress,
|
|
179
179
|
raw_config=raw_pipeline_config,
|
|
180
180
|
)
|
|
181
|
-
|
|
182
|
-
ret = run_pipeline_to_completion(pipeline)
|
|
183
|
-
|
|
184
|
-
# The main ingestion has completed. If it was successful, potentially show an upgrade nudge message.
|
|
185
|
-
if ret == 0:
|
|
186
|
-
upgrade.check_upgrade_post(
|
|
187
|
-
main_method_runtime=timer.elapsed_seconds(), graph=pipeline.ctx.graph
|
|
188
|
-
)
|
|
181
|
+
ret = run_pipeline_to_completion(pipeline)
|
|
189
182
|
|
|
190
183
|
if ret:
|
|
191
184
|
sys.exit(ret)
|
|
@@ -193,8 +186,6 @@ def run(
|
|
|
193
186
|
|
|
194
187
|
|
|
195
188
|
@ingest.command()
|
|
196
|
-
@upgrade.check_upgrade
|
|
197
|
-
@telemetry.with_telemetry()
|
|
198
189
|
@click.option(
|
|
199
190
|
"-n",
|
|
200
191
|
"--name",
|
|
@@ -252,6 +243,7 @@ def run(
|
|
|
252
243
|
required=False,
|
|
253
244
|
default=None,
|
|
254
245
|
)
|
|
246
|
+
@upgrade.check_upgrade
|
|
255
247
|
def deploy(
|
|
256
248
|
name: Optional[str],
|
|
257
249
|
config: str,
|
|
@@ -386,7 +378,6 @@ def mcps(path: str) -> None:
|
|
|
386
378
|
"--source", type=str, default=None, help="Filter by ingestion source name."
|
|
387
379
|
)
|
|
388
380
|
@upgrade.check_upgrade
|
|
389
|
-
@telemetry.with_telemetry()
|
|
390
381
|
def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) -> None:
|
|
391
382
|
"""
|
|
392
383
|
List ingestion source runs with their details, optionally filtered by URN or source.
|
|
@@ -515,7 +506,6 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
|
|
|
515
506
|
help="If enabled, will list ingestion runs which have been soft deleted",
|
|
516
507
|
)
|
|
517
508
|
@upgrade.check_upgrade
|
|
518
|
-
@telemetry.with_telemetry()
|
|
519
509
|
def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> None:
|
|
520
510
|
"""List recent ingestion runs to datahub"""
|
|
521
511
|
|
|
@@ -565,7 +555,6 @@ def list_runs(page_offset: int, page_size: int, include_soft_deletes: bool) -> N
|
|
|
565
555
|
)
|
|
566
556
|
@click.option("-a", "--show-aspect", required=False, is_flag=True)
|
|
567
557
|
@upgrade.check_upgrade
|
|
568
|
-
@telemetry.with_telemetry()
|
|
569
558
|
def show(
|
|
570
559
|
run_id: str, start: int, count: int, include_soft_deletes: bool, show_aspect: bool
|
|
571
560
|
) -> None:
|
|
@@ -615,7 +604,6 @@ def show(
|
|
|
615
604
|
help="Path to directory where rollback reports will be saved to",
|
|
616
605
|
)
|
|
617
606
|
@upgrade.check_upgrade
|
|
618
|
-
@telemetry.with_telemetry()
|
|
619
607
|
def rollback(
|
|
620
608
|
run_id: str, force: bool, dry_run: bool, safe: bool, report_dir: str
|
|
621
609
|
) -> None:
|
datahub/cli/migrate.py
CHANGED
|
@@ -37,6 +37,7 @@ from datahub.metadata.schema_classes import (
|
|
|
37
37
|
SystemMetadataClass,
|
|
38
38
|
)
|
|
39
39
|
from datahub.telemetry import telemetry
|
|
40
|
+
from datahub.upgrade import upgrade
|
|
40
41
|
from datahub.utilities.urns.urn import Urn
|
|
41
42
|
|
|
42
43
|
log = logging.getLogger(__name__)
|
|
@@ -119,6 +120,7 @@ def _get_type_from_urn(urn: str) -> str:
|
|
|
119
120
|
help="When enabled, will not delete (hard/soft) the previous entities.",
|
|
120
121
|
)
|
|
121
122
|
@telemetry.with_telemetry()
|
|
123
|
+
@upgrade.check_upgrade
|
|
122
124
|
def dataplatform2instance(
|
|
123
125
|
instance: str,
|
|
124
126
|
platform: str,
|
datahub/cli/put_cli.py
CHANGED
|
@@ -14,7 +14,6 @@ from datahub.metadata.schema_classes import (
|
|
|
14
14
|
PlatformTypeClass,
|
|
15
15
|
SystemMetadataClass,
|
|
16
16
|
)
|
|
17
|
-
from datahub.telemetry import telemetry
|
|
18
17
|
from datahub.upgrade import upgrade
|
|
19
18
|
from datahub.utilities.urns.data_platform_urn import DataPlatformUrn
|
|
20
19
|
from datahub.utilities.urns.urn import guess_entity_type
|
|
@@ -45,7 +44,6 @@ def put() -> None:
|
|
|
45
44
|
help="Run ID into which we should log the aspect.",
|
|
46
45
|
)
|
|
47
46
|
@upgrade.check_upgrade
|
|
48
|
-
@telemetry.with_telemetry()
|
|
49
47
|
def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> None:
|
|
50
48
|
"""Update a single aspect of an entity"""
|
|
51
49
|
|
|
@@ -75,8 +73,6 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
|
|
|
75
73
|
|
|
76
74
|
@put.command()
|
|
77
75
|
@click.pass_context
|
|
78
|
-
@upgrade.check_upgrade
|
|
79
|
-
@telemetry.with_telemetry()
|
|
80
76
|
@click.option(
|
|
81
77
|
"--name",
|
|
82
78
|
type=str,
|
|
@@ -98,6 +94,7 @@ def aspect(urn: str, aspect: str, aspect_data: str, run_id: Optional[str]) -> No
|
|
|
98
94
|
@click.option(
|
|
99
95
|
"--run-id", type=str, help="Run ID into which we should log the platform."
|
|
100
96
|
)
|
|
97
|
+
@upgrade.check_upgrade
|
|
101
98
|
def platform(
|
|
102
99
|
ctx: click.Context, name: str, display_name: Optional[str], logo: str, run_id: str
|
|
103
100
|
) -> None:
|
|
@@ -1,27 +1,50 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
-
import os
|
|
4
3
|
import os.path
|
|
5
4
|
import re
|
|
6
5
|
from typing import Dict, Optional
|
|
7
6
|
|
|
8
7
|
import click
|
|
8
|
+
import packaging
|
|
9
9
|
import requests
|
|
10
10
|
import yaml
|
|
11
11
|
from packaging.version import parse
|
|
12
12
|
from pydantic import BaseModel
|
|
13
13
|
|
|
14
|
+
from datahub._version import nice_version_name
|
|
15
|
+
from datahub.configuration.env_vars import get_force_local_quickstart_mapping
|
|
16
|
+
|
|
14
17
|
logger = logging.getLogger(__name__)
|
|
15
18
|
|
|
16
|
-
LOCAL_QUICKSTART_MAPPING_FILE =
|
|
19
|
+
LOCAL_QUICKSTART_MAPPING_FILE = get_force_local_quickstart_mapping()
|
|
17
20
|
DEFAULT_LOCAL_CONFIG_PATH = "~/.datahub/quickstart/quickstart_version_mapping.yaml"
|
|
18
21
|
DEFAULT_REMOTE_CONFIG_PATH = "https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/quickstart_version_mapping.yaml"
|
|
19
22
|
|
|
23
|
+
MINIMUM_SUPPORTED_VERSION = "v1.1.0"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_minimum_supported_version_message(version: str) -> str:
|
|
27
|
+
MINIMUM_SUPPORTED_VERSION_MESSAGE = f"""
|
|
28
|
+
DataHub CLI Version Compatibility Issue
|
|
29
|
+
|
|
30
|
+
You're trying to install DataHub server version {version} which is not supported by this CLI version.
|
|
31
|
+
|
|
32
|
+
This CLI (version {nice_version_name()}) only supports installing DataHub server versions {MINIMUM_SUPPORTED_VERSION} and above.
|
|
33
|
+
|
|
34
|
+
To install older server versions:
|
|
35
|
+
1. Uninstall current CLI: pip uninstall acryl-datahub
|
|
36
|
+
2. Install older CLI: pip install acryl-datahub==1.1
|
|
37
|
+
3. Run quickstart with your desired version: datahub docker quickstart --version <version>
|
|
38
|
+
|
|
39
|
+
For more information: https://docs.datahub.com/docs/quickstart#install-datahub-server
|
|
40
|
+
"""
|
|
41
|
+
return MINIMUM_SUPPORTED_VERSION_MESSAGE
|
|
42
|
+
|
|
20
43
|
|
|
21
44
|
class QuickstartExecutionPlan(BaseModel):
|
|
22
45
|
composefile_git_ref: str
|
|
23
46
|
docker_tag: str
|
|
24
|
-
mysql_tag: Optional[str]
|
|
47
|
+
mysql_tag: Optional[str] = None
|
|
25
48
|
|
|
26
49
|
|
|
27
50
|
def _is_it_a_version(version: str) -> bool:
|
|
@@ -126,15 +149,23 @@ class QuickstartVersionMappingConfig(BaseModel):
|
|
|
126
149
|
mysql_tag=str(mysql_tag),
|
|
127
150
|
),
|
|
128
151
|
)
|
|
152
|
+
|
|
153
|
+
if not is_minimum_supported_version(requested_version):
|
|
154
|
+
click.secho(
|
|
155
|
+
get_minimum_supported_version_message(version=requested_version),
|
|
156
|
+
fg="red",
|
|
157
|
+
)
|
|
158
|
+
raise click.ClickException("Minimum supported version not met")
|
|
159
|
+
|
|
129
160
|
# new CLI version is downloading the composefile corresponding to the requested version
|
|
130
|
-
# if the version is older than
|
|
131
|
-
#
|
|
132
|
-
# the
|
|
161
|
+
# if the version is older than <MINIMUM_SUPPORTED_VERSION>, it doesn't contain the
|
|
162
|
+
# docker compose based resolved compose file. In those cases, we pick up the composefile from
|
|
163
|
+
# MINIMUM_SUPPORTED_VERSION which contains the compose file.
|
|
133
164
|
if _is_it_a_version(result.composefile_git_ref):
|
|
134
|
-
if parse("
|
|
135
|
-
# The merge commit where
|
|
136
|
-
# https://github.com/datahub-project/datahub/pull/
|
|
137
|
-
result.composefile_git_ref = "
|
|
165
|
+
if parse("v1.2.0") > parse(result.composefile_git_ref):
|
|
166
|
+
# The merge commit where profiles based resolved compose file was added.
|
|
167
|
+
# https://github.com/datahub-project/datahub/pull/13566
|
|
168
|
+
result.composefile_git_ref = "21726bc3341490f4182b904626c793091ac95edd"
|
|
138
169
|
|
|
139
170
|
return result
|
|
140
171
|
|
|
@@ -148,3 +179,15 @@ def save_quickstart_config(
|
|
|
148
179
|
with open(path, "w") as f:
|
|
149
180
|
yaml.dump(config.dict(), f)
|
|
150
181
|
logger.info(f"Saved quickstart config to {path}.")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def is_minimum_supported_version(version: str) -> bool:
|
|
185
|
+
if not _is_it_a_version(version):
|
|
186
|
+
return True
|
|
187
|
+
|
|
188
|
+
requested_version = packaging.version.parse(version)
|
|
189
|
+
minimum_supported_version = packaging.version.parse(MINIMUM_SUPPORTED_VERSION)
|
|
190
|
+
if requested_version < minimum_supported_version:
|
|
191
|
+
return False
|
|
192
|
+
|
|
193
|
+
return True
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DEPRECATED: This assertions CLI is no longer supported and will be removed in a future version.
|
|
3
|
+
Please use alternative methods for managing assertions in DataHub.
|
|
4
|
+
"""
|
|
5
|
+
|
|
1
6
|
import logging
|
|
2
7
|
import os
|
|
3
8
|
from pathlib import Path
|
|
@@ -17,7 +22,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
|
|
17
22
|
from datahub.ingestion.graph.client import get_default_graph
|
|
18
23
|
from datahub.ingestion.graph.config import ClientMode
|
|
19
24
|
from datahub.integrations.assertion.registry import ASSERTION_PLATFORMS
|
|
20
|
-
from datahub.telemetry import telemetry
|
|
21
25
|
from datahub.upgrade import upgrade
|
|
22
26
|
|
|
23
27
|
logger = logging.getLogger(__name__)
|
|
@@ -27,16 +31,35 @@ REPORT_FILE_NAME = "compile_report.json"
|
|
|
27
31
|
|
|
28
32
|
@click.group(cls=DefaultGroup, default="upsert")
|
|
29
33
|
def assertions() -> None:
|
|
30
|
-
"""A group of commands to interact with the Assertion entity in DataHub.
|
|
34
|
+
"""A group of commands to interact with the Assertion entity in DataHub.
|
|
35
|
+
|
|
36
|
+
⚠️ DEPRECATED: This assertions CLI is no longer supported and will be removed
|
|
37
|
+
in a future version. Please use alternative methods for managing assertions in DataHub.
|
|
38
|
+
"""
|
|
39
|
+
click.secho(
|
|
40
|
+
"⚠️ WARNING: The assertions CLI is deprecated and no longer supported. "
|
|
41
|
+
"It may be removed in a future version. Please use alternative methods for managing assertions in DataHub.",
|
|
42
|
+
fg="yellow",
|
|
43
|
+
bold=True,
|
|
44
|
+
err=True,
|
|
45
|
+
)
|
|
31
46
|
pass
|
|
32
47
|
|
|
33
48
|
|
|
34
49
|
@assertions.command()
|
|
35
50
|
@click.option("-f", "--file", required=True, type=click.Path(exists=True))
|
|
36
51
|
@upgrade.check_upgrade
|
|
37
|
-
@telemetry.with_telemetry()
|
|
38
52
|
def upsert(file: str) -> None:
|
|
39
|
-
"""Upsert (create or update) a set of assertions in DataHub.
|
|
53
|
+
"""Upsert (create or update) a set of assertions in DataHub.
|
|
54
|
+
|
|
55
|
+
⚠️ DEPRECATED: This command is deprecated and no longer supported.
|
|
56
|
+
"""
|
|
57
|
+
click.secho(
|
|
58
|
+
"⚠️ WARNING: The 'upsert' command is deprecated and no longer supported.",
|
|
59
|
+
fg="yellow",
|
|
60
|
+
bold=True,
|
|
61
|
+
err=True,
|
|
62
|
+
)
|
|
40
63
|
|
|
41
64
|
assertions_spec: AssertionsConfigSpec = AssertionsConfigSpec.from_yaml(file)
|
|
42
65
|
|
|
@@ -71,8 +94,6 @@ def upsert(file: str) -> None:
|
|
|
71
94
|
default=[],
|
|
72
95
|
help="Platform-specific extra key-value inputs in form key=value",
|
|
73
96
|
)
|
|
74
|
-
@upgrade.check_upgrade
|
|
75
|
-
@telemetry.with_telemetry()
|
|
76
97
|
def compile(
|
|
77
98
|
file: str, platform: str, output_to: Optional[str], extras: List[str]
|
|
78
99
|
) -> None:
|
|
@@ -82,7 +103,15 @@ def compile(
|
|
|
82
103
|
In future, we may introduce separate command to automatically apply these compiled changes
|
|
83
104
|
in assertion platform. Currently, generated result artifacts are stored in target folder
|
|
84
105
|
unless another folder is specified using option `--output-to <folder>`.
|
|
106
|
+
|
|
107
|
+
⚠️ DEPRECATED: This command is deprecated and no longer supported.
|
|
85
108
|
"""
|
|
109
|
+
click.secho(
|
|
110
|
+
"⚠️ WARNING: The 'compile' command is deprecated and no longer supported.",
|
|
111
|
+
fg="yellow",
|
|
112
|
+
bold=True,
|
|
113
|
+
err=True,
|
|
114
|
+
)
|
|
86
115
|
|
|
87
116
|
if platform not in ASSERTION_PLATFORMS:
|
|
88
117
|
click.secho(
|
|
@@ -150,3 +179,5 @@ def extras_list_to_dict(extras: List[str]) -> Dict[str, str]:
|
|
|
150
179
|
# Later:
|
|
151
180
|
# 3. execute compiled assertions on assertion platform (Later, requires connection details to platform),
|
|
152
181
|
# 4. cleanup assertions from assertion platform (generate artifacts. optionally execute)
|
|
182
|
+
#
|
|
183
|
+
# NOTE: This entire assertions CLI is deprecated and these TODOs will not be implemented.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
import warnings
|
|
2
3
|
from typing import Optional
|
|
3
4
|
|
|
4
5
|
import click
|
|
@@ -7,7 +8,6 @@ from click_default_group import DefaultGroup
|
|
|
7
8
|
from datahub.api.entities.datacontract.datacontract import DataContract
|
|
8
9
|
from datahub.ingestion.graph.client import get_default_graph
|
|
9
10
|
from datahub.ingestion.graph.config import ClientMode
|
|
10
|
-
from datahub.telemetry import telemetry
|
|
11
11
|
from datahub.upgrade import upgrade
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
@@ -15,16 +15,52 @@ logger = logging.getLogger(__name__)
|
|
|
15
15
|
|
|
16
16
|
@click.group(cls=DefaultGroup, default="upsert")
|
|
17
17
|
def datacontract() -> None:
|
|
18
|
-
"""
|
|
19
|
-
|
|
18
|
+
"""
|
|
19
|
+
A group of commands to interact with the DataContract entity in DataHub.
|
|
20
|
+
|
|
21
|
+
WARNING: This CLI is DEPRECATED and no longer supported.
|
|
22
|
+
Please migrate to alternative data contract solutions.
|
|
23
|
+
"""
|
|
24
|
+
# Issue deprecation warning
|
|
25
|
+
warnings.warn(
|
|
26
|
+
"The datacontract CLI is deprecated and no longer supported. "
|
|
27
|
+
"Please migrate to alternative data contract solutions.",
|
|
28
|
+
DeprecationWarning,
|
|
29
|
+
stacklevel=2,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# Log deprecation message for runtime visibility
|
|
33
|
+
logger.warning(
|
|
34
|
+
"DEPRECATED: The datacontract CLI is no longer supported and will be removed in a future version. "
|
|
35
|
+
"Please migrate to alternative data contract solutions."
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# Display deprecation message to user
|
|
39
|
+
click.secho(
|
|
40
|
+
"⚠️ WARNING: This datacontract CLI is DEPRECATED and no longer supported.",
|
|
41
|
+
fg="yellow",
|
|
42
|
+
bold=True,
|
|
43
|
+
)
|
|
44
|
+
click.secho("Please migrate to alternative data contract solutions.", fg="yellow")
|
|
20
45
|
|
|
21
46
|
|
|
22
47
|
@datacontract.command()
|
|
23
48
|
@click.option("-f", "--file", required=True, type=click.Path(exists=True))
|
|
24
49
|
@upgrade.check_upgrade
|
|
25
|
-
@telemetry.with_telemetry()
|
|
26
50
|
def upsert(file: str) -> None:
|
|
27
|
-
"""
|
|
51
|
+
"""
|
|
52
|
+
Upsert (create or update) a Data Contract in DataHub.
|
|
53
|
+
|
|
54
|
+
WARNING: This command is DEPRECATED and no longer supported.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
click.secho(
|
|
58
|
+
"⚠️ WARNING: The 'upsert' command is deprecated and no longer supported.",
|
|
59
|
+
fg="yellow",
|
|
60
|
+
bold=True,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
logger.warning("DEPRECATED: datacontract upsert command is no longer supported")
|
|
28
64
|
|
|
29
65
|
data_contract: DataContract = DataContract.from_yaml(file)
|
|
30
66
|
urn = data_contract.urn
|
|
@@ -60,9 +96,20 @@ def upsert(file: str) -> None:
|
|
|
60
96
|
)
|
|
61
97
|
@click.option("--hard/--soft", required=False, is_flag=True, default=False)
|
|
62
98
|
@upgrade.check_upgrade
|
|
63
|
-
@telemetry.with_telemetry()
|
|
64
99
|
def delete(urn: Optional[str], file: Optional[str], hard: bool) -> None:
|
|
65
|
-
"""
|
|
100
|
+
"""
|
|
101
|
+
Delete a Data Contract in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata.
|
|
102
|
+
|
|
103
|
+
WARNING: This command is DEPRECATED and no longer supported.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
click.secho(
|
|
107
|
+
"⚠️ WARNING: The 'delete' command is deprecated and no longer supported.",
|
|
108
|
+
fg="yellow",
|
|
109
|
+
bold=True,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
logger.warning("DEPRECATED: datacontract delete command is no longer supported")
|
|
66
113
|
|
|
67
114
|
if not urn:
|
|
68
115
|
if not file:
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import difflib
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
-
import os
|
|
5
4
|
import pathlib
|
|
6
5
|
import sys
|
|
7
6
|
from pathlib import Path
|
|
@@ -14,6 +13,7 @@ from click_default_group import DefaultGroup
|
|
|
14
13
|
|
|
15
14
|
from datahub.api.entities.dataproduct.dataproduct import DataProduct
|
|
16
15
|
from datahub.cli.specific.file_loader import load_file
|
|
16
|
+
from datahub.configuration.env_vars import get_dataproduct_external_url
|
|
17
17
|
from datahub.emitter.mce_builder import (
|
|
18
18
|
make_group_urn,
|
|
19
19
|
make_user_urn,
|
|
@@ -23,7 +23,6 @@ from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
|
|
|
23
23
|
from datahub.ingestion.graph.config import ClientMode
|
|
24
24
|
from datahub.metadata.schema_classes import OwnerClass, OwnershipTypeClass
|
|
25
25
|
from datahub.specific.dataproduct import DataProductPatchBuilder
|
|
26
|
-
from datahub.telemetry import telemetry
|
|
27
26
|
from datahub.upgrade import upgrade
|
|
28
27
|
from datahub.utilities.urns.urn import Urn
|
|
29
28
|
|
|
@@ -85,9 +84,7 @@ def mutate(file: Path, validate_assets: bool, external_url: str, upsert: bool) -
|
|
|
85
84
|
with get_default_graph(ClientMode.CLI) as graph:
|
|
86
85
|
data_product: DataProduct = DataProduct.from_yaml(file, graph)
|
|
87
86
|
external_url_override = (
|
|
88
|
-
external_url
|
|
89
|
-
or os.getenv("DATAHUB_DATAPRODUCT_EXTERNAL_URL")
|
|
90
|
-
or data_product.external_url
|
|
87
|
+
external_url or get_dataproduct_external_url() or data_product.external_url
|
|
91
88
|
)
|
|
92
89
|
data_product.external_url = external_url_override
|
|
93
90
|
if upsert and not graph.exists(data_product.urn):
|
|
@@ -130,7 +127,6 @@ def mutate(file: Path, validate_assets: bool, external_url: str, upsert: bool) -
|
|
|
130
127
|
)
|
|
131
128
|
@click.option("--external-url", required=False, type=str)
|
|
132
129
|
@upgrade.check_upgrade
|
|
133
|
-
@telemetry.with_telemetry()
|
|
134
130
|
def update(file: Path, validate_assets: bool, external_url: str) -> None:
|
|
135
131
|
"""Create or Update a Data Product in DataHub. Use upsert if you want to apply partial updates."""
|
|
136
132
|
|
|
@@ -146,7 +142,6 @@ def update(file: Path, validate_assets: bool, external_url: str) -> None:
|
|
|
146
142
|
)
|
|
147
143
|
@click.option("--external-url", required=False, type=str)
|
|
148
144
|
@upgrade.check_upgrade
|
|
149
|
-
@telemetry.with_telemetry()
|
|
150
145
|
def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
|
|
151
146
|
"""Upsert attributes to a Data Product in DataHub."""
|
|
152
147
|
|
|
@@ -159,7 +154,6 @@ def upsert(file: Path, validate_assets: bool, external_url: str) -> None:
|
|
|
159
154
|
@click.option("-f", "--file", required=True, type=click.Path(exists=True))
|
|
160
155
|
@click.option("--update", required=False, is_flag=True, default=False)
|
|
161
156
|
@upgrade.check_upgrade
|
|
162
|
-
@telemetry.with_telemetry()
|
|
163
157
|
def diff(file: Path, update: bool) -> None:
|
|
164
158
|
"""Diff a Data Product file with its twin in DataHub"""
|
|
165
159
|
|
|
@@ -206,7 +200,6 @@ def diff(file: Path, update: bool) -> None:
|
|
|
206
200
|
)
|
|
207
201
|
@click.option("--hard/--soft", required=False, is_flag=True, default=False)
|
|
208
202
|
@upgrade.check_upgrade
|
|
209
|
-
@telemetry.with_telemetry()
|
|
210
203
|
def delete(urn: str, file: Path, hard: bool) -> None:
|
|
211
204
|
"""Delete a Data Product in DataHub. Defaults to a soft-delete. Use --hard to completely erase metadata."""
|
|
212
205
|
|
|
@@ -242,7 +235,6 @@ def delete(urn: str, file: Path, hard: bool) -> None:
|
|
|
242
235
|
@click.option("--urn", required=True, type=str)
|
|
243
236
|
@click.option("--to-file", required=False, type=str)
|
|
244
237
|
@upgrade.check_upgrade
|
|
245
|
-
@telemetry.with_telemetry()
|
|
246
238
|
def get(urn: str, to_file: str) -> None:
|
|
247
239
|
"""Get a Data Product from DataHub"""
|
|
248
240
|
|
|
@@ -279,7 +271,6 @@ def get(urn: str, to_file: str) -> None:
|
|
|
279
271
|
help="A markdown file that contains documentation for this data product",
|
|
280
272
|
)
|
|
281
273
|
@upgrade.check_upgrade
|
|
282
|
-
@telemetry.with_telemetry()
|
|
283
274
|
def set_description(urn: str, description: str, md_file: Path) -> None:
|
|
284
275
|
"""Set description for a Data Product in DataHub"""
|
|
285
276
|
|
|
@@ -330,7 +321,6 @@ def set_description(urn: str, description: str, md_file: Path) -> None:
|
|
|
330
321
|
default=OwnershipTypeClass.TECHNICAL_OWNER,
|
|
331
322
|
)
|
|
332
323
|
@upgrade.check_upgrade
|
|
333
|
-
@telemetry.with_telemetry()
|
|
334
324
|
def add_owner(urn: str, owner: str, owner_type: str) -> None:
|
|
335
325
|
"""Add owner for a Data Product in DataHub"""
|
|
336
326
|
|
|
@@ -353,7 +343,6 @@ def add_owner(urn: str, owner: str, owner_type: str) -> None:
|
|
|
353
343
|
@click.option("--urn", required=True, type=str)
|
|
354
344
|
@click.argument("owner_urn", required=True, type=str)
|
|
355
345
|
@upgrade.check_upgrade
|
|
356
|
-
@telemetry.with_telemetry()
|
|
357
346
|
def remove_owner(urn: str, owner_urn: str) -> None:
|
|
358
347
|
"""Remove owner for a Data Product in DataHub"""
|
|
359
348
|
|
|
@@ -375,7 +364,6 @@ def remove_owner(urn: str, owner_urn: str) -> None:
|
|
|
375
364
|
"--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
|
|
376
365
|
)
|
|
377
366
|
@upgrade.check_upgrade
|
|
378
|
-
@telemetry.with_telemetry()
|
|
379
367
|
def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
|
|
380
368
|
"""Add asset for a Data Product in DataHub"""
|
|
381
369
|
|
|
@@ -402,7 +390,6 @@ def add_asset(urn: str, asset: str, validate_assets: bool) -> None:
|
|
|
402
390
|
"--validate-assets/--no-validate-assets", required=False, is_flag=True, default=True
|
|
403
391
|
)
|
|
404
392
|
@upgrade.check_upgrade
|
|
405
|
-
@telemetry.with_telemetry()
|
|
406
393
|
def remove_asset(urn: str, asset: str, validate_assets: bool) -> None:
|
|
407
394
|
"""Remove asset for a Data Product in DataHub"""
|
|
408
395
|
|
|
@@ -14,7 +14,6 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
|
|
|
14
14
|
from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
|
|
15
15
|
from datahub.ingestion.graph.config import ClientMode
|
|
16
16
|
from datahub.metadata.com.linkedin.pegasus2avro.common import Siblings
|
|
17
|
-
from datahub.telemetry import telemetry
|
|
18
17
|
from datahub.upgrade import upgrade
|
|
19
18
|
|
|
20
19
|
logger = logging.getLogger(__name__)
|
|
@@ -33,8 +32,6 @@ def dataset() -> None:
|
|
|
33
32
|
@click.option(
|
|
34
33
|
"-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
|
|
35
34
|
)
|
|
36
|
-
@upgrade.check_upgrade
|
|
37
|
-
@telemetry.with_telemetry()
|
|
38
35
|
def upsert(file: Path, dry_run: bool) -> None:
|
|
39
36
|
"""Upsert attributes to a Dataset in DataHub."""
|
|
40
37
|
# Call the sync command with to_datahub=True to perform the upsert operation
|
|
@@ -48,7 +45,6 @@ def upsert(file: Path, dry_run: bool) -> None:
|
|
|
48
45
|
@click.option("--urn", required=True, type=str)
|
|
49
46
|
@click.option("--to-file", required=False, type=str)
|
|
50
47
|
@upgrade.check_upgrade
|
|
51
|
-
@telemetry.with_telemetry()
|
|
52
48
|
def get(urn: str, to_file: str) -> None:
|
|
53
49
|
"""Get a Dataset from DataHub"""
|
|
54
50
|
|
|
@@ -77,7 +73,7 @@ def get(urn: str, to_file: str) -> None:
|
|
|
77
73
|
help="URN of secondary sibling(s)",
|
|
78
74
|
multiple=True,
|
|
79
75
|
)
|
|
80
|
-
@
|
|
76
|
+
@upgrade.check_upgrade
|
|
81
77
|
def add_sibling(urn: str, sibling_urns: Tuple[str]) -> None:
|
|
82
78
|
all_urns = set()
|
|
83
79
|
all_urns.add(urn)
|
|
@@ -117,8 +113,6 @@ def _get_existing_siblings(graph: DataHubGraph, urn: str) -> Set[str]:
|
|
|
117
113
|
@click.option("--lintCheck", required=False, is_flag=True)
|
|
118
114
|
@click.option("--lintFix", required=False, is_flag=True)
|
|
119
115
|
@click.argument("file", type=click.Path(exists=True))
|
|
120
|
-
@upgrade.check_upgrade
|
|
121
|
-
@telemetry.with_telemetry()
|
|
122
116
|
def file(lintcheck: bool, lintfix: bool, file: str) -> None:
|
|
123
117
|
"""Operate on a Dataset file"""
|
|
124
118
|
|
|
@@ -175,7 +169,6 @@ def file(lintcheck: bool, lintfix: bool, file: str) -> None:
|
|
|
175
169
|
"-n", "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry run"
|
|
176
170
|
)
|
|
177
171
|
@upgrade.check_upgrade
|
|
178
|
-
@telemetry.with_telemetry()
|
|
179
172
|
def sync(file: str, to_datahub: bool, dry_run: bool) -> None:
|
|
180
173
|
"""Sync a Dataset file to/from DataHub"""
|
|
181
174
|
|
|
@@ -8,7 +8,6 @@ from click_default_group import DefaultGroup
|
|
|
8
8
|
from datahub.api.entities.forms.forms import Forms
|
|
9
9
|
from datahub.ingestion.graph.client import get_default_graph
|
|
10
10
|
from datahub.ingestion.graph.config import ClientMode
|
|
11
|
-
from datahub.telemetry import telemetry
|
|
12
11
|
from datahub.upgrade import upgrade
|
|
13
12
|
|
|
14
13
|
logger = logging.getLogger(__name__)
|
|
@@ -24,8 +23,6 @@ def forms() -> None:
|
|
|
24
23
|
name="upsert",
|
|
25
24
|
)
|
|
26
25
|
@click.option("-f", "--file", required=True, type=click.Path(exists=True))
|
|
27
|
-
@upgrade.check_upgrade
|
|
28
|
-
@telemetry.with_telemetry()
|
|
29
26
|
def upsert(file: Path) -> None:
|
|
30
27
|
"""Upsert forms in DataHub."""
|
|
31
28
|
|
|
@@ -38,7 +35,6 @@ def upsert(file: Path) -> None:
|
|
|
38
35
|
@click.option("--urn", required=True, type=str)
|
|
39
36
|
@click.option("--to-file", required=False, type=str)
|
|
40
37
|
@upgrade.check_upgrade
|
|
41
|
-
@telemetry.with_telemetry()
|
|
42
38
|
def get(urn: str, to_file: str) -> None:
|
|
43
39
|
"""Get form from DataHub"""
|
|
44
40
|
with get_default_graph(ClientMode.CLI) as graph:
|
|
@@ -11,7 +11,6 @@ from datahub.api.entities.corpgroup.corpgroup import (
|
|
|
11
11
|
from datahub.cli.specific.file_loader import load_file
|
|
12
12
|
from datahub.ingestion.graph.client import get_default_graph
|
|
13
13
|
from datahub.ingestion.graph.config import ClientMode
|
|
14
|
-
from datahub.telemetry import telemetry
|
|
15
14
|
from datahub.upgrade import upgrade
|
|
16
15
|
|
|
17
16
|
logger = logging.getLogger(__name__)
|
|
@@ -35,7 +34,6 @@ def group() -> None:
|
|
|
35
34
|
help="When set, writes to the editable section of the metadata graph, overwriting writes from the UI",
|
|
36
35
|
)
|
|
37
36
|
@upgrade.check_upgrade
|
|
38
|
-
@telemetry.with_telemetry()
|
|
39
37
|
def upsert(file: Path, override_editable: bool) -> None:
|
|
40
38
|
"""Create or Update a Group with embedded Users"""
|
|
41
39
|
|
|
@@ -12,7 +12,6 @@ from datahub.api.entities.structuredproperties.structuredproperties import (
|
|
|
12
12
|
)
|
|
13
13
|
from datahub.ingestion.graph.client import get_default_graph
|
|
14
14
|
from datahub.ingestion.graph.config import ClientMode
|
|
15
|
-
from datahub.telemetry import telemetry
|
|
16
15
|
from datahub.upgrade import upgrade
|
|
17
16
|
from datahub.utilities.urns.urn import Urn
|
|
18
17
|
|
|
@@ -30,7 +29,6 @@ def properties() -> None:
|
|
|
30
29
|
)
|
|
31
30
|
@click.option("-f", "--file", required=True, type=click.Path(exists=True))
|
|
32
31
|
@upgrade.check_upgrade
|
|
33
|
-
@telemetry.with_telemetry()
|
|
34
32
|
def upsert(file: Path) -> None:
|
|
35
33
|
"""Upsert structured properties in DataHub."""
|
|
36
34
|
|
|
@@ -44,7 +42,6 @@ def upsert(file: Path) -> None:
|
|
|
44
42
|
@click.option("--urn", required=True, type=str)
|
|
45
43
|
@click.option("--to-file", required=False, type=str)
|
|
46
44
|
@upgrade.check_upgrade
|
|
47
|
-
@telemetry.with_telemetry()
|
|
48
45
|
def get(urn: str, to_file: str) -> None:
|
|
49
46
|
"""Get structured properties from DataHub"""
|
|
50
47
|
urn = Urn.make_structured_property_urn(urn)
|
|
@@ -71,7 +68,7 @@ def get(urn: str, to_file: str) -> None:
|
|
|
71
68
|
)
|
|
72
69
|
@click.option("--details/--no-details", is_flag=True, default=True)
|
|
73
70
|
@click.option("--to-file", required=False, type=str)
|
|
74
|
-
@
|
|
71
|
+
@upgrade.check_upgrade
|
|
75
72
|
def list(details: bool, to_file: str) -> None:
|
|
76
73
|
"""List structured properties in DataHub"""
|
|
77
74
|
|