acryl-datahub 1.0.0rc18__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/METADATA +2686 -2563
- {acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/RECORD +499 -392
- {acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/WHEEL +1 -1
- {acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/entry_points.txt +7 -1
- datahub/_version.py +1 -1
- datahub/api/circuit_breaker/operation_circuit_breaker.py +2 -2
- datahub/api/entities/assertion/assertion.py +1 -1
- datahub/api/entities/common/serialized_value.py +1 -1
- datahub/api/entities/corpgroup/corpgroup.py +1 -1
- datahub/api/entities/datacontract/datacontract.py +35 -3
- datahub/api/entities/datajob/dataflow.py +18 -3
- datahub/api/entities/datajob/datajob.py +24 -4
- datahub/api/entities/dataprocess/dataprocess_instance.py +4 -0
- datahub/api/entities/dataproduct/dataproduct.py +32 -3
- datahub/api/entities/dataset/dataset.py +47 -72
- datahub/api/entities/external/__init__.py +0 -0
- datahub/api/entities/external/external_entities.py +724 -0
- datahub/api/entities/external/external_tag.py +147 -0
- datahub/api/entities/external/lake_formation_external_entites.py +162 -0
- datahub/api/entities/external/restricted_text.py +172 -0
- datahub/api/entities/external/unity_catalog_external_entites.py +172 -0
- datahub/api/entities/forms/forms.py +37 -37
- datahub/api/entities/structuredproperties/structuredproperties.py +6 -6
- datahub/api/graphql/assertion.py +1 -1
- datahub/api/graphql/base.py +8 -6
- datahub/api/graphql/operation.py +14 -10
- datahub/cli/check_cli.py +91 -9
- datahub/cli/cli_utils.py +63 -0
- datahub/cli/config_utils.py +20 -12
- datahub/cli/container_cli.py +5 -0
- datahub/cli/delete_cli.py +133 -34
- datahub/cli/docker_check.py +110 -14
- datahub/cli/docker_cli.py +155 -231
- datahub/cli/exists_cli.py +2 -3
- datahub/cli/get_cli.py +2 -3
- datahub/cli/graphql_cli.py +1422 -0
- datahub/cli/iceberg_cli.py +11 -5
- datahub/cli/ingest_cli.py +25 -26
- datahub/cli/migrate.py +12 -9
- datahub/cli/migration_utils.py +4 -3
- datahub/cli/put_cli.py +4 -6
- datahub/cli/quickstart_versioning.py +53 -10
- datahub/cli/specific/assertions_cli.py +39 -7
- datahub/cli/specific/datacontract_cli.py +57 -9
- datahub/cli/specific/dataproduct_cli.py +12 -24
- datahub/cli/specific/dataset_cli.py +31 -21
- datahub/cli/specific/forms_cli.py +2 -5
- datahub/cli/specific/group_cli.py +2 -3
- datahub/cli/specific/structuredproperties_cli.py +5 -7
- datahub/cli/specific/user_cli.py +174 -4
- datahub/cli/state_cli.py +2 -3
- datahub/cli/timeline_cli.py +2 -3
- datahub/configuration/common.py +46 -2
- datahub/configuration/connection_resolver.py +5 -2
- datahub/configuration/env_vars.py +331 -0
- datahub/configuration/import_resolver.py +7 -4
- datahub/configuration/kafka.py +21 -1
- datahub/configuration/pydantic_migration_helpers.py +6 -13
- datahub/configuration/source_common.py +4 -3
- datahub/configuration/validate_field_deprecation.py +5 -2
- datahub/configuration/validate_field_removal.py +8 -2
- datahub/configuration/validate_field_rename.py +6 -5
- datahub/configuration/validate_multiline_string.py +5 -2
- datahub/emitter/mce_builder.py +12 -8
- datahub/emitter/mcp.py +20 -5
- datahub/emitter/mcp_builder.py +12 -0
- datahub/emitter/request_helper.py +138 -15
- datahub/emitter/response_helper.py +111 -19
- datahub/emitter/rest_emitter.py +399 -163
- datahub/entrypoints.py +10 -5
- datahub/errors.py +12 -0
- datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py +299 -2
- datahub/ingestion/api/auto_work_units/auto_validate_input_fields.py +87 -0
- datahub/ingestion/api/common.py +9 -0
- datahub/ingestion/api/decorators.py +15 -3
- datahub/ingestion/api/report.py +381 -3
- datahub/ingestion/api/sink.py +27 -2
- datahub/ingestion/api/source.py +174 -62
- datahub/ingestion/api/source_helpers.py +41 -3
- datahub/ingestion/api/source_protocols.py +23 -0
- datahub/ingestion/autogenerated/__init__.py +0 -0
- datahub/ingestion/autogenerated/capability_summary.json +3652 -0
- datahub/ingestion/autogenerated/lineage.json +402 -0
- datahub/ingestion/autogenerated/lineage_helper.py +177 -0
- datahub/ingestion/extractor/schema_util.py +31 -5
- datahub/ingestion/glossary/classification_mixin.py +9 -2
- datahub/ingestion/graph/client.py +492 -55
- datahub/ingestion/graph/config.py +18 -2
- datahub/ingestion/graph/filters.py +96 -32
- datahub/ingestion/graph/links.py +55 -0
- datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +21 -11
- datahub/ingestion/run/pipeline.py +90 -23
- datahub/ingestion/run/pipeline_config.py +3 -3
- datahub/ingestion/sink/datahub_kafka.py +1 -0
- datahub/ingestion/sink/datahub_rest.py +31 -23
- datahub/ingestion/sink/file.py +1 -0
- datahub/ingestion/source/abs/config.py +1 -1
- datahub/ingestion/source/abs/datalake_profiler_config.py +1 -1
- datahub/ingestion/source/abs/source.py +15 -30
- datahub/ingestion/source/apply/datahub_apply.py +6 -5
- datahub/ingestion/source/aws/aws_common.py +185 -13
- datahub/ingestion/source/aws/glue.py +517 -244
- datahub/ingestion/source/aws/platform_resource_repository.py +30 -0
- datahub/ingestion/source/aws/s3_boto_utils.py +100 -5
- datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py +1 -1
- datahub/ingestion/source/aws/sagemaker_processors/models.py +4 -4
- datahub/ingestion/source/aws/tag_entities.py +270 -0
- datahub/ingestion/source/azure/azure_common.py +3 -3
- datahub/ingestion/source/bigquery_v2/bigquery.py +51 -7
- datahub/ingestion/source/bigquery_v2/bigquery_config.py +51 -81
- datahub/ingestion/source/bigquery_v2/bigquery_connection.py +81 -0
- datahub/ingestion/source/bigquery_v2/bigquery_queries.py +6 -1
- datahub/ingestion/source/bigquery_v2/bigquery_report.py +0 -2
- datahub/ingestion/source/bigquery_v2/bigquery_schema.py +23 -16
- datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +20 -5
- datahub/ingestion/source/bigquery_v2/common.py +1 -1
- datahub/ingestion/source/bigquery_v2/lineage.py +1 -1
- datahub/ingestion/source/bigquery_v2/profiler.py +4 -2
- datahub/ingestion/source/bigquery_v2/queries.py +3 -3
- datahub/ingestion/source/bigquery_v2/queries_extractor.py +45 -9
- datahub/ingestion/source/cassandra/cassandra.py +7 -18
- datahub/ingestion/source/cassandra/cassandra_api.py +36 -0
- datahub/ingestion/source/cassandra/cassandra_config.py +20 -0
- datahub/ingestion/source/cassandra/cassandra_profiling.py +26 -24
- datahub/ingestion/source/cassandra/cassandra_utils.py +1 -2
- datahub/ingestion/source/common/data_platforms.py +23 -0
- datahub/ingestion/source/common/gcp_credentials_config.py +9 -1
- datahub/ingestion/source/common/subtypes.py +73 -1
- datahub/ingestion/source/data_lake_common/data_lake_utils.py +59 -10
- datahub/ingestion/source/data_lake_common/object_store.py +732 -0
- datahub/ingestion/source/data_lake_common/path_spec.py +87 -38
- datahub/ingestion/source/datahub/config.py +19 -5
- datahub/ingestion/source/datahub/datahub_database_reader.py +205 -36
- datahub/ingestion/source/datahub/datahub_source.py +11 -1
- datahub/ingestion/source/dbt/dbt_cloud.py +17 -10
- datahub/ingestion/source/dbt/dbt_common.py +270 -26
- datahub/ingestion/source/dbt/dbt_core.py +88 -47
- datahub/ingestion/source/dbt/dbt_tests.py +8 -6
- datahub/ingestion/source/debug/__init__.py +0 -0
- datahub/ingestion/source/debug/datahub_debug.py +300 -0
- datahub/ingestion/source/delta_lake/config.py +9 -5
- datahub/ingestion/source/delta_lake/source.py +8 -0
- datahub/ingestion/source/dremio/dremio_api.py +114 -73
- datahub/ingestion/source/dremio/dremio_aspects.py +3 -2
- datahub/ingestion/source/dremio/dremio_config.py +5 -4
- datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py +1 -1
- datahub/ingestion/source/dremio/dremio_entities.py +6 -5
- datahub/ingestion/source/dremio/dremio_reporting.py +22 -3
- datahub/ingestion/source/dremio/dremio_source.py +228 -215
- datahub/ingestion/source/dremio/dremio_sql_queries.py +82 -21
- datahub/ingestion/source/dynamodb/dynamodb.py +19 -13
- datahub/ingestion/source/excel/__init__.py +0 -0
- datahub/ingestion/source/excel/config.py +92 -0
- datahub/ingestion/source/excel/excel_file.py +539 -0
- datahub/ingestion/source/excel/profiling.py +308 -0
- datahub/ingestion/source/excel/report.py +49 -0
- datahub/ingestion/source/excel/source.py +662 -0
- datahub/ingestion/source/excel/util.py +18 -0
- datahub/ingestion/source/feast.py +12 -14
- datahub/ingestion/source/file.py +3 -0
- datahub/ingestion/source/fivetran/config.py +67 -8
- datahub/ingestion/source/fivetran/fivetran.py +228 -43
- datahub/ingestion/source/fivetran/fivetran_log_api.py +42 -9
- datahub/ingestion/source/fivetran/fivetran_query.py +58 -36
- datahub/ingestion/source/fivetran/fivetran_rest_api.py +65 -0
- datahub/ingestion/source/fivetran/response_models.py +97 -0
- datahub/ingestion/source/gc/datahub_gc.py +0 -2
- datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +101 -104
- datahub/ingestion/source/gcs/gcs_source.py +53 -10
- datahub/ingestion/source/gcs/gcs_utils.py +36 -9
- datahub/ingestion/source/ge_data_profiler.py +146 -33
- datahub/ingestion/source/ge_profiling_config.py +26 -11
- datahub/ingestion/source/grafana/entity_mcp_builder.py +272 -0
- datahub/ingestion/source/grafana/field_utils.py +307 -0
- datahub/ingestion/source/grafana/grafana_api.py +142 -0
- datahub/ingestion/source/grafana/grafana_config.py +104 -0
- datahub/ingestion/source/grafana/grafana_source.py +522 -84
- datahub/ingestion/source/grafana/lineage.py +202 -0
- datahub/ingestion/source/grafana/models.py +137 -0
- datahub/ingestion/source/grafana/report.py +90 -0
- datahub/ingestion/source/grafana/types.py +16 -0
- datahub/ingestion/source/hex/__init__.py +0 -0
- datahub/ingestion/source/hex/api.py +402 -0
- datahub/ingestion/source/hex/constants.py +8 -0
- datahub/ingestion/source/hex/hex.py +311 -0
- datahub/ingestion/source/hex/mapper.py +412 -0
- datahub/ingestion/source/hex/model.py +78 -0
- datahub/ingestion/source/hex/query_fetcher.py +307 -0
- datahub/ingestion/source/iceberg/iceberg.py +385 -164
- datahub/ingestion/source/iceberg/iceberg_common.py +2 -2
- datahub/ingestion/source/iceberg/iceberg_profiler.py +25 -20
- datahub/ingestion/source/identity/azure_ad.py +1 -1
- datahub/ingestion/source/identity/okta.py +1 -14
- datahub/ingestion/source/kafka/kafka.py +28 -71
- datahub/ingestion/source/kafka/kafka_config.py +78 -0
- datahub/ingestion/source/kafka_connect/common.py +2 -2
- datahub/ingestion/source/kafka_connect/sink_connectors.py +157 -48
- datahub/ingestion/source/kafka_connect/source_connectors.py +63 -5
- datahub/ingestion/source/ldap.py +1 -1
- datahub/ingestion/source/looker/looker_common.py +216 -86
- datahub/ingestion/source/looker/looker_config.py +15 -4
- datahub/ingestion/source/looker/looker_constant.py +4 -0
- datahub/ingestion/source/looker/looker_lib_wrapper.py +37 -4
- datahub/ingestion/source/looker/looker_liquid_tag.py +56 -5
- datahub/ingestion/source/looker/looker_source.py +539 -555
- datahub/ingestion/source/looker/looker_view_id_cache.py +1 -1
- datahub/ingestion/source/looker/lookml_concept_context.py +1 -1
- datahub/ingestion/source/looker/lookml_config.py +31 -3
- datahub/ingestion/source/looker/lookml_refinement.py +1 -1
- datahub/ingestion/source/looker/lookml_source.py +103 -118
- datahub/ingestion/source/looker/view_upstream.py +494 -1
- datahub/ingestion/source/metabase.py +32 -6
- datahub/ingestion/source/metadata/business_glossary.py +7 -7
- datahub/ingestion/source/metadata/lineage.py +11 -10
- datahub/ingestion/source/mlflow.py +254 -23
- datahub/ingestion/source/mock_data/__init__.py +0 -0
- datahub/ingestion/source/mock_data/datahub_mock_data.py +533 -0
- datahub/ingestion/source/mock_data/datahub_mock_data_report.py +12 -0
- datahub/ingestion/source/mock_data/table_naming_helper.py +97 -0
- datahub/ingestion/source/mode.py +359 -181
- datahub/ingestion/source/mongodb.py +11 -1
- datahub/ingestion/source/neo4j/neo4j_source.py +122 -153
- datahub/ingestion/source/nifi.py +5 -5
- datahub/ingestion/source/openapi.py +85 -38
- datahub/ingestion/source/openapi_parser.py +59 -40
- datahub/ingestion/source/powerbi/config.py +92 -27
- datahub/ingestion/source/powerbi/m_query/data_classes.py +3 -0
- datahub/ingestion/source/powerbi/m_query/odbc.py +185 -0
- datahub/ingestion/source/powerbi/m_query/parser.py +2 -2
- datahub/ingestion/source/powerbi/m_query/pattern_handler.py +358 -14
- datahub/ingestion/source/powerbi/m_query/resolver.py +10 -0
- datahub/ingestion/source/powerbi/powerbi.py +66 -32
- datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +2 -2
- datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +11 -12
- datahub/ingestion/source/powerbi_report_server/report_server.py +0 -23
- datahub/ingestion/source/powerbi_report_server/report_server_domain.py +2 -4
- datahub/ingestion/source/preset.py +3 -3
- datahub/ingestion/source/qlik_sense/data_classes.py +28 -8
- datahub/ingestion/source/qlik_sense/qlik_sense.py +2 -1
- datahub/ingestion/source/redash.py +1 -1
- datahub/ingestion/source/redshift/config.py +15 -9
- datahub/ingestion/source/redshift/datashares.py +1 -1
- datahub/ingestion/source/redshift/lineage.py +386 -687
- datahub/ingestion/source/redshift/profile.py +2 -2
- datahub/ingestion/source/redshift/query.py +24 -20
- datahub/ingestion/source/redshift/redshift.py +52 -111
- datahub/ingestion/source/redshift/redshift_schema.py +17 -12
- datahub/ingestion/source/redshift/report.py +0 -2
- datahub/ingestion/source/redshift/usage.py +13 -11
- datahub/ingestion/source/s3/report.py +4 -2
- datahub/ingestion/source/s3/source.py +515 -244
- datahub/ingestion/source/sac/sac.py +3 -1
- datahub/ingestion/source/salesforce.py +28 -13
- datahub/ingestion/source/schema/json_schema.py +14 -14
- datahub/ingestion/source/schema_inference/object.py +22 -6
- datahub/ingestion/source/sigma/config.py +75 -8
- datahub/ingestion/source/sigma/data_classes.py +3 -0
- datahub/ingestion/source/sigma/sigma.py +36 -7
- datahub/ingestion/source/sigma/sigma_api.py +99 -58
- datahub/ingestion/source/slack/slack.py +403 -140
- datahub/ingestion/source/snaplogic/__init__.py +0 -0
- datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
- datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
- datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
- datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
- datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
- datahub/ingestion/source/snowflake/constants.py +4 -0
- datahub/ingestion/source/snowflake/snowflake_config.py +103 -34
- datahub/ingestion/source/snowflake/snowflake_connection.py +47 -25
- datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +25 -6
- datahub/ingestion/source/snowflake/snowflake_profiler.py +1 -6
- datahub/ingestion/source/snowflake/snowflake_queries.py +511 -107
- datahub/ingestion/source/snowflake/snowflake_query.py +100 -72
- datahub/ingestion/source/snowflake/snowflake_report.py +4 -2
- datahub/ingestion/source/snowflake/snowflake_schema.py +381 -16
- datahub/ingestion/source/snowflake/snowflake_schema_gen.py +163 -52
- datahub/ingestion/source/snowflake/snowflake_summary.py +7 -1
- datahub/ingestion/source/snowflake/snowflake_tag.py +4 -1
- datahub/ingestion/source/snowflake/snowflake_usage_v2.py +8 -2
- datahub/ingestion/source/snowflake/snowflake_utils.py +62 -17
- datahub/ingestion/source/snowflake/snowflake_v2.py +56 -10
- datahub/ingestion/source/snowflake/stored_proc_lineage.py +143 -0
- datahub/ingestion/source/sql/athena.py +219 -26
- datahub/ingestion/source/sql/athena_properties_extractor.py +795 -0
- datahub/ingestion/source/sql/clickhouse.py +29 -9
- datahub/ingestion/source/sql/cockroachdb.py +5 -4
- datahub/ingestion/source/sql/druid.py +9 -4
- datahub/ingestion/source/sql/hana.py +3 -1
- datahub/ingestion/source/sql/hive.py +28 -8
- datahub/ingestion/source/sql/hive_metastore.py +24 -25
- datahub/ingestion/source/sql/mariadb.py +0 -1
- datahub/ingestion/source/sql/mssql/job_models.py +18 -2
- datahub/ingestion/source/sql/mssql/source.py +376 -62
- datahub/ingestion/source/sql/mysql.py +154 -4
- datahub/ingestion/source/sql/oracle.py +62 -11
- datahub/ingestion/source/sql/postgres.py +142 -6
- datahub/ingestion/source/sql/presto.py +20 -2
- datahub/ingestion/source/sql/sql_common.py +281 -49
- datahub/ingestion/source/sql/sql_config.py +1 -34
- datahub/ingestion/source/sql/sql_generic_profiler.py +2 -1
- datahub/ingestion/source/sql/sql_types.py +27 -2
- datahub/ingestion/source/sql/sqlalchemy_uri.py +68 -0
- datahub/ingestion/source/sql/stored_procedures/__init__.py +0 -0
- datahub/ingestion/source/sql/stored_procedures/base.py +253 -0
- datahub/ingestion/source/sql/{mssql/stored_procedure_lineage.py → stored_procedures/lineage.py} +2 -29
- datahub/ingestion/source/sql/teradata.py +1028 -245
- datahub/ingestion/source/sql/trino.py +43 -10
- datahub/ingestion/source/sql/two_tier_sql_source.py +3 -4
- datahub/ingestion/source/sql/vertica.py +14 -7
- datahub/ingestion/source/sql_queries.py +219 -121
- datahub/ingestion/source/state/checkpoint.py +8 -29
- datahub/ingestion/source/state/entity_removal_state.py +5 -2
- datahub/ingestion/source/state/redundant_run_skip_handler.py +21 -0
- datahub/ingestion/source/state/stale_entity_removal_handler.py +0 -1
- datahub/ingestion/source/state/stateful_ingestion_base.py +36 -11
- datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider.py +2 -1
- datahub/ingestion/source/superset.py +810 -126
- datahub/ingestion/source/tableau/tableau.py +172 -69
- datahub/ingestion/source/tableau/tableau_common.py +11 -4
- datahub/ingestion/source/tableau/tableau_constant.py +1 -4
- datahub/ingestion/source/tableau/tableau_server_wrapper.py +3 -0
- datahub/ingestion/source/tableau/tableau_validation.py +1 -1
- datahub/ingestion/source/unity/config.py +161 -40
- datahub/ingestion/source/unity/connection.py +61 -0
- datahub/ingestion/source/unity/connection_test.py +1 -0
- datahub/ingestion/source/unity/platform_resource_repository.py +19 -0
- datahub/ingestion/source/unity/proxy.py +794 -51
- datahub/ingestion/source/unity/proxy_patch.py +321 -0
- datahub/ingestion/source/unity/proxy_types.py +36 -2
- datahub/ingestion/source/unity/report.py +15 -3
- datahub/ingestion/source/unity/source.py +465 -131
- datahub/ingestion/source/unity/tag_entities.py +197 -0
- datahub/ingestion/source/unity/usage.py +46 -4
- datahub/ingestion/source/usage/clickhouse_usage.py +11 -4
- datahub/ingestion/source/usage/starburst_trino_usage.py +10 -5
- datahub/ingestion/source/usage/usage_common.py +4 -68
- datahub/ingestion/source/vertexai/__init__.py +0 -0
- datahub/ingestion/source/vertexai/vertexai.py +1367 -0
- datahub/ingestion/source/vertexai/vertexai_config.py +29 -0
- datahub/ingestion/source/vertexai/vertexai_result_type_utils.py +89 -0
- datahub/ingestion/source_config/pulsar.py +3 -1
- datahub/ingestion/source_report/ingestion_stage.py +50 -11
- datahub/ingestion/transformer/add_dataset_dataproduct.py +1 -1
- datahub/ingestion/transformer/add_dataset_ownership.py +19 -3
- datahub/ingestion/transformer/base_transformer.py +8 -5
- datahub/ingestion/transformer/dataset_domain.py +1 -1
- datahub/ingestion/transformer/set_browse_path.py +112 -0
- datahub/integrations/assertion/common.py +3 -2
- datahub/integrations/assertion/snowflake/compiler.py +4 -3
- datahub/lite/lite_util.py +2 -2
- datahub/metadata/{_schema_classes.py → _internal_schema_classes.py} +3095 -631
- datahub/metadata/_urns/urn_defs.py +1866 -1582
- datahub/metadata/com/linkedin/pegasus2avro/application/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/dataplatform/slack/__init__.py +15 -0
- datahub/metadata/com/linkedin/pegasus2avro/event/__init__.py +11 -0
- datahub/metadata/com/linkedin/pegasus2avro/event/notification/__init__.py +15 -0
- datahub/metadata/com/linkedin/pegasus2avro/event/notification/settings/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/file/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/logical/__init__.py +15 -0
- datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +8 -0
- datahub/metadata/com/linkedin/pegasus2avro/module/__init__.py +31 -0
- datahub/metadata/com/linkedin/pegasus2avro/platform/event/v1/__init__.py +4 -0
- datahub/metadata/com/linkedin/pegasus2avro/role/__init__.py +2 -0
- datahub/metadata/com/linkedin/pegasus2avro/settings/asset/__init__.py +19 -0
- datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py +8 -0
- datahub/metadata/com/linkedin/pegasus2avro/template/__init__.py +31 -0
- datahub/metadata/schema.avsc +18404 -16617
- datahub/metadata/schema_classes.py +3 -3
- datahub/metadata/schemas/Actors.avsc +38 -1
- datahub/metadata/schemas/ApplicationKey.avsc +31 -0
- datahub/metadata/schemas/ApplicationProperties.avsc +72 -0
- datahub/metadata/schemas/Applications.avsc +38 -0
- datahub/metadata/schemas/AssetSettings.avsc +63 -0
- datahub/metadata/schemas/ChartInfo.avsc +2 -1
- datahub/metadata/schemas/ChartKey.avsc +1 -0
- datahub/metadata/schemas/ContainerKey.avsc +1 -0
- datahub/metadata/schemas/ContainerProperties.avsc +8 -0
- datahub/metadata/schemas/CorpUserEditableInfo.avsc +15 -1
- datahub/metadata/schemas/CorpUserKey.avsc +2 -1
- datahub/metadata/schemas/CorpUserSettings.avsc +145 -0
- datahub/metadata/schemas/DashboardKey.avsc +1 -0
- datahub/metadata/schemas/DataContractKey.avsc +2 -1
- datahub/metadata/schemas/DataFlowInfo.avsc +8 -0
- datahub/metadata/schemas/DataFlowKey.avsc +1 -0
- datahub/metadata/schemas/DataHubFileInfo.avsc +230 -0
- datahub/metadata/schemas/DataHubFileKey.avsc +21 -0
- datahub/metadata/schemas/DataHubIngestionSourceKey.avsc +2 -1
- datahub/metadata/schemas/DataHubOpenAPISchemaKey.avsc +22 -0
- datahub/metadata/schemas/DataHubPageModuleKey.avsc +21 -0
- datahub/metadata/schemas/DataHubPageModuleProperties.avsc +298 -0
- datahub/metadata/schemas/DataHubPageTemplateKey.avsc +21 -0
- datahub/metadata/schemas/DataHubPageTemplateProperties.avsc +251 -0
- datahub/metadata/schemas/DataHubPolicyInfo.avsc +12 -1
- datahub/metadata/schemas/DataJobInfo.avsc +8 -0
- datahub/metadata/schemas/DataJobInputOutput.avsc +8 -0
- datahub/metadata/schemas/DataJobKey.avsc +1 -0
- datahub/metadata/schemas/DataProcessInstanceInput.avsc +2 -1
- datahub/metadata/schemas/DataProcessInstanceOutput.avsc +2 -1
- datahub/metadata/schemas/DataProcessKey.avsc +8 -0
- datahub/metadata/schemas/DataProductKey.avsc +3 -1
- datahub/metadata/schemas/DataProductProperties.avsc +1 -1
- datahub/metadata/schemas/DataTransformLogic.avsc +4 -2
- datahub/metadata/schemas/DatasetKey.avsc +11 -1
- datahub/metadata/schemas/DatasetUsageStatistics.avsc +8 -0
- datahub/metadata/schemas/Deprecation.avsc +2 -0
- datahub/metadata/schemas/DomainKey.avsc +2 -1
- datahub/metadata/schemas/ExecutionRequestInput.avsc +5 -0
- datahub/metadata/schemas/FormInfo.avsc +5 -0
- datahub/metadata/schemas/GlobalSettingsInfo.avsc +134 -0
- datahub/metadata/schemas/GlossaryNodeKey.avsc +2 -1
- datahub/metadata/schemas/GlossaryTermKey.avsc +3 -1
- datahub/metadata/schemas/IcebergWarehouseInfo.avsc +8 -0
- datahub/metadata/schemas/IncidentInfo.avsc +3 -3
- datahub/metadata/schemas/InstitutionalMemory.avsc +31 -0
- datahub/metadata/schemas/LogicalParent.avsc +145 -0
- datahub/metadata/schemas/MLFeatureKey.avsc +1 -0
- datahub/metadata/schemas/MLFeatureTableKey.avsc +1 -0
- datahub/metadata/schemas/MLModelDeploymentKey.avsc +8 -0
- datahub/metadata/schemas/MLModelDeploymentProperties.avsc +3 -0
- datahub/metadata/schemas/MLModelGroupKey.avsc +11 -1
- datahub/metadata/schemas/MLModelGroupProperties.avsc +16 -0
- datahub/metadata/schemas/MLModelKey.avsc +9 -0
- datahub/metadata/schemas/MLPrimaryKeyKey.avsc +1 -0
- datahub/metadata/schemas/MetadataChangeEvent.avsc +189 -47
- datahub/metadata/schemas/MetadataChangeLog.avsc +65 -44
- datahub/metadata/schemas/MetadataChangeProposal.avsc +64 -0
- datahub/metadata/schemas/NotebookKey.avsc +1 -0
- datahub/metadata/schemas/Operation.avsc +21 -2
- datahub/metadata/schemas/Ownership.avsc +69 -0
- datahub/metadata/schemas/QueryProperties.avsc +24 -2
- datahub/metadata/schemas/QuerySubjects.avsc +1 -12
- datahub/metadata/schemas/RelationshipChangeEvent.avsc +215 -0
- datahub/metadata/schemas/SchemaFieldKey.avsc +4 -1
- datahub/metadata/schemas/Siblings.avsc +2 -0
- datahub/metadata/schemas/SlackUserInfo.avsc +160 -0
- datahub/metadata/schemas/StructuredProperties.avsc +69 -0
- datahub/metadata/schemas/StructuredPropertySettings.avsc +9 -0
- datahub/metadata/schemas/SystemMetadata.avsc +147 -0
- datahub/metadata/schemas/UpstreamLineage.avsc +9 -0
- datahub/metadata/schemas/__init__.py +3 -3
- datahub/sdk/__init__.py +7 -0
- datahub/sdk/_all_entities.py +15 -0
- datahub/sdk/_shared.py +393 -10
- datahub/sdk/_utils.py +4 -0
- datahub/sdk/chart.py +386 -0
- datahub/sdk/container.py +7 -0
- datahub/sdk/dashboard.py +453 -0
- datahub/sdk/dataflow.py +309 -0
- datahub/sdk/datajob.py +367 -0
- datahub/sdk/dataset.py +180 -4
- datahub/sdk/entity.py +99 -3
- datahub/sdk/entity_client.py +154 -12
- datahub/sdk/lineage_client.py +943 -0
- datahub/sdk/main_client.py +83 -8
- datahub/sdk/mlmodel.py +383 -0
- datahub/sdk/mlmodelgroup.py +240 -0
- datahub/sdk/search_client.py +85 -8
- datahub/sdk/search_filters.py +393 -68
- datahub/secret/datahub_secret_store.py +5 -1
- datahub/secret/environment_secret_store.py +29 -0
- datahub/secret/file_secret_store.py +49 -0
- datahub/specific/aspect_helpers/fine_grained_lineage.py +76 -0
- datahub/specific/aspect_helpers/siblings.py +73 -0
- datahub/specific/aspect_helpers/structured_properties.py +27 -0
- datahub/specific/chart.py +1 -1
- datahub/specific/datajob.py +15 -1
- datahub/specific/dataproduct.py +4 -0
- datahub/specific/dataset.py +51 -59
- datahub/sql_parsing/_sqlglot_patch.py +1 -2
- datahub/sql_parsing/fingerprint_utils.py +6 -0
- datahub/sql_parsing/split_statements.py +30 -3
- datahub/sql_parsing/sql_parsing_aggregator.py +144 -63
- datahub/sql_parsing/sqlglot_lineage.py +517 -44
- datahub/sql_parsing/sqlglot_utils.py +30 -18
- datahub/sql_parsing/tool_meta_extractor.py +25 -2
- datahub/telemetry/telemetry.py +30 -16
- datahub/testing/check_imports.py +1 -1
- datahub/testing/docker_utils.py +8 -2
- datahub/testing/mce_helpers.py +421 -0
- datahub/testing/mcp_diff.py +17 -21
- datahub/testing/sdk_v2_helpers.py +18 -0
- datahub/upgrade/upgrade.py +86 -30
- datahub/utilities/file_backed_collections.py +14 -15
- datahub/utilities/hive_schema_to_avro.py +2 -2
- datahub/utilities/ingest_utils.py +2 -2
- datahub/utilities/is_pytest.py +3 -2
- datahub/utilities/logging_manager.py +30 -7
- datahub/utilities/mapping.py +29 -2
- datahub/utilities/sample_data.py +5 -4
- datahub/utilities/server_config_util.py +298 -10
- datahub/utilities/sqlalchemy_query_combiner.py +6 -4
- datahub/utilities/stats_collections.py +4 -0
- datahub/utilities/threaded_iterator_executor.py +16 -3
- datahub/utilities/urn_encoder.py +1 -1
- datahub/utilities/urns/urn.py +41 -2
- datahub/emitter/sql_parsing_builder.py +0 -306
- datahub/ingestion/source/redshift/lineage_v2.py +0 -458
- datahub/ingestion/source/vertexai.py +0 -697
- datahub/ingestion/transformer/system_metadata_transformer.py +0 -45
- {acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info/licenses}/LICENSE +0 -0
- {acryl_datahub-1.0.0rc18.dist-info → acryl_datahub-1.3.0.1rc9.dist-info}/top_level.txt +0 -0
datahub/sdk/dashboard.py
ADDED
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Dict, List, Optional, Sequence, Type, Union
|
|
5
|
+
|
|
6
|
+
from deprecated.sphinx import deprecated
|
|
7
|
+
from typing_extensions import Self
|
|
8
|
+
|
|
9
|
+
import datahub.metadata.schema_classes as models
|
|
10
|
+
from datahub.metadata.urns import ChartUrn, DashboardUrn, DatasetUrn, Urn
|
|
11
|
+
from datahub.sdk._shared import (
|
|
12
|
+
ActorUrnOrStr,
|
|
13
|
+
ChangeAuditStampsMixin,
|
|
14
|
+
ChartUrnOrStr,
|
|
15
|
+
DashboardUrnOrStr,
|
|
16
|
+
DataPlatformInstanceUrnOrStr,
|
|
17
|
+
DataPlatformUrnOrStr,
|
|
18
|
+
DatasetUrnOrStr,
|
|
19
|
+
DomainInputType,
|
|
20
|
+
HasContainer,
|
|
21
|
+
HasDomain,
|
|
22
|
+
HasInstitutionalMemory,
|
|
23
|
+
HasOwnership,
|
|
24
|
+
HasPlatformInstance,
|
|
25
|
+
HasSubtype,
|
|
26
|
+
HasTags,
|
|
27
|
+
HasTerms,
|
|
28
|
+
LinksInputType,
|
|
29
|
+
OwnersInputType,
|
|
30
|
+
ParentContainerInputType,
|
|
31
|
+
TagsInputType,
|
|
32
|
+
TermsInputType,
|
|
33
|
+
)
|
|
34
|
+
from datahub.sdk.chart import Chart
|
|
35
|
+
from datahub.sdk.dataset import Dataset
|
|
36
|
+
from datahub.sdk.entity import Entity, ExtraAspectsType
|
|
37
|
+
from datahub.utilities.sentinels import Unset, unset
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Dashboard(
|
|
41
|
+
ChangeAuditStampsMixin,
|
|
42
|
+
HasPlatformInstance,
|
|
43
|
+
HasSubtype,
|
|
44
|
+
HasOwnership,
|
|
45
|
+
HasContainer,
|
|
46
|
+
HasInstitutionalMemory,
|
|
47
|
+
HasTags,
|
|
48
|
+
HasTerms,
|
|
49
|
+
HasDomain,
|
|
50
|
+
Entity,
|
|
51
|
+
):
|
|
52
|
+
"""Represents a dashboard in DataHub."""
|
|
53
|
+
|
|
54
|
+
__slots__ = ()
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def get_urn_type(cls) -> Type[DashboardUrn]:
|
|
58
|
+
"""Get the URN type for dashboards.
|
|
59
|
+
Returns:
|
|
60
|
+
The DashboardUrn class.
|
|
61
|
+
"""
|
|
62
|
+
return DashboardUrn
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
*,
|
|
67
|
+
# Identity.
|
|
68
|
+
name: str,
|
|
69
|
+
platform: DataPlatformUrnOrStr,
|
|
70
|
+
display_name: Optional[str] = None,
|
|
71
|
+
platform_instance: Optional[DataPlatformInstanceUrnOrStr] = None,
|
|
72
|
+
# Dashboard properties.
|
|
73
|
+
description: Optional[str] = None,
|
|
74
|
+
external_url: Optional[str] = None,
|
|
75
|
+
dashboard_url: Optional[str] = None,
|
|
76
|
+
custom_properties: Optional[Dict[str, str]] = None,
|
|
77
|
+
last_modified: Optional[datetime] = None,
|
|
78
|
+
last_modified_by: Optional[ActorUrnOrStr] = None,
|
|
79
|
+
created_at: Optional[datetime] = None,
|
|
80
|
+
created_by: Optional[ActorUrnOrStr] = None,
|
|
81
|
+
deleted_on: Optional[datetime] = None,
|
|
82
|
+
deleted_by: Optional[ActorUrnOrStr] = None,
|
|
83
|
+
last_refreshed: Optional[datetime] = None,
|
|
84
|
+
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]] = None,
|
|
85
|
+
charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]] = None,
|
|
86
|
+
dashboards: Optional[Sequence[Union[DashboardUrnOrStr, Dashboard]]] = None,
|
|
87
|
+
# Standard aspects.
|
|
88
|
+
parent_container: ParentContainerInputType | Unset = unset,
|
|
89
|
+
subtype: Optional[str] = None,
|
|
90
|
+
owners: Optional[OwnersInputType] = None,
|
|
91
|
+
links: Optional[LinksInputType] = None,
|
|
92
|
+
tags: Optional[TagsInputType] = None,
|
|
93
|
+
terms: Optional[TermsInputType] = None,
|
|
94
|
+
domain: Optional[DomainInputType] = None,
|
|
95
|
+
extra_aspects: ExtraAspectsType = None,
|
|
96
|
+
):
|
|
97
|
+
"""Initialize a new Dashboard instance."""
|
|
98
|
+
urn = DashboardUrn.create_from_ids(
|
|
99
|
+
platform=str(platform),
|
|
100
|
+
name=name,
|
|
101
|
+
platform_instance=str(platform_instance) if platform_instance else None,
|
|
102
|
+
)
|
|
103
|
+
super().__init__(urn)
|
|
104
|
+
self._set_extra_aspects(extra_aspects)
|
|
105
|
+
|
|
106
|
+
self._set_platform_instance(platform, platform_instance)
|
|
107
|
+
self._ensure_dashboard_props(display_name=display_name)
|
|
108
|
+
|
|
109
|
+
self._init_dashboard_properties(
|
|
110
|
+
description,
|
|
111
|
+
display_name,
|
|
112
|
+
external_url,
|
|
113
|
+
dashboard_url,
|
|
114
|
+
custom_properties,
|
|
115
|
+
last_modified,
|
|
116
|
+
last_modified_by,
|
|
117
|
+
created_at,
|
|
118
|
+
created_by,
|
|
119
|
+
last_refreshed,
|
|
120
|
+
deleted_on,
|
|
121
|
+
deleted_by,
|
|
122
|
+
input_datasets,
|
|
123
|
+
charts,
|
|
124
|
+
dashboards,
|
|
125
|
+
)
|
|
126
|
+
self._init_standard_aspects(
|
|
127
|
+
parent_container, subtype, owners, links, tags, terms, domain
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def _init_dashboard_properties(
|
|
131
|
+
self,
|
|
132
|
+
description: Optional[str],
|
|
133
|
+
display_name: Optional[str],
|
|
134
|
+
external_url: Optional[str],
|
|
135
|
+
dashboard_url: Optional[str],
|
|
136
|
+
custom_properties: Optional[Dict[str, str]],
|
|
137
|
+
last_modified: Optional[datetime],
|
|
138
|
+
last_modified_by: Optional[ActorUrnOrStr],
|
|
139
|
+
created_at: Optional[datetime],
|
|
140
|
+
created_by: Optional[ActorUrnOrStr],
|
|
141
|
+
last_refreshed: Optional[datetime],
|
|
142
|
+
deleted_on: Optional[datetime],
|
|
143
|
+
deleted_by: Optional[ActorUrnOrStr],
|
|
144
|
+
input_datasets: Optional[Sequence[Union[DatasetUrnOrStr, Dataset]]],
|
|
145
|
+
charts: Optional[Sequence[Union[ChartUrnOrStr, Chart]]],
|
|
146
|
+
dashboards: Optional[Sequence[Union[DashboardUrnOrStr, Dashboard]]],
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Initialize dashboard-specific properties."""
|
|
149
|
+
if description is not None:
|
|
150
|
+
self.set_description(description)
|
|
151
|
+
if display_name is not None:
|
|
152
|
+
self.set_display_name(display_name)
|
|
153
|
+
if external_url is not None:
|
|
154
|
+
self.set_external_url(external_url)
|
|
155
|
+
if dashboard_url is not None:
|
|
156
|
+
self.set_dashboard_url(dashboard_url)
|
|
157
|
+
if custom_properties is not None:
|
|
158
|
+
self.set_custom_properties(custom_properties)
|
|
159
|
+
if last_modified is not None:
|
|
160
|
+
self.set_last_modified(last_modified)
|
|
161
|
+
if last_modified_by is not None:
|
|
162
|
+
self.set_last_modified_by(last_modified_by)
|
|
163
|
+
if created_at is not None:
|
|
164
|
+
self.set_created_at(created_at)
|
|
165
|
+
if created_by is not None:
|
|
166
|
+
self.set_created_by(created_by)
|
|
167
|
+
if deleted_on is not None:
|
|
168
|
+
self.set_deleted_on(deleted_on)
|
|
169
|
+
if deleted_by is not None:
|
|
170
|
+
self.set_deleted_by(deleted_by)
|
|
171
|
+
if last_refreshed is not None:
|
|
172
|
+
self.set_last_refreshed(last_refreshed)
|
|
173
|
+
if input_datasets is not None:
|
|
174
|
+
self.set_input_datasets(input_datasets)
|
|
175
|
+
if charts is not None:
|
|
176
|
+
self.set_charts(charts)
|
|
177
|
+
if dashboards is not None:
|
|
178
|
+
self.set_dashboards(dashboards)
|
|
179
|
+
|
|
180
|
+
def _init_standard_aspects(
|
|
181
|
+
self,
|
|
182
|
+
parent_container: ParentContainerInputType | Unset,
|
|
183
|
+
subtype: Optional[str],
|
|
184
|
+
owners: Optional[OwnersInputType],
|
|
185
|
+
links: Optional[LinksInputType],
|
|
186
|
+
tags: Optional[TagsInputType],
|
|
187
|
+
terms: Optional[TermsInputType],
|
|
188
|
+
domain: Optional[DomainInputType],
|
|
189
|
+
) -> None:
|
|
190
|
+
"""Initialize standard aspects."""
|
|
191
|
+
if parent_container is not unset:
|
|
192
|
+
self._set_container(parent_container)
|
|
193
|
+
if subtype is not None:
|
|
194
|
+
self.set_subtype(subtype)
|
|
195
|
+
if owners is not None:
|
|
196
|
+
self.set_owners(owners)
|
|
197
|
+
if links is not None:
|
|
198
|
+
self.set_links(links)
|
|
199
|
+
if tags is not None:
|
|
200
|
+
self.set_tags(tags)
|
|
201
|
+
if terms is not None:
|
|
202
|
+
self.set_terms(terms)
|
|
203
|
+
if domain is not None:
|
|
204
|
+
self.set_domain(domain)
|
|
205
|
+
|
|
206
|
+
@classmethod
|
|
207
|
+
def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
|
|
208
|
+
assert isinstance(urn, DashboardUrn)
|
|
209
|
+
entity = cls(
|
|
210
|
+
platform=urn.dashboard_tool,
|
|
211
|
+
name=urn.dashboard_id,
|
|
212
|
+
)
|
|
213
|
+
return entity._init_from_graph(current_aspects)
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def urn(self) -> DashboardUrn:
|
|
217
|
+
assert isinstance(self._urn, DashboardUrn)
|
|
218
|
+
return self._urn
|
|
219
|
+
|
|
220
|
+
def _ensure_dashboard_props(
|
|
221
|
+
self, display_name: Optional[str] = None
|
|
222
|
+
) -> models.DashboardInfoClass:
|
|
223
|
+
"""Get the dashboard properties safely."""
|
|
224
|
+
return self._setdefault_aspect(
|
|
225
|
+
models.DashboardInfoClass(
|
|
226
|
+
title=display_name or self.urn.dashboard_id,
|
|
227
|
+
description="",
|
|
228
|
+
lastModified=models.ChangeAuditStampsClass(),
|
|
229
|
+
customProperties={},
|
|
230
|
+
dashboards=[],
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
def _get_audit_stamps(self) -> models.ChangeAuditStampsClass:
|
|
235
|
+
"""Get the audit stamps from the dashboard properties."""
|
|
236
|
+
return self._ensure_dashboard_props().lastModified
|
|
237
|
+
|
|
238
|
+
def _set_audit_stamps(self, audit_stamps: models.ChangeAuditStampsClass) -> None:
|
|
239
|
+
"""Set the audit stamps on the dashboard properties."""
|
|
240
|
+
self._ensure_dashboard_props().lastModified = audit_stamps
|
|
241
|
+
|
|
242
|
+
@property
|
|
243
|
+
def name(self) -> str:
|
|
244
|
+
"""Get the name of the dashboard."""
|
|
245
|
+
return self.urn.dashboard_id
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
@deprecated("Use display_name instead", version="1.2.0.7")
|
|
249
|
+
def title(self) -> str:
|
|
250
|
+
"""Get the display name of the dashboard."""
|
|
251
|
+
return self.display_name
|
|
252
|
+
|
|
253
|
+
@deprecated("Use set_display_name instead", version="1.2.0.7")
|
|
254
|
+
def set_title(self, title: str) -> None:
|
|
255
|
+
"""Set the display name of the dashboard."""
|
|
256
|
+
self.set_display_name(title)
|
|
257
|
+
|
|
258
|
+
@property
|
|
259
|
+
def description(self) -> Optional[str]:
|
|
260
|
+
"""Get the description of the dashboard."""
|
|
261
|
+
# Because description is a required field, we treat "" as None.
|
|
262
|
+
return self._ensure_dashboard_props().description or None
|
|
263
|
+
|
|
264
|
+
def set_description(self, description: str) -> None:
|
|
265
|
+
"""Set the description of the dashboard."""
|
|
266
|
+
self._ensure_dashboard_props().description = description
|
|
267
|
+
|
|
268
|
+
@property
|
|
269
|
+
def display_name(self) -> str:
|
|
270
|
+
"""Get the display name of the dashboard."""
|
|
271
|
+
return self._ensure_dashboard_props().title
|
|
272
|
+
|
|
273
|
+
def set_display_name(self, display_name: str) -> None:
|
|
274
|
+
"""Set the display name of the dashboard."""
|
|
275
|
+
self._ensure_dashboard_props().title = display_name
|
|
276
|
+
|
|
277
|
+
@property
|
|
278
|
+
def external_url(self) -> Optional[str]:
|
|
279
|
+
"""Get the external URL of the dashboard."""
|
|
280
|
+
return self._ensure_dashboard_props().externalUrl
|
|
281
|
+
|
|
282
|
+
def set_external_url(self, external_url: str) -> None:
|
|
283
|
+
"""Set the external URL of the dashboard."""
|
|
284
|
+
self._ensure_dashboard_props().externalUrl = external_url
|
|
285
|
+
|
|
286
|
+
@property
|
|
287
|
+
def dashboard_url(self) -> Optional[str]:
|
|
288
|
+
"""Get the dashboard URL."""
|
|
289
|
+
return self._ensure_dashboard_props().dashboardUrl
|
|
290
|
+
|
|
291
|
+
def set_dashboard_url(self, dashboard_url: str) -> None:
|
|
292
|
+
"""Set the dashboard URL."""
|
|
293
|
+
self._ensure_dashboard_props().dashboardUrl = dashboard_url
|
|
294
|
+
|
|
295
|
+
@property
|
|
296
|
+
def custom_properties(self) -> Dict[str, str]:
|
|
297
|
+
"""Get the custom properties of the dashboard."""
|
|
298
|
+
props = self._ensure_dashboard_props()
|
|
299
|
+
return props.customProperties or {}
|
|
300
|
+
|
|
301
|
+
def set_custom_properties(self, custom_properties: Dict[str, str]) -> None:
|
|
302
|
+
"""Set the custom properties of the dashboard."""
|
|
303
|
+
self._ensure_dashboard_props().customProperties = custom_properties
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def last_refreshed(self) -> Optional[datetime]:
|
|
307
|
+
"""Get the last refresh timestamp of the dashboard."""
|
|
308
|
+
props = self._ensure_dashboard_props()
|
|
309
|
+
return (
|
|
310
|
+
datetime.fromtimestamp(props.lastRefreshed)
|
|
311
|
+
if props.lastRefreshed is not None
|
|
312
|
+
else None
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
def set_last_refreshed(self, last_refreshed: datetime) -> None:
|
|
316
|
+
"""Set the last refresh timestamp of the dashboard."""
|
|
317
|
+
self._ensure_dashboard_props().lastRefreshed = int(last_refreshed.timestamp())
|
|
318
|
+
|
|
319
|
+
@property
|
|
320
|
+
def input_datasets(self) -> List[DatasetUrn]:
|
|
321
|
+
"""Get the input datasets of the dashboard."""
|
|
322
|
+
props = self._ensure_dashboard_props()
|
|
323
|
+
return [
|
|
324
|
+
DatasetUrn.from_string(edge.destinationUrn)
|
|
325
|
+
for edge in (props.datasetEdges or [])
|
|
326
|
+
]
|
|
327
|
+
|
|
328
|
+
def set_input_datasets(
|
|
329
|
+
self, input_datasets: Sequence[Union[DatasetUrnOrStr, Dataset]]
|
|
330
|
+
) -> None:
|
|
331
|
+
"""Set the input datasets of the dashboard."""
|
|
332
|
+
props = self._ensure_dashboard_props()
|
|
333
|
+
dataset_edges = props.datasetEdges or []
|
|
334
|
+
for dataset in input_datasets:
|
|
335
|
+
if isinstance(dataset, Dataset):
|
|
336
|
+
dataset_urn = dataset.urn
|
|
337
|
+
else:
|
|
338
|
+
dataset_urn = DatasetUrn.from_string(dataset)
|
|
339
|
+
dataset_edges.append(models.EdgeClass(destinationUrn=str(dataset_urn)))
|
|
340
|
+
props.datasetEdges = dataset_edges
|
|
341
|
+
|
|
342
|
+
def add_input_dataset(self, input_dataset: Union[DatasetUrnOrStr, Dataset]) -> None:
|
|
343
|
+
"""Add an input dataset to the dashboard."""
|
|
344
|
+
if isinstance(input_dataset, Dataset):
|
|
345
|
+
input_dataset_urn = input_dataset.urn
|
|
346
|
+
else:
|
|
347
|
+
input_dataset_urn = DatasetUrn.from_string(input_dataset)
|
|
348
|
+
props = self._ensure_dashboard_props()
|
|
349
|
+
dataset_edges = props.datasetEdges or []
|
|
350
|
+
existing_urns = [edge.destinationUrn for edge in dataset_edges]
|
|
351
|
+
if str(input_dataset_urn) not in existing_urns:
|
|
352
|
+
dataset_edges.append(
|
|
353
|
+
models.EdgeClass(destinationUrn=str(input_dataset_urn))
|
|
354
|
+
)
|
|
355
|
+
props.datasetEdges = dataset_edges
|
|
356
|
+
|
|
357
|
+
def remove_input_dataset(
|
|
358
|
+
self, input_dataset: Union[DatasetUrnOrStr, Dataset]
|
|
359
|
+
) -> None:
|
|
360
|
+
"""Remove an input dataset from the dashboard."""
|
|
361
|
+
if isinstance(input_dataset, Dataset):
|
|
362
|
+
input_dataset_urn = input_dataset.urn
|
|
363
|
+
else:
|
|
364
|
+
input_dataset_urn = DatasetUrn.from_string(input_dataset)
|
|
365
|
+
props = self._ensure_dashboard_props()
|
|
366
|
+
props.datasetEdges = [
|
|
367
|
+
edge
|
|
368
|
+
for edge in (props.datasetEdges or [])
|
|
369
|
+
if edge.destinationUrn != str(input_dataset_urn)
|
|
370
|
+
]
|
|
371
|
+
|
|
372
|
+
@property
|
|
373
|
+
def charts(self) -> List[ChartUrn]:
|
|
374
|
+
"""Get the charts of the dashboard."""
|
|
375
|
+
chart_edges = self._ensure_dashboard_props().chartEdges
|
|
376
|
+
if chart_edges is None:
|
|
377
|
+
return []
|
|
378
|
+
return [ChartUrn.from_string(edge.destinationUrn) for edge in chart_edges]
|
|
379
|
+
|
|
380
|
+
def set_charts(self, charts: Sequence[Union[ChartUrnOrStr, Chart]]) -> None:
|
|
381
|
+
"""Set the charts of the dashboard."""
|
|
382
|
+
props = self._ensure_dashboard_props()
|
|
383
|
+
chart_edges = props.chartEdges or []
|
|
384
|
+
for chart in charts:
|
|
385
|
+
if isinstance(chart, Chart):
|
|
386
|
+
chart_urn = chart.urn
|
|
387
|
+
else:
|
|
388
|
+
chart_urn = ChartUrn.from_string(chart)
|
|
389
|
+
chart_edges.append(models.EdgeClass(destinationUrn=str(chart_urn)))
|
|
390
|
+
props.chartEdges = chart_edges
|
|
391
|
+
|
|
392
|
+
def add_chart(self, chart: Union[ChartUrnOrStr, Chart]) -> None:
|
|
393
|
+
"""Add a chart to the dashboard."""
|
|
394
|
+
if isinstance(chart, Chart):
|
|
395
|
+
chart_urn = chart.urn
|
|
396
|
+
else:
|
|
397
|
+
chart_urn = ChartUrn.from_string(chart)
|
|
398
|
+
props = self._ensure_dashboard_props()
|
|
399
|
+
chart_edges = props.chartEdges or []
|
|
400
|
+
existing_urns = [
|
|
401
|
+
edge.destinationUrn
|
|
402
|
+
for edge in chart_edges
|
|
403
|
+
if edge.destinationUrn is not None
|
|
404
|
+
]
|
|
405
|
+
if str(chart_urn) not in existing_urns:
|
|
406
|
+
chart_edges.append(models.EdgeClass(destinationUrn=str(chart_urn)))
|
|
407
|
+
props.chartEdges = chart_edges
|
|
408
|
+
|
|
409
|
+
def remove_chart(self, chart: Union[ChartUrnOrStr, Chart]) -> None:
|
|
410
|
+
"""Remove a chart from the dashboard."""
|
|
411
|
+
if isinstance(chart, Chart):
|
|
412
|
+
chart_urn = chart.urn
|
|
413
|
+
else:
|
|
414
|
+
chart_urn = ChartUrn.from_string(chart)
|
|
415
|
+
props = self._ensure_dashboard_props()
|
|
416
|
+
props.chartEdges = [
|
|
417
|
+
edge
|
|
418
|
+
for edge in (props.chartEdges or [])
|
|
419
|
+
if edge.destinationUrn != str(chart_urn)
|
|
420
|
+
]
|
|
421
|
+
|
|
422
|
+
@property
|
|
423
|
+
def dashboards(self) -> List[DashboardUrn]:
|
|
424
|
+
"""Get the dashboards of the dashboard."""
|
|
425
|
+
props = self._ensure_dashboard_props()
|
|
426
|
+
return [
|
|
427
|
+
DashboardUrn.from_string(dashboard.destinationUrn)
|
|
428
|
+
for dashboard in (props.dashboards or [])
|
|
429
|
+
]
|
|
430
|
+
|
|
431
|
+
def set_dashboards(
|
|
432
|
+
self, dashboards: Sequence[Union[DashboardUrnOrStr, Dashboard]]
|
|
433
|
+
) -> None:
|
|
434
|
+
"""Set the dashboards of the dashboard."""
|
|
435
|
+
props = self._ensure_dashboard_props()
|
|
436
|
+
for dashboard in dashboards:
|
|
437
|
+
if isinstance(dashboard, Dashboard):
|
|
438
|
+
dashboard_urn = dashboard.urn
|
|
439
|
+
else:
|
|
440
|
+
dashboard_urn = DashboardUrn.from_string(dashboard)
|
|
441
|
+
props.dashboards.append(models.EdgeClass(destinationUrn=str(dashboard_urn)))
|
|
442
|
+
|
|
443
|
+
def add_dashboard(self, dashboard: Union[DashboardUrnOrStr, Dashboard]) -> None:
|
|
444
|
+
"""Add a dashboard to the dashboard."""
|
|
445
|
+
if isinstance(dashboard, Dashboard):
|
|
446
|
+
dashboard_urn = dashboard.urn
|
|
447
|
+
else:
|
|
448
|
+
dashboard_urn = DashboardUrn.from_string(dashboard)
|
|
449
|
+
props = self._ensure_dashboard_props()
|
|
450
|
+
dashboards = props.dashboards or []
|
|
451
|
+
existing_urns = [dashboard.destinationUrn for dashboard in dashboards]
|
|
452
|
+
if str(dashboard_urn) not in existing_urns:
|
|
453
|
+
dashboards.append(models.EdgeClass(destinationUrn=str(dashboard_urn)))
|