acceldata-sdk-python 26.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acceldata_sdk_python-26.7.0/.pytest_cache/README.md +8 -0
- acceldata_sdk_python-26.7.0/LICENSE +22 -0
- acceldata_sdk_python-26.7.0/MANIFEST.in +2 -0
- acceldata_sdk_python-26.7.0/PKG-INFO +26 -0
- acceldata_sdk_python-26.7.0/README-catalog.md +37 -0
- acceldata_sdk_python-26.7.0/README-datasources-assets.md +347 -0
- acceldata_sdk_python-26.7.0/README-historical-pipeline.md +192 -0
- acceldata_sdk_python-26.7.0/README-pipeline-continuation-id.md +108 -0
- acceldata_sdk_python-26.7.0/README-pipeline-end-to-end.md +161 -0
- acceldata_sdk_python-26.7.0/README-pipeline.md +433 -0
- acceldata_sdk_python-26.7.0/README-policies.md +552 -0
- acceldata_sdk_python-26.7.0/README-release.md +31 -0
- acceldata_sdk_python-26.7.0/README-tags.md +228 -0
- acceldata_sdk_python-26.7.0/README-transient-retry.md +171 -0
- acceldata_sdk_python-26.7.0/README.md +3 -0
- acceldata_sdk_python-26.7.0/VERSION +1 -0
- acceldata_sdk_python-26.7.0/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/acceldata/__init__.py +7 -0
- acceldata_sdk_python-26.7.0/acceldata/client/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/acceldata/client/adoc_client.py +666 -0
- acceldata_sdk_python-26.7.0/acceldata/client/http_client.py +346 -0
- acceldata_sdk_python-26.7.0/acceldata/client/transient_retry.py +142 -0
- acceldata_sdk_python-26.7.0/acceldata/constants/__init__.py +1 -0
- acceldata_sdk_python-26.7.0/acceldata/constants/constants.py +13 -0
- acceldata_sdk_python-26.7.0/acceldata/exceptions/__init__.py +52 -0
- acceldata_sdk_python-26.7.0/acceldata/exceptions/api_errors.py +85 -0
- acceldata_sdk_python-26.7.0/acceldata/exceptions/exceptions.py +174 -0
- acceldata_sdk_python-26.7.0/acceldata/exceptions/sdk_exceptions.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/mappers/__init__.py +21 -0
- acceldata_sdk_python-26.7.0/acceldata/mappers/catalog_executions.py +107 -0
- acceldata_sdk_python-26.7.0/acceldata/models/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/above_configs.py +99 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/above_configs_success.py +138 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/absolute_configs.py +197 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/absolute_configs_fallback.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/absolute_threshold_config.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/analysis_job_context.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/analytics_pipeline.py +385 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/analytics_pipeline_endpoint.py +136 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/anomaly_config.py +95 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/anomaly_details.py +119 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/anomaly_threshold_config.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/artifact.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/assembly.py +171 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset.py +162 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_catalog_listing.py +159 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_catalog_listing_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_label_request.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_progress.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_progress_details.py +117 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_tag.py +124 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_tag_info.py +140 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_tag_request.py +95 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_tag_response.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_tags_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/asset_type.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/auto_anomaly_feature_analysis.py +160 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/auto_anomaly_item_execution.py +133 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/auto_anomaly_meta_data.py +162 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/auto_anomaly_rule_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/auto_anomaly_summary.py +124 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/backing_asset.py +126 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/below_configs.py +99 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/big_decimal.py +100 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/big_integer.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/bounds_date_time_marker_config.py +110 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/bounds_file_event_marker_config.py +108 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/bounds_id_marker_config.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/catalog_asset_meta_data.py +128 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/changes_by_config.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/column_mapping.py +139 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/column_metadata.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/column_metric_with_anomaly_details.py +119 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/column_variable.py +111 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/composite_policy_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/composite_policy_item_execution.py +104 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/connection.py +118 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/connection_test_result.py +106 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/connection_type.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/connection_types_response.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/crawler.py +88 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/crawler_action_response.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/crawler_execution.py +147 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/crawler_execution_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/crawler_status_response.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/create_profile.py +194 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/custom_sql_config.py +121 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_bricks_spark_resource_config.py +104 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_cadence_rule.py +236 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_cadence_rule_details.py +125 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_cadence_rule_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_cadence_rule_item.py +104 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_cadence_rule_item_execution.py +158 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_cadence_rule_response.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_drift_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_drift_item_execution.py +121 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_quality_engine.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_quality_execution.py +106 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_quality_rule.py +323 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_quality_rule_details.py +186 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_quality_rule_response.py +120 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/data_quality_rules_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/databricks_compute_config.py +113 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataplane_engine.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataplane_health_metrics.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataplane_job.py +216 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataplane_job_status.py +113 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataplane_request.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataproc_compute_config.py +106 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dataproc_spark_resource_config.py +104 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/date_based_partition_marker_config.py +116 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/date_partitioning_marker_config.py +113 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/date_time_marker_config.py +109 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/day_month_year_based_partition_marker_config.py +124 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/decreased_by_config.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/delete_artifact_response.py +88 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/dq_rule_execution.py +248 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/drift.py +121 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/enable_rule_request.py +88 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/engine.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/execution_cancellation_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/execution_response.py +144 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/execution_result.py +126 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/executions_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/failed_asset_detail.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/failed_summary.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/file.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/file_marker_config.py +99 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/generic_config_fallback.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/generic_execution.py +221 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/generic_rule_execution.py +137 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/global_storage_config_dto.py +172 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/hosted_dp_deployment_response.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/id_marker_config.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/increased_by_config.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/item.py +160 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/item_execution.py +192 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/item_segment.py +131 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/job_schedule.py +118 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/job_spec.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/kubernetes_metrics.py +91 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/label.py +97 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/livy_compute_config.py +103 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/livy_spark_resource_config.py +106 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/look_back_config.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/manual_tag_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/mapping.py +97 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/marker.py +88 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/marker_config.py +349 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/marker_config_fallback.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/meta.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/meta_data.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/meta_data_item.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/mini_profile_request.py +112 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/notification_payload.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/offset_based_marker_config.py +97 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/operational_config.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/pattern_configuration.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/pipe_line_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/pipeline_compute_config.py +110 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/policy_execution_marker_config.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/policy_execution_request.py +229 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/policy_group.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/policy_transform_udf.py +113 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/process_metrics.py +91 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_anomaly_details_response.py +109 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_anomaly_item_execution.py +127 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_anomaly_rule_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_anomaly_rule_item.py +109 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_cancellation_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_request.py +211 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/profile_request_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/property_template.py +178 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/property_templates.py +123 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/pub_sub_lookback_marker_config.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/pub_sub_timestamp_marker_config.py +108 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/quality_rule.py +381 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/quality_rule_execution_metrics.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/quality_rule_with_latest_execution.py +115 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/range_configs.py +109 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_item.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_item_execution.py +157 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_rule.py +339 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_rule_details.py +225 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_rule_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_rule_response.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/reconciliation_rules_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/relative_config.py +197 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/relative_config_fallback.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/relative_threshold_config.py +111 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/root_cause_analysis.py +108 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_execution.py +327 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_execution_result.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_item_result.py +125 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_meta_data.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_spark_sql_dynamic_filter_variable_mapping.py +100 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_tag.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_threshold_configuration.py +236 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rule_threshold_configuration_fallback.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/rules_with_latest_execution_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/sample_data_api_response.py +122 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/schema_drift_item_execution.py +131 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/schema_drift_rule_execution.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/secret_manager_configuration.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/secret_manager_configuration_with_option.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/security_config.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/segment_metric_json.py +142 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/segments_analysis.py +145 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/short_segment.py +97 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/simple_quality_rule_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/snapshot_data.py +117 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/source_model.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/source_type.py +100 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/source_types_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/spark_job_status.py +113 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/spark_resource_config.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_assembly.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_crawler_action_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_crawler_status_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_delete_artifact_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_list.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_map.py +88 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_meta_data.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_mini_profile_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/standard_response_profile_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/start_profiling_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/tag_domain.py +116 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/threshold_breach_details.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/time_based_config.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/time_based_threshold_config.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/timestamp_based_marker_config.py +115 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/udf_variable.py +97 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/yunikorn_compute_config.py +133 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/catalog/yunikorn_spark_resource_config.py +154 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/big_decimal.py +100 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/big_integer.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/child_span_response.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/create_span_event.py +109 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/edge.py +108 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/event_alert_object.py +95 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/generic_config_fallback.py +101 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/identifier.py +146 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/job_request.py +130 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/meta.py +107 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/meta_data.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/metric.py +103 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/node.py +201 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/node_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/notification_payload.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline.py +152 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_baseline_metric.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_details.py +118 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_details_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_input_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_listing_info.py +118 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_run.py +136 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_run_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_runs_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipeline_summary.py +98 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/pipelines_listing_response.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/policy_summary.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/span.py +115 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/span_event.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/span_event_request.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/span_event_response.py +92 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/span_request.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/spans_response.py +96 -0
- acceldata_sdk_python-26.7.0/acceldata/models/api/pipeline/tag.py +97 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/__init__.py +105 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/asset_resource.py +112 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/asset_source_type.py +59 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/custom_asset_metadata.py +11 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/datasource_resource.py +88 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/executor.py +524 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/policy_execution_request.py +114 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/policy_execution_type.py +16 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/policy_filter.py +94 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/policy_rule_resources.py +388 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/profile_request_resource.py +77 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/profiling.py +82 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/catalog/profiling_configuration_error.py +5 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/__init__.py +62 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/create_job_input.py +108 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/create_pipeline_input_request.py +72 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/events/__init__.py +5 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/events/generic_event.py +31 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/events/log_event.py +34 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/events/span_event.py +8 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/pipeline_resource.py +171 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/pipeline_run_input.py +84 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/pipeline_run_resource.py +208 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/pipeline_source_type.py +20 -0
- acceldata_sdk_python-26.7.0/acceldata/models/sdk/pipeline/span_resource.py +327 -0
- acceldata_sdk_python-26.7.0/acceldata/openapi_deserialization_patches.py +161 -0
- acceldata_sdk_python-26.7.0/acceldata/openapi_repr_patches.py +62 -0
- acceldata_sdk_python-26.7.0/acceldata/services/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/acceldata/services/api_timestamp.py +31 -0
- acceldata_sdk_python-26.7.0/acceldata/services/asset_service.py +651 -0
- acceldata_sdk_python-26.7.0/acceldata/services/datasource_service.py +120 -0
- acceldata_sdk_python-26.7.0/acceldata/services/generic_service.py +132 -0
- acceldata_sdk_python-26.7.0/acceldata/services/marker_config.py +232 -0
- acceldata_sdk_python-26.7.0/acceldata/services/pipeline_big_decimal_api.py +102 -0
- acceldata_sdk_python-26.7.0/acceldata/services/pipeline_response_parse.py +209 -0
- acceldata_sdk_python-26.7.0/acceldata/services/pipeline_service.py +494 -0
- acceldata_sdk_python-26.7.0/acceldata/services/rule_service.py +620 -0
- acceldata_sdk_python-26.7.0/acceldata/urls/__init__.py +1 -0
- acceldata_sdk_python-26.7.0/acceldata/urls/catalog_urls.py +137 -0
- acceldata_sdk_python-26.7.0/acceldata/urls/pipeline_urls.py +55 -0
- acceldata_sdk_python-26.7.0/acceldata/utils/__init__.py +6 -0
- acceldata_sdk_python-26.7.0/acceldata/utils/datetime_utils.py +16 -0
- acceldata_sdk_python-26.7.0/acceldata/utils/operation_ack.py +22 -0
- acceldata_sdk_python-26.7.0/acceldata/utils/span_helpers.py +95 -0
- acceldata_sdk_python-26.7.0/acceldata/utils/time_range_utils.py +90 -0
- acceldata_sdk_python-26.7.0/acceldata_sdk_python.egg-info/PKG-INFO +26 -0
- acceldata_sdk_python-26.7.0/acceldata_sdk_python.egg-info/SOURCES.txt +355 -0
- acceldata_sdk_python-26.7.0/acceldata_sdk_python.egg-info/dependency_links.txt +1 -0
- acceldata_sdk_python-26.7.0/acceldata_sdk_python.egg-info/requires.txt +8 -0
- acceldata_sdk_python-26.7.0/acceldata_sdk_python.egg-info/top_level.txt +5 -0
- acceldata_sdk_python-26.7.0/pyproject.toml +47 -0
- acceldata_sdk_python-26.7.0/setup.cfg +4 -0
- acceldata_sdk_python-26.7.0/tests/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/tests/conftest.py +70 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/__init__.py +0 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_assets_service.py +415 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_constants.py +147 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_datasource_service.py +109 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_dq_policy_execution.py +172 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_etl_pipeline_flow.py +201 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_executor.py +491 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_explicit_time_pipeline.py +291 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_generic_service.py +60 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_new_sdk_pipeline.py +167 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_pipeline_spans_jobs.py +113 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_reconciliation_policy_execution.py +150 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_rule_service.py +164 -0
- acceldata_sdk_python-26.7.0/tests/integration_tests/test_tags_and_labels_integration.py +328 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_api_errors.py +58 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_asset_service_profile_sync.py +167 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_asset_service_sample_data.py +87 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_asset_tags_and_labels.py +131 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_executor.py +393 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_get_policy.py +85 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_http_client_cancel_profile.py +89 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_marker_config_deserialization.py +515 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_marker_config_service.py +271 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_openapi_deserialization_patches.py +91 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_openapi_repr_patches.py +89 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_pipeline_resources.py +369 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_pipeline_response_parse.py +115 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_pipeline_service_spans.py +95 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_policy_executions.py +582 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_policy_resource_aliases.py +56 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_polymorphic_fallbacks.py +216 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_span_context.py +126 -0
- acceldata_sdk_python-26.7.0/tests/unit/test_transient_retry.py +63 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# pytest cache directory #
|
|
2
|
+
|
|
3
|
+
This directory contains data from the pytest's cache plugin,
|
|
4
|
+
which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
|
|
5
|
+
|
|
6
|
+
**Do not** commit this to version control.
|
|
7
|
+
|
|
8
|
+
See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 - present Acceldata Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to
|
|
7
|
+
deal in the Software without restriction, including without limitation the
|
|
8
|
+
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
9
|
+
sell copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
18
|
+
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: acceldata_sdk_python
|
|
3
|
+
Version: 26.7.0
|
|
4
|
+
Summary: Acceldata SDK Python
|
|
5
|
+
Author-email: acceldata <apisupport@acceldata.io>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: acceldata-sdk-python
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: pydantic>=2
|
|
16
|
+
Requires-Dist: requests
|
|
17
|
+
Requires-Dist: requests-toolbelt
|
|
18
|
+
Requires-Dist: semantic-version
|
|
19
|
+
Provides-Extra: release
|
|
20
|
+
Requires-Dist: build>=1; extra == "release"
|
|
21
|
+
Requires-Dist: twine>=5; extra == "release"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# Acceldata SDK for Python
|
|
25
|
+
|
|
26
|
+
Documentation: https://docs.acceldata.io/documentation/acceldata-sdk-python
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Catalog guide
|
|
2
|
+
|
|
3
|
+
Catalog APIs discover and monitor the data estate in ADOC. The SDK targets **Python 3.10+**; see the main [README](README.md) for install, `AdocClient` setup, and the short overview of generated versus hand-written packages.
|
|
4
|
+
|
|
5
|
+
## Capabilities
|
|
6
|
+
|
|
7
|
+
- **Datasources and crawlers** — list or load assemblies by name, id, or `AssetSourceType`; start crawlers; read crawler status.
|
|
8
|
+
- **Assets** — resolve by numeric id or string uid; list `AssetType` definitions; read and extend metadata; fetch sample data.
|
|
9
|
+
- **Profiling** — full, incremental, and selective runs with `AssetProfilingParams` and catalog `MarkerConfig` (ID, date-time, file-event bounds, and other wire variants); poll, cancel, and re-fetch profile requests.
|
|
10
|
+
- **Policies** (separate from profiling jobs) — execute and monitor data quality, reconciliation, and cadence rules: [README-policies.md](README-policies.md).
|
|
11
|
+
|
|
12
|
+
## Quick example
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from acceldata.client.adoc_client import AdocClient
|
|
16
|
+
|
|
17
|
+
client = AdocClient(
|
|
18
|
+
url="https://<your-adoc-url>",
|
|
19
|
+
access_key="<your-access-key>",
|
|
20
|
+
secret_key="<your-secret-key>",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
datasources = client.get_all_datasources()
|
|
24
|
+
asset = client.get_asset("<asset_uid>")
|
|
25
|
+
sample = asset.sample_data()
|
|
26
|
+
|
|
27
|
+
print(asset.to_dict())
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Pick a task
|
|
31
|
+
|
|
32
|
+
- **Comprehensive** catalog: datasources, filters, crawlers, assets, metadata, custom metadata, sample, profiling: [README-datasources-assets.md](README-datasources-assets.md)
|
|
33
|
+
- **Policies** and execution: [README-policies.md](README-policies.md)
|
|
34
|
+
|
|
35
|
+
## Next step
|
|
36
|
+
|
|
37
|
+
Open the task-specific guide that matches the workflow; the same `AdocClient` instance can be reused across scripts.
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
# Datasources, assets, metadata, and profiling
|
|
2
|
+
|
|
3
|
+
This guide covers catalog operations with `AdocClient`: discover connections, run crawlers, resolve assets, work with metadata and sample data, and run profiling (full, incremental, and selective with marker configuration).
|
|
4
|
+
|
|
5
|
+
- **Prerequisites:** [README](README.md) (install and client setup). Requires **Python 3.10+**.
|
|
6
|
+
- **Data policies** (data quality, reconciliation, cadence): [README-policies.md](README-policies.md)
|
|
7
|
+
|
|
8
|
+
ADOC can register many data source types (warehouses, lakes, messaging, and more). In code, use `AssetSourceType` to filter or interpret catalog responses.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1) Datasources
|
|
13
|
+
|
|
14
|
+
Load one datasource by **name** or **numeric id**, optionally with extended **properties** on the returned object (or pass `properties=True` when loading).
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from acceldata.client.adoc_client import AdocClient
|
|
18
|
+
from acceldata.models.sdk.catalog import AssetSourceType
|
|
19
|
+
|
|
20
|
+
client = AdocClient(
|
|
21
|
+
url="https://<your-adoc-url>",
|
|
22
|
+
access_key="<your-access-key>",
|
|
23
|
+
secret_key="<your-secret-key>",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# By name, optional properties payload
|
|
27
|
+
ds = client.get_datasource("snowflake_prod")
|
|
28
|
+
ds = client.get_datasource("snowflake_prod", properties=True)
|
|
29
|
+
ds = client.get_datasource(5) # same as get_datasource_by_id(5)
|
|
30
|
+
|
|
31
|
+
# All datasources
|
|
32
|
+
all_sources = client.get_all_datasources() # alias: get_datasources() with no filter
|
|
33
|
+
|
|
34
|
+
# Filter by source type
|
|
35
|
+
snowflake_only = client.get_datasources(AssetSourceType.SNOWFLAKE)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
A fetch returns a `DatasourceResource` with `id`, `name`, and `source_type`. For the full API-shaped payload (for logging or debugging), use `to_dict()` rather than reaching into internal fields. Use `refresh(properties=...)` to reload from the catalog.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
print(ds.to_dict())
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## 2) Crawler operations
|
|
47
|
+
|
|
48
|
+
Crawler start and status are available from the **datasource object** (preferred) or by **name** on the client. Both support optional `RetryConfig` for transient HTTP/transport issues.
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
# Using a DatasourceResource
|
|
52
|
+
ds = client.get_datasource("my_datasource")
|
|
53
|
+
start = ds.start_crawler()
|
|
54
|
+
status = ds.get_crawler_status()
|
|
55
|
+
|
|
56
|
+
# Or by name on the client
|
|
57
|
+
client.start_crawler("my_datasource")
|
|
58
|
+
client.get_crawler_status("my_datasource")
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 3) Assets and asset types
|
|
64
|
+
|
|
65
|
+
### Resolve an asset
|
|
66
|
+
|
|
67
|
+
Pass a **string uid** (for example `Feature_bag_datasource.feature_1`) or a **numeric id** (`int`, or a string of digits, which is treated as an id, not a uid).
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
# By numeric id
|
|
71
|
+
asset = client.get_asset(1)
|
|
72
|
+
|
|
73
|
+
# By uid
|
|
74
|
+
asset = client.get_asset("Feature_bag_datasource.feature_1")
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The result is an `AssetResource`. `asset.to_dict()` returns the full payload as plain data.
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
print(asset.to_dict())
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### List asset type definitions
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
types = client.get_all_asset_types() # alias of get_asset_types()
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Each entry is a catalog `AssetType` (name, id, and related fields as returned by the API).
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## 4) Metadata, custom metadata, and sample data
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from acceldata.models.sdk.catalog import CustomAssetMetadata
|
|
97
|
+
|
|
98
|
+
asset = client.get_asset("my_datasource.my_table")
|
|
99
|
+
|
|
100
|
+
# Fetched metadata (as stored for the asset)
|
|
101
|
+
md = asset.get_metadata()
|
|
102
|
+
# Or: client.get_asset_metadata(asset_id)
|
|
103
|
+
|
|
104
|
+
# Merge custom key/value pairs (string values; merged then POSTed)
|
|
105
|
+
asset.add_custom_metadata(
|
|
106
|
+
[
|
|
107
|
+
CustomAssetMetadata("owner", "analytics"),
|
|
108
|
+
CustomAssetMetadata("pii", "true"),
|
|
109
|
+
]
|
|
110
|
+
)
|
|
111
|
+
# Or pass a flat mapping from the client: client.add_custom_metadata(asset_id, {"owner": "analytics"})
|
|
112
|
+
|
|
113
|
+
# Sample data (default): trigger async job and block by polling until not IN_PROGRESS
|
|
114
|
+
sample = asset.sample_data(sync=True) # sync=True is the default
|
|
115
|
+
# Or: client.sample_data(asset_id, sync=True)
|
|
116
|
+
|
|
117
|
+
# Non-blocking mode: trigger async job and return immediately with request_id
|
|
118
|
+
initial = asset.sample_data(sync=False)
|
|
119
|
+
# Or: initial = client.sample_data(asset_id, sync=False)
|
|
120
|
+
request_id = initial.request_id if initial is not None else None
|
|
121
|
+
|
|
122
|
+
# With an async `request_id`, poll the result
|
|
123
|
+
if request_id:
|
|
124
|
+
result = client.get_sample_data_result(request_id)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## 5) Profiling: parameters and modes
|
|
130
|
+
|
|
131
|
+
Catalog profiling is driven by `AssetProfilingParams`, which only sets the catalog profiling type and optional `markerConfig` (other start-profile fields use catalog defaults inside `to_start_profiling_request()`).
|
|
132
|
+
|
|
133
|
+
| Field | Type | When required |
|
|
134
|
+
|-------|------|---------------|
|
|
135
|
+
| `profiling_type` | `ProfilingType` | Always (`FULL`, `INCREMENTAL`, or `SELECTIVE`). |
|
|
136
|
+
| `marker_config` | `MarkerConfig` (optional) | Required for `SELECTIVE`. Also required for `INCREMENTAL` (per validation: incremental runs need a marker on the request). Omitted for most `FULL` runs. |
|
|
137
|
+
|
|
138
|
+
`ExecutionType` (from the same profiling module as `ProfilingType`) mirrors the same string values as `ProfilingType` for execution-oriented typing in application code.
|
|
139
|
+
|
|
140
|
+
**Preferred pattern for starting a run:** use `AdocClient.profile_asset` to obtain a `ProfileRequestResource` with `get_status()` and `cancel()`. Pass optional `transient_retry=RetryConfig(...)` to retry transient HTTP/transport failures; default retry status codes include **HTTP 409** (for example, profile already running).
|
|
141
|
+
|
|
142
|
+
`profile_asset(..., sync=False)` (default) is fire-and-forget. It triggers profiling and immediately returns the initial request (with `id`).
|
|
143
|
+
|
|
144
|
+
`profile_asset(..., sync=True)` uses the returned request id to keep checking profile status until it reaches a terminal state.
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from acceldata.client.transient_retry import RetryConfig
|
|
148
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
149
|
+
|
|
150
|
+
params = AssetProfilingParams(profiling_type=ProfilingType.FULL)
|
|
151
|
+
# Async fire-and-forget (default)
|
|
152
|
+
profile = client.profile_asset(
|
|
153
|
+
123,
|
|
154
|
+
params,
|
|
155
|
+
sync=False,
|
|
156
|
+
# transient_retry=RetryConfig(max_attempts=6, initial_interval_seconds=30.0, max_interval_seconds=30.0),
|
|
157
|
+
)
|
|
158
|
+
print(profile.to_dict())
|
|
159
|
+
|
|
160
|
+
# Sync mode: block until terminal status by polling with request id
|
|
161
|
+
completed_profile = client.profile_asset(
|
|
162
|
+
123,
|
|
163
|
+
params,
|
|
164
|
+
sync=True,
|
|
165
|
+
)
|
|
166
|
+
print(completed_profile.to_dict())
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
`AssetResource.start_profile(profiling_params=..., sync=...)` returns the same `ProfileRequestResource` shape (status and cancel helpers):
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
173
|
+
|
|
174
|
+
res = asset.start_profile(
|
|
175
|
+
AssetProfilingParams(profiling_type=ProfilingType.FULL),
|
|
176
|
+
sync=False, # default: async fire-and-forget
|
|
177
|
+
)
|
|
178
|
+
print(res.to_dict())
|
|
179
|
+
details = res.get_status()
|
|
180
|
+
res.cancel() # may return None for terminal/no-op cases, see API behavior
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
**Latest status and lookup by request id**
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
# Latest status for the asset — response type is MiniProfileRequest
|
|
187
|
+
latest = client.get_latest_profile_status(asset_id=123)
|
|
188
|
+
# Or: asset.get_latest_profile_status() on an AssetResource
|
|
189
|
+
|
|
190
|
+
# Specific profile request by (asset_id, req_id) — also exposed as get_profile_status
|
|
191
|
+
by_req = client.get_profile_request_details(asset_id=123, req_id=456)
|
|
192
|
+
# Or: client.get_profile_status(123, 456)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
**Cancel** by request id (when only the id is available, not a resource instance):
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
client.cancel_profile(profile_req_id)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
`ProfileRequestResource` delegates cancel and status to these client/service methods. Transient read retries for crawler and profile calls can be configured with `RetryConfig` where the method allows it.
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## 5a) Trigger full profiling
|
|
206
|
+
|
|
207
|
+
Runs the configured full scan for the asset. The snippet below assumes a `client` from [section 1](#1-datasources).
|
|
208
|
+
|
|
209
|
+
```python
|
|
210
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
211
|
+
|
|
212
|
+
asset = client.get_asset("my_datasource.my_table")
|
|
213
|
+
params = AssetProfilingParams(profiling_type=ProfilingType.FULL)
|
|
214
|
+
client.profile_asset(asset.id, params)
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
---
|
|
218
|
+
|
|
219
|
+
## 5b) Trigger incremental profiling
|
|
220
|
+
|
|
221
|
+
Use when the asset’s catalog configuration defines an incremental strategy; a **marker is required** on the request. For an id-column incremental strategy, use **`id_marker_config`**. For other strategies, use the **`*_marker_config` helper** in `acceldata.services.marker_config` that matches the generated class you need (e.g. **`date_time_marker_config`** for `DateTimeMarkerConfig`, **`file_marker_config`** for `FileMarkerConfig`—same naming pattern as `BoundsDateTimeMarkerConfig` → **`bounds_date_time_marker_config`**). You can also set `type` yourself using **`marker_type_for`** or **`API_TYPE_BY_MARKER_CLASS`**, or call **`as_marker_config`**, which can fix a wrong `type` on a typed model instance. `AssetProfilingParams.marker_config` also accepts a generated branch instance or a `MarkerConfig` wrapper; the client normalizes them when the request is sent.
|
|
222
|
+
|
|
223
|
+
The snippet assumes a configured `AdocClient` `client` (see the start of this guide). As with full profiling ([section 5](#5-profiling-parameters-and-modes)), `profile_asset` accepts `sync=True` to poll until a terminal status and `transient_retry=RetryConfig(...)` for transient HTTP/transport retries (including while polling in sync mode).
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
from acceldata.client.transient_retry import RetryConfig
|
|
227
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
228
|
+
from acceldata.services.marker_config import id_marker_config
|
|
229
|
+
|
|
230
|
+
asset = client.get_asset("my_datasource.my_table")
|
|
231
|
+
params = AssetProfilingParams(
|
|
232
|
+
profiling_type=ProfilingType.INCREMENTAL,
|
|
233
|
+
marker_config=id_marker_config(
|
|
234
|
+
id_column_name="pk_col",
|
|
235
|
+
initial_offset=0,
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
client.profile_asset(
|
|
239
|
+
asset.id,
|
|
240
|
+
params,
|
|
241
|
+
sync=False, # default: async; use sync=True to block until terminal
|
|
242
|
+
# transient_retry=RetryConfig(max_attempts=6, initial_interval_seconds=30.0, max_interval_seconds=30.0),
|
|
243
|
+
)
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
---
|
|
247
|
+
|
|
248
|
+
## 5c) Trigger selective profiling (bounds-style markers)
|
|
249
|
+
|
|
250
|
+
Selective profiling must include a **marker**. For id-bounded slices use `bounds_id_marker_config`; for date- or file-event–bounded slices use **`bounds_date_time_marker_config`** or **`bounds_file_event_marker_config`**. Each example below assumes a configured `client` from [section 1](#1-datasources). The same `sync` and `transient_retry` options as in [section 5](#5-profiling-parameters-and-modes) apply to `profile_asset` for selective runs.
|
|
251
|
+
|
|
252
|
+
**ID-bounded (monotonic / range column)**
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
from acceldata.client.transient_retry import RetryConfig
|
|
256
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
257
|
+
from acceldata.services.marker_config import bounds_id_marker_config
|
|
258
|
+
|
|
259
|
+
asset = client.get_asset("my_datasource.my_table")
|
|
260
|
+
params = AssetProfilingParams(
|
|
261
|
+
profiling_type=ProfilingType.SELECTIVE,
|
|
262
|
+
marker_config=bounds_id_marker_config(
|
|
263
|
+
id_column_name="ID",
|
|
264
|
+
from_id=0,
|
|
265
|
+
to_id=1000,
|
|
266
|
+
),
|
|
267
|
+
)
|
|
268
|
+
client.profile_asset(
|
|
269
|
+
asset.id,
|
|
270
|
+
params,
|
|
271
|
+
sync=False,
|
|
272
|
+
# transient_retry=RetryConfig(max_attempts=6, initial_interval_seconds=30.0, max_interval_seconds=30.0),
|
|
273
|
+
)
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
**DateTime-bounded (column, format, timezone, optional range strings)**
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
from acceldata.client.transient_retry import RetryConfig
|
|
280
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
281
|
+
from acceldata.services.marker_config import bounds_date_time_marker_config
|
|
282
|
+
|
|
283
|
+
asset = client.get_asset("my_datasource.my_table")
|
|
284
|
+
params = AssetProfilingParams(
|
|
285
|
+
profiling_type=ProfilingType.SELECTIVE,
|
|
286
|
+
marker_config=bounds_date_time_marker_config(
|
|
287
|
+
date_column_name="TO_DATE",
|
|
288
|
+
format="yyyy-MM-dd",
|
|
289
|
+
time_zone_id="Asia/Calcutta",
|
|
290
|
+
from_date="2023-07-01 00:00:00.000",
|
|
291
|
+
to_date="2024-07-14 23:59:59.999",
|
|
292
|
+
),
|
|
293
|
+
)
|
|
294
|
+
client.profile_asset(
|
|
295
|
+
asset.id,
|
|
296
|
+
params,
|
|
297
|
+
sync=False,
|
|
298
|
+
# transient_retry=RetryConfig(max_attempts=6, initial_interval_seconds=30.0, max_interval_seconds=30.0),
|
|
299
|
+
)
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
**File event–based bounds**
|
|
303
|
+
|
|
304
|
+
```python
|
|
305
|
+
from acceldata.client.transient_retry import RetryConfig
|
|
306
|
+
from acceldata.models.sdk.catalog import AssetProfilingParams, ProfilingType
|
|
307
|
+
from acceldata.services.marker_config import bounds_file_event_marker_config
|
|
308
|
+
|
|
309
|
+
asset = client.get_asset("my_datasource.my_table")
|
|
310
|
+
params = AssetProfilingParams(
|
|
311
|
+
profiling_type=ProfilingType.SELECTIVE,
|
|
312
|
+
marker_config=bounds_file_event_marker_config(
|
|
313
|
+
date_column_name="ingest_date",
|
|
314
|
+
time_zone_id="Asia/Calcutta",
|
|
315
|
+
from_date="2019-04-01 00:00:00.000",
|
|
316
|
+
to_date="2024-07-16 23:59:59.999",
|
|
317
|
+
),
|
|
318
|
+
)
|
|
319
|
+
client.profile_asset(
|
|
320
|
+
asset.id,
|
|
321
|
+
params,
|
|
322
|
+
sync=False,
|
|
323
|
+
# transient_retry=RetryConfig(max_attempts=6, initial_interval_seconds=30.0, max_interval_seconds=30.0),
|
|
324
|
+
)
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
For partitioning, file-based markers, Pub/Sub, offsets, and similar cases, use the matching `*_marker_config` helper in `acceldata.services.marker_config` (each pairs with a generated `*MarkerConfig` class—see **`API_TYPE_BY_MARKER_CLASS`**), or build the model with an explicit `type=`, then pass the instance the same way as in the examples above.
|
|
328
|
+
|
|
329
|
+
---
|
|
330
|
+
|
|
331
|
+
## 6) Relationship to other SDK areas
|
|
332
|
+
|
|
333
|
+
- **Data quality, reconciliation, and cadence (freshness) policies** are separate catalog rule executions; start and poll them with `execute_policy`, `Executor`, and the helpers in [README-policies.md](README-policies.md).
|
|
334
|
+
|
|
335
|
+
## Next step
|
|
336
|
+
|
|
337
|
+
- Policy execution: [README-policies.md](README-policies.md)
|
|
338
|
+
- Catalog index: [README-catalog.md](README-catalog.md)
|
|
339
|
+
|
|
340
|
+
---
|
|
341
|
+
|
|
342
|
+
## Operational notes
|
|
343
|
+
|
|
344
|
+
- For **long or flaky** catalog calls, pass `transient_retry=RetryConfig(...)` where the client method supports it.
|
|
345
|
+
- Use **defaults** and match **marker** discriminators to the target ADOC environment; invalid combinations raise validation errors (for example, missing `markerConfig` for `SELECTIVE`).
|
|
346
|
+
|
|
347
|
+
When wiring transient retries, import `RetryConfig` next to the `AdocClient` setup imports.
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# Historical pipeline load
|
|
2
|
+
|
|
3
|
+
This guide covers backfilling past pipeline executions with explicit timestamps.
|
|
4
|
+
|
|
5
|
+
For install and `AdocClient` setup, see [README](README.md). For normal real-time execution, use the [Pipelines guide](README-pipeline.md).
|
|
6
|
+
|
|
7
|
+
## When this guide helps
|
|
8
|
+
|
|
9
|
+
- Replaying past runs into ADOC
|
|
10
|
+
- Migrating existing pipeline history
|
|
11
|
+
- Aligning event and run times with original execution time
|
|
12
|
+
|
|
13
|
+
## Historical flow
|
|
14
|
+
|
|
15
|
+
1. Create pipeline with explicit creation time
|
|
16
|
+
2. Create run with explicit `started_at`
|
|
17
|
+
3. Create span with `with_explicit_time=True`
|
|
18
|
+
4. Create a job bounded by span and fetch that span by UID
|
|
19
|
+
5. Create child spans under the fetched job span
|
|
20
|
+
6. Start, send events, and end spans with explicit timestamps
|
|
21
|
+
7. Complete run with explicit `finished_at`
|
|
22
|
+
|
|
23
|
+
## Minimal example
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from datetime import datetime, timedelta
|
|
27
|
+
|
|
28
|
+
from acceldata.client.adoc_client import AdocClient
|
|
29
|
+
from acceldata.models.api.pipeline.meta import Meta
|
|
30
|
+
from acceldata.models.sdk.pipeline.create_job_input import CreateJobInput, JobInputOutputRef
|
|
31
|
+
from acceldata.models.sdk.pipeline import GenericEvent, PipelineRunResult, PipelineRunStatus
|
|
32
|
+
from acceldata.models.sdk.pipeline.create_pipeline_input_request import (
|
|
33
|
+
CreatePipelineInputRequest,
|
|
34
|
+
)
|
|
35
|
+
from acceldata.models.sdk.pipeline.pipeline_run_input import PipelineRunInput
|
|
36
|
+
|
|
37
|
+
client = AdocClient(
|
|
38
|
+
url="https://<your-adoc-url>",
|
|
39
|
+
access_key="<your-access-key>",
|
|
40
|
+
secret_key="<your-secret-key>",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
base_time = datetime(2024, 1, 15, 10, 0, 0)
|
|
44
|
+
|
|
45
|
+
# 1) Create pipeline with historical createdAt
|
|
46
|
+
pipeline = client.create_pipeline(
|
|
47
|
+
CreatePipelineInputRequest(
|
|
48
|
+
uid="historical_customers_etl",
|
|
49
|
+
name="Historical customers ETL",
|
|
50
|
+
meta=Meta(owner="data-team", team="analytics", code_location="..."),
|
|
51
|
+
createdAt=base_time,
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# 2) Create run with historical start time
|
|
56
|
+
run_start = base_time + timedelta(hours=1)
|
|
57
|
+
run = pipeline.create_pipeline_run(PipelineRunInput(started_at=run_start))
|
|
58
|
+
# `run` is a PipelineRunResource.
|
|
59
|
+
print(run.to_dict())
|
|
60
|
+
|
|
61
|
+
# 3) Create span with explicit-time mode (no auto-start)
|
|
62
|
+
root_span = run.create_root_span(
|
|
63
|
+
uid="historical_customers_root",
|
|
64
|
+
with_explicit_time=True,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# 4) Create a bounded job span and load it by UID
|
|
68
|
+
job = run.create_job(
|
|
69
|
+
CreateJobInput(
|
|
70
|
+
uid="historical_customers_transform",
|
|
71
|
+
name="Historical customers transform",
|
|
72
|
+
description="Backfilled transform job with explicit span timing.",
|
|
73
|
+
pipeline_run_id=run.id,
|
|
74
|
+
bounded_by_span=True,
|
|
75
|
+
with_explicit_time=True,
|
|
76
|
+
span_uid="historical_customers_transform_span",
|
|
77
|
+
inputs=[JobInputOutputRef(asset_uid="s3_ds.staging.customers_raw_snapshot")],
|
|
78
|
+
outputs=[JobInputOutputRef(asset_uid="snowflake_ds.warehouse.customers_curated")],
|
|
79
|
+
meta=Meta(owner="data-team", team="analytics", code_location="..."),
|
|
80
|
+
context={"replay": "historical_customers_etl"},
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
job_span = run.get_span("historical_customers_transform_span") # explicit source for child spans below
|
|
84
|
+
|
|
85
|
+
alternative_job = run.create_job(
|
|
86
|
+
CreateJobInput(
|
|
87
|
+
uid="historical_customers_enrich",
|
|
88
|
+
name="Historical customers enrich",
|
|
89
|
+
description="Unbounded job bound to a manually created child span.",
|
|
90
|
+
pipeline_run_id=run.id,
|
|
91
|
+
bounded_by_span=False,
|
|
92
|
+
with_explicit_time=True,
|
|
93
|
+
inputs=[JobInputOutputRef(asset_uid="snowflake_ds.warehouse.customers_curated")],
|
|
94
|
+
outputs=[JobInputOutputRef(asset_uid="snowflake_ds.warehouse.customers_enriched")],
|
|
95
|
+
meta=Meta(owner="data-team", team="analytics", code_location="..."),
|
|
96
|
+
context={"replay": "historical_customers_etl", "variant": "manual_span"},
|
|
97
|
+
)
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# Alternative with bounded_by_span=False: create and bind span explicitly.
|
|
101
|
+
alternative_job_span = job_span.create_child_span(
|
|
102
|
+
uid="historical_customers_enrich_span",
|
|
103
|
+
associated_job_uids=[alternative_job.uid],
|
|
104
|
+
with_explicit_time=True,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# 5) Start, send events, and end spans at chosen timestamps
|
|
108
|
+
root_span_start = run_start + timedelta(seconds=10)
|
|
109
|
+
job_span_start = run_start + timedelta(minutes=1)
|
|
110
|
+
alternative_start = run_start + timedelta(minutes=1, seconds=30)
|
|
111
|
+
alternative_event = run_start + timedelta(minutes=1, seconds=45)
|
|
112
|
+
alternative_end = run_start + timedelta(minutes=1, seconds=55)
|
|
113
|
+
validate_start = run_start + timedelta(minutes=2)
|
|
114
|
+
validate_event = run_start + timedelta(minutes=3)
|
|
115
|
+
validate_end = run_start + timedelta(minutes=4)
|
|
116
|
+
publish_start = run_start + timedelta(minutes=5)
|
|
117
|
+
publish_event = run_start + timedelta(minutes=6)
|
|
118
|
+
publish_end = run_start + timedelta(minutes=8)
|
|
119
|
+
job_span_end = run_start + timedelta(minutes=9)
|
|
120
|
+
root_span_end = run_start + timedelta(minutes=15)
|
|
121
|
+
run_end = run_start + timedelta(minutes=20)
|
|
122
|
+
|
|
123
|
+
root_span.start(created_at=root_span_start)
|
|
124
|
+
job_span.start(created_at=job_span_start)
|
|
125
|
+
alternative_job_span.start(created_at=alternative_start)
|
|
126
|
+
alternative_job_span.send_event(
|
|
127
|
+
GenericEvent(
|
|
128
|
+
event_uid="historical_enrich_completed",
|
|
129
|
+
context_data={"rows_enriched": 11800},
|
|
130
|
+
created_at=alternative_event,
|
|
131
|
+
)
|
|
132
|
+
)
|
|
133
|
+
alternative_job_span.end(created_at=alternative_end)
|
|
134
|
+
|
|
135
|
+
validate_span = job_span.create_child_span(
|
|
136
|
+
uid="historical_customers_validate_span",
|
|
137
|
+
associated_job_uids=[job.uid],
|
|
138
|
+
with_explicit_time=True,
|
|
139
|
+
)
|
|
140
|
+
validate_span.start(created_at=validate_start)
|
|
141
|
+
validate_span.send_event(
|
|
142
|
+
GenericEvent(
|
|
143
|
+
event_uid="historical_validate_completed",
|
|
144
|
+
context_data={"rows_validated": 12000},
|
|
145
|
+
created_at=validate_event,
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
validate_span.end(created_at=validate_end)
|
|
149
|
+
|
|
150
|
+
publish_span = job_span.create_child_span(
|
|
151
|
+
uid="historical_customers_publish_span",
|
|
152
|
+
associated_job_uids=[job.uid],
|
|
153
|
+
with_explicit_time=True,
|
|
154
|
+
)
|
|
155
|
+
publish_span.start(created_at=publish_start)
|
|
156
|
+
publish_span.send_event(
|
|
157
|
+
GenericEvent(
|
|
158
|
+
event_uid="historical_publish_completed",
|
|
159
|
+
context_data={"rows_loaded": 11800},
|
|
160
|
+
created_at=publish_event,
|
|
161
|
+
)
|
|
162
|
+
)
|
|
163
|
+
publish_span.end(created_at=publish_end)
|
|
164
|
+
job_span.end(created_at=job_span_end)
|
|
165
|
+
root_span.end(created_at=root_span_end)
|
|
166
|
+
|
|
167
|
+
# 5) Complete run with explicit finish time
|
|
168
|
+
run.update_pipeline_run(
|
|
169
|
+
result=PipelineRunResult.SUCCESS,
|
|
170
|
+
status=PipelineRunStatus.COMPLETED,
|
|
171
|
+
finished_at=run_end,
|
|
172
|
+
)
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
`JobInputOutputRef(asset_uid=...)` must use dot notation (`<data_source_uid>.<asset_uid>`). The value is available from the Asset page in the ADOC UI.
|
|
176
|
+
|
|
177
|
+
## Critical rules
|
|
178
|
+
|
|
179
|
+
- Always set `with_explicit_time=True` for spans in historical mode
|
|
180
|
+
- Always call `start(created_at=...)` explicitly for those spans
|
|
181
|
+
- Always set `created_at` for important events
|
|
182
|
+
- Always set `finished_at` when completing the run
|
|
183
|
+
|
|
184
|
+
## Common mistakes
|
|
185
|
+
|
|
186
|
+
- Forgetting to call `start(created_at=...)` before sending events or ending spans
|
|
187
|
+
- Mixing historical and current-time values in one run
|
|
188
|
+
- Completing the run without `finished_at`
|
|
189
|
+
|
|
190
|
+
## Next step
|
|
191
|
+
|
|
192
|
+
When multiple scripts contribute to the same historical run, combine this flow with [Pipeline continuation ID](README-pipeline-continuation-id.md).
|