acceldata-sdk-python 26.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acceldata/__init__.py +7 -0
- acceldata/client/__init__.py +0 -0
- acceldata/client/adoc_client.py +666 -0
- acceldata/client/http_client.py +346 -0
- acceldata/client/transient_retry.py +142 -0
- acceldata/constants/__init__.py +1 -0
- acceldata/constants/constants.py +13 -0
- acceldata/exceptions/__init__.py +52 -0
- acceldata/exceptions/api_errors.py +85 -0
- acceldata/exceptions/exceptions.py +174 -0
- acceldata/exceptions/sdk_exceptions.py +98 -0
- acceldata/mappers/__init__.py +21 -0
- acceldata/mappers/catalog_executions.py +107 -0
- acceldata/models/__init__.py +0 -0
- acceldata/models/api/catalog/above_configs.py +99 -0
- acceldata/models/api/catalog/above_configs_success.py +138 -0
- acceldata/models/api/catalog/absolute_configs.py +197 -0
- acceldata/models/api/catalog/absolute_configs_fallback.py +101 -0
- acceldata/models/api/catalog/absolute_threshold_config.py +105 -0
- acceldata/models/api/catalog/analysis_job_context.py +102 -0
- acceldata/models/api/catalog/analytics_pipeline.py +385 -0
- acceldata/models/api/catalog/analytics_pipeline_endpoint.py +136 -0
- acceldata/models/api/catalog/anomaly_config.py +95 -0
- acceldata/models/api/catalog/anomaly_details.py +119 -0
- acceldata/models/api/catalog/anomaly_threshold_config.py +105 -0
- acceldata/models/api/catalog/artifact.py +92 -0
- acceldata/models/api/catalog/assembly.py +171 -0
- acceldata/models/api/catalog/asset.py +162 -0
- acceldata/models/api/catalog/asset_catalog_listing.py +159 -0
- acceldata/models/api/catalog/asset_catalog_listing_response.py +102 -0
- acceldata/models/api/catalog/asset_label_request.py +96 -0
- acceldata/models/api/catalog/asset_progress.py +96 -0
- acceldata/models/api/catalog/asset_progress_details.py +117 -0
- acceldata/models/api/catalog/asset_tag.py +124 -0
- acceldata/models/api/catalog/asset_tag_info.py +140 -0
- acceldata/models/api/catalog/asset_tag_request.py +95 -0
- acceldata/models/api/catalog/asset_tag_response.py +96 -0
- acceldata/models/api/catalog/asset_tags_response.py +102 -0
- acceldata/models/api/catalog/asset_type.py +96 -0
- acceldata/models/api/catalog/auto_anomaly_feature_analysis.py +160 -0
- acceldata/models/api/catalog/auto_anomaly_item_execution.py +133 -0
- acceldata/models/api/catalog/auto_anomaly_meta_data.py +162 -0
- acceldata/models/api/catalog/auto_anomaly_rule_execution.py +102 -0
- acceldata/models/api/catalog/auto_anomaly_summary.py +124 -0
- acceldata/models/api/catalog/backing_asset.py +126 -0
- acceldata/models/api/catalog/below_configs.py +99 -0
- acceldata/models/api/catalog/big_decimal.py +100 -0
- acceldata/models/api/catalog/big_integer.py +98 -0
- acceldata/models/api/catalog/bounds_date_time_marker_config.py +110 -0
- acceldata/models/api/catalog/bounds_file_event_marker_config.py +108 -0
- acceldata/models/api/catalog/bounds_id_marker_config.py +96 -0
- acceldata/models/api/catalog/catalog_asset_meta_data.py +128 -0
- acceldata/models/api/catalog/changes_by_config.py +105 -0
- acceldata/models/api/catalog/column_mapping.py +139 -0
- acceldata/models/api/catalog/column_metadata.py +90 -0
- acceldata/models/api/catalog/column_metric_with_anomaly_details.py +119 -0
- acceldata/models/api/catalog/column_variable.py +111 -0
- acceldata/models/api/catalog/composite_policy_execution.py +102 -0
- acceldata/models/api/catalog/composite_policy_item_execution.py +104 -0
- acceldata/models/api/catalog/connection.py +118 -0
- acceldata/models/api/catalog/connection_test_result.py +106 -0
- acceldata/models/api/catalog/connection_type.py +90 -0
- acceldata/models/api/catalog/connection_types_response.py +96 -0
- acceldata/models/api/catalog/crawler.py +88 -0
- acceldata/models/api/catalog/crawler_action_response.py +90 -0
- acceldata/models/api/catalog/crawler_execution.py +147 -0
- acceldata/models/api/catalog/crawler_execution_response.py +102 -0
- acceldata/models/api/catalog/crawler_status_response.py +90 -0
- acceldata/models/api/catalog/create_profile.py +194 -0
- acceldata/models/api/catalog/custom_sql_config.py +121 -0
- acceldata/models/api/catalog/data_bricks_spark_resource_config.py +104 -0
- acceldata/models/api/catalog/data_cadence_rule.py +236 -0
- acceldata/models/api/catalog/data_cadence_rule_details.py +125 -0
- acceldata/models/api/catalog/data_cadence_rule_execution.py +102 -0
- acceldata/models/api/catalog/data_cadence_rule_item.py +104 -0
- acceldata/models/api/catalog/data_cadence_rule_item_execution.py +158 -0
- acceldata/models/api/catalog/data_cadence_rule_response.py +98 -0
- acceldata/models/api/catalog/data_drift_execution.py +102 -0
- acceldata/models/api/catalog/data_drift_item_execution.py +121 -0
- acceldata/models/api/catalog/data_quality_engine.py +90 -0
- acceldata/models/api/catalog/data_quality_execution.py +106 -0
- acceldata/models/api/catalog/data_quality_rule.py +323 -0
- acceldata/models/api/catalog/data_quality_rule_details.py +186 -0
- acceldata/models/api/catalog/data_quality_rule_response.py +120 -0
- acceldata/models/api/catalog/data_quality_rules_response.py +102 -0
- acceldata/models/api/catalog/databricks_compute_config.py +113 -0
- acceldata/models/api/catalog/dataplane_engine.py +105 -0
- acceldata/models/api/catalog/dataplane_health_metrics.py +101 -0
- acceldata/models/api/catalog/dataplane_job.py +216 -0
- acceldata/models/api/catalog/dataplane_job_status.py +113 -0
- acceldata/models/api/catalog/dataplane_request.py +96 -0
- acceldata/models/api/catalog/dataproc_compute_config.py +106 -0
- acceldata/models/api/catalog/dataproc_spark_resource_config.py +104 -0
- acceldata/models/api/catalog/date_based_partition_marker_config.py +116 -0
- acceldata/models/api/catalog/date_partitioning_marker_config.py +113 -0
- acceldata/models/api/catalog/date_time_marker_config.py +109 -0
- acceldata/models/api/catalog/day_month_year_based_partition_marker_config.py +124 -0
- acceldata/models/api/catalog/decreased_by_config.py +105 -0
- acceldata/models/api/catalog/delete_artifact_response.py +88 -0
- acceldata/models/api/catalog/dq_rule_execution.py +248 -0
- acceldata/models/api/catalog/drift.py +121 -0
- acceldata/models/api/catalog/enable_rule_request.py +88 -0
- acceldata/models/api/catalog/engine.py +90 -0
- acceldata/models/api/catalog/execution_cancellation_response.py +92 -0
- acceldata/models/api/catalog/execution_response.py +144 -0
- acceldata/models/api/catalog/execution_result.py +126 -0
- acceldata/models/api/catalog/executions_response.py +102 -0
- acceldata/models/api/catalog/failed_asset_detail.py +90 -0
- acceldata/models/api/catalog/failed_summary.py +98 -0
- acceldata/models/api/catalog/file.py +94 -0
- acceldata/models/api/catalog/file_marker_config.py +99 -0
- acceldata/models/api/catalog/generic_config_fallback.py +101 -0
- acceldata/models/api/catalog/generic_execution.py +221 -0
- acceldata/models/api/catalog/generic_rule_execution.py +137 -0
- acceldata/models/api/catalog/global_storage_config_dto.py +172 -0
- acceldata/models/api/catalog/hosted_dp_deployment_response.py +98 -0
- acceldata/models/api/catalog/id_marker_config.py +94 -0
- acceldata/models/api/catalog/increased_by_config.py +105 -0
- acceldata/models/api/catalog/item.py +160 -0
- acceldata/models/api/catalog/item_execution.py +192 -0
- acceldata/models/api/catalog/item_segment.py +131 -0
- acceldata/models/api/catalog/job_schedule.py +118 -0
- acceldata/models/api/catalog/job_spec.py +96 -0
- acceldata/models/api/catalog/kubernetes_metrics.py +91 -0
- acceldata/models/api/catalog/label.py +97 -0
- acceldata/models/api/catalog/livy_compute_config.py +103 -0
- acceldata/models/api/catalog/livy_spark_resource_config.py +106 -0
- acceldata/models/api/catalog/look_back_config.py +90 -0
- acceldata/models/api/catalog/manual_tag_request.py +92 -0
- acceldata/models/api/catalog/mapping.py +97 -0
- acceldata/models/api/catalog/marker.py +88 -0
- acceldata/models/api/catalog/marker_config.py +349 -0
- acceldata/models/api/catalog/marker_config_fallback.py +101 -0
- acceldata/models/api/catalog/meta.py +92 -0
- acceldata/models/api/catalog/meta_data.py +92 -0
- acceldata/models/api/catalog/meta_data_item.py +94 -0
- acceldata/models/api/catalog/mini_profile_request.py +112 -0
- acceldata/models/api/catalog/notification_payload.py +114 -0
- acceldata/models/api/catalog/offset_based_marker_config.py +97 -0
- acceldata/models/api/catalog/operational_config.py +92 -0
- acceldata/models/api/catalog/pattern_configuration.py +90 -0
- acceldata/models/api/catalog/pipe_line_request.py +92 -0
- acceldata/models/api/catalog/pipeline_compute_config.py +110 -0
- acceldata/models/api/catalog/policy_execution_marker_config.py +94 -0
- acceldata/models/api/catalog/policy_execution_request.py +229 -0
- acceldata/models/api/catalog/policy_group.py +114 -0
- acceldata/models/api/catalog/policy_transform_udf.py +113 -0
- acceldata/models/api/catalog/process_metrics.py +91 -0
- acceldata/models/api/catalog/profile_anomaly_details_response.py +109 -0
- acceldata/models/api/catalog/profile_anomaly_item_execution.py +127 -0
- acceldata/models/api/catalog/profile_anomaly_rule_execution.py +102 -0
- acceldata/models/api/catalog/profile_anomaly_rule_item.py +109 -0
- acceldata/models/api/catalog/profile_cancellation_response.py +92 -0
- acceldata/models/api/catalog/profile_request.py +211 -0
- acceldata/models/api/catalog/profile_request_response.py +92 -0
- acceldata/models/api/catalog/property_template.py +178 -0
- acceldata/models/api/catalog/property_templates.py +123 -0
- acceldata/models/api/catalog/pub_sub_lookback_marker_config.py +96 -0
- acceldata/models/api/catalog/pub_sub_timestamp_marker_config.py +108 -0
- acceldata/models/api/catalog/quality_rule.py +381 -0
- acceldata/models/api/catalog/quality_rule_execution_metrics.py +114 -0
- acceldata/models/api/catalog/quality_rule_with_latest_execution.py +115 -0
- acceldata/models/api/catalog/range_configs.py +109 -0
- acceldata/models/api/catalog/reconciliation_item.py +102 -0
- acceldata/models/api/catalog/reconciliation_item_execution.py +157 -0
- acceldata/models/api/catalog/reconciliation_rule.py +339 -0
- acceldata/models/api/catalog/reconciliation_rule_details.py +225 -0
- acceldata/models/api/catalog/reconciliation_rule_execution.py +102 -0
- acceldata/models/api/catalog/reconciliation_rule_response.py +98 -0
- acceldata/models/api/catalog/reconciliation_rules_response.py +102 -0
- acceldata/models/api/catalog/relative_config.py +197 -0
- acceldata/models/api/catalog/relative_config_fallback.py +101 -0
- acceldata/models/api/catalog/relative_threshold_config.py +111 -0
- acceldata/models/api/catalog/root_cause_analysis.py +108 -0
- acceldata/models/api/catalog/rule_execution.py +327 -0
- acceldata/models/api/catalog/rule_execution_result.py +90 -0
- acceldata/models/api/catalog/rule_item_result.py +125 -0
- acceldata/models/api/catalog/rule_meta_data.py +94 -0
- acceldata/models/api/catalog/rule_spark_sql_dynamic_filter_variable_mapping.py +100 -0
- acceldata/models/api/catalog/rule_tag.py +101 -0
- acceldata/models/api/catalog/rule_threshold_configuration.py +236 -0
- acceldata/models/api/catalog/rule_threshold_configuration_fallback.py +101 -0
- acceldata/models/api/catalog/rules_with_latest_execution_response.py +102 -0
- acceldata/models/api/catalog/sample_data_api_response.py +122 -0
- acceldata/models/api/catalog/schema_drift_item_execution.py +131 -0
- acceldata/models/api/catalog/schema_drift_rule_execution.py +102 -0
- acceldata/models/api/catalog/secret_manager_configuration.py +90 -0
- acceldata/models/api/catalog/secret_manager_configuration_with_option.py +98 -0
- acceldata/models/api/catalog/security_config.py +90 -0
- acceldata/models/api/catalog/segment_metric_json.py +142 -0
- acceldata/models/api/catalog/segments_analysis.py +145 -0
- acceldata/models/api/catalog/short_segment.py +97 -0
- acceldata/models/api/catalog/simple_quality_rule_response.py +92 -0
- acceldata/models/api/catalog/snapshot_data.py +117 -0
- acceldata/models/api/catalog/source_model.py +90 -0
- acceldata/models/api/catalog/source_type.py +100 -0
- acceldata/models/api/catalog/source_types_response.py +102 -0
- acceldata/models/api/catalog/spark_job_status.py +113 -0
- acceldata/models/api/catalog/spark_resource_config.py +114 -0
- acceldata/models/api/catalog/standard_response_assembly.py +92 -0
- acceldata/models/api/catalog/standard_response_crawler_action_response.py +92 -0
- acceldata/models/api/catalog/standard_response_crawler_status_response.py +92 -0
- acceldata/models/api/catalog/standard_response_delete_artifact_response.py +92 -0
- acceldata/models/api/catalog/standard_response_list.py +96 -0
- acceldata/models/api/catalog/standard_response_map.py +88 -0
- acceldata/models/api/catalog/standard_response_meta_data.py +92 -0
- acceldata/models/api/catalog/standard_response_mini_profile_request.py +92 -0
- acceldata/models/api/catalog/standard_response_profile_request.py +92 -0
- acceldata/models/api/catalog/start_profiling_request.py +92 -0
- acceldata/models/api/catalog/tag_domain.py +116 -0
- acceldata/models/api/catalog/threshold_breach_details.py +101 -0
- acceldata/models/api/catalog/time_based_config.py +92 -0
- acceldata/models/api/catalog/time_based_threshold_config.py +105 -0
- acceldata/models/api/catalog/timestamp_based_marker_config.py +115 -0
- acceldata/models/api/catalog/udf_variable.py +97 -0
- acceldata/models/api/catalog/yunikorn_compute_config.py +133 -0
- acceldata/models/api/catalog/yunikorn_spark_resource_config.py +154 -0
- acceldata/models/api/pipeline/big_decimal.py +100 -0
- acceldata/models/api/pipeline/big_integer.py +98 -0
- acceldata/models/api/pipeline/child_span_response.py +96 -0
- acceldata/models/api/pipeline/create_span_event.py +109 -0
- acceldata/models/api/pipeline/edge.py +108 -0
- acceldata/models/api/pipeline/event_alert_object.py +95 -0
- acceldata/models/api/pipeline/generic_config_fallback.py +101 -0
- acceldata/models/api/pipeline/identifier.py +146 -0
- acceldata/models/api/pipeline/job_request.py +130 -0
- acceldata/models/api/pipeline/meta.py +107 -0
- acceldata/models/api/pipeline/meta_data.py +92 -0
- acceldata/models/api/pipeline/metric.py +103 -0
- acceldata/models/api/pipeline/node.py +201 -0
- acceldata/models/api/pipeline/node_response.py +92 -0
- acceldata/models/api/pipeline/notification_payload.py +114 -0
- acceldata/models/api/pipeline/pipeline.py +152 -0
- acceldata/models/api/pipeline/pipeline_baseline_metric.py +92 -0
- acceldata/models/api/pipeline/pipeline_details.py +118 -0
- acceldata/models/api/pipeline/pipeline_details_response.py +102 -0
- acceldata/models/api/pipeline/pipeline_input_request.py +92 -0
- acceldata/models/api/pipeline/pipeline_listing_info.py +118 -0
- acceldata/models/api/pipeline/pipeline_response.py +92 -0
- acceldata/models/api/pipeline/pipeline_run.py +136 -0
- acceldata/models/api/pipeline/pipeline_run_request.py +92 -0
- acceldata/models/api/pipeline/pipeline_runs_response.py +102 -0
- acceldata/models/api/pipeline/pipeline_summary.py +98 -0
- acceldata/models/api/pipeline/pipelines_listing_response.py +102 -0
- acceldata/models/api/pipeline/policy_summary.py +102 -0
- acceldata/models/api/pipeline/span.py +115 -0
- acceldata/models/api/pipeline/span_event.py +114 -0
- acceldata/models/api/pipeline/span_event_request.py +92 -0
- acceldata/models/api/pipeline/span_event_response.py +92 -0
- acceldata/models/api/pipeline/span_request.py +94 -0
- acceldata/models/api/pipeline/spans_response.py +96 -0
- acceldata/models/api/pipeline/tag.py +97 -0
- acceldata/models/sdk/__init__.py +0 -0
- acceldata/models/sdk/catalog/__init__.py +105 -0
- acceldata/models/sdk/catalog/asset_resource.py +112 -0
- acceldata/models/sdk/catalog/asset_source_type.py +59 -0
- acceldata/models/sdk/catalog/custom_asset_metadata.py +11 -0
- acceldata/models/sdk/catalog/datasource_resource.py +88 -0
- acceldata/models/sdk/catalog/executor.py +524 -0
- acceldata/models/sdk/catalog/policy_execution_request.py +114 -0
- acceldata/models/sdk/catalog/policy_execution_type.py +16 -0
- acceldata/models/sdk/catalog/policy_filter.py +94 -0
- acceldata/models/sdk/catalog/policy_rule_resources.py +388 -0
- acceldata/models/sdk/catalog/profile_request_resource.py +77 -0
- acceldata/models/sdk/catalog/profiling.py +82 -0
- acceldata/models/sdk/catalog/profiling_configuration_error.py +5 -0
- acceldata/models/sdk/pipeline/__init__.py +62 -0
- acceldata/models/sdk/pipeline/create_job_input.py +108 -0
- acceldata/models/sdk/pipeline/create_pipeline_input_request.py +72 -0
- acceldata/models/sdk/pipeline/events/__init__.py +5 -0
- acceldata/models/sdk/pipeline/events/generic_event.py +31 -0
- acceldata/models/sdk/pipeline/events/log_event.py +34 -0
- acceldata/models/sdk/pipeline/events/span_event.py +8 -0
- acceldata/models/sdk/pipeline/pipeline_resource.py +171 -0
- acceldata/models/sdk/pipeline/pipeline_run_input.py +84 -0
- acceldata/models/sdk/pipeline/pipeline_run_resource.py +208 -0
- acceldata/models/sdk/pipeline/pipeline_source_type.py +20 -0
- acceldata/models/sdk/pipeline/span_resource.py +327 -0
- acceldata/openapi_deserialization_patches.py +161 -0
- acceldata/openapi_repr_patches.py +62 -0
- acceldata/services/__init__.py +0 -0
- acceldata/services/api_timestamp.py +31 -0
- acceldata/services/asset_service.py +651 -0
- acceldata/services/datasource_service.py +120 -0
- acceldata/services/generic_service.py +132 -0
- acceldata/services/marker_config.py +232 -0
- acceldata/services/pipeline_big_decimal_api.py +102 -0
- acceldata/services/pipeline_response_parse.py +209 -0
- acceldata/services/pipeline_service.py +494 -0
- acceldata/services/rule_service.py +620 -0
- acceldata/urls/__init__.py +1 -0
- acceldata/urls/catalog_urls.py +137 -0
- acceldata/urls/pipeline_urls.py +55 -0
- acceldata/utils/__init__.py +6 -0
- acceldata/utils/datetime_utils.py +16 -0
- acceldata/utils/operation_ack.py +22 -0
- acceldata/utils/span_helpers.py +95 -0
- acceldata/utils/time_range_utils.py +90 -0
- acceldata_sdk_python-26.7.0.dist-info/METADATA +26 -0
- acceldata_sdk_python-26.7.0.dist-info/RECORD +303 -0
- acceldata_sdk_python-26.7.0.dist-info/WHEEL +5 -0
- acceldata_sdk_python-26.7.0.dist-info/licenses/LICENSE +22 -0
- acceldata_sdk_python-26.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,524 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from enum import IntEnum
|
|
5
|
+
from time import sleep
|
|
6
|
+
from typing import Callable
|
|
7
|
+
|
|
8
|
+
from acceldata.client.transient_retry import (
|
|
9
|
+
RetryConfig,
|
|
10
|
+
effective_transient_retry,
|
|
11
|
+
transient_retry_call,
|
|
12
|
+
)
|
|
13
|
+
from acceldata.exceptions import (
|
|
14
|
+
APIError,
|
|
15
|
+
AcceldataSdkException,
|
|
16
|
+
ApiException,
|
|
17
|
+
InvalidRuleTypeError,
|
|
18
|
+
PolicyExecutionCompletedWithErrorResultError,
|
|
19
|
+
PolicyExecutionCompletedWithWarningError,
|
|
20
|
+
)
|
|
21
|
+
from acceldata.mappers.catalog_executions import (
|
|
22
|
+
execution_status_pair_from_result,
|
|
23
|
+
rule_execution_error_message_from_result,
|
|
24
|
+
rule_execution_id_from_generic_post_response,
|
|
25
|
+
)
|
|
26
|
+
from acceldata.utils.operation_ack import ack_failure, ack_success
|
|
27
|
+
from acceldata.models.api.catalog.data_quality_execution import DataQualityExecution
|
|
28
|
+
from acceldata.models.api.catalog.execution_cancellation_response import ExecutionCancellationResponse
|
|
29
|
+
from acceldata.models.api.catalog.execution_result import ExecutionResult
|
|
30
|
+
from acceldata.models.api.catalog.generic_execution import GenericExecution
|
|
31
|
+
from acceldata.models.api.catalog.generic_rule_execution import GenericRuleExecution
|
|
32
|
+
from acceldata.models.api.catalog.reconciliation_rule_execution import ReconciliationRuleExecution
|
|
33
|
+
from acceldata.models.api.catalog.quality_rule import QualityRule
|
|
34
|
+
from acceldata.models.api.catalog.rule_execution import RuleExecution
|
|
35
|
+
from acceldata.models.api.catalog.rule_execution_result import RuleExecutionResult
|
|
36
|
+
from acceldata.models.sdk.catalog.policy_execution_request import PolicyExecutionRequestInput
|
|
37
|
+
from acceldata.models.sdk.catalog.policy_filter import RuleExecutionStatus, RuleType
|
|
38
|
+
LOGGER = logging.getLogger("sdk.executor")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _execution_result_for_log(er: ExecutionResult) -> str:
|
|
42
|
+
"""Compact log string for :class:`ExecutionResult` without full model serialization."""
|
|
43
|
+
try:
|
|
44
|
+
es, rs = execution_status_pair_from_result(er)
|
|
45
|
+
msg = rule_execution_error_message_from_result(er)
|
|
46
|
+
parts = [f"executionStatus={es}", f"resultStatus={rs}"]
|
|
47
|
+
if msg:
|
|
48
|
+
parts.append(f"executionError={msg}")
|
|
49
|
+
return ", ".join(parts)
|
|
50
|
+
except Exception:
|
|
51
|
+
LOGGER.debug("ExecutionResult compact formatting failed; using str()", exc_info=True)
|
|
52
|
+
return str(er)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _execution_progress_for_log(er: ExecutionResult) -> str | None:
|
|
56
|
+
"""Compact progress summary from `ExecutionResult.items` for poll logs."""
|
|
57
|
+
items = getattr(er, "items", None)
|
|
58
|
+
if not items:
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
terminal_item_statuses = {
|
|
62
|
+
RuleExecutionStatus.SUCCESSFUL.value,
|
|
63
|
+
RuleExecutionStatus.WARNING.value,
|
|
64
|
+
RuleExecutionStatus.ERRORED.value,
|
|
65
|
+
RuleExecutionStatus.ABORTED.value,
|
|
66
|
+
RuleExecutionStatus.SKIPPED.value,
|
|
67
|
+
}
|
|
68
|
+
status_counts: dict[str, int] = {}
|
|
69
|
+
completed = 0
|
|
70
|
+
result_percents: list[float] = []
|
|
71
|
+
|
|
72
|
+
for item in items:
|
|
73
|
+
st = getattr(item, "status", None) or "UNKNOWN"
|
|
74
|
+
status_counts[st] = status_counts.get(st, 0) + 1
|
|
75
|
+
if st in terminal_item_statuses:
|
|
76
|
+
completed += 1
|
|
77
|
+
rp = getattr(item, "result_percent", None)
|
|
78
|
+
if isinstance(rp, (int, float)):
|
|
79
|
+
result_percents.append(float(rp))
|
|
80
|
+
|
|
81
|
+
total = len(items)
|
|
82
|
+
percent_done = (completed / total) * 100.0 if total else 0.0
|
|
83
|
+
status_parts = ",".join(f"{k}:{status_counts[k]}" for k in sorted(status_counts))
|
|
84
|
+
msg = f"items={completed}/{total} ({percent_done:.1f}%), itemStatuses=[{status_parts}]"
|
|
85
|
+
if result_percents:
|
|
86
|
+
avg_percent = sum(result_percents) / len(result_percents)
|
|
87
|
+
msg = f"{msg}, avgItemResultPercent={avg_percent:.2f}"
|
|
88
|
+
return msg
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _errored_execution_result(message: str) -> ExecutionResult:
|
|
92
|
+
es = RuleExecutionStatus.ERRORED.value
|
|
93
|
+
re = RuleExecution.model_construct(
|
|
94
|
+
execution_status=es,
|
|
95
|
+
result_status=es,
|
|
96
|
+
execution_error=message,
|
|
97
|
+
)
|
|
98
|
+
ge = GenericRuleExecution(re)
|
|
99
|
+
rer = RuleExecutionResult.model_construct(
|
|
100
|
+
rule_execution_type="DQ_RULE_TYPE_EXECUTION",
|
|
101
|
+
status=es,
|
|
102
|
+
)
|
|
103
|
+
return ExecutionResult.model_construct(execution=ge, items=[], meta={}, result=rer)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class FailureStrategy(IntEnum):
|
|
107
|
+
DoNotFail = 0
|
|
108
|
+
FailOnWarning = 1
|
|
109
|
+
FailOnError = 2
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# Poll loops: ``total_retries == POLICY_POLL_UNLIMITED_RETRIES`` (-1) means no cap on RUNNING/WAITING rounds.
|
|
113
|
+
POLICY_POLL_UNLIMITED_RETRIES: int = -1
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# ``Executor`` / ``execute_policy`` — catalog POST + poll helpers wired for these kinds only.
|
|
117
|
+
_EXECUTOR_RULE_TYPES: tuple[RuleType, ...] = (
|
|
118
|
+
RuleType.DATA_QUALITY,
|
|
119
|
+
RuleType.RECONCILIATION,
|
|
120
|
+
RuleType.DATA_CADENCE,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def get_callables(rule_type: RuleType, adoc_client) -> tuple[Callable, Callable]:
|
|
125
|
+
if rule_type == RuleType.RECONCILIATION:
|
|
126
|
+
return adoc_client.execute_reconciliation_rule, adoc_client.get_reconciliation_rule_result
|
|
127
|
+
if rule_type == RuleType.DATA_QUALITY:
|
|
128
|
+
return adoc_client.execute_dq_rule, adoc_client.get_dq_rule_result
|
|
129
|
+
if rule_type == RuleType.DATA_CADENCE:
|
|
130
|
+
return adoc_client.execute_freshness_rule, adoc_client.get_freshness_rule_result
|
|
131
|
+
allowed_s = ", ".join(t.name for t in _EXECUTOR_RULE_TYPES)
|
|
132
|
+
raise InvalidRuleTypeError(
|
|
133
|
+
f"Rule type {rule_type!r} is not supported for execution. Use one of: {allowed_s}.",
|
|
134
|
+
allowed_rule_types=tuple(t.name for t in _EXECUTOR_RULE_TYPES),
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def get_result_callable_for_policy_polling(rule_type: RuleType, adoc_client) -> Callable[..., ExecutionResult]:
|
|
139
|
+
"""GET ``.../executions/{{id}}/result`` for kinds that expose that contract (not used by :class:`Executor` alone)."""
|
|
140
|
+
if rule_type == RuleType.RECONCILIATION:
|
|
141
|
+
return adoc_client.get_reconciliation_rule_result
|
|
142
|
+
if rule_type == RuleType.DATA_QUALITY:
|
|
143
|
+
return adoc_client.get_dq_rule_result
|
|
144
|
+
if rule_type == RuleType.DATA_CADENCE:
|
|
145
|
+
return adoc_client.get_freshness_rule_result
|
|
146
|
+
allowed_poll = ("DATA_QUALITY", "RECONCILIATION", "DATA_CADENCE")
|
|
147
|
+
raise InvalidRuleTypeError(
|
|
148
|
+
f"policy execution status/result polling is not implemented for {rule_type.name}; "
|
|
149
|
+
f"supported kinds: {', '.join(allowed_poll)}. For other rule types use policy_executions().",
|
|
150
|
+
allowed_rule_types=allowed_poll,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def policy_execution_status_via_result(
|
|
155
|
+
result_call: Callable[..., ExecutionResult],
|
|
156
|
+
execution_id: int | str,
|
|
157
|
+
retry_cfg: RetryConfig | None,
|
|
158
|
+
) -> RuleExecutionStatus:
|
|
159
|
+
"""One-shot status from the catalog execution result payload (same behavior as :meth:`Executor.get_execution_status`)."""
|
|
160
|
+
try:
|
|
161
|
+
details = transient_retry_call(
|
|
162
|
+
lambda: result_call(execution_id=execution_id),
|
|
163
|
+
retry_cfg,
|
|
164
|
+
logger=LOGGER,
|
|
165
|
+
operation_name="get_execution_status",
|
|
166
|
+
)
|
|
167
|
+
try:
|
|
168
|
+
_, rs = execution_status_pair_from_result(details)
|
|
169
|
+
return RuleExecutionStatus(rs)
|
|
170
|
+
except ValueError:
|
|
171
|
+
return RuleExecutionStatus.ERRORED
|
|
172
|
+
except APIError as e:
|
|
173
|
+
ack_failure(
|
|
174
|
+
LOGGER,
|
|
175
|
+
"get_execution_status",
|
|
176
|
+
"http_error",
|
|
177
|
+
status_code=e.status_code,
|
|
178
|
+
message=e.message,
|
|
179
|
+
execution_id=execution_id,
|
|
180
|
+
)
|
|
181
|
+
return RuleExecutionStatus.ERRORED
|
|
182
|
+
except ApiException as e:
|
|
183
|
+
ack_failure(
|
|
184
|
+
LOGGER,
|
|
185
|
+
"get_execution_status",
|
|
186
|
+
"transport_error",
|
|
187
|
+
reason=e.reason,
|
|
188
|
+
execution_id=execution_id,
|
|
189
|
+
)
|
|
190
|
+
return RuleExecutionStatus.ERRORED
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def policy_execution_result_via_poll(
|
|
194
|
+
result_call: Callable[..., ExecutionResult],
|
|
195
|
+
execution_id: int | str,
|
|
196
|
+
*,
|
|
197
|
+
sleep_interval: float,
|
|
198
|
+
failure_strategy: FailureStrategy,
|
|
199
|
+
retry_cfg: RetryConfig | None,
|
|
200
|
+
total_retries: int = POLICY_POLL_UNLIMITED_RETRIES,
|
|
201
|
+
) -> ExecutionResult:
|
|
202
|
+
"""Poll until terminal (same behavior as :meth:`Executor.get_execution_result`).
|
|
203
|
+
|
|
204
|
+
``total_retries`` — max non-terminal (RUNNING/WAITING) poll rounds after which polling stops
|
|
205
|
+
with a synthetic ERRORED result. Default ``POLICY_POLL_UNLIMITED_RETRIES`` (``-1``) means no cap;
|
|
206
|
+
an integer ``>= 1`` sets that cap.
|
|
207
|
+
"""
|
|
208
|
+
retry_count = 0
|
|
209
|
+
execution_details: ExecutionResult | None = None
|
|
210
|
+
try:
|
|
211
|
+
while True:
|
|
212
|
+
execution_details = transient_retry_call(
|
|
213
|
+
lambda: result_call(execution_id=execution_id),
|
|
214
|
+
retry_cfg,
|
|
215
|
+
logger=LOGGER,
|
|
216
|
+
operation_name="get_execution_result",
|
|
217
|
+
)
|
|
218
|
+
st, rs = execution_status_pair_from_result(execution_details)
|
|
219
|
+
progress = _execution_progress_for_log(execution_details)
|
|
220
|
+
if progress:
|
|
221
|
+
LOGGER.info(
|
|
222
|
+
"Policy execution %s poll #%s: executionStatus=%s resultStatus=%s, %s",
|
|
223
|
+
execution_id,
|
|
224
|
+
retry_count + 1,
|
|
225
|
+
st,
|
|
226
|
+
rs,
|
|
227
|
+
progress,
|
|
228
|
+
)
|
|
229
|
+
else:
|
|
230
|
+
LOGGER.info(
|
|
231
|
+
"Policy execution %s poll #%s: executionStatus=%s resultStatus=%s",
|
|
232
|
+
execution_id,
|
|
233
|
+
retry_count + 1,
|
|
234
|
+
st,
|
|
235
|
+
rs,
|
|
236
|
+
)
|
|
237
|
+
if st in (
|
|
238
|
+
RuleExecutionStatus.RUNNING.value,
|
|
239
|
+
RuleExecutionStatus.WAITING.value,
|
|
240
|
+
):
|
|
241
|
+
sleep(sleep_interval)
|
|
242
|
+
retry_count += 1
|
|
243
|
+
if total_retries == POLICY_POLL_UNLIMITED_RETRIES or retry_count < total_retries:
|
|
244
|
+
continue
|
|
245
|
+
msg = f"Exiting after {total_retries} non-terminal poll rounds."
|
|
246
|
+
ack_failure(
|
|
247
|
+
LOGGER,
|
|
248
|
+
"get_execution_result",
|
|
249
|
+
"poll_cap_exhausted",
|
|
250
|
+
execution_id=execution_id,
|
|
251
|
+
total_retries=total_retries,
|
|
252
|
+
)
|
|
253
|
+
execution_details = _errored_execution_result(msg)
|
|
254
|
+
break
|
|
255
|
+
ack_success(
|
|
256
|
+
LOGGER,
|
|
257
|
+
"get_execution_result",
|
|
258
|
+
execution_id=execution_id,
|
|
259
|
+
execution_status=st,
|
|
260
|
+
result_status=rs,
|
|
261
|
+
)
|
|
262
|
+
break
|
|
263
|
+
except (APIError, ApiException) as e:
|
|
264
|
+
kind = "http error" if isinstance(e, APIError) else "transport error"
|
|
265
|
+
ack_failure(
|
|
266
|
+
LOGGER,
|
|
267
|
+
"get_execution_result",
|
|
268
|
+
"transport_or_http_error",
|
|
269
|
+
kind=kind,
|
|
270
|
+
execution_id=execution_id,
|
|
271
|
+
error=str(e),
|
|
272
|
+
)
|
|
273
|
+
LOGGER.error("Rule execution failed (%s): %s", kind, e)
|
|
274
|
+
execution_details = _errored_execution_result(
|
|
275
|
+
f"Rule execution failed due to {kind}. Details:{e}"
|
|
276
|
+
)
|
|
277
|
+
if failure_strategy >= FailureStrategy.FailOnError:
|
|
278
|
+
em = rule_execution_error_message_from_result(execution_details)
|
|
279
|
+
raise AcceldataSdkException(em or str(e)) from e
|
|
280
|
+
|
|
281
|
+
assert execution_details is not None
|
|
282
|
+
|
|
283
|
+
_, rs = execution_status_pair_from_result(execution_details)
|
|
284
|
+
if rs == RuleExecutionStatus.SUCCESSFUL.value:
|
|
285
|
+
ack_success(
|
|
286
|
+
LOGGER,
|
|
287
|
+
"policy_execution_outcome",
|
|
288
|
+
execution_id=execution_id,
|
|
289
|
+
result_status=rs,
|
|
290
|
+
)
|
|
291
|
+
elif rs == RuleExecutionStatus.WARNING.value:
|
|
292
|
+
ack_success(
|
|
293
|
+
LOGGER,
|
|
294
|
+
"policy_execution_outcome",
|
|
295
|
+
execution_id=execution_id,
|
|
296
|
+
result_status=rs,
|
|
297
|
+
note="warning",
|
|
298
|
+
)
|
|
299
|
+
if failure_strategy == FailureStrategy.FailOnWarning:
|
|
300
|
+
raise PolicyExecutionCompletedWithWarningError(
|
|
301
|
+
f"Execution completed with warning. Details: {execution_details}",
|
|
302
|
+
details=execution_details,
|
|
303
|
+
)
|
|
304
|
+
else:
|
|
305
|
+
ack_failure(
|
|
306
|
+
LOGGER,
|
|
307
|
+
"policy_execution_outcome",
|
|
308
|
+
"terminal_error_result",
|
|
309
|
+
execution_id=execution_id,
|
|
310
|
+
result_status=rs,
|
|
311
|
+
detail=_execution_result_for_log(execution_details),
|
|
312
|
+
)
|
|
313
|
+
if failure_strategy >= FailureStrategy.FailOnError:
|
|
314
|
+
raise PolicyExecutionCompletedWithErrorResultError(
|
|
315
|
+
f"Rule execution completed with errors. Details: {execution_details}",
|
|
316
|
+
details=execution_details,
|
|
317
|
+
)
|
|
318
|
+
return execution_details
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
class Executor:
|
|
322
|
+
def __init__(
|
|
323
|
+
self,
|
|
324
|
+
rule_type: RuleType,
|
|
325
|
+
adoc_client,
|
|
326
|
+
sync: bool = True,
|
|
327
|
+
*,
|
|
328
|
+
transient_retry: RetryConfig | None = None,
|
|
329
|
+
):
|
|
330
|
+
self.id: int | None = None
|
|
331
|
+
self.error_message: str | None = None
|
|
332
|
+
self.message: str | None = None
|
|
333
|
+
self.sync = sync
|
|
334
|
+
self.client = adoc_client
|
|
335
|
+
self._transient_retry = transient_retry
|
|
336
|
+
self._exec_call, self._result_call = get_callables(rule_type, adoc_client)
|
|
337
|
+
|
|
338
|
+
def _resolved_poll_retry(self, explicit: RetryConfig | None) -> RetryConfig | None:
|
|
339
|
+
"""Per-call ``explicit`` overrides the executor-level default from ``Executor(..., transient_retry=...)``."""
|
|
340
|
+
return effective_transient_retry(explicit, self._transient_retry)
|
|
341
|
+
|
|
342
|
+
def get_status(self) -> RuleExecutionStatus:
|
|
343
|
+
if self.id is None:
|
|
344
|
+
raise ValueError(
|
|
345
|
+
"A valid execution id is required: run execute() first, or set executor.id to an existing execution id."
|
|
346
|
+
)
|
|
347
|
+
return self.get_execution_status(self.id)
|
|
348
|
+
|
|
349
|
+
def get_execution_status(
|
|
350
|
+
self,
|
|
351
|
+
execution_id: int | str,
|
|
352
|
+
*,
|
|
353
|
+
transient_retry: RetryConfig | None = None,
|
|
354
|
+
) -> RuleExecutionStatus:
|
|
355
|
+
cfg = self._resolved_poll_retry(transient_retry)
|
|
356
|
+
status = policy_execution_status_via_result(self._result_call, execution_id, cfg)
|
|
357
|
+
self.message = (
|
|
358
|
+
f"Policy execution {execution_id} current status is {status.value}."
|
|
359
|
+
)
|
|
360
|
+
return status
|
|
361
|
+
|
|
362
|
+
def get_result(
|
|
363
|
+
self,
|
|
364
|
+
sleep_interval: float = 5.0,
|
|
365
|
+
total_retries: int = POLICY_POLL_UNLIMITED_RETRIES,
|
|
366
|
+
failure_strategy: FailureStrategy = FailureStrategy.DoNotFail,
|
|
367
|
+
*,
|
|
368
|
+
transient_retry: RetryConfig | None = None,
|
|
369
|
+
) -> ExecutionResult:
|
|
370
|
+
"""Poll until this executor's run is terminal; same as :meth:`get_execution_result` for ``execution_id=self.id``.
|
|
371
|
+
|
|
372
|
+
Call after :meth:`execute` or ``AdocClient.execute_policy`` (``sync=False``) once ``self.id`` is set.
|
|
373
|
+
When ``sync=True`` (the default), this is already invoked inside :meth:`execute`.
|
|
374
|
+
|
|
375
|
+
``sleep_interval`` — seconds between polls while status is RUNNING or WAITING.
|
|
376
|
+
|
|
377
|
+
``total_retries`` — max non-terminal poll rounds; default ``POLICY_POLL_UNLIMITED_RETRIES`` (``-1``) polls
|
|
378
|
+
until terminal or HTTP failure; an integer ``>= 1`` stops early with a synthetic ERRORED payload
|
|
379
|
+
(see :meth:`get_execution_result`).
|
|
380
|
+
|
|
381
|
+
``failure_strategy`` — whether to raise when the finished run has WARNING or ERROR result status.
|
|
382
|
+
|
|
383
|
+
``transient_retry`` — optional per-call :class:`~acceldata.client.transient_retry.RetryConfig` for
|
|
384
|
+
each GET; overrides ``Executor(..., transient_retry=...)``.
|
|
385
|
+
"""
|
|
386
|
+
if self.id is None:
|
|
387
|
+
raise ValueError(
|
|
388
|
+
"A valid execution id is required: run execute() first, or set executor.id to an existing execution id."
|
|
389
|
+
)
|
|
390
|
+
return self.get_execution_result(
|
|
391
|
+
execution_id=self.id,
|
|
392
|
+
sleep_interval=sleep_interval,
|
|
393
|
+
total_retries=total_retries,
|
|
394
|
+
failure_strategy=failure_strategy,
|
|
395
|
+
transient_retry=transient_retry,
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
def get_execution_result(
|
|
399
|
+
self,
|
|
400
|
+
execution_id: int | str,
|
|
401
|
+
sleep_interval: float = 5.0,
|
|
402
|
+
total_retries: int = POLICY_POLL_UNLIMITED_RETRIES,
|
|
403
|
+
failure_strategy: FailureStrategy = FailureStrategy.DoNotFail,
|
|
404
|
+
*,
|
|
405
|
+
transient_retry: RetryConfig | None = None,
|
|
406
|
+
) -> ExecutionResult:
|
|
407
|
+
"""Wait until the execution leaves RUNNING/WAITING, then return :class:`~acceldata.models.api.catalog.execution_result.ExecutionResult`.
|
|
408
|
+
|
|
409
|
+
``total_retries`` limits how many *RUNNING/WAITING* poll rounds apply before giving up; default
|
|
410
|
+
``POLICY_POLL_UNLIMITED_RETRIES`` (``-1``) means no limit until terminal.
|
|
411
|
+
On HTTP or other errors while waiting, an ERRORED result is produced and ``failure_strategy`` may raise.
|
|
412
|
+
|
|
413
|
+
When ``transient_retry`` is set (or inherited from ``Executor(..., transient_retry=...)``),
|
|
414
|
+
each GET retries transient HTTP/transport failures before treating the request as failed.
|
|
415
|
+
"""
|
|
416
|
+
cfg = self._resolved_poll_retry(transient_retry)
|
|
417
|
+
result = policy_execution_result_via_poll(
|
|
418
|
+
self._result_call,
|
|
419
|
+
execution_id,
|
|
420
|
+
sleep_interval=sleep_interval,
|
|
421
|
+
failure_strategy=failure_strategy,
|
|
422
|
+
retry_cfg=cfg,
|
|
423
|
+
total_retries=total_retries,
|
|
424
|
+
)
|
|
425
|
+
_, rs = execution_status_pair_from_result(result)
|
|
426
|
+
self.message = (
|
|
427
|
+
f"Policy execution {execution_id} completed with result status {rs}."
|
|
428
|
+
)
|
|
429
|
+
return result
|
|
430
|
+
|
|
431
|
+
def execute(
|
|
432
|
+
self,
|
|
433
|
+
rule_id: int | str,
|
|
434
|
+
policy_execution_request: PolicyExecutionRequestInput,
|
|
435
|
+
sleep_interval: float = 5.0,
|
|
436
|
+
total_retries: int = POLICY_POLL_UNLIMITED_RETRIES,
|
|
437
|
+
failure_strategy: FailureStrategy = FailureStrategy.DoNotFail,
|
|
438
|
+
*,
|
|
439
|
+
transient_retry: RetryConfig | None = None,
|
|
440
|
+
) -> Executor:
|
|
441
|
+
"""When ``transient_retry`` is set (or inherited from ``Executor(..., transient_retry=...)``),
|
|
442
|
+
the initial start request (POST) uses the same retry policy as result retrieval in ``sync=True`` mode.
|
|
443
|
+
|
|
444
|
+
``total_retries`` is passed to :meth:`get_execution_result` when ``sync=True``; see that method and
|
|
445
|
+
``POLICY_POLL_UNLIMITED_RETRIES`` for semantics.
|
|
446
|
+
"""
|
|
447
|
+
cfg = self._resolved_poll_retry(transient_retry)
|
|
448
|
+
try:
|
|
449
|
+
execution_obj: GenericExecution = transient_retry_call(
|
|
450
|
+
lambda: self._exec_call(
|
|
451
|
+
rule_id=rule_id,
|
|
452
|
+
policy_execution_request=policy_execution_request,
|
|
453
|
+
),
|
|
454
|
+
cfg,
|
|
455
|
+
logger=LOGGER,
|
|
456
|
+
operation_name="start_policy_execution",
|
|
457
|
+
)
|
|
458
|
+
self.id = rule_execution_id_from_generic_post_response(execution_obj)
|
|
459
|
+
if self.id is None:
|
|
460
|
+
raise ValueError(
|
|
461
|
+
"The catalog response did not include an execution id after starting the rule. "
|
|
462
|
+
"Try execute_*_rule directly and inspect the response."
|
|
463
|
+
)
|
|
464
|
+
ack_success(
|
|
465
|
+
LOGGER,
|
|
466
|
+
"start_policy_execution",
|
|
467
|
+
rule_id=rule_id,
|
|
468
|
+
execution_id=self.id,
|
|
469
|
+
)
|
|
470
|
+
if self.sync:
|
|
471
|
+
self.get_execution_result(
|
|
472
|
+
self.id,
|
|
473
|
+
sleep_interval=sleep_interval,
|
|
474
|
+
total_retries=total_retries,
|
|
475
|
+
failure_strategy=failure_strategy,
|
|
476
|
+
transient_retry=transient_retry,
|
|
477
|
+
)
|
|
478
|
+
if self.message is None:
|
|
479
|
+
self.message = (
|
|
480
|
+
f"Policy execution {self.id} completed."
|
|
481
|
+
)
|
|
482
|
+
else:
|
|
483
|
+
self.message = (
|
|
484
|
+
f"Policy execution triggered successfully with execution id {self.id}."
|
|
485
|
+
)
|
|
486
|
+
return self
|
|
487
|
+
except (APIError, ApiException, ValueError) as e:
|
|
488
|
+
self.error_message = f"Rule execution failed: {e}"
|
|
489
|
+
self.message = self.error_message
|
|
490
|
+
LOGGER.warning(
|
|
491
|
+
"Executor.execute failed (rule_id=%r, sync=%s): %s",
|
|
492
|
+
rule_id,
|
|
493
|
+
self.sync,
|
|
494
|
+
e,
|
|
495
|
+
exc_info=LOGGER.isEnabledFor(logging.DEBUG),
|
|
496
|
+
)
|
|
497
|
+
if failure_strategy >= FailureStrategy.FailOnError:
|
|
498
|
+
raise
|
|
499
|
+
return self
|
|
500
|
+
|
|
501
|
+
def cancel(self) -> ExecutionCancellationResponse:
|
|
502
|
+
if self.id is None:
|
|
503
|
+
raise ValueError("A valid execution id is required before cancellation.")
|
|
504
|
+
out = self.client.cancel_rule_execution(self.id)
|
|
505
|
+
msg = getattr(getattr(out, "result", None), "message", None)
|
|
506
|
+
self.message = msg or f"Cancellation requested for policy execution {self.id}."
|
|
507
|
+
return out
|
|
508
|
+
|
|
509
|
+
def get_dq_rule_execution_details(self, execution_id: str | int) -> DataQualityExecution:
|
|
510
|
+
return self.client.get_dq_rule_execution_details(execution_id)
|
|
511
|
+
|
|
512
|
+
def get_reconciliation_rule_execution_details(
|
|
513
|
+
self, execution_id: str | int
|
|
514
|
+
) -> ReconciliationRuleExecution:
|
|
515
|
+
return self.client.get_reconciliation_rule_execution_details(execution_id)
|
|
516
|
+
|
|
517
|
+
def cancel_rule_execution(self, execution_id: str | int) -> ExecutionCancellationResponse:
|
|
518
|
+
return self.client.cancel_rule_execution(execution_id)
|
|
519
|
+
|
|
520
|
+
def enable_rule(self, rule_id: str | int) -> QualityRule:
|
|
521
|
+
return self.client.enable_rule(rule_id)
|
|
522
|
+
|
|
523
|
+
def disable_rule(self, rule_id: str | int) -> QualityRule:
|
|
524
|
+
return self.client.disable_rule(rule_id)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, List, Optional, TypeAlias, Union
|
|
4
|
+
|
|
5
|
+
from acceldata.models.api.catalog.column_variable import ColumnVariable
|
|
6
|
+
from acceldata.models.api.catalog.policy_execution_marker_config import PolicyExecutionMarkerConfig
|
|
7
|
+
from acceldata.models.api.catalog.policy_execution_request import (
|
|
8
|
+
PolicyExecutionRequest as CatalogPolicyExecutionRequest,
|
|
9
|
+
)
|
|
10
|
+
from acceldata.models.api.catalog.rule_spark_sql_dynamic_filter_variable_mapping import (
|
|
11
|
+
RuleSparkSQLDynamicFilterVariableMapping,
|
|
12
|
+
)
|
|
13
|
+
from acceldata.models.api.catalog.spark_resource_config import SparkResourceConfig
|
|
14
|
+
from acceldata.models.sdk.catalog.policy_execution_type import (
|
|
15
|
+
PolicyExecutionType,
|
|
16
|
+
policy_execution_type_wire,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# Accept either the SDK convenience wrapper or the generated catalog request model.
|
|
20
|
+
PolicyExecutionRequestInput: TypeAlias = Union["PolicyExecutionInput", CatalogPolicyExecutionRequest]
|
|
21
|
+
|
|
22
|
+
# Catalog wire value for policy execution requests built from :class:`PolicyExecutionInput`.
|
|
23
|
+
_SDK_POLICY_EXECUTION_RESOURCE_STRATEGY = "INVENTORY"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ensure_catalog_policy_execution_request(
|
|
27
|
+
req: PolicyExecutionRequestInput,
|
|
28
|
+
) -> CatalogPolicyExecutionRequest:
|
|
29
|
+
"""Normalize execution payloads to :class:`CatalogPolicyExecutionRequest`."""
|
|
30
|
+
if isinstance(req, CatalogPolicyExecutionRequest):
|
|
31
|
+
return req
|
|
32
|
+
if isinstance(req, PolicyExecutionInput):
|
|
33
|
+
return req.to_api()
|
|
34
|
+
raise TypeError(
|
|
35
|
+
"policy_execution_request must be PolicyExecutionInput (PolicyExecutionRequest) or "
|
|
36
|
+
f"acceldata.models.api.catalog.policy_execution_request.PolicyExecutionRequest, got {type(req)!r}"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def policy_request_with_pipeline_run_id(
|
|
41
|
+
req: PolicyExecutionRequestInput,
|
|
42
|
+
pipeline_run_id: int | None,
|
|
43
|
+
) -> PolicyExecutionRequestInput:
|
|
44
|
+
"""Return ``req`` unchanged when ``pipeline_run_id`` is ``None``.
|
|
45
|
+
|
|
46
|
+
Otherwise return a :class:`CatalogPolicyExecutionRequest` copy with
|
|
47
|
+
``pipeline_run_id`` set; an explicit value overrides any pipeline id already on ``req``.
|
|
48
|
+
"""
|
|
49
|
+
if pipeline_run_id is None:
|
|
50
|
+
return req
|
|
51
|
+
api = ensure_catalog_policy_execution_request(req)
|
|
52
|
+
return api.model_copy(update={"pipeline_run_id": pipeline_run_id})
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class PolicyExecutionInput:
|
|
56
|
+
"""Convenience wrapper over :class:`CatalogPolicyExecutionRequest`.
|
|
57
|
+
|
|
58
|
+
Uses **camelCase** parameter and attribute names matching the catalog execution JSON
|
|
59
|
+
(``executionType``, ``markerConfigs``, ``ruleItemSelections``, ``includeInQualityScore``,
|
|
60
|
+
``pipelineRunId``, ``sparkSQLDynamicFilterVariableMapping``, ``sparkFilterSelectedColumns``,
|
|
61
|
+
``sparkResourceConfig``).
|
|
62
|
+
|
|
63
|
+
The catalog request also requires **column variables** on the wire. This class defaults to
|
|
64
|
+
``columnVariables=[]``. Resource strategy is fixed for SDK-originated runs; use
|
|
65
|
+
:class:`CatalogPolicyExecutionRequest` directly if you need full control of every field.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
executionType: PolicyExecutionType | str,
|
|
71
|
+
*,
|
|
72
|
+
markerConfigs: Optional[List[PolicyExecutionMarkerConfig]] = None,
|
|
73
|
+
ruleItemSelections: Optional[List[int]] = None,
|
|
74
|
+
includeInQualityScore: bool = True,
|
|
75
|
+
pipelineRunId: Optional[int] = None,
|
|
76
|
+
columnVariables: Optional[List[ColumnVariable]] = None,
|
|
77
|
+
sparkSQLDynamicFilterVariableMapping: Optional[
|
|
78
|
+
List[RuleSparkSQLDynamicFilterVariableMapping]
|
|
79
|
+
] = None,
|
|
80
|
+
sparkFilterSelectedColumns: Optional[List[str]] = None,
|
|
81
|
+
sparkResourceConfig: Optional[SparkResourceConfig] = None,
|
|
82
|
+
) -> None:
|
|
83
|
+
self.executionType = executionType
|
|
84
|
+
self.markerConfigs = markerConfigs
|
|
85
|
+
self.ruleItemSelections = ruleItemSelections
|
|
86
|
+
self.includeInQualityScore = includeInQualityScore
|
|
87
|
+
self.pipelineRunId = pipelineRunId
|
|
88
|
+
self.columnVariables = columnVariables
|
|
89
|
+
self.sparkSQLDynamicFilterVariableMapping = (
|
|
90
|
+
sparkSQLDynamicFilterVariableMapping
|
|
91
|
+
if sparkSQLDynamicFilterVariableMapping is not None
|
|
92
|
+
else []
|
|
93
|
+
)
|
|
94
|
+
self.sparkFilterSelectedColumns = sparkFilterSelectedColumns
|
|
95
|
+
self.sparkResourceConfig = sparkResourceConfig
|
|
96
|
+
|
|
97
|
+
def to_api(self) -> CatalogPolicyExecutionRequest:
|
|
98
|
+
exec_wire = policy_execution_type_wire(self.executionType)
|
|
99
|
+
mapping = self.sparkSQLDynamicFilterVariableMapping
|
|
100
|
+
cols = self.columnVariables if self.columnVariables is not None else []
|
|
101
|
+
data: dict[str, Any] = {
|
|
102
|
+
# Use wire aliases to support generated models that validate by alias only.
|
|
103
|
+
"executionType": exec_wire,
|
|
104
|
+
"markerConfigs": self.markerConfigs,
|
|
105
|
+
"ruleItemSelections": self.ruleItemSelections,
|
|
106
|
+
"includeInQualityScore": self.includeInQualityScore,
|
|
107
|
+
"pipelineRunId": self.pipelineRunId,
|
|
108
|
+
"columnVariables": cols,
|
|
109
|
+
"resourceStrategyType": _SDK_POLICY_EXECUTION_RESOURCE_STRATEGY,
|
|
110
|
+
"sparkSQLDynamicFilterVariableMapping": mapping if mapping else None,
|
|
111
|
+
"sparkFilterSelectedColumns": self.sparkFilterSelectedColumns,
|
|
112
|
+
"sparkResourceConfig": self.sparkResourceConfig,
|
|
113
|
+
}
|
|
114
|
+
return CatalogPolicyExecutionRequest.model_validate(data)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from enum import Enum
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class PolicyExecutionType(str, Enum):
|
|
7
|
+
"""Wire values for ``executionType`` on policy execution APIs (DQ, recon, freshness, etc.)."""
|
|
8
|
+
|
|
9
|
+
SELECTIVE = "SELECTIVE"
|
|
10
|
+
FULL = "FULL"
|
|
11
|
+
INCREMENTAL = "INCREMENTAL"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def policy_execution_type_wire(value: PolicyExecutionType | str) -> str:
|
|
15
|
+
"""Return the API string for ``executionType``."""
|
|
16
|
+
return value.value if isinstance(value, PolicyExecutionType) else value
|