precis-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +16 -0
- app/api/__init__.py +32 -0
- app/api/dependencies.py +185 -0
- app/api/main.py +301 -0
- app/api/middleware/__init__.py +34 -0
- app/api/middleware/exception_handler.py +126 -0
- app/api/middleware/request_logging.py +97 -0
- app/api/middleware/token_auth.py +215 -0
- app/api/models/__init__.py +151 -0
- app/api/models/connection_rules.py +98 -0
- app/api/models/files.py +99 -0
- app/api/models/full_validation.py +554 -0
- app/api/models/project.py +117 -0
- app/api/models/projects.py +80 -0
- app/api/models/schema.py +121 -0
- app/api/models/v2_responses.py +46 -0
- app/api/models/validation.py +303 -0
- app/api/models/workspace.py +118 -0
- app/api/routers/__init__.py +56 -0
- app/api/routers/ai/__init__.py +36 -0
- app/api/routers/ai/chat.py +243 -0
- app/api/routers/ai/generate.py +156 -0
- app/api/routers/ai/hardware.py +199 -0
- app/api/routers/ai/jobs.py +544 -0
- app/api/routers/ai/migrate.py +471 -0
- app/api/routers/ai/models.py +352 -0
- app/api/routers/ai/ollama.py +108 -0
- app/api/routers/ai/providers.py +491 -0
- app/api/routers/ai/router.py +44 -0
- app/api/routers/ai/stream.py +281 -0
- app/api/routers/ai/utils.py +182 -0
- app/api/routers/core/__init__.py +34 -0
- app/api/routers/core/connection_rules.py +188 -0
- app/api/routers/core/data_sources.py +374 -0
- app/api/routers/core/regex.py +333 -0
- app/api/routers/core/reporting.py +263 -0
- app/api/routers/files/__init__.py +24 -0
- app/api/routers/files/ops.py +179 -0
- app/api/routers/files/transfer.py +101 -0
- app/api/routers/preview/__init__.py +35 -0
- app/api/routers/preview/content_mode.py +264 -0
- app/api/routers/preview/header_row.py +154 -0
- app/api/routers/preview/models.py +81 -0
- app/api/routers/preview/path_mode.py +440 -0
- app/api/routers/preview/router.py +30 -0
- app/api/routers/project/__init__.py +61 -0
- app/api/routers/project/base.py +111 -0
- app/api/routers/project/constraint.py +345 -0
- app/api/routers/project/full_config.py +411 -0
- app/api/routers/project/full_config_writer.py +327 -0
- app/api/routers/project/helpers.py +167 -0
- app/api/routers/project/inspection_fix.py +330 -0
- app/api/routers/project/manifest.py +605 -0
- app/api/routers/project/models.py +177 -0
- app/api/routers/project/pattern.py +268 -0
- app/api/routers/project/regex.py +358 -0
- app/api/routers/project/scanner.py +157 -0
- app/api/routers/project/schema.py +567 -0
- app/api/routers/project/schema_helpers.py +149 -0
- app/api/routers/project/settings.py +399 -0
- app/api/routers/project/template.py +325 -0
- app/api/routers/project/validation.py +227 -0
- app/api/routers/project/view.py +177 -0
- app/api/routers/project/workspaces.py +167 -0
- app/api/routers/projects/__init__.py +25 -0
- app/api/routers/projects/create.py +121 -0
- app/api/routers/projects/open.py +103 -0
- app/api/routers/projects/scan.py +100 -0
- app/api/routers/validation/__init__.py +39 -0
- app/api/routers/validation/common.py +263 -0
- app/api/routers/validation/content_mode.py +401 -0
- app/api/routers/validation/history.py +235 -0
- app/api/routers/validation/inline_mode.py +196 -0
- app/api/routers/validation/path_mode.py +314 -0
- app/api/routers/validation/router.py +51 -0
- app/api/services/full_validation_response_builder.py +412 -0
- app/api/services/io_error_messages.py +41 -0
- app/api/services/preview_service.py +248 -0
- app/cli/__init__.py +18 -0
- app/cli/__main__.py +30 -0
- app/cli/shared_services/__init__.py +42 -0
- app/cli/shared_services/config_ops.py +581 -0
- app/cli/shared_services/generation_ops.py +160 -0
- app/cli/shared_services/project_ops.py +288 -0
- app/cli/shell/__init__.py +48 -0
- app/cli/shell/commands/__init__.py +63 -0
- app/cli/shell/commands/ai/__init__.py +260 -0
- app/cli/shell/commands/ai/chat.py +331 -0
- app/cli/shell/commands/ai/delete.py +165 -0
- app/cli/shell/commands/ai/diff.py +70 -0
- app/cli/shell/commands/ai/display.py +428 -0
- app/cli/shell/commands/ai/executor.py +225 -0
- app/cli/shell/commands/ai/executor_utils.py +271 -0
- app/cli/shell/commands/ai/generate.py +285 -0
- app/cli/shell/commands/ai/interaction.py +262 -0
- app/cli/shell/commands/ai/migrate.py +281 -0
- app/cli/shell/commands/ai/resolver.py +224 -0
- app/cli/shell/commands/ai/status.py +101 -0
- app/cli/shell/commands/ai/switch.py +158 -0
- app/cli/shell/commands/ai/utils.py +52 -0
- app/cli/shell/commands/base.py +343 -0
- app/cli/shell/commands/config/__init__.py +131 -0
- app/cli/shell/commands/config/base.py +80 -0
- app/cli/shell/commands/config/check.py +246 -0
- app/cli/shell/commands/config/edit.py +148 -0
- app/cli/shell/commands/config/get.py +111 -0
- app/cli/shell/commands/config/init.py +123 -0
- app/cli/shell/commands/config/inspect.py +164 -0
- app/cli/shell/commands/config/list.py +104 -0
- app/cli/shell/commands/config/set.py +119 -0
- app/cli/shell/commands/config/show.py +150 -0
- app/cli/shell/commands/exit.py +67 -0
- app/cli/shell/commands/help.py +104 -0
- app/cli/shell/commands/infer_schema.py +156 -0
- app/cli/shell/commands/open.py +201 -0
- app/cli/shell/commands/project.py +215 -0
- app/cli/shell/commands/provider.py +619 -0
- app/cli/shell/commands/system.py +170 -0
- app/cli/shell/commands/validate.py +455 -0
- app/cli/shell/completer.py +231 -0
- app/cli/shell/config_storage.py +191 -0
- app/cli/shell/exceptions.py +90 -0
- app/cli/shell/formatter.py +371 -0
- app/cli/shell/interactive_menu.py +348 -0
- app/cli/shell/main.py +286 -0
- app/cli/shell/parser.py +266 -0
- app/cli/start.py +180 -0
- app/cli_main.py +52 -0
- app/mcp_server.py +311 -0
- app/shared/core/__init__.py +18 -0
- app/shared/core/app_version.py +59 -0
- app/shared/core/config/__init__.py +119 -0
- app/shared/core/config/server.py +193 -0
- app/shared/core/data_source/__init__.py +106 -0
- app/shared/core/data_source/loader.py +348 -0
- app/shared/core/data_source/loaders/__init__.py +213 -0
- app/shared/core/data_source/loaders/base.py +215 -0
- app/shared/core/data_source/loaders/converter.py +337 -0
- app/shared/core/data_source/loaders/csv_loader.py +252 -0
- app/shared/core/data_source/loaders/excel_loader.py +666 -0
- app/shared/core/data_source/loaders/extractor.py +268 -0
- app/shared/core/data_source/loaders/json_loader.py +354 -0
- app/shared/core/data_source/loaders/registry.py +88 -0
- app/shared/core/data_source/loaders/sql_loader.py +313 -0
- app/shared/core/data_source/loaders/strategies/__init__.py +100 -0
- app/shared/core/data_source/loaders/strategies/array_parser.py +277 -0
- app/shared/core/data_source/loaders/strategies/lines_parser.py +355 -0
- app/shared/core/data_source/loaders/strategies/object_parser.py +278 -0
- app/shared/core/data_source/schema_info.py +50 -0
- app/shared/core/data_source/specs/__init__.py +128 -0
- app/shared/core/data_source/specs/base.py +258 -0
- app/shared/core/data_source/specs/csv_source.py +137 -0
- app/shared/core/data_source/specs/excel_source.py +141 -0
- app/shared/core/data_source/specs/file_base.py +252 -0
- app/shared/core/data_source/specs/json_source.py +226 -0
- app/shared/core/data_source/specs/sql_source.py +161 -0
- app/shared/core/encoding.py +36 -0
- app/shared/core/io/__init__.py +24 -0
- app/shared/core/io/yaml.py +379 -0
- app/shared/core/manifest_schema/__init__.py +52 -0
- app/shared/core/manifest_schema/types.py +99 -0
- app/shared/core/manifest_schema/version.py +175 -0
- app/shared/core/patterns/__init__.py +24 -0
- app/shared/core/patterns/loader.py +134 -0
- app/shared/core/patterns/writer.py +244 -0
- app/shared/core/project/__init__.py +47 -0
- app/shared/core/project/constraint/builders/__init__.py +50 -0
- app/shared/core/project/constraint/builders/base.py +100 -0
- app/shared/core/project/constraint/builders/composite.py +77 -0
- app/shared/core/project/constraint/builders/conditional.py +67 -0
- app/shared/core/project/constraint/builders/foreign_key.py +53 -0
- app/shared/core/project/constraint/builders/registry.py +64 -0
- app/shared/core/project/constraint/builders/scripted.py +51 -0
- app/shared/core/project/constraint/builders/single_column.py +86 -0
- app/shared/core/project/constraint/builders/unique.py +53 -0
- app/shared/core/project/constraint/factory.py +170 -0
- app/shared/core/project/constraint/reader.py +214 -0
- app/shared/core/project/constraint/registry.py +233 -0
- app/shared/core/project/constraint/types/__init__.py +63 -0
- app/shared/core/project/constraint/types/constraint_file.py +261 -0
- app/shared/core/project/constraint/types/refs.py +460 -0
- app/shared/core/project/constraint/types.py +28 -0
- app/shared/core/project/constraint/writer.py +181 -0
- app/shared/core/project/loader/__init__.py +56 -0
- app/shared/core/project/loader/loader.py +30 -0
- app/shared/core/project/loader/loader_parts/config_inspector.py +137 -0
- app/shared/core/project/loader/loader_parts/embedded_constraints.py +224 -0
- app/shared/core/project/loader/loader_parts/file_loaders.py +58 -0
- app/shared/core/project/loader/loader_parts/inspection_ids.py +85 -0
- app/shared/core/project/loader/loader_parts/inspector_helpers.py +312 -0
- app/shared/core/project/loader/loader_parts/inspector_id_checks.py +274 -0
- app/shared/core/project/loader/loader_parts/inspector_reference_checks.py +690 -0
- app/shared/core/project/loader/loader_parts/inspector_uniqueness_checks.py +286 -0
- app/shared/core/project/loader/loader_parts/loading_error_messages.py +298 -0
- app/shared/core/project/loader/loader_parts/main.py +432 -0
- app/shared/core/project/loader/loader_parts/path_validation.py +117 -0
- app/shared/core/project/loader/loader_parts/runtime.py +125 -0
- app/shared/core/project/loader/types.py +246 -0
- app/shared/core/project/manifest/coverage.py +391 -0
- app/shared/core/project/manifest/reader.py +260 -0
- app/shared/core/project/manifest/types.py +91 -0
- app/shared/core/project/manifest/types_parts/__init__.py +60 -0
- app/shared/core/project/manifest/types_parts/constants.py +40 -0
- app/shared/core/project/manifest/types_parts/data_source.py +68 -0
- app/shared/core/project/manifest/types_parts/info.py +59 -0
- app/shared/core/project/manifest/types_parts/manifest.py +294 -0
- app/shared/core/project/manifest/types_parts/refs.py +153 -0
- app/shared/core/project/manifest/types_parts/settings.py +92 -0
- app/shared/core/project/manifest/types_parts/settings_file_processing.py +64 -0
- app/shared/core/project/manifest/types_parts/settings_script_security.py +78 -0
- app/shared/core/project/manifest/types_parts/settings_validation.py +83 -0
- app/shared/core/project/manifest/types_parts/template.py +66 -0
- app/shared/core/project/manifest/writer.py +262 -0
- app/shared/core/project/manual_data/__init__.py +27 -0
- app/shared/core/project/manual_data/types.py +75 -0
- app/shared/core/project/regex/reader.py +197 -0
- app/shared/core/project/regex/types.py +405 -0
- app/shared/core/project/regex/writer.py +123 -0
- app/shared/core/project/schema/reader.py +170 -0
- app/shared/core/project/schema/types.py +47 -0
- app/shared/core/project/schema/types_parts/__init__.py +36 -0
- app/shared/core/project/schema/types_parts/column.py +174 -0
- app/shared/core/project/schema/types_parts/column_utils.py +72 -0
- app/shared/core/project/schema/types_parts/constraint.py +165 -0
- app/shared/core/project/schema/types_parts/schema_id.py +66 -0
- app/shared/core/project/schema/types_parts/source.py +255 -0
- app/shared/core/project/schema/types_parts/source_options.py +347 -0
- app/shared/core/project/schema/types_parts/table.py +230 -0
- app/shared/core/project/schema/writer.py +139 -0
- app/shared/core/project/schema_ref_check.py +95 -0
- app/shared/core/project/template/__init__.py +27 -0
- app/shared/core/project/template/expander.py +263 -0
- app/shared/core/project/template/reader.py +120 -0
- app/shared/core/project/template/types.py +114 -0
- app/shared/core/project/transform/reader.py +76 -0
- app/shared/core/project/transform/types.py +116 -0
- app/shared/core/project/transform/writer.py +84 -0
- app/shared/core/pydantic_messages.py +59 -0
- app/shared/core/reporter/__init__.py +46 -0
- app/shared/core/reporter/reporter.py +220 -0
- app/shared/core/reporter/reporters/__init__.py +65 -0
- app/shared/core/reporter/reporters/base.py +188 -0
- app/shared/core/reporter/reporters/dingtalk_app_reporter.py +274 -0
- app/shared/core/reporter/reporters/email_reporter.py +271 -0
- app/shared/core/reporter/reporters/feishu_app_reporter.py +467 -0
- app/shared/core/reporter/reporters/local_file_reporter.py +208 -0
- app/shared/core/reporter/reporters/wecom_app_reporter.py +268 -0
- app/shared/core/utils/__init__.py +18 -0
- app/shared/core/utils/path_utils.py +60 -0
- app/shared/core/utils/regex_extract.py +113 -0
- app/shared/domain/__init__.py +114 -0
- app/shared/domain/constraints/__init__.py +76 -0
- app/shared/domain/constraints/allowed_values.py +211 -0
- app/shared/domain/constraints/base.py +141 -0
- app/shared/domain/constraints/charset.py +337 -0
- app/shared/domain/constraints/composite.py +174 -0
- app/shared/domain/constraints/condition_registry.py +130 -0
- app/shared/domain/constraints/conditional.py +629 -0
- app/shared/domain/constraints/date_logic.py +731 -0
- app/shared/domain/constraints/foreign_key.py +261 -0
- app/shared/domain/constraints/key_normalization.py +56 -0
- app/shared/domain/constraints/not_null.py +185 -0
- app/shared/domain/constraints/range.py +360 -0
- app/shared/domain/constraints/regex.py +218 -0
- app/shared/domain/constraints/scripted.py +276 -0
- app/shared/domain/constraints/unique.py +233 -0
- app/shared/domain/data_engine.py +384 -0
- app/shared/domain/data_types.py +113 -0
- app/shared/domain/data_types_parts/__init__.py +67 -0
- app/shared/domain/data_types_parts/base.py +204 -0
- app/shared/domain/data_types_parts/composite.py +250 -0
- app/shared/domain/data_types_parts/expression.py +202 -0
- app/shared/domain/data_types_parts/extracted.py +106 -0
- app/shared/domain/data_types_parts/json_types.py +234 -0
- app/shared/domain/data_types_parts/scalars.py +719 -0
- app/shared/domain/data_types_parts/sequence.py +129 -0
- app/shared/domain/dataset_schema.py +48 -0
- app/shared/domain/eval_sandbox.py +63 -0
- app/shared/domain/expression_system.py +366 -0
- app/shared/domain/regex_flags.py +53 -0
- app/shared/domain/schema/builder.py +223 -0
- app/shared/domain/schema/models.py +339 -0
- app/shared/domain/transforms/__init__.py +28 -0
- app/shared/domain/transforms/aggregate.py +141 -0
- app/shared/domain/transforms/base.py +171 -0
- app/shared/domain/transforms/cast_type.py +120 -0
- app/shared/domain/transforms/concat.py +103 -0
- app/shared/domain/transforms/conditional_assign.py +114 -0
- app/shared/domain/transforms/date_format.py +81 -0
- app/shared/domain/transforms/digits.py +77 -0
- app/shared/domain/transforms/drop_duplicates.py +94 -0
- app/shared/domain/transforms/fill_na.py +94 -0
- app/shared/domain/transforms/filter_rows.py +97 -0
- app/shared/domain/transforms/lookup.py +78 -0
- app/shared/domain/transforms/lower_case.py +70 -0
- app/shared/domain/transforms/map_value.py +90 -0
- app/shared/domain/transforms/math_expr.py +112 -0
- app/shared/domain/transforms/modulo.py +82 -0
- app/shared/domain/transforms/regex_extract.py +103 -0
- app/shared/domain/transforms/registry.py +95 -0
- app/shared/domain/transforms/replace.py +88 -0
- app/shared/domain/transforms/sort_rows.py +94 -0
- app/shared/domain/transforms/string_split.py +84 -0
- app/shared/domain/transforms/strip.py +73 -0
- app/shared/domain/transforms/substring.py +92 -0
- app/shared/domain/transforms/upper_case.py +70 -0
- app/shared/domain/transforms/weighted_sum.py +104 -0
- app/shared/domain/validation_constraints.py +69 -0
- app/shared/services/__init__.py +62 -0
- app/shared/services/ai/__init__.py +48 -0
- app/shared/services/ai/agent/__init__.py +40 -0
- app/shared/services/ai/agent/chat_tools/__init__.py +47 -0
- app/shared/services/ai/agent/chat_tools/apply_actions.py +592 -0
- app/shared/services/ai/agent/chat_tools/ask_user.py +237 -0
- app/shared/services/ai/agent/chat_tools/read_canvas.py +140 -0
- app/shared/services/ai/agent/chat_tools/read_project.py +160 -0
- app/shared/services/ai/agent/chat_tools/read_table.py +301 -0
- app/shared/services/ai/agent/chat_tools/schemas.py +105 -0
- app/shared/services/ai/agent/chat_tools/validate_table.py +131 -0
- app/shared/services/ai/agent/executor.py +554 -0
- app/shared/services/ai/agent/memory.py +256 -0
- app/shared/services/ai/agent/planner.py +282 -0
- app/shared/services/ai/agent/tool_registry.py +296 -0
- app/shared/services/ai/agent/tools/__init__.py +32 -0
- app/shared/services/ai/agent/tools/config_generate.py +133 -0
- app/shared/services/ai/agent/tools/config_refine.py +109 -0
- app/shared/services/ai/agent/tools/config_validate.py +403 -0
- app/shared/services/ai/agent/tools/merge_results.py +136 -0
- app/shared/services/ai/agent/tools/plan_chunks.py +79 -0
- app/shared/services/ai/agent/tools/script_parse.py +359 -0
- app/shared/services/ai/agent/types.py +138 -0
- app/shared/services/ai/chat_agent_runner.py +596 -0
- app/shared/services/ai/chat_orchestrator.py +725 -0
- app/shared/services/ai/failure_messages.py +65 -0
- app/shared/services/ai/job_storage.py +274 -0
- app/shared/services/ai/migrate_service.py +550 -0
- app/shared/services/ai/streaming/__init__.py +65 -0
- app/shared/services/ai/streaming/event_journal.py +197 -0
- app/shared/services/ai/streaming/orchestrator.py +262 -0
- app/shared/services/ai/streaming/pending_interaction_store.py +227 -0
- app/shared/services/ai/streaming/sse_response.py +148 -0
- app/shared/services/ai/streaming/types.py +73 -0
- app/shared/services/ai/types.py +213 -0
- app/shared/services/ai/utils.py +322 -0
- app/shared/services/diff/config_diff.py +320 -0
- app/shared/services/hardware.py +237 -0
- app/shared/services/llm/__init__.py +62 -0
- app/shared/services/llm/actions/__init__.py +42 -0
- app/shared/services/llm/actions/_canvas_validator.py +147 -0
- app/shared/services/llm/actions/_constraint_validator.py +401 -0
- app/shared/services/llm/actions/_regex_validator.py +85 -0
- app/shared/services/llm/actions/_schema_validator.py +82 -0
- app/shared/services/llm/actions/_settings_validator.py +76 -0
- app/shared/services/llm/actions/_transform_validator.py +78 -0
- app/shared/services/llm/actions/action_handlers.py +400 -0
- app/shared/services/llm/actions/action_parser.py +63 -0
- app/shared/services/llm/actions/action_processor.py +468 -0
- app/shared/services/llm/actions/action_validator.py +311 -0
- app/shared/services/llm/actions/diff_compute.py +195 -0
- app/shared/services/llm/actions/regex_handlers.py +284 -0
- app/shared/services/llm/actions/registry.py +336 -0
- app/shared/services/llm/actions/schema_handlers.py +335 -0
- app/shared/services/llm/actions/settings_handlers.py +180 -0
- app/shared/services/llm/actions/specs.py +259 -0
- app/shared/services/llm/actions/transform_handlers.py +279 -0
- app/shared/services/llm/actions/validation_types.py +147 -0
- app/shared/services/llm/cache/__init__.py +18 -0
- app/shared/services/llm/cache/response_cache.py +80 -0
- app/shared/services/llm/chat/__init__.py +35 -0
- app/shared/services/llm/chat/chat_system_prompt.py +561 -0
- app/shared/services/llm/chat/response_parser.py +296 -0
- app/shared/services/llm/config/__init__.py +45 -0
- app/shared/services/llm/config/crypto.py +135 -0
- app/shared/services/llm/config/loader.py +258 -0
- app/shared/services/llm/config/models.py +202 -0
- app/shared/services/llm/config/presets.py +128 -0
- app/shared/services/llm/config_generator.py +163 -0
- app/shared/services/llm/constraints/__init__.py +41 -0
- app/shared/services/llm/constraints/constraint_builder.py +177 -0
- app/shared/services/llm/constraints/constraint_deletion.py +84 -0
- app/shared/services/llm/constraints/constraint_id.py +174 -0
- app/shared/services/llm/constraints/frontend_instructions.py +333 -0
- app/shared/services/llm/constraints/inline_batch.py +279 -0
- app/shared/services/llm/discovery/__init__.py +35 -0
- app/shared/services/llm/discovery/scanner.py +181 -0
- app/shared/services/llm/generation/__init__.py +42 -0
- app/shared/services/llm/generation/agent_wiring.py +156 -0
- app/shared/services/llm/generation/config_builder.py +386 -0
- app/shared/services/llm/generation/errors.py +36 -0
- app/shared/services/llm/generation/existing_config.py +99 -0
- app/shared/services/llm/generation/profiler.py +279 -0
- app/shared/services/llm/generation/prompt_builder.py +199 -0
- app/shared/services/llm/generation/response_parser.py +144 -0
- app/shared/services/llm/generation/service.py +622 -0
- app/shared/services/llm/models.py +104 -0
- app/shared/services/llm/providers/__init__.py +48 -0
- app/shared/services/llm/providers/base.py +310 -0
- app/shared/services/llm/providers/cached_provider.py +90 -0
- app/shared/services/llm/providers/ollama.py +498 -0
- app/shared/services/llm/providers/openai.py +334 -0
- app/shared/services/llm/providers/registry.py +108 -0
- app/shared/services/llm/schema_resolver.py +141 -0
- app/shared/services/llm/suggestion_utils.py +219 -0
- app/shared/services/llm/validate_executor.py +163 -0
- app/shared/services/llm/yaml_io.py +406 -0
- app/shared/services/preview/__init__.py +18 -0
- app/shared/services/preview/loader.py +145 -0
- app/shared/services/preview/path_validation.py +138 -0
- app/shared/services/project_loader.py +57 -0
- app/shared/services/schema_inference.py +260 -0
- app/shared/services/schema_runtime_builder.py +120 -0
- app/shared/services/validation/__init__.py +52 -0
- app/shared/services/validation/chunked_loader.py +509 -0
- app/shared/services/validation/dag/__init__.py +35 -0
- app/shared/services/validation/dag/builder.py +141 -0
- app/shared/services/validation/dag/executor.py +225 -0
- app/shared/services/validation/dag/sorter.py +77 -0
- app/shared/services/validation/data_loader.py +231 -0
- app/shared/services/validation/engine.py +446 -0
- app/shared/services/validation/executor.py +1030 -0
- app/shared/services/validation/extractors.py +266 -0
- app/shared/services/validation/history.py +209 -0
- app/shared/services/validation/json_payload.py +136 -0
- app/shared/services/validation/loader.py +152 -0
- app/shared/services/validation/memory_monitor.py +179 -0
- app/shared/services/validation/postprocess.py +208 -0
- app/shared/services/validation/progress.py +64 -0
- app/shared/services/validation/report_export.py +222 -0
- app/shared/services/validation/resolver.py +180 -0
- app/shared/services/validation/service.py +418 -0
- app/shared/services/validation/types.py +238 -0
- app/shared/services/validation/validators/__init__.py +36 -0
- app/shared/services/validation/validators/adapter.py +182 -0
- app/shared/services/validation/validators/base.py +332 -0
- app/shared/services/validation/validators/composite.py +233 -0
- app/shared/services/validation/validators/date_logic.py +276 -0
- app/start_server.py +133 -0
- precis_cli-0.1.3.dist-info/METADATA +180 -0
- precis_cli-0.1.3.dist-info/RECORD +442 -0
- precis_cli-0.1.3.dist-info/WHEEL +5 -0
- precis_cli-0.1.3.dist-info/entry_points.txt +4 -0
- precis_cli-0.1.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 条件约束模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 实现 ConditionalConstraint 类
|
|
21
|
+
- 实现条件逻辑: 当 A 条件满足时,B 列必须满足 C 条件
|
|
22
|
+
- 支持简单条件和复合条件 (AND/OR)
|
|
23
|
+
|
|
24
|
+
架构设计:
|
|
25
|
+
- 继承基类: 继承自 Constraint 抽象类
|
|
26
|
+
- 条件解析: 支持 DSL 字典和注册的条件函数
|
|
27
|
+
- 条件类型: not_null, greater_than, in, eq, neq 等
|
|
28
|
+
|
|
29
|
+
输入示例:
|
|
30
|
+
# 约束配置 - 简单条件: 当 status="VIP" 时, credit_limit 必须 > 1000
|
|
31
|
+
constraint1 = ConditionalConstraint(
|
|
32
|
+
table="customers",
|
|
33
|
+
if_column="status",
|
|
34
|
+
if_value="VIP",
|
|
35
|
+
then_column="credit_limit",
|
|
36
|
+
then_condition={"operator": "greater_than", "value": 1000}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
# 约束配置 - 复合条件: 当 (age >= 18 AND country="CN") 时, id_card 必须 not_null
|
|
40
|
+
constraint2 = ConditionalConstraint(
|
|
41
|
+
table="users",
|
|
42
|
+
if_conditions=[
|
|
43
|
+
{"column": "age", "operator": "greater_than", "value": 18},
|
|
44
|
+
{"column": "country", "operator": "eq", "value": "CN"}
|
|
45
|
+
],
|
|
46
|
+
if_logic="and",
|
|
47
|
+
then_column="id_card",
|
|
48
|
+
then_condition={"operator": "not_null"}
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# 数据集输入
|
|
52
|
+
datasets = {
|
|
53
|
+
"customers": DataFrame({
|
|
54
|
+
"id": [1, 2, 3],
|
|
55
|
+
"status": ["VIP", "Normal", "VIP"],
|
|
56
|
+
"credit_limit": [500, 2000, 800] # 第1行和第3行不满足条件
|
|
57
|
+
})
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
输出示例:
|
|
61
|
+
# 验证失败 (第1行 VIP 但 credit_limit=500, 第3行 VIP 但 credit_limit=800)
|
|
62
|
+
{
|
|
63
|
+
"errors": [
|
|
64
|
+
{
|
|
65
|
+
"error_type": "ConditionalViolation",
|
|
66
|
+
"table": "customers",
|
|
67
|
+
"row_index": 0,
|
|
68
|
+
"value": {"status": "VIP", "credit_limit": 500},
|
|
69
|
+
"message": "条件约束冲突: 当满足条件时, 'credit_limit' 的值 '500' 不满足要求 (满足DSL {'operator': 'greater_than', 'value': 1000})."
|
|
70
|
+
}
|
|
71
|
+
],
|
|
72
|
+
"info": {...}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
DSL 操作符说明:
|
|
76
|
+
- not_null: 值不能为空
|
|
77
|
+
- greater_than: 值必须大于 threshold
|
|
78
|
+
- less_than: 值必须小于 threshold
|
|
79
|
+
- in: 值必须在列表中
|
|
80
|
+
- eq: 值必须等于指定值
|
|
81
|
+
- neq: 值必须不等于指定值
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
from __future__ import annotations
|
|
85
|
+
|
|
86
|
+
# 1. 标准库导入
|
|
87
|
+
import logging
|
|
88
|
+
import time
|
|
89
|
+
from collections.abc import Callable
|
|
90
|
+
from typing import Any
|
|
91
|
+
|
|
92
|
+
# 2. 第三方库导入
|
|
93
|
+
import pandas as pd
|
|
94
|
+
|
|
95
|
+
# 3. 项目内部导入
|
|
96
|
+
from app.shared.domain.constraints.base import Constraint
|
|
97
|
+
from app.shared.domain.constraints.condition_registry import CONDITION_REGISTRY
|
|
98
|
+
|
|
99
|
+
logger = logging.getLogger(__name__)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# if 条件(composite/if_conditions)支持的操作符白名单:未知操作符 fail-fast(见 _apply_if_condition),
|
|
103
|
+
# 防止拼写错误(如 not_equal)被默认分支静默按 eq 执行导致语义反转。
|
|
104
|
+
_KNOWN_IF_OPERATORS = frozenset({"eq", "neq", "in", "not_null", "greater_than", "less_than"})
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class ConditionalConstraint(Constraint):
|
|
108
|
+
"""
|
|
109
|
+
@classdesc 条件约束
|
|
110
|
+
|
|
111
|
+
实现"当满足某条件时,另一列必须满足某规则"的业务逻辑。
|
|
112
|
+
支持两种触发条件模式:
|
|
113
|
+
1. 简单条件: 当 if_column == if_value 时触发
|
|
114
|
+
2. 复合条件: 当多个条件组合满足时触发(支持 AND/OR 逻辑)
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
def __init__(
|
|
118
|
+
self,
|
|
119
|
+
table: str,
|
|
120
|
+
if_column: str = "",
|
|
121
|
+
if_value: Any = None,
|
|
122
|
+
then_column: str = "",
|
|
123
|
+
then_condition: dict | str | None = None,
|
|
124
|
+
if_conditions: list[dict] | None = None,
|
|
125
|
+
if_logic: str = "and",
|
|
126
|
+
):
|
|
127
|
+
"""
|
|
128
|
+
@methoddesc 初始化条件约束
|
|
129
|
+
|
|
130
|
+
参数:
|
|
131
|
+
table: 目标表名
|
|
132
|
+
if_column: 简单条件模式下的触发列名(与 if_conditions 互斥)
|
|
133
|
+
if_value: 简单条件模式下的触发值(与 if_conditions 互斥)
|
|
134
|
+
then_column: 需要被验证的列名
|
|
135
|
+
then_condition: 对 then_column 的验证要求,可以是:
|
|
136
|
+
- 字典(DSL): 如 {"operator": "greater_than", "value": 1000}
|
|
137
|
+
- 字符串: 已注册的条件函数名,如 "is_not_empty"
|
|
138
|
+
if_conditions: 复合条件列表,每个元素是条件字典,如:
|
|
139
|
+
[{"column": "age", "operator": "greater_than", "value": 18}]
|
|
140
|
+
if_logic: 复合条件的逻辑组合方式,"and" 或 "or",默认为 "and"
|
|
141
|
+
|
|
142
|
+
属性:
|
|
143
|
+
_condition_func: 解析后的条件验证函数(Callable),在初始化时通过 _parse_condition() 生成
|
|
144
|
+
_condition_str: 条件的可读描述字符串
|
|
145
|
+
"""
|
|
146
|
+
self.table = table
|
|
147
|
+
self.if_column = if_column
|
|
148
|
+
self.if_value = if_value
|
|
149
|
+
self.if_conditions = if_conditions or []
|
|
150
|
+
self.if_logic = if_logic or "and"
|
|
151
|
+
self.then_column = then_column
|
|
152
|
+
self.then_condition_config = then_condition
|
|
153
|
+
# 解析 then_condition,生成可执行的验证函数
|
|
154
|
+
self._condition_func = self._parse_condition()
|
|
155
|
+
# 生成条件的可读描述字符串
|
|
156
|
+
self._condition_str = self._condition_to_string()
|
|
157
|
+
|
|
158
|
+
def _parse_condition(self) -> Callable[[Any, dict[str, Any] | None], bool | None]:
|
|
159
|
+
"""
|
|
160
|
+
@methoddesc 解析 then_condition,生成验证函数
|
|
161
|
+
|
|
162
|
+
根据 then_condition 的类型(字典 DSL 或字符串函数名),
|
|
163
|
+
生成并返回一个接收单个值、返回布尔结果的验证函数。
|
|
164
|
+
|
|
165
|
+
返回:
|
|
166
|
+
验证函数: Callable[[Any], bool]
|
|
167
|
+
|
|
168
|
+
抛出:
|
|
169
|
+
ValueError: 当 DSL 操作符不支持,或字符串函数名未注册时抛出
|
|
170
|
+
TypeError: 当 then_condition 类型既不是字典也不是字符串时抛出
|
|
171
|
+
"""
|
|
172
|
+
then_config = self.then_condition_config
|
|
173
|
+
if isinstance(then_config, dict):
|
|
174
|
+
operator = then_config.get("operator")
|
|
175
|
+
ref_column = then_config.get("ref_column")
|
|
176
|
+
|
|
177
|
+
# 辅助函数:获取比较值(固定值或引用列)
|
|
178
|
+
def _get_compared_value(x: Any, row: dict[str, Any] | None) -> Any:
|
|
179
|
+
if ref_column and row is not None:
|
|
180
|
+
return row.get(ref_column)
|
|
181
|
+
return then_config.get("value")
|
|
182
|
+
|
|
183
|
+
if operator == "not_null":
|
|
184
|
+
|
|
185
|
+
def _not_null(x: Any, row: dict[str, Any] | None = None) -> bool:
|
|
186
|
+
return bool(pd.notna(x)) and x != ""
|
|
187
|
+
|
|
188
|
+
return _not_null
|
|
189
|
+
if operator == "greater_than":
|
|
190
|
+
threshold = then_config.get("value")
|
|
191
|
+
|
|
192
|
+
# THEN 序比较操作数域=仅数值。固定阈值的构建期校验在 validate() 内
|
|
193
|
+
# 以 ConstraintConfigError 条目呈现(见 validate 前置检查),构造器
|
|
194
|
+
# 不抛异常;此处仅做行级三态跳过。日期比较引导 DateLogic——旧实现的
|
|
195
|
+
# 字符串字典序仅对 ISO 格式碰巧正确('6/1/2024' vs '10/1/2024' 文本
|
|
196
|
+
# 序即错),不属承诺行为。
|
|
197
|
+
|
|
198
|
+
def _safe_greater_than(x: Any, row: dict[str, Any] | None = None) -> bool | None:
|
|
199
|
+
compared = _get_compared_value(x, row) if ref_column else threshold
|
|
200
|
+
if pd.isna(x) or x == "":
|
|
201
|
+
return False
|
|
202
|
+
if compared is None or compared == "":
|
|
203
|
+
return False
|
|
204
|
+
# a14ef85c 修复字典序误判;NaN 改返回 None(无法判定→跳过该行)而非
|
|
205
|
+
# False(判违规):日期字符串等不可数值化的值不该被报"不满足条件",
|
|
206
|
+
# 脏数据由列类型约束负责报告。与 IF 侧 NaN 不触发的 UNKNOWN 语义对称。
|
|
207
|
+
try:
|
|
208
|
+
x_num = pd.to_numeric(x, errors="coerce")
|
|
209
|
+
compared_num = pd.to_numeric(compared, errors="coerce")
|
|
210
|
+
if pd.isna(x_num) or pd.isna(compared_num):
|
|
211
|
+
return None
|
|
212
|
+
return bool(x_num > compared_num)
|
|
213
|
+
except Exception:
|
|
214
|
+
logger.debug(
|
|
215
|
+
f"greater_than 比较失败: x={x!r}, compared={compared!r}, "
|
|
216
|
+
f"table={self.table}, then_column={self.then_column}"
|
|
217
|
+
)
|
|
218
|
+
return None
|
|
219
|
+
|
|
220
|
+
return _safe_greater_than
|
|
221
|
+
if operator == "in":
|
|
222
|
+
# 回归: 容器键兼容 + fail-closed。原实现只认 "values" 键,用户按习惯写
|
|
223
|
+
# "value"(列表)时取到空列表 → 所有触发行误报违规。现同时接受两种键
|
|
224
|
+
# (values 优先);容器非 list 时按配置错误 fail-fast(与本函数对不支持
|
|
225
|
+
# 操作符抛 ValueError 的守卫先例一致),不再静默按空列表处理。
|
|
226
|
+
values = then_config.get("values")
|
|
227
|
+
if values is None:
|
|
228
|
+
values = then_config.get("value")
|
|
229
|
+
if not isinstance(values, list):
|
|
230
|
+
raise ValueError(
|
|
231
|
+
f"'in' 操作符要求 'values'(或 'value')为列表,实际为 {type(values).__name__}: {values!r}"
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
def _safe_in(x: Any, row: dict[str, Any] | None = None) -> bool:
|
|
235
|
+
if pd.isna(x) or x == "":
|
|
236
|
+
return False
|
|
237
|
+
try:
|
|
238
|
+
return x in values
|
|
239
|
+
except Exception:
|
|
240
|
+
logger.debug(
|
|
241
|
+
f"in 操作失败: x={x!r}, values={values!r}, "
|
|
242
|
+
f"table={self.table}, then_column={self.then_column}"
|
|
243
|
+
)
|
|
244
|
+
return False
|
|
245
|
+
|
|
246
|
+
return _safe_in
|
|
247
|
+
if operator == "less_than":
|
|
248
|
+
threshold = then_config.get("value")
|
|
249
|
+
|
|
250
|
+
# 同 greater_than:阈值校验在 validate() 前置检查,此处仅行级三态
|
|
251
|
+
|
|
252
|
+
def _safe_less_than(x: Any, row: dict[str, Any] | None = None) -> bool | None:
|
|
253
|
+
compared = _get_compared_value(x, row) if ref_column else threshold
|
|
254
|
+
if pd.isna(x) or x == "":
|
|
255
|
+
return False
|
|
256
|
+
if compared is None or compared == "":
|
|
257
|
+
return False
|
|
258
|
+
# A4 修复:与 greater_than 同理,先双侧数值转换再比较,避免 str
|
|
259
|
+
# 字典序比较(9 < "10" → False 误判);不可数值化改返回 None
|
|
260
|
+
# (无法判定→跳过该行)而非 False(判违规),脏数据由列类型约束报告。
|
|
261
|
+
try:
|
|
262
|
+
x_num = pd.to_numeric(x, errors="coerce")
|
|
263
|
+
compared_num = pd.to_numeric(compared, errors="coerce")
|
|
264
|
+
if pd.isna(x_num) or pd.isna(compared_num):
|
|
265
|
+
return None
|
|
266
|
+
return bool(x_num < compared_num)
|
|
267
|
+
except Exception:
|
|
268
|
+
logger.debug(
|
|
269
|
+
f"less_than 比较失败: x={x!r}, compared={compared!r}, "
|
|
270
|
+
f"table={self.table}, then_column={self.then_column}"
|
|
271
|
+
)
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
return _safe_less_than
|
|
275
|
+
if operator == "eq":
|
|
276
|
+
expected = then_config.get("value")
|
|
277
|
+
|
|
278
|
+
def _safe_eq(x: Any, row: dict[str, Any] | None = None) -> bool:
|
|
279
|
+
compared = _get_compared_value(x, row) if ref_column else expected
|
|
280
|
+
# §1.23: 双侧皆 NaN/None 视为满足(SQL NULL=NULL 是 UNKNOWN 不判违规)——
|
|
281
|
+
# 原实现 compared 为 NaN(float) 时落到 x == compared → False → 误报
|
|
282
|
+
if pd.isna(x) and (compared is None or (isinstance(compared, float) and pd.isna(compared))):
|
|
283
|
+
return True
|
|
284
|
+
return bool(x == compared)
|
|
285
|
+
|
|
286
|
+
return _safe_eq
|
|
287
|
+
if operator == "neq":
|
|
288
|
+
expected = then_config.get("value")
|
|
289
|
+
|
|
290
|
+
def _safe_neq(x: Any, row: dict[str, Any] | None = None) -> bool:
|
|
291
|
+
compared = _get_compared_value(x, row) if ref_column else expected
|
|
292
|
+
# §1.23: 与 _safe_eq 对称——双侧皆 NaN/None 时"不等于"不成立(不违规)
|
|
293
|
+
if pd.isna(x) and (compared is None or (isinstance(compared, float) and pd.isna(compared))):
|
|
294
|
+
return False
|
|
295
|
+
return bool(x != compared)
|
|
296
|
+
|
|
297
|
+
return _safe_neq
|
|
298
|
+
raise ValueError(f"不支持的DSL操作符: '{operator}'")
|
|
299
|
+
|
|
300
|
+
if isinstance(self.then_condition_config, str):
|
|
301
|
+
# 从全局注册表中查找已注册的条件函数
|
|
302
|
+
if self.then_condition_config in CONDITION_REGISTRY:
|
|
303
|
+
original = CONDITION_REGISTRY[self.then_condition_config]
|
|
304
|
+
|
|
305
|
+
# 包装为兼容新签名 (value, row=None) 的函数
|
|
306
|
+
def _wrapped_condition(x: Any, row: dict[str, Any] | None = None) -> bool:
|
|
307
|
+
return bool(original(x))
|
|
308
|
+
|
|
309
|
+
return _wrapped_condition
|
|
310
|
+
raise ValueError(
|
|
311
|
+
f"未注册的条件函数名: '{self.then_condition_config}'. 可用: {list(CONDITION_REGISTRY.keys())}"
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
raise TypeError(f"then_condition配置类型不支持: {type(self.then_condition_config)}")
|
|
315
|
+
|
|
316
|
+
def _condition_to_string(self) -> str:
|
|
317
|
+
"""将 then_condition 转换为可读描述字符串"""
|
|
318
|
+
if isinstance(self.then_condition_config, dict):
|
|
319
|
+
return f"满足DSL {self.then_condition_config}"
|
|
320
|
+
if isinstance(self.then_condition_config, str):
|
|
321
|
+
return f"满足已注册的规则 '{self.then_condition_config}'"
|
|
322
|
+
return "满足未知规则"
|
|
323
|
+
|
|
324
|
+
def _get_description(self) -> str:
|
|
325
|
+
"""生成约束描述"""
|
|
326
|
+
if self.if_conditions:
|
|
327
|
+
return f"条件约束: {self.table} 当满足条件时 {self.then_column} 必须 {self._condition_str}"
|
|
328
|
+
return f"条件约束: {self.table} 当 {self.if_column}={self.if_value} 时 {self.then_column} 必须 {self._condition_str}"
|
|
329
|
+
|
|
330
|
+
def validate(self, datasets: dict[str, pd.DataFrame], **kwargs: Any) -> dict[str, Any]:
|
|
331
|
+
"""
|
|
332
|
+
@methoddesc 执行条件约束验证
|
|
333
|
+
|
|
334
|
+
参数:
|
|
335
|
+
datasets: 数据集字典,键为表名,值为 pandas DataFrame
|
|
336
|
+
**kwargs: 额外的关键字参数(本方法不使用)
|
|
337
|
+
|
|
338
|
+
返回:
|
|
339
|
+
验证结果字典,包含 errors(错误列表)和 info(约束信息)
|
|
340
|
+
|
|
341
|
+
逻辑说明:
|
|
342
|
+
1. 检查目标表是否存在
|
|
343
|
+
2. 检查 then_column 是否存在
|
|
344
|
+
3. 根据条件模式(简单/复合)筛选触发行
|
|
345
|
+
4. 对每个触发行检查 then_column 是否满足条件
|
|
346
|
+
5. 为不满足条件的行生成错误记录
|
|
347
|
+
"""
|
|
348
|
+
errors: list[dict[str, Any]] = []
|
|
349
|
+
|
|
350
|
+
# 检查目标表是否存在
|
|
351
|
+
if self.table not in datasets:
|
|
352
|
+
errors.append(
|
|
353
|
+
{
|
|
354
|
+
"error_type": "ConstraintConfigError",
|
|
355
|
+
"table": self.table,
|
|
356
|
+
"column": self.then_column,
|
|
357
|
+
"message": f"条件约束失败: 表 '{self.table}' 不在数据集中。",
|
|
358
|
+
}
|
|
359
|
+
)
|
|
360
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
361
|
+
|
|
362
|
+
df = datasets[self.table]
|
|
363
|
+
|
|
364
|
+
# 检查目标列是否存在
|
|
365
|
+
if self.then_column not in df.columns:
|
|
366
|
+
errors.append(
|
|
367
|
+
{
|
|
368
|
+
"error_type": "ConstraintConfigError",
|
|
369
|
+
"table": self.table,
|
|
370
|
+
"column": self.then_column,
|
|
371
|
+
"message": f"条件约束失败: 列 '{self.then_column}' 不在表 '{self.table}' 中。",
|
|
372
|
+
}
|
|
373
|
+
)
|
|
374
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
375
|
+
|
|
376
|
+
# 回归 D1: THEN 侧用 ref_column 做列间比较时,ref_column 必须存在。原实现 IF 侧有
|
|
377
|
+
# 列校验、THEN 侧没有(不对称),ref_column 缺失时 row.get(缺失列)=None → 比较全 False
|
|
378
|
+
# → 每个触发行误报 ConditionalViolation,且无配置错误提示。这里补上对称校验。
|
|
379
|
+
if isinstance(self.then_condition_config, dict):
|
|
380
|
+
ref_col = self.then_condition_config.get("ref_column")
|
|
381
|
+
if ref_col and ref_col not in df.columns:
|
|
382
|
+
errors.append(
|
|
383
|
+
{
|
|
384
|
+
"error_type": "ConstraintConfigError",
|
|
385
|
+
"table": self.table,
|
|
386
|
+
"column": ref_col,
|
|
387
|
+
"message": f"条件约束失败: 引用列 '{ref_col}' 不在表 '{self.table}' 中。",
|
|
388
|
+
}
|
|
389
|
+
)
|
|
390
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
391
|
+
|
|
392
|
+
# THEN 序比较(greater_than/less_than)固定阈值前置校验:操作数域=仅数值
|
|
393
|
+
# (用户拍板)。阈值不可转数值(如 'abc'、日期字符串)报配置错误并引导
|
|
394
|
+
# DateLogic,而非逐行误报 ConditionalViolation;字符串数值("1000")兼容。
|
|
395
|
+
# ref_column 列间比较的参照值来自行数据,无法前置校验,由行级三态跳过兜底。
|
|
396
|
+
if isinstance(self.then_condition_config, dict):
|
|
397
|
+
_then_op = self.then_condition_config.get("operator")
|
|
398
|
+
if _then_op in ("greater_than", "less_than") and not self.then_condition_config.get("ref_column"):
|
|
399
|
+
_then_threshold = self.then_condition_config.get("value")
|
|
400
|
+
if _then_threshold is not None and _then_threshold != "":
|
|
401
|
+
try:
|
|
402
|
+
_threshold_num = pd.to_numeric(_then_threshold)
|
|
403
|
+
except (ValueError, TypeError):
|
|
404
|
+
_threshold_num = None
|
|
405
|
+
if _threshold_num is None or pd.isna(_threshold_num):
|
|
406
|
+
errors.append(
|
|
407
|
+
{
|
|
408
|
+
"error_type": "ConstraintConfigError",
|
|
409
|
+
"table": self.table,
|
|
410
|
+
"column": self.then_column,
|
|
411
|
+
"message": (
|
|
412
|
+
f"条件约束失败: '{_then_op}' 的 THEN 阈值必须为数值,"
|
|
413
|
+
f"实际: {_then_threshold!r};日期比较请使用 DateLogic 约束。"
|
|
414
|
+
),
|
|
415
|
+
}
|
|
416
|
+
)
|
|
417
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
418
|
+
|
|
419
|
+
def _apply_if_condition(cond: dict) -> pd.Series:
|
|
420
|
+
"""
|
|
421
|
+
将单个条件应用到 DataFrame,返回布尔 Series
|
|
422
|
+
|
|
423
|
+
参数:
|
|
424
|
+
cond: 条件字典,包含 column/operator/value/values 等字段
|
|
425
|
+
|
|
426
|
+
返回:
|
|
427
|
+
pandas Series,表示每行是否满足该条件
|
|
428
|
+
|
|
429
|
+
抛出:
|
|
430
|
+
KeyError: 当条件引用的列不存在时抛出
|
|
431
|
+
"""
|
|
432
|
+
# 支持两种字段名: if_column(旧)和 column(新)
|
|
433
|
+
col = cond.get("if_column") or cond.get("column") or ""
|
|
434
|
+
op = cond.get("operator") or "eq"
|
|
435
|
+
value = cond.get("value")
|
|
436
|
+
values = cond.get("values", [])
|
|
437
|
+
if col not in df.columns:
|
|
438
|
+
raise KeyError(col)
|
|
439
|
+
# 未知操作符不再静默落入默认 eq 分支(语义反转且无告警),与 then 侧
|
|
440
|
+
# "不支持的DSL操作符" 守卫(见 _parse_condition)保持一致的 fail-fast 语义。
|
|
441
|
+
if op not in _KNOWN_IF_OPERATORS:
|
|
442
|
+
raise ValueError(f"不支持的 if 条件操作符: '{op}'(支持: {sorted(_KNOWN_IF_OPERATORS)})")
|
|
443
|
+
s = df[col]
|
|
444
|
+
if op == "not_null":
|
|
445
|
+
# 非空: 既不是 NaN 也不是空字符串
|
|
446
|
+
return pd.notna(s) & (s != "")
|
|
447
|
+
if op == "in":
|
|
448
|
+
# 回归: 容器非 list 时必须按配置错误处理。原实现静默按空列表处理 →
|
|
449
|
+
# 条件永不触发 → 违规被静默吞掉(fail-open)。抛 ValueError 后由
|
|
450
|
+
# validate() 的 except ValueError 统一转成 ConstraintConfigError 条目。
|
|
451
|
+
if not isinstance(values, list):
|
|
452
|
+
raise ValueError(f"'in' 条件要求 'values' 为列表,实际为 {type(values).__name__}: {values!r}")
|
|
453
|
+
# 值在指定列表中
|
|
454
|
+
return s.isin(values)
|
|
455
|
+
if op == "greater_than":
|
|
456
|
+
# 大于: 先转为数值再比较
|
|
457
|
+
# §1.5: 阈值缺省不再静默按 0.0,转换失败不再静默全 False——都报配置错误
|
|
458
|
+
if value is None:
|
|
459
|
+
raise ValueError("'greater_than' 条件缺少阈值 value")
|
|
460
|
+
try:
|
|
461
|
+
threshold = float(value)
|
|
462
|
+
except Exception as e:
|
|
463
|
+
raise ValueError(f"'greater_than' 条件阈值 '{value}' 无法转换为数值") from e
|
|
464
|
+
return pd.to_numeric(s, errors="coerce") > threshold
|
|
465
|
+
if op == "neq":
|
|
466
|
+
# 不等于: 空值不触发(对标 SQL 语义 NULL != X 为 UNKNOWN,与 Unique 的
|
|
467
|
+
# NULL 豁免口径一致;原 `s != value` 会把 NaN != X 判为 True 导致空值行误报)
|
|
468
|
+
return s.ne(value) & s.notna()
|
|
469
|
+
if op == "less_than":
|
|
470
|
+
# 小于: 先转为数值再比较
|
|
471
|
+
# §1.5: 阈值缺省不再静默按 0.0,转换失败不再静默全 False——都报配置错误
|
|
472
|
+
if value is None:
|
|
473
|
+
raise ValueError("'less_than' 条件缺少阈值 value")
|
|
474
|
+
try:
|
|
475
|
+
threshold = float(value)
|
|
476
|
+
except Exception as e:
|
|
477
|
+
raise ValueError(f"'less_than' 条件阈值 '{value}' 无法转换为数值") from e
|
|
478
|
+
return pd.to_numeric(s, errors="coerce") < threshold
|
|
479
|
+
# 默认: 等于(op == "eq")
|
|
480
|
+
# §1.2: eq 条件 value 留空 → 配置错误(原实现整列和 None 比较恒 False,零触发静默失效)
|
|
481
|
+
if value is None:
|
|
482
|
+
raise ValueError("'eq' 条件缺少 value(if_value 留空),请检查约束配置")
|
|
483
|
+
return s == value
|
|
484
|
+
|
|
485
|
+
# ============================================================================
|
|
486
|
+
# 筛选触发条件的行
|
|
487
|
+
# ============================================================================
|
|
488
|
+
if self.if_conditions:
|
|
489
|
+
# 复合条件模式: 使用多个条件组合筛选
|
|
490
|
+
# if_logic 归一大小写不敏感("OR"/"And" 均合法),未知值 fail-fast 报配置错误,
|
|
491
|
+
# 不再静默按 AND 组合缩小触发行集——与未知 if 操作符的守卫同一语义
|
|
492
|
+
logic = (self.if_logic or "and").lower()
|
|
493
|
+
if logic not in ("and", "or"):
|
|
494
|
+
errors.append(
|
|
495
|
+
{
|
|
496
|
+
"error_type": "ConstraintConfigError",
|
|
497
|
+
"table": self.table,
|
|
498
|
+
"message": f"条件约束失败: 未知的 if_logic '{self.if_logic}',支持的值为: and, or。",
|
|
499
|
+
}
|
|
500
|
+
)
|
|
501
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
502
|
+
try:
|
|
503
|
+
masks = [_apply_if_condition(c) for c in self.if_conditions]
|
|
504
|
+
except KeyError as e:
|
|
505
|
+
missing = str(e).strip("'")
|
|
506
|
+
errors.append(
|
|
507
|
+
{
|
|
508
|
+
"error_type": "ConstraintConfigError",
|
|
509
|
+
"table": self.table,
|
|
510
|
+
"column": missing,
|
|
511
|
+
"message": f"条件约束失败: 列 '{missing}' 不在表 '{self.table}' 中。",
|
|
512
|
+
}
|
|
513
|
+
)
|
|
514
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
515
|
+
except ValueError as e:
|
|
516
|
+
errors.append(
|
|
517
|
+
{
|
|
518
|
+
"error_type": "ConstraintConfigError",
|
|
519
|
+
"table": self.table,
|
|
520
|
+
"message": f"条件约束失败: {e}",
|
|
521
|
+
}
|
|
522
|
+
)
|
|
523
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
524
|
+
|
|
525
|
+
# 根据逻辑运算符组合多个条件的布尔掩码(使用已归一化的 logic)
|
|
526
|
+
if logic == "or":
|
|
527
|
+
# OR 逻辑: 任一条件满足即触发
|
|
528
|
+
mask = masks[0].copy()
|
|
529
|
+
for m in masks[1:]:
|
|
530
|
+
mask = mask | m
|
|
531
|
+
else:
|
|
532
|
+
# AND 逻辑: 所有条件都满足才触发
|
|
533
|
+
mask = masks[0].copy()
|
|
534
|
+
for m in masks[1:]:
|
|
535
|
+
mask = mask & m
|
|
536
|
+
triggered_rows = df[mask]
|
|
537
|
+
elif self.if_column:
|
|
538
|
+
# 简单条件模式: 当 if_column == if_value 时触发
|
|
539
|
+
if self.if_column not in df.columns:
|
|
540
|
+
errors.append(
|
|
541
|
+
{
|
|
542
|
+
"error_type": "ConstraintConfigError",
|
|
543
|
+
"table": self.table,
|
|
544
|
+
"column": self.if_column,
|
|
545
|
+
"message": f"条件约束失败: 列 '{self.if_column}' 不在表 '{self.table}' 中。",
|
|
546
|
+
}
|
|
547
|
+
)
|
|
548
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
549
|
+
# §1.2: if_value 留空(None)视为配置错误——原实现拿整列和 None 比较恒 False,
|
|
550
|
+
# 零触发行让约束形同虚设且无提示。想"无条件触发"的正确写法是不配 if_column。
|
|
551
|
+
if self.if_value is None:
|
|
552
|
+
errors.append(
|
|
553
|
+
{
|
|
554
|
+
"error_type": "ConstraintConfigError",
|
|
555
|
+
"table": self.table,
|
|
556
|
+
"column": self.if_column,
|
|
557
|
+
"message": (
|
|
558
|
+
"条件约束失败: if_value 未配置(留空),简单条件模式必须指定触发值 if_value;"
|
|
559
|
+
"如需对所有行触发 THEN 检查,请移除 if_column 配置。"
|
|
560
|
+
),
|
|
561
|
+
}
|
|
562
|
+
)
|
|
563
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
564
|
+
triggered_rows = df[df[self.if_column] == self.if_value]
|
|
565
|
+
else:
|
|
566
|
+
# 无条件模式: if_conditions 为空且 if_column 未设置,对所有行触发 THEN 检查
|
|
567
|
+
triggered_rows = df
|
|
568
|
+
|
|
569
|
+
# ============================================================================
|
|
570
|
+
# 检查触发的行是否满足 then 条件
|
|
571
|
+
# ============================================================================
|
|
572
|
+
# D6: deadline 检查点。逐行约束内部必须有中断点,否则百万行表跑很久,
|
|
573
|
+
# timeout_seconds 形同虚设(原 engine 只在约束之间检查)。每 200 行查一次,
|
|
574
|
+
# 超时则中断循环并返回 Timeout 标记。
|
|
575
|
+
deadline = kwargs.get("deadline")
|
|
576
|
+
check_interval = 200
|
|
577
|
+
triggered_dict = triggered_rows.to_dict("index")
|
|
578
|
+
for i, (index, row) in enumerate(triggered_dict.items()):
|
|
579
|
+
if deadline is not None and i % check_interval == 0 and time.monotonic() > deadline:
|
|
580
|
+
errors.append(
|
|
581
|
+
{
|
|
582
|
+
"error_type": "Timeout",
|
|
583
|
+
"stage": "constraint",
|
|
584
|
+
"table": self.table,
|
|
585
|
+
"message": (
|
|
586
|
+
f"条件约束执行超时,已在第 {i} 行(共 {len(triggered_dict)} 触发行)中断,剩余触发行未校验。"
|
|
587
|
+
),
|
|
588
|
+
}
|
|
589
|
+
)
|
|
590
|
+
logger.warning(f"条件约束超时中断: 已处理 {i}/{len(triggered_dict)} 触发行")
|
|
591
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|
|
592
|
+
value_to_check = row[self.then_column]
|
|
593
|
+
# 使用解析出的验证函数检查值
|
|
594
|
+
# 如果配置了 ref_column,传入整行数据以支持列间比较
|
|
595
|
+
ref_column = (
|
|
596
|
+
self.then_condition_config.get("ref_column") if isinstance(self.then_condition_config, dict) else None
|
|
597
|
+
)
|
|
598
|
+
if ref_column:
|
|
599
|
+
check_result = self._condition_func(value_to_check, row)
|
|
600
|
+
else:
|
|
601
|
+
check_result = self._condition_func(value_to_check, None)
|
|
602
|
+
# 序比较(greater_than/less_than)的行级三态:值或参照不可数值化时
|
|
603
|
+
# predicate 返回 None——无法判定不等于违反,跳过该行(脏数据由列
|
|
604
|
+
# 类型约束负责报告),不再误报 ConditionalViolation。
|
|
605
|
+
if check_result is None:
|
|
606
|
+
continue
|
|
607
|
+
if not check_result:
|
|
608
|
+
# 构建触发条件的信息,用于错误消息
|
|
609
|
+
if_value_payload: dict[str, Any] = {}
|
|
610
|
+
if self.if_conditions:
|
|
611
|
+
for c in self.if_conditions:
|
|
612
|
+
col = c.get("if_column") or c.get("column")
|
|
613
|
+
if col and col in row:
|
|
614
|
+
if_value_payload[col] = row[col]
|
|
615
|
+
else:
|
|
616
|
+
if_value_payload[self.if_column] = self.if_value
|
|
617
|
+
|
|
618
|
+
row_index = int(index) if index is not None else 0
|
|
619
|
+
errors.append(
|
|
620
|
+
{
|
|
621
|
+
"error_type": "ConditionalViolation",
|
|
622
|
+
"table": self.table,
|
|
623
|
+
"row_index": row_index,
|
|
624
|
+
"value": {**if_value_payload, self.then_column: value_to_check},
|
|
625
|
+
"message": f"条件约束冲突: 当满足条件时, '{self.then_column}' 的值 '{value_to_check}' 不满足要求 ({self._condition_str}).",
|
|
626
|
+
}
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
return {"errors": errors, "info": self.get_constraint_info()}
|