precis-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +16 -0
- app/api/__init__.py +32 -0
- app/api/dependencies.py +185 -0
- app/api/main.py +301 -0
- app/api/middleware/__init__.py +34 -0
- app/api/middleware/exception_handler.py +126 -0
- app/api/middleware/request_logging.py +97 -0
- app/api/middleware/token_auth.py +215 -0
- app/api/models/__init__.py +151 -0
- app/api/models/connection_rules.py +98 -0
- app/api/models/files.py +99 -0
- app/api/models/full_validation.py +554 -0
- app/api/models/project.py +117 -0
- app/api/models/projects.py +80 -0
- app/api/models/schema.py +121 -0
- app/api/models/v2_responses.py +46 -0
- app/api/models/validation.py +303 -0
- app/api/models/workspace.py +118 -0
- app/api/routers/__init__.py +56 -0
- app/api/routers/ai/__init__.py +36 -0
- app/api/routers/ai/chat.py +243 -0
- app/api/routers/ai/generate.py +156 -0
- app/api/routers/ai/hardware.py +199 -0
- app/api/routers/ai/jobs.py +544 -0
- app/api/routers/ai/migrate.py +471 -0
- app/api/routers/ai/models.py +352 -0
- app/api/routers/ai/ollama.py +108 -0
- app/api/routers/ai/providers.py +491 -0
- app/api/routers/ai/router.py +44 -0
- app/api/routers/ai/stream.py +281 -0
- app/api/routers/ai/utils.py +182 -0
- app/api/routers/core/__init__.py +34 -0
- app/api/routers/core/connection_rules.py +188 -0
- app/api/routers/core/data_sources.py +374 -0
- app/api/routers/core/regex.py +333 -0
- app/api/routers/core/reporting.py +263 -0
- app/api/routers/files/__init__.py +24 -0
- app/api/routers/files/ops.py +179 -0
- app/api/routers/files/transfer.py +101 -0
- app/api/routers/preview/__init__.py +35 -0
- app/api/routers/preview/content_mode.py +264 -0
- app/api/routers/preview/header_row.py +154 -0
- app/api/routers/preview/models.py +81 -0
- app/api/routers/preview/path_mode.py +440 -0
- app/api/routers/preview/router.py +30 -0
- app/api/routers/project/__init__.py +61 -0
- app/api/routers/project/base.py +111 -0
- app/api/routers/project/constraint.py +345 -0
- app/api/routers/project/full_config.py +411 -0
- app/api/routers/project/full_config_writer.py +327 -0
- app/api/routers/project/helpers.py +167 -0
- app/api/routers/project/inspection_fix.py +330 -0
- app/api/routers/project/manifest.py +605 -0
- app/api/routers/project/models.py +177 -0
- app/api/routers/project/pattern.py +268 -0
- app/api/routers/project/regex.py +358 -0
- app/api/routers/project/scanner.py +157 -0
- app/api/routers/project/schema.py +567 -0
- app/api/routers/project/schema_helpers.py +149 -0
- app/api/routers/project/settings.py +399 -0
- app/api/routers/project/template.py +325 -0
- app/api/routers/project/validation.py +227 -0
- app/api/routers/project/view.py +177 -0
- app/api/routers/project/workspaces.py +167 -0
- app/api/routers/projects/__init__.py +25 -0
- app/api/routers/projects/create.py +121 -0
- app/api/routers/projects/open.py +103 -0
- app/api/routers/projects/scan.py +100 -0
- app/api/routers/validation/__init__.py +39 -0
- app/api/routers/validation/common.py +263 -0
- app/api/routers/validation/content_mode.py +401 -0
- app/api/routers/validation/history.py +235 -0
- app/api/routers/validation/inline_mode.py +196 -0
- app/api/routers/validation/path_mode.py +314 -0
- app/api/routers/validation/router.py +51 -0
- app/api/services/full_validation_response_builder.py +412 -0
- app/api/services/io_error_messages.py +41 -0
- app/api/services/preview_service.py +248 -0
- app/cli/__init__.py +18 -0
- app/cli/__main__.py +30 -0
- app/cli/shared_services/__init__.py +42 -0
- app/cli/shared_services/config_ops.py +581 -0
- app/cli/shared_services/generation_ops.py +160 -0
- app/cli/shared_services/project_ops.py +288 -0
- app/cli/shell/__init__.py +48 -0
- app/cli/shell/commands/__init__.py +63 -0
- app/cli/shell/commands/ai/__init__.py +260 -0
- app/cli/shell/commands/ai/chat.py +331 -0
- app/cli/shell/commands/ai/delete.py +165 -0
- app/cli/shell/commands/ai/diff.py +70 -0
- app/cli/shell/commands/ai/display.py +428 -0
- app/cli/shell/commands/ai/executor.py +225 -0
- app/cli/shell/commands/ai/executor_utils.py +271 -0
- app/cli/shell/commands/ai/generate.py +285 -0
- app/cli/shell/commands/ai/interaction.py +262 -0
- app/cli/shell/commands/ai/migrate.py +281 -0
- app/cli/shell/commands/ai/resolver.py +224 -0
- app/cli/shell/commands/ai/status.py +101 -0
- app/cli/shell/commands/ai/switch.py +158 -0
- app/cli/shell/commands/ai/utils.py +52 -0
- app/cli/shell/commands/base.py +343 -0
- app/cli/shell/commands/config/__init__.py +131 -0
- app/cli/shell/commands/config/base.py +80 -0
- app/cli/shell/commands/config/check.py +246 -0
- app/cli/shell/commands/config/edit.py +148 -0
- app/cli/shell/commands/config/get.py +111 -0
- app/cli/shell/commands/config/init.py +123 -0
- app/cli/shell/commands/config/inspect.py +164 -0
- app/cli/shell/commands/config/list.py +104 -0
- app/cli/shell/commands/config/set.py +119 -0
- app/cli/shell/commands/config/show.py +150 -0
- app/cli/shell/commands/exit.py +67 -0
- app/cli/shell/commands/help.py +104 -0
- app/cli/shell/commands/infer_schema.py +156 -0
- app/cli/shell/commands/open.py +201 -0
- app/cli/shell/commands/project.py +215 -0
- app/cli/shell/commands/provider.py +619 -0
- app/cli/shell/commands/system.py +170 -0
- app/cli/shell/commands/validate.py +455 -0
- app/cli/shell/completer.py +231 -0
- app/cli/shell/config_storage.py +191 -0
- app/cli/shell/exceptions.py +90 -0
- app/cli/shell/formatter.py +371 -0
- app/cli/shell/interactive_menu.py +348 -0
- app/cli/shell/main.py +286 -0
- app/cli/shell/parser.py +266 -0
- app/cli/start.py +180 -0
- app/cli_main.py +52 -0
- app/mcp_server.py +311 -0
- app/shared/core/__init__.py +18 -0
- app/shared/core/app_version.py +59 -0
- app/shared/core/config/__init__.py +119 -0
- app/shared/core/config/server.py +193 -0
- app/shared/core/data_source/__init__.py +106 -0
- app/shared/core/data_source/loader.py +348 -0
- app/shared/core/data_source/loaders/__init__.py +213 -0
- app/shared/core/data_source/loaders/base.py +215 -0
- app/shared/core/data_source/loaders/converter.py +337 -0
- app/shared/core/data_source/loaders/csv_loader.py +252 -0
- app/shared/core/data_source/loaders/excel_loader.py +666 -0
- app/shared/core/data_source/loaders/extractor.py +268 -0
- app/shared/core/data_source/loaders/json_loader.py +354 -0
- app/shared/core/data_source/loaders/registry.py +88 -0
- app/shared/core/data_source/loaders/sql_loader.py +313 -0
- app/shared/core/data_source/loaders/strategies/__init__.py +100 -0
- app/shared/core/data_source/loaders/strategies/array_parser.py +277 -0
- app/shared/core/data_source/loaders/strategies/lines_parser.py +355 -0
- app/shared/core/data_source/loaders/strategies/object_parser.py +278 -0
- app/shared/core/data_source/schema_info.py +50 -0
- app/shared/core/data_source/specs/__init__.py +128 -0
- app/shared/core/data_source/specs/base.py +258 -0
- app/shared/core/data_source/specs/csv_source.py +137 -0
- app/shared/core/data_source/specs/excel_source.py +141 -0
- app/shared/core/data_source/specs/file_base.py +252 -0
- app/shared/core/data_source/specs/json_source.py +226 -0
- app/shared/core/data_source/specs/sql_source.py +161 -0
- app/shared/core/encoding.py +36 -0
- app/shared/core/io/__init__.py +24 -0
- app/shared/core/io/yaml.py +379 -0
- app/shared/core/manifest_schema/__init__.py +52 -0
- app/shared/core/manifest_schema/types.py +99 -0
- app/shared/core/manifest_schema/version.py +175 -0
- app/shared/core/patterns/__init__.py +24 -0
- app/shared/core/patterns/loader.py +134 -0
- app/shared/core/patterns/writer.py +244 -0
- app/shared/core/project/__init__.py +47 -0
- app/shared/core/project/constraint/builders/__init__.py +50 -0
- app/shared/core/project/constraint/builders/base.py +100 -0
- app/shared/core/project/constraint/builders/composite.py +77 -0
- app/shared/core/project/constraint/builders/conditional.py +67 -0
- app/shared/core/project/constraint/builders/foreign_key.py +53 -0
- app/shared/core/project/constraint/builders/registry.py +64 -0
- app/shared/core/project/constraint/builders/scripted.py +51 -0
- app/shared/core/project/constraint/builders/single_column.py +86 -0
- app/shared/core/project/constraint/builders/unique.py +53 -0
- app/shared/core/project/constraint/factory.py +170 -0
- app/shared/core/project/constraint/reader.py +214 -0
- app/shared/core/project/constraint/registry.py +233 -0
- app/shared/core/project/constraint/types/__init__.py +63 -0
- app/shared/core/project/constraint/types/constraint_file.py +261 -0
- app/shared/core/project/constraint/types/refs.py +460 -0
- app/shared/core/project/constraint/types.py +28 -0
- app/shared/core/project/constraint/writer.py +181 -0
- app/shared/core/project/loader/__init__.py +56 -0
- app/shared/core/project/loader/loader.py +30 -0
- app/shared/core/project/loader/loader_parts/config_inspector.py +137 -0
- app/shared/core/project/loader/loader_parts/embedded_constraints.py +224 -0
- app/shared/core/project/loader/loader_parts/file_loaders.py +58 -0
- app/shared/core/project/loader/loader_parts/inspection_ids.py +85 -0
- app/shared/core/project/loader/loader_parts/inspector_helpers.py +312 -0
- app/shared/core/project/loader/loader_parts/inspector_id_checks.py +274 -0
- app/shared/core/project/loader/loader_parts/inspector_reference_checks.py +690 -0
- app/shared/core/project/loader/loader_parts/inspector_uniqueness_checks.py +286 -0
- app/shared/core/project/loader/loader_parts/loading_error_messages.py +298 -0
- app/shared/core/project/loader/loader_parts/main.py +432 -0
- app/shared/core/project/loader/loader_parts/path_validation.py +117 -0
- app/shared/core/project/loader/loader_parts/runtime.py +125 -0
- app/shared/core/project/loader/types.py +246 -0
- app/shared/core/project/manifest/coverage.py +391 -0
- app/shared/core/project/manifest/reader.py +260 -0
- app/shared/core/project/manifest/types.py +91 -0
- app/shared/core/project/manifest/types_parts/__init__.py +60 -0
- app/shared/core/project/manifest/types_parts/constants.py +40 -0
- app/shared/core/project/manifest/types_parts/data_source.py +68 -0
- app/shared/core/project/manifest/types_parts/info.py +59 -0
- app/shared/core/project/manifest/types_parts/manifest.py +294 -0
- app/shared/core/project/manifest/types_parts/refs.py +153 -0
- app/shared/core/project/manifest/types_parts/settings.py +92 -0
- app/shared/core/project/manifest/types_parts/settings_file_processing.py +64 -0
- app/shared/core/project/manifest/types_parts/settings_script_security.py +78 -0
- app/shared/core/project/manifest/types_parts/settings_validation.py +83 -0
- app/shared/core/project/manifest/types_parts/template.py +66 -0
- app/shared/core/project/manifest/writer.py +262 -0
- app/shared/core/project/manual_data/__init__.py +27 -0
- app/shared/core/project/manual_data/types.py +75 -0
- app/shared/core/project/regex/reader.py +197 -0
- app/shared/core/project/regex/types.py +405 -0
- app/shared/core/project/regex/writer.py +123 -0
- app/shared/core/project/schema/reader.py +170 -0
- app/shared/core/project/schema/types.py +47 -0
- app/shared/core/project/schema/types_parts/__init__.py +36 -0
- app/shared/core/project/schema/types_parts/column.py +174 -0
- app/shared/core/project/schema/types_parts/column_utils.py +72 -0
- app/shared/core/project/schema/types_parts/constraint.py +165 -0
- app/shared/core/project/schema/types_parts/schema_id.py +66 -0
- app/shared/core/project/schema/types_parts/source.py +255 -0
- app/shared/core/project/schema/types_parts/source_options.py +347 -0
- app/shared/core/project/schema/types_parts/table.py +230 -0
- app/shared/core/project/schema/writer.py +139 -0
- app/shared/core/project/schema_ref_check.py +95 -0
- app/shared/core/project/template/__init__.py +27 -0
- app/shared/core/project/template/expander.py +263 -0
- app/shared/core/project/template/reader.py +120 -0
- app/shared/core/project/template/types.py +114 -0
- app/shared/core/project/transform/reader.py +76 -0
- app/shared/core/project/transform/types.py +116 -0
- app/shared/core/project/transform/writer.py +84 -0
- app/shared/core/pydantic_messages.py +59 -0
- app/shared/core/reporter/__init__.py +46 -0
- app/shared/core/reporter/reporter.py +220 -0
- app/shared/core/reporter/reporters/__init__.py +65 -0
- app/shared/core/reporter/reporters/base.py +188 -0
- app/shared/core/reporter/reporters/dingtalk_app_reporter.py +274 -0
- app/shared/core/reporter/reporters/email_reporter.py +271 -0
- app/shared/core/reporter/reporters/feishu_app_reporter.py +467 -0
- app/shared/core/reporter/reporters/local_file_reporter.py +208 -0
- app/shared/core/reporter/reporters/wecom_app_reporter.py +268 -0
- app/shared/core/utils/__init__.py +18 -0
- app/shared/core/utils/path_utils.py +60 -0
- app/shared/core/utils/regex_extract.py +113 -0
- app/shared/domain/__init__.py +114 -0
- app/shared/domain/constraints/__init__.py +76 -0
- app/shared/domain/constraints/allowed_values.py +211 -0
- app/shared/domain/constraints/base.py +141 -0
- app/shared/domain/constraints/charset.py +337 -0
- app/shared/domain/constraints/composite.py +174 -0
- app/shared/domain/constraints/condition_registry.py +130 -0
- app/shared/domain/constraints/conditional.py +629 -0
- app/shared/domain/constraints/date_logic.py +731 -0
- app/shared/domain/constraints/foreign_key.py +261 -0
- app/shared/domain/constraints/key_normalization.py +56 -0
- app/shared/domain/constraints/not_null.py +185 -0
- app/shared/domain/constraints/range.py +360 -0
- app/shared/domain/constraints/regex.py +218 -0
- app/shared/domain/constraints/scripted.py +276 -0
- app/shared/domain/constraints/unique.py +233 -0
- app/shared/domain/data_engine.py +384 -0
- app/shared/domain/data_types.py +113 -0
- app/shared/domain/data_types_parts/__init__.py +67 -0
- app/shared/domain/data_types_parts/base.py +204 -0
- app/shared/domain/data_types_parts/composite.py +250 -0
- app/shared/domain/data_types_parts/expression.py +202 -0
- app/shared/domain/data_types_parts/extracted.py +106 -0
- app/shared/domain/data_types_parts/json_types.py +234 -0
- app/shared/domain/data_types_parts/scalars.py +719 -0
- app/shared/domain/data_types_parts/sequence.py +129 -0
- app/shared/domain/dataset_schema.py +48 -0
- app/shared/domain/eval_sandbox.py +63 -0
- app/shared/domain/expression_system.py +366 -0
- app/shared/domain/regex_flags.py +53 -0
- app/shared/domain/schema/builder.py +223 -0
- app/shared/domain/schema/models.py +339 -0
- app/shared/domain/transforms/__init__.py +28 -0
- app/shared/domain/transforms/aggregate.py +141 -0
- app/shared/domain/transforms/base.py +171 -0
- app/shared/domain/transforms/cast_type.py +120 -0
- app/shared/domain/transforms/concat.py +103 -0
- app/shared/domain/transforms/conditional_assign.py +114 -0
- app/shared/domain/transforms/date_format.py +81 -0
- app/shared/domain/transforms/digits.py +77 -0
- app/shared/domain/transforms/drop_duplicates.py +94 -0
- app/shared/domain/transforms/fill_na.py +94 -0
- app/shared/domain/transforms/filter_rows.py +97 -0
- app/shared/domain/transforms/lookup.py +78 -0
- app/shared/domain/transforms/lower_case.py +70 -0
- app/shared/domain/transforms/map_value.py +90 -0
- app/shared/domain/transforms/math_expr.py +112 -0
- app/shared/domain/transforms/modulo.py +82 -0
- app/shared/domain/transforms/regex_extract.py +103 -0
- app/shared/domain/transforms/registry.py +95 -0
- app/shared/domain/transforms/replace.py +88 -0
- app/shared/domain/transforms/sort_rows.py +94 -0
- app/shared/domain/transforms/string_split.py +84 -0
- app/shared/domain/transforms/strip.py +73 -0
- app/shared/domain/transforms/substring.py +92 -0
- app/shared/domain/transforms/upper_case.py +70 -0
- app/shared/domain/transforms/weighted_sum.py +104 -0
- app/shared/domain/validation_constraints.py +69 -0
- app/shared/services/__init__.py +62 -0
- app/shared/services/ai/__init__.py +48 -0
- app/shared/services/ai/agent/__init__.py +40 -0
- app/shared/services/ai/agent/chat_tools/__init__.py +47 -0
- app/shared/services/ai/agent/chat_tools/apply_actions.py +592 -0
- app/shared/services/ai/agent/chat_tools/ask_user.py +237 -0
- app/shared/services/ai/agent/chat_tools/read_canvas.py +140 -0
- app/shared/services/ai/agent/chat_tools/read_project.py +160 -0
- app/shared/services/ai/agent/chat_tools/read_table.py +301 -0
- app/shared/services/ai/agent/chat_tools/schemas.py +105 -0
- app/shared/services/ai/agent/chat_tools/validate_table.py +131 -0
- app/shared/services/ai/agent/executor.py +554 -0
- app/shared/services/ai/agent/memory.py +256 -0
- app/shared/services/ai/agent/planner.py +282 -0
- app/shared/services/ai/agent/tool_registry.py +296 -0
- app/shared/services/ai/agent/tools/__init__.py +32 -0
- app/shared/services/ai/agent/tools/config_generate.py +133 -0
- app/shared/services/ai/agent/tools/config_refine.py +109 -0
- app/shared/services/ai/agent/tools/config_validate.py +403 -0
- app/shared/services/ai/agent/tools/merge_results.py +136 -0
- app/shared/services/ai/agent/tools/plan_chunks.py +79 -0
- app/shared/services/ai/agent/tools/script_parse.py +359 -0
- app/shared/services/ai/agent/types.py +138 -0
- app/shared/services/ai/chat_agent_runner.py +596 -0
- app/shared/services/ai/chat_orchestrator.py +725 -0
- app/shared/services/ai/failure_messages.py +65 -0
- app/shared/services/ai/job_storage.py +274 -0
- app/shared/services/ai/migrate_service.py +550 -0
- app/shared/services/ai/streaming/__init__.py +65 -0
- app/shared/services/ai/streaming/event_journal.py +197 -0
- app/shared/services/ai/streaming/orchestrator.py +262 -0
- app/shared/services/ai/streaming/pending_interaction_store.py +227 -0
- app/shared/services/ai/streaming/sse_response.py +148 -0
- app/shared/services/ai/streaming/types.py +73 -0
- app/shared/services/ai/types.py +213 -0
- app/shared/services/ai/utils.py +322 -0
- app/shared/services/diff/config_diff.py +320 -0
- app/shared/services/hardware.py +237 -0
- app/shared/services/llm/__init__.py +62 -0
- app/shared/services/llm/actions/__init__.py +42 -0
- app/shared/services/llm/actions/_canvas_validator.py +147 -0
- app/shared/services/llm/actions/_constraint_validator.py +401 -0
- app/shared/services/llm/actions/_regex_validator.py +85 -0
- app/shared/services/llm/actions/_schema_validator.py +82 -0
- app/shared/services/llm/actions/_settings_validator.py +76 -0
- app/shared/services/llm/actions/_transform_validator.py +78 -0
- app/shared/services/llm/actions/action_handlers.py +400 -0
- app/shared/services/llm/actions/action_parser.py +63 -0
- app/shared/services/llm/actions/action_processor.py +468 -0
- app/shared/services/llm/actions/action_validator.py +311 -0
- app/shared/services/llm/actions/diff_compute.py +195 -0
- app/shared/services/llm/actions/regex_handlers.py +284 -0
- app/shared/services/llm/actions/registry.py +336 -0
- app/shared/services/llm/actions/schema_handlers.py +335 -0
- app/shared/services/llm/actions/settings_handlers.py +180 -0
- app/shared/services/llm/actions/specs.py +259 -0
- app/shared/services/llm/actions/transform_handlers.py +279 -0
- app/shared/services/llm/actions/validation_types.py +147 -0
- app/shared/services/llm/cache/__init__.py +18 -0
- app/shared/services/llm/cache/response_cache.py +80 -0
- app/shared/services/llm/chat/__init__.py +35 -0
- app/shared/services/llm/chat/chat_system_prompt.py +561 -0
- app/shared/services/llm/chat/response_parser.py +296 -0
- app/shared/services/llm/config/__init__.py +45 -0
- app/shared/services/llm/config/crypto.py +135 -0
- app/shared/services/llm/config/loader.py +258 -0
- app/shared/services/llm/config/models.py +202 -0
- app/shared/services/llm/config/presets.py +128 -0
- app/shared/services/llm/config_generator.py +163 -0
- app/shared/services/llm/constraints/__init__.py +41 -0
- app/shared/services/llm/constraints/constraint_builder.py +177 -0
- app/shared/services/llm/constraints/constraint_deletion.py +84 -0
- app/shared/services/llm/constraints/constraint_id.py +174 -0
- app/shared/services/llm/constraints/frontend_instructions.py +333 -0
- app/shared/services/llm/constraints/inline_batch.py +279 -0
- app/shared/services/llm/discovery/__init__.py +35 -0
- app/shared/services/llm/discovery/scanner.py +181 -0
- app/shared/services/llm/generation/__init__.py +42 -0
- app/shared/services/llm/generation/agent_wiring.py +156 -0
- app/shared/services/llm/generation/config_builder.py +386 -0
- app/shared/services/llm/generation/errors.py +36 -0
- app/shared/services/llm/generation/existing_config.py +99 -0
- app/shared/services/llm/generation/profiler.py +279 -0
- app/shared/services/llm/generation/prompt_builder.py +199 -0
- app/shared/services/llm/generation/response_parser.py +144 -0
- app/shared/services/llm/generation/service.py +622 -0
- app/shared/services/llm/models.py +104 -0
- app/shared/services/llm/providers/__init__.py +48 -0
- app/shared/services/llm/providers/base.py +310 -0
- app/shared/services/llm/providers/cached_provider.py +90 -0
- app/shared/services/llm/providers/ollama.py +498 -0
- app/shared/services/llm/providers/openai.py +334 -0
- app/shared/services/llm/providers/registry.py +108 -0
- app/shared/services/llm/schema_resolver.py +141 -0
- app/shared/services/llm/suggestion_utils.py +219 -0
- app/shared/services/llm/validate_executor.py +163 -0
- app/shared/services/llm/yaml_io.py +406 -0
- app/shared/services/preview/__init__.py +18 -0
- app/shared/services/preview/loader.py +145 -0
- app/shared/services/preview/path_validation.py +138 -0
- app/shared/services/project_loader.py +57 -0
- app/shared/services/schema_inference.py +260 -0
- app/shared/services/schema_runtime_builder.py +120 -0
- app/shared/services/validation/__init__.py +52 -0
- app/shared/services/validation/chunked_loader.py +509 -0
- app/shared/services/validation/dag/__init__.py +35 -0
- app/shared/services/validation/dag/builder.py +141 -0
- app/shared/services/validation/dag/executor.py +225 -0
- app/shared/services/validation/dag/sorter.py +77 -0
- app/shared/services/validation/data_loader.py +231 -0
- app/shared/services/validation/engine.py +446 -0
- app/shared/services/validation/executor.py +1030 -0
- app/shared/services/validation/extractors.py +266 -0
- app/shared/services/validation/history.py +209 -0
- app/shared/services/validation/json_payload.py +136 -0
- app/shared/services/validation/loader.py +152 -0
- app/shared/services/validation/memory_monitor.py +179 -0
- app/shared/services/validation/postprocess.py +208 -0
- app/shared/services/validation/progress.py +64 -0
- app/shared/services/validation/report_export.py +222 -0
- app/shared/services/validation/resolver.py +180 -0
- app/shared/services/validation/service.py +418 -0
- app/shared/services/validation/types.py +238 -0
- app/shared/services/validation/validators/__init__.py +36 -0
- app/shared/services/validation/validators/adapter.py +182 -0
- app/shared/services/validation/validators/base.py +332 -0
- app/shared/services/validation/validators/composite.py +233 -0
- app/shared/services/validation/validators/date_logic.py +276 -0
- app/start_server.py +133 -0
- precis_cli-0.1.3.dist-info/METADATA +180 -0
- precis_cli-0.1.3.dist-info/RECORD +442 -0
- precis_cli-0.1.3.dist-info/WHEEL +5 -0
- precis_cli-0.1.3.dist-info/entry_points.txt +4 -0
- precis_cli-0.1.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 全量校验响应构建器
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 将 ValidationExecutor 的输出转换为 FullValidationResponse
|
|
21
|
+
- 封装错误分类、ID -> Name 映射、通过项构造、统计聚合、覆盖率组装
|
|
22
|
+
- 使 API 路由层仅负责参数校验与执行器调用
|
|
23
|
+
|
|
24
|
+
输入示例:
|
|
25
|
+
builder = FullValidationResponseBuilder(executor, started=time.monotonic(), coverage=coverage)
|
|
26
|
+
response = builder.build_from_result(result)
|
|
27
|
+
|
|
28
|
+
输出示例:
|
|
29
|
+
FullValidationResponse 实例,包含 success/summary/errors/passed_items/statistics 等字段
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import logging
|
|
35
|
+
import time
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
from app.api.models import (
|
|
39
|
+
FullValidationErrorItem,
|
|
40
|
+
FullValidationResponse,
|
|
41
|
+
FullValidationSummary,
|
|
42
|
+
ValidationPassedItem,
|
|
43
|
+
ValidationStatistics,
|
|
44
|
+
)
|
|
45
|
+
from app.shared.services.validation.executor import ValidationExecutor
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class FullValidationResponseBuilder:
|
|
51
|
+
"""
|
|
52
|
+
全量校验响应构建器。
|
|
53
|
+
|
|
54
|
+
负责把执行器返回的原始结果字典转换为前端需要的 Pydantic 响应模型,
|
|
55
|
+
同时支持执行器初始化失败或执行异常时的降级响应构建。
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
# 回归 #10: 单次响应返回的错误条目上限。超过此数时截断 errors 列表(保留真实总数),
|
|
59
|
+
# 防止百万级错误导致响应数百 MB、前端卡死。5000 条足以覆盖典型排查需求。
|
|
60
|
+
MAX_ERRORS = 5000
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
executor: ValidationExecutor | None,
|
|
65
|
+
started: float,
|
|
66
|
+
coverage: Any | None = None,
|
|
67
|
+
):
|
|
68
|
+
"""
|
|
69
|
+
初始化响应构建器。
|
|
70
|
+
|
|
71
|
+
参数:
|
|
72
|
+
executor: 校验执行器实例;初始化失败时可为 None
|
|
73
|
+
started: 路由层记录的起始时间(time.monotonic())
|
|
74
|
+
coverage: 预计算的覆盖率 payload(可选)
|
|
75
|
+
"""
|
|
76
|
+
self.executor = executor
|
|
77
|
+
self.started = started
|
|
78
|
+
self.coverage = coverage
|
|
79
|
+
|
|
80
|
+
def _duration_ms(self) -> int:
|
|
81
|
+
"""计算从 started 到当前的耗时(毫秒)。"""
|
|
82
|
+
return int((time.monotonic() - self.started) * 1000)
|
|
83
|
+
|
|
84
|
+
def _build_id_to_name(self) -> dict[str, str]:
|
|
85
|
+
"""构建 Schema ID -> 显示名称 的映射。"""
|
|
86
|
+
id_to_name: dict[str, str] = {}
|
|
87
|
+
if not self.executor or not self.executor.dataset_schema:
|
|
88
|
+
return id_to_name
|
|
89
|
+
|
|
90
|
+
for tid, schema in self.executor.dataset_schema.tables.items():
|
|
91
|
+
target_name = schema.name or tid
|
|
92
|
+
if schema.id:
|
|
93
|
+
id_to_name[schema.id] = target_name
|
|
94
|
+
id_to_name[tid] = target_name
|
|
95
|
+
|
|
96
|
+
return id_to_name
|
|
97
|
+
|
|
98
|
+
def _build_errors(self, result: dict[str, Any]) -> list[FullValidationErrorItem]:
|
|
99
|
+
"""将 executor 结果中的 loading_errors 与 errors 转换为响应错误项。"""
|
|
100
|
+
errors: list[FullValidationErrorItem] = []
|
|
101
|
+
|
|
102
|
+
# 加载阶段错误
|
|
103
|
+
for le in result.get("loading_errors", []):
|
|
104
|
+
error_type = str(le.get("error_type") or "LoadError")
|
|
105
|
+
# message 为空的加载错误(如 SchemaIdDuplicate 只填 title/description)回退到
|
|
106
|
+
# 描述/标题,避免用户只看到一句孤零零的"建议:"而不知道问题本身是什么
|
|
107
|
+
message = str(le.get("message") or le.get("description") or le.get("title") or "")
|
|
108
|
+
suggestion = le.get("suggestion", "")
|
|
109
|
+
if suggestion:
|
|
110
|
+
message = f"{message}\n建议: {suggestion}"
|
|
111
|
+
|
|
112
|
+
errors.append(
|
|
113
|
+
FullValidationErrorItem(
|
|
114
|
+
stage="loading",
|
|
115
|
+
error_type=error_type,
|
|
116
|
+
check_type="DataLoad",
|
|
117
|
+
message=message,
|
|
118
|
+
table=le.get("table"),
|
|
119
|
+
source_path=le.get("file_path"),
|
|
120
|
+
source_file=le.get("source_file"),
|
|
121
|
+
source_sheet=le.get("source_sheet"),
|
|
122
|
+
)
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# 校验错误
|
|
126
|
+
for err in result.get("errors", []):
|
|
127
|
+
# 跳过重复的超时错误
|
|
128
|
+
if err.get("error_type") == "Timeout" and any(e.error_type == "Timeout" for e in errors):
|
|
129
|
+
continue
|
|
130
|
+
|
|
131
|
+
stage = str(err.get("stage") or "constraint")
|
|
132
|
+
message = str(err.get("message") or err.get("error_message") or "")
|
|
133
|
+
row_index_raw = err.get("row_index")
|
|
134
|
+
row_index = None if row_index_raw is None or row_index_raw != row_index_raw else row_index_raw
|
|
135
|
+
|
|
136
|
+
errors.append(
|
|
137
|
+
FullValidationErrorItem(
|
|
138
|
+
stage=stage,
|
|
139
|
+
error_type=str(err.get("error_type") or "ValidationError"),
|
|
140
|
+
check_type=err.get("check_type"),
|
|
141
|
+
message=message,
|
|
142
|
+
table=err.get("table"),
|
|
143
|
+
table_id=err.get("table_id"),
|
|
144
|
+
column=err.get("column"),
|
|
145
|
+
column_id=err.get("column_id"),
|
|
146
|
+
row_index=row_index,
|
|
147
|
+
# 回归 C3: 用 is not None 而非 falsy 判断,避免 value=0/False/"" 被吞成 None,
|
|
148
|
+
# 妨碍用户定位(如 Range 对值 0 的违规)。
|
|
149
|
+
value=str(err.get("value")) if err.get("value") is not None else None,
|
|
150
|
+
source_path=err.get("source_path"),
|
|
151
|
+
source_file=err.get("source_file"),
|
|
152
|
+
source_sheet=err.get("source_sheet"),
|
|
153
|
+
)
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
return errors
|
|
157
|
+
|
|
158
|
+
def _build_passed_items(self, result: dict[str, Any], id_to_name: dict[str, str]) -> list[ValidationPassedItem]:
|
|
159
|
+
"""从 executor 结果中构造所有通过项。"""
|
|
160
|
+
passed_items: list[ValidationPassedItem] = []
|
|
161
|
+
raw_datasets = result.get("raw_datasets", {})
|
|
162
|
+
|
|
163
|
+
# 1. 成功加载的数据表
|
|
164
|
+
for table_id in raw_datasets.keys():
|
|
165
|
+
table_name = id_to_name.get(table_id, table_id)
|
|
166
|
+
dataset_info = raw_datasets.get(table_id, {})
|
|
167
|
+
source_file = None
|
|
168
|
+
source_sheet = None
|
|
169
|
+
if isinstance(dataset_info, dict):
|
|
170
|
+
source_file = dataset_info.get("source_file")
|
|
171
|
+
source_sheet = dataset_info.get("source_sheet")
|
|
172
|
+
|
|
173
|
+
passed_items.append(
|
|
174
|
+
ValidationPassedItem(
|
|
175
|
+
stage="loading",
|
|
176
|
+
check_type="DataLoad",
|
|
177
|
+
message=f"数据表 '{table_name}' 加载成功",
|
|
178
|
+
table=table_name,
|
|
179
|
+
source_file=source_file,
|
|
180
|
+
source_sheet=source_sheet,
|
|
181
|
+
)
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
validation_details = result.get("validation_details", {})
|
|
185
|
+
|
|
186
|
+
# 2. 格式校验通过项
|
|
187
|
+
for format_check in validation_details.get("format_checks", []):
|
|
188
|
+
if not format_check.get("passed", False):
|
|
189
|
+
continue
|
|
190
|
+
passed_items.append(
|
|
191
|
+
ValidationPassedItem(
|
|
192
|
+
stage="format",
|
|
193
|
+
check_type="FormatValidation",
|
|
194
|
+
message=f"数据表 '{format_check['table']}' 格式校验通过",
|
|
195
|
+
table=format_check["table"],
|
|
196
|
+
source_file=format_check.get("source_file"),
|
|
197
|
+
source_sheet=format_check.get("source_sheet"),
|
|
198
|
+
)
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# 3. 约束校验通过项
|
|
202
|
+
for constraint_check in validation_details.get("constraint_checks", []):
|
|
203
|
+
if not constraint_check.get("passed", False):
|
|
204
|
+
continue
|
|
205
|
+
constraint_type = constraint_check.get("constraint_type", "Unknown")
|
|
206
|
+
passed_items.append(
|
|
207
|
+
ValidationPassedItem(
|
|
208
|
+
stage="constraint",
|
|
209
|
+
check_type=constraint_type,
|
|
210
|
+
message=constraint_check.get("description", "") or f"{constraint_type} 约束校验通过",
|
|
211
|
+
table=constraint_check.get("table"),
|
|
212
|
+
source_file=constraint_check.get("source_file"),
|
|
213
|
+
source_sheet=constraint_check.get("source_sheet"),
|
|
214
|
+
)
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
return passed_items
|
|
218
|
+
|
|
219
|
+
def _build_statistics(
|
|
220
|
+
self,
|
|
221
|
+
errors: list[FullValidationErrorItem],
|
|
222
|
+
passed_items: list[ValidationPassedItem],
|
|
223
|
+
) -> ValidationStatistics:
|
|
224
|
+
"""按类型和表聚合统计信息。
|
|
225
|
+
|
|
226
|
+
回归 C1: 顶层 total_checks/passed_count/failed_count/pass_rate 必须按"检查项"粒度
|
|
227
|
+
计算(与 passed_items 同粒度),不能分子按项、分母按错误行混算。原实现
|
|
228
|
+
failed_count=len(errors)(行数),导致 1 个约束在 N 行违规 → 通过率≈1/N,即使
|
|
229
|
+
其余检查全过也显示灾难性低通过率,污染历史趋势。
|
|
230
|
+
- 顶层计数:每个不同的 (stage, check_type, table) 失败检查记 1(与通过的检查项同粒度)。
|
|
231
|
+
- by_type/by_table 明细:保留按错误条目计数(便于用户看每类/每表的具体错误规模)。
|
|
232
|
+
"""
|
|
233
|
+
by_type: dict[str, dict[str, int]] = {}
|
|
234
|
+
by_table: dict[str, dict[str, int]] = {}
|
|
235
|
+
|
|
236
|
+
def _ensure_bucket(container: dict[str, dict[str, int]], key: Any) -> dict[str, int]:
|
|
237
|
+
target_key = str(key or "Unknown")
|
|
238
|
+
if target_key not in container:
|
|
239
|
+
container[target_key] = {"total": 0, "passed": 0, "failed": 0}
|
|
240
|
+
return container[target_key]
|
|
241
|
+
|
|
242
|
+
for item in passed_items:
|
|
243
|
+
type_bucket = _ensure_bucket(by_type, item.check_type)
|
|
244
|
+
type_bucket["total"] += 1
|
|
245
|
+
type_bucket["passed"] += 1
|
|
246
|
+
if item.table:
|
|
247
|
+
table_bucket = _ensure_bucket(by_table, item.table)
|
|
248
|
+
table_bucket["total"] += 1
|
|
249
|
+
table_bucket["passed"] += 1
|
|
250
|
+
|
|
251
|
+
# 收集"失败检查项"的唯一标识(检查项粒度),与 passed_items 对齐。
|
|
252
|
+
# 标识 = (stage, check_type/error_type, table);同一检查在多行违规只算 1 个失败检查。
|
|
253
|
+
failed_check_keys: set[tuple[str, str, str]] = set()
|
|
254
|
+
for error_item in errors:
|
|
255
|
+
check_key = (
|
|
256
|
+
error_item.stage,
|
|
257
|
+
str(error_item.check_type or error_item.error_type or "Unknown"),
|
|
258
|
+
str(error_item.table or ""),
|
|
259
|
+
)
|
|
260
|
+
failed_check_keys.add(check_key)
|
|
261
|
+
|
|
262
|
+
type_bucket = _ensure_bucket(by_type, error_item.check_type or error_item.error_type)
|
|
263
|
+
type_bucket["total"] += 1
|
|
264
|
+
type_bucket["failed"] += 1
|
|
265
|
+
if error_item.table:
|
|
266
|
+
table_bucket = _ensure_bucket(by_table, error_item.table)
|
|
267
|
+
table_bucket["total"] += 1
|
|
268
|
+
table_bucket["failed"] += 1
|
|
269
|
+
|
|
270
|
+
passed_count = len(passed_items)
|
|
271
|
+
failed_count = len(failed_check_keys)
|
|
272
|
+
total_checks = passed_count + failed_count
|
|
273
|
+
pass_rate = 100.0 if total_checks == 0 else (passed_count / total_checks * 100)
|
|
274
|
+
|
|
275
|
+
return ValidationStatistics(
|
|
276
|
+
total_checks=total_checks,
|
|
277
|
+
passed_count=passed_count,
|
|
278
|
+
failed_count=failed_count,
|
|
279
|
+
pass_rate=pass_rate,
|
|
280
|
+
by_type=by_type,
|
|
281
|
+
by_table=by_table,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
def _build_summary(
|
|
285
|
+
self,
|
|
286
|
+
result: dict[str, Any],
|
|
287
|
+
errors: list[FullValidationErrorItem],
|
|
288
|
+
*,
|
|
289
|
+
total_error_count_real: int | None = None,
|
|
290
|
+
) -> FullValidationSummary:
|
|
291
|
+
"""构造摘要统计。
|
|
292
|
+
|
|
293
|
+
参数:
|
|
294
|
+
total_error_count_real: 真实错误总数(截断前)。回归 #10: 当 errors 被截断时,
|
|
295
|
+
len(errors) < 真实总数,total_error_count 必须用真实值让用户了解问题规模。
|
|
296
|
+
"""
|
|
297
|
+
loading_error_count = len([e for e in errors if e.stage == "loading"])
|
|
298
|
+
format_error_count = len([e for e in errors if e.stage == "format"])
|
|
299
|
+
constraint_error_count = len([e for e in errors if e.stage == "constraint"])
|
|
300
|
+
|
|
301
|
+
raw_datasets = result.get("raw_datasets", {})
|
|
302
|
+
files_loaded = len(set(raw_datasets.keys()))
|
|
303
|
+
|
|
304
|
+
files_total = 0
|
|
305
|
+
if self.executor and self.executor.dataset_schema:
|
|
306
|
+
files_total = len(self.executor.dataset_schema.tables)
|
|
307
|
+
|
|
308
|
+
# total_error_count 用真实总数(截断前),供用户了解问题规模与前端显示"X/Y 条"。
|
|
309
|
+
total_for_summary = total_error_count_real if total_error_count_real is not None else len(errors)
|
|
310
|
+
|
|
311
|
+
return FullValidationSummary(
|
|
312
|
+
files_total=files_total,
|
|
313
|
+
files_loaded=files_loaded,
|
|
314
|
+
tables_loaded=len(raw_datasets),
|
|
315
|
+
loading_error_count=loading_error_count,
|
|
316
|
+
format_error_count=format_error_count,
|
|
317
|
+
constraint_error_count=constraint_error_count,
|
|
318
|
+
total_error_count=total_for_summary,
|
|
319
|
+
duration_ms=result.get("duration_ms", self._duration_ms()),
|
|
320
|
+
# C6: 透出遇错即停的中断状态(executor 在 result["interrupted"] 标记)
|
|
321
|
+
interrupted=bool(result.get("interrupted", False)),
|
|
322
|
+
# D6: 透出超时中断状态(executor 在 result["timeout_occurred"] 标记,原为死字段)
|
|
323
|
+
timed_out=bool(result.get("timeout_occurred", False)),
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
def build_from_result(self, result: dict[str, Any]) -> FullValidationResponse:
|
|
327
|
+
"""
|
|
328
|
+
基于 executor 成功返回的结果字典构建完整响应。
|
|
329
|
+
|
|
330
|
+
参数:
|
|
331
|
+
result: ValidationExecutor.execute 返回的字典
|
|
332
|
+
|
|
333
|
+
返回:
|
|
334
|
+
FullValidationResponse 实例
|
|
335
|
+
"""
|
|
336
|
+
errors = self._build_errors(result)
|
|
337
|
+
id_to_name = self._build_id_to_name()
|
|
338
|
+
passed_items = self._build_passed_items(result, id_to_name)
|
|
339
|
+
|
|
340
|
+
# 统计必须基于完整错误列表(by_type/by_table 聚合 + failed_count 检查项粒度),
|
|
341
|
+
# 在截断之前计算。
|
|
342
|
+
statistics = self._build_statistics(errors, passed_items)
|
|
343
|
+
|
|
344
|
+
# 回归 #10: 错误列表截断保护。整列格式错/大规模约束违规时,百万级错误 dict 会让
|
|
345
|
+
# 响应数百 MB、前端卡死。这里在响应层截断 errors 到上限,但 total_error_count 保留
|
|
346
|
+
# 真实总数(供用户了解问题规模)、errors_truncated 标记供前端显示"显示 X/Y 条"。
|
|
347
|
+
total_error_count_real = len(errors)
|
|
348
|
+
truncated = total_error_count_real > self.MAX_ERRORS
|
|
349
|
+
if truncated:
|
|
350
|
+
errors = errors[: self.MAX_ERRORS]
|
|
351
|
+
|
|
352
|
+
summary = self._build_summary(result, errors, total_error_count_real=total_error_count_real)
|
|
353
|
+
summary.errors_truncated = truncated
|
|
354
|
+
|
|
355
|
+
return FullValidationResponse(
|
|
356
|
+
success=(total_error_count_real == 0),
|
|
357
|
+
summary=summary,
|
|
358
|
+
errors=errors,
|
|
359
|
+
passed_items=passed_items,
|
|
360
|
+
statistics=statistics,
|
|
361
|
+
error=None,
|
|
362
|
+
warnings=result.get("warnings", []),
|
|
363
|
+
coverage=self.coverage,
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
def build_error(
|
|
367
|
+
self,
|
|
368
|
+
error: Exception,
|
|
369
|
+
*,
|
|
370
|
+
files_total: int = 0,
|
|
371
|
+
) -> FullValidationResponse:
|
|
372
|
+
"""
|
|
373
|
+
构建执行器初始化失败或执行异常时的降级响应。
|
|
374
|
+
|
|
375
|
+
参数:
|
|
376
|
+
error: 异常对象
|
|
377
|
+
files_total: 扫描到的源文件总数
|
|
378
|
+
|
|
379
|
+
返回:
|
|
380
|
+
FullValidationResponse 实例(success=False)
|
|
381
|
+
"""
|
|
382
|
+
summary = FullValidationSummary(
|
|
383
|
+
files_total=files_total,
|
|
384
|
+
files_loaded=0,
|
|
385
|
+
tables_loaded=0,
|
|
386
|
+
loading_error_count=0,
|
|
387
|
+
format_error_count=0,
|
|
388
|
+
constraint_error_count=0,
|
|
389
|
+
total_error_count=0,
|
|
390
|
+
duration_ms=self._duration_ms(),
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
return FullValidationResponse(
|
|
394
|
+
success=False,
|
|
395
|
+
summary=summary,
|
|
396
|
+
errors=[],
|
|
397
|
+
passed_items=[],
|
|
398
|
+
statistics=ValidationStatistics(
|
|
399
|
+
total_checks=0,
|
|
400
|
+
passed_count=0,
|
|
401
|
+
failed_count=0,
|
|
402
|
+
# 回归 #9: 执行失败时 pass_rate 必须为 0.0 而非 100.0。原实现返回 100.0,
|
|
403
|
+
# 与 success=False 矛盾,前端会把崩溃的运行以"满分通过"写入历史趋势,
|
|
404
|
+
# 污染纵向对比(通过率趋势)。失败即 0% 通过。
|
|
405
|
+
pass_rate=0.0,
|
|
406
|
+
by_type={},
|
|
407
|
+
by_table={},
|
|
408
|
+
),
|
|
409
|
+
error=str(error),
|
|
410
|
+
warnings=[],
|
|
411
|
+
coverage=self.coverage,
|
|
412
|
+
)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 文件读写异常的用户可读描述
|
|
18
|
+
|
|
19
|
+
视图/工作区/报告配置等 JSON 文件的读写异常原文(OSError 英文)
|
|
20
|
+
直接拼进 HTTP detail 会成为用户看不懂的提示。本模块把常见 OS
|
|
21
|
+
错误翻译为中文原因短语,未识别的异常保留原文。
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def describe_io_error(exc: BaseException) -> str:
|
|
28
|
+
"""把文件读写异常翻译成用户可读的原因短语。
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
exc: 读写文件时捕获的异常
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
中文原因短语;未识别的异常返回原文
|
|
35
|
+
"""
|
|
36
|
+
if isinstance(exc, PermissionError):
|
|
37
|
+
return "文件正被其他程序占用,或没有读写权限"
|
|
38
|
+
if isinstance(exc, FileNotFoundError):
|
|
39
|
+
return "文件不存在,可能已被移动或删除"
|
|
40
|
+
text = str(exc).strip()
|
|
41
|
+
return text or "未知原因"
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 预览数据处理服务模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 提取 DataFrame 到二维列表的公共转换逻辑
|
|
21
|
+
- 封装文件类型检测
|
|
22
|
+
- 封装临时文件清理
|
|
23
|
+
- 提供基于路径和基于内容(上传)的统一预览接口
|
|
24
|
+
|
|
25
|
+
输入示例:
|
|
26
|
+
data, sheets = preview_from_path("/path/to/file.xlsx", max_rows=100, max_cols=20)
|
|
27
|
+
data, sheets = preview_from_content(content, ".xlsx", max_rows=100, max_cols=20, sheet_name="Sheet2")
|
|
28
|
+
|
|
29
|
+
输出示例:
|
|
30
|
+
([["col1", "col2"], ["a", "b"]], ["Sheet1", "Sheet2"])
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import logging
|
|
36
|
+
import os
|
|
37
|
+
import tempfile
|
|
38
|
+
import time
|
|
39
|
+
|
|
40
|
+
import pandas as pd
|
|
41
|
+
from fastapi import HTTPException
|
|
42
|
+
|
|
43
|
+
from app.shared.core.data_source.loaders.excel_loader import get_excel_sheet_names
|
|
44
|
+
from app.shared.services.preview.loader import load_preview_data
|
|
45
|
+
|
|
46
|
+
logger = logging.getLogger(__name__)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def detect_file_type(file_ext: str) -> str:
|
|
50
|
+
"""根据文件扩展名判断文件类型。
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
file_ext: 文件扩展名(包含点号,如 .xlsx)
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
"excel" | "csv" | "json"
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
HTTPException: 不支持的文件类型
|
|
60
|
+
"""
|
|
61
|
+
if file_ext in [".xlsx", ".xls"]:
|
|
62
|
+
return "excel"
|
|
63
|
+
if file_ext == ".csv":
|
|
64
|
+
return "csv"
|
|
65
|
+
if file_ext in [".json", ".jsonl", ".ndjson"]:
|
|
66
|
+
return "json"
|
|
67
|
+
raise HTTPException(status_code=400, detail=f"不支持的文件类型: {file_ext}")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def df_to_list(df: pd.DataFrame, max_cols: int) -> list[list[str]]:
|
|
71
|
+
"""将 DataFrame 转为二维字符串列表,限制列数。
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
df: pandas DataFrame
|
|
75
|
+
max_cols: 最大列数
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
二维字符串列表
|
|
79
|
+
"""
|
|
80
|
+
data: list[list[str]] = []
|
|
81
|
+
for _, row in df.iterrows():
|
|
82
|
+
row_data = [str(cell) for cell in row.tolist()[:max_cols]]
|
|
83
|
+
data.append(row_data)
|
|
84
|
+
return data
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cleanup_temp_file(tmp_path: str) -> None:
|
|
88
|
+
"""清理临时文件,最多重试 3 次。
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
tmp_path: 临时文件路径
|
|
92
|
+
"""
|
|
93
|
+
if not os.path.exists(tmp_path):
|
|
94
|
+
return
|
|
95
|
+
for _attempt in range(3):
|
|
96
|
+
try:
|
|
97
|
+
os.unlink(tmp_path)
|
|
98
|
+
logger.info(f"[PREVIEW] 已删除临时文件: {tmp_path}")
|
|
99
|
+
break
|
|
100
|
+
except PermissionError:
|
|
101
|
+
time.sleep(0.1)
|
|
102
|
+
except Exception as e:
|
|
103
|
+
logger.info(f"[PREVIEW] 无法删除临时文件: {e}")
|
|
104
|
+
break
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def preview_from_path(
|
|
108
|
+
file_path: str,
|
|
109
|
+
max_rows: int,
|
|
110
|
+
max_cols: int,
|
|
111
|
+
sheet_name: str | None = None,
|
|
112
|
+
) -> tuple[list[list[str]], str, int, list[str] | None, str | None]:
|
|
113
|
+
"""基于文件路径加载预览数据。
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
file_path: 文件路径
|
|
117
|
+
max_rows: 最大行数
|
|
118
|
+
max_cols: 最大列数
|
|
119
|
+
sheet_name: 指定工作表名称(仅 Excel)
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
(data, file_type, total_rows, sheets, current_sheet)
|
|
123
|
+
|
|
124
|
+
Raises:
|
|
125
|
+
HTTPException: 文件不存在或读取失败
|
|
126
|
+
"""
|
|
127
|
+
if not os.path.exists(file_path):
|
|
128
|
+
raise HTTPException(status_code=404, detail=f"文件未找到: {file_path}")
|
|
129
|
+
|
|
130
|
+
file_ext = os.path.splitext(file_path)[1].lower()
|
|
131
|
+
file_type = detect_file_type(file_ext)
|
|
132
|
+
|
|
133
|
+
if file_type == "excel":
|
|
134
|
+
try:
|
|
135
|
+
if sheet_name is None:
|
|
136
|
+
# 引擎按扩展名自适应(.xls→xlrd)、句柄在助手内确保关闭
|
|
137
|
+
sheets = get_excel_sheet_names(file_path)
|
|
138
|
+
sheet_name = sheets[0] if sheets else "Sheet1"
|
|
139
|
+
|
|
140
|
+
df, sheet_names = load_preview_data(
|
|
141
|
+
file_path=file_path,
|
|
142
|
+
file_type="excel",
|
|
143
|
+
max_rows=max_rows,
|
|
144
|
+
sheet_name=sheet_name,
|
|
145
|
+
)
|
|
146
|
+
data = df_to_list(df, max_cols)
|
|
147
|
+
return data, file_type, len(df), sheet_names, sheet_name
|
|
148
|
+
except ValueError as e:
|
|
149
|
+
if "Worksheet" in str(e):
|
|
150
|
+
raise HTTPException(status_code=404, detail=f"工作表 '{sheet_name}' 不存在") from e
|
|
151
|
+
raise HTTPException(status_code=500, detail=f"读取工作表失败: {str(e)}") from e
|
|
152
|
+
except Exception as e:
|
|
153
|
+
raise HTTPException(status_code=500, detail=f"读取Excel文件失败: {str(e)}") from e
|
|
154
|
+
|
|
155
|
+
# JSON
|
|
156
|
+
if file_type == "json":
|
|
157
|
+
try:
|
|
158
|
+
df, _ = load_preview_data(file_path=file_path, file_type="json", max_rows=max_rows)
|
|
159
|
+
data = df_to_list(df, max_cols)
|
|
160
|
+
return data, file_type, len(df), None, None
|
|
161
|
+
except Exception as e:
|
|
162
|
+
raise HTTPException(status_code=500, detail=f"读取JSON文件失败: {str(e)}") from e
|
|
163
|
+
|
|
164
|
+
# CSV
|
|
165
|
+
try:
|
|
166
|
+
df, _ = load_preview_data(file_path=file_path, file_type="csv", max_rows=max_rows)
|
|
167
|
+
data = df_to_list(df, max_cols)
|
|
168
|
+
return data, file_type, len(df), None, None
|
|
169
|
+
except Exception as e:
|
|
170
|
+
raise HTTPException(status_code=500, detail=f"读取CSV文件失败: {str(e)}") from e
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def preview_from_content(
|
|
174
|
+
content: bytes,
|
|
175
|
+
file_ext: str,
|
|
176
|
+
max_rows: int,
|
|
177
|
+
max_cols: int,
|
|
178
|
+
sheet_name: str | None = None,
|
|
179
|
+
) -> tuple[list[list[str]], str, int, list[str] | None, str | None]:
|
|
180
|
+
"""基于文件内容(上传)加载预览数据。
|
|
181
|
+
|
|
182
|
+
将内容写入临时文件后解析,处理完成后自动清理。
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
content: 文件二进制内容
|
|
186
|
+
file_ext: 文件扩展名
|
|
187
|
+
max_rows: 最大行数
|
|
188
|
+
max_cols: 最大列数
|
|
189
|
+
sheet_name: 指定工作表名称(仅 Excel)
|
|
190
|
+
|
|
191
|
+
Returns:
|
|
192
|
+
(data, file_type, total_rows, sheets, current_sheet)
|
|
193
|
+
|
|
194
|
+
Raises:
|
|
195
|
+
HTTPException: 内容为空或读取失败
|
|
196
|
+
"""
|
|
197
|
+
if len(content) == 0:
|
|
198
|
+
raise HTTPException(status_code=400, detail="文件内容为空")
|
|
199
|
+
|
|
200
|
+
file_type = detect_file_type(file_ext)
|
|
201
|
+
tmp_path = ""
|
|
202
|
+
|
|
203
|
+
try:
|
|
204
|
+
with tempfile.NamedTemporaryFile(suffix=file_ext, delete=False) as tmp:
|
|
205
|
+
tmp.write(content)
|
|
206
|
+
tmp_path = tmp.name
|
|
207
|
+
|
|
208
|
+
if file_type == "excel":
|
|
209
|
+
try:
|
|
210
|
+
# 引擎按扩展名自适应(.xls→xlrd)、句柄在助手内确保关闭
|
|
211
|
+
sheets = get_excel_sheet_names(tmp_path)
|
|
212
|
+
actual_sheet = sheet_name or (sheets[0] if sheets else "Sheet1")
|
|
213
|
+
|
|
214
|
+
df, sheet_names = load_preview_data(
|
|
215
|
+
file_path=tmp_path,
|
|
216
|
+
file_type="excel",
|
|
217
|
+
max_rows=max_rows,
|
|
218
|
+
sheet_name=actual_sheet,
|
|
219
|
+
)
|
|
220
|
+
data = df_to_list(df, max_cols)
|
|
221
|
+
return data, file_type, len(df), sheet_names, actual_sheet
|
|
222
|
+
except Exception as e:
|
|
223
|
+
raise HTTPException(status_code=500, detail=f"读取Excel文件失败: {str(e)}") from e
|
|
224
|
+
finally:
|
|
225
|
+
import gc
|
|
226
|
+
|
|
227
|
+
gc.collect()
|
|
228
|
+
|
|
229
|
+
# JSON
|
|
230
|
+
if file_type == "json":
|
|
231
|
+
try:
|
|
232
|
+
df, _ = load_preview_data(file_path=tmp_path, file_type="json", max_rows=max_rows)
|
|
233
|
+
data = df_to_list(df, max_cols)
|
|
234
|
+
return data, file_type, len(df), None, None
|
|
235
|
+
except Exception as e:
|
|
236
|
+
raise HTTPException(status_code=500, detail=f"读取JSON文件失败: {str(e)}") from e
|
|
237
|
+
|
|
238
|
+
# CSV
|
|
239
|
+
try:
|
|
240
|
+
df, _ = load_preview_data(file_path=tmp_path, file_type="csv", max_rows=max_rows)
|
|
241
|
+
data = df_to_list(df, max_cols)
|
|
242
|
+
return data, file_type, len(df), None, None
|
|
243
|
+
except Exception as e:
|
|
244
|
+
raise HTTPException(status_code=500, detail=f"读取CSV文件失败: {str(e)}") from e
|
|
245
|
+
|
|
246
|
+
finally:
|
|
247
|
+
if tmp_path:
|
|
248
|
+
cleanup_temp_file(tmp_path)
|