precis-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +16 -0
- app/api/__init__.py +32 -0
- app/api/dependencies.py +185 -0
- app/api/main.py +301 -0
- app/api/middleware/__init__.py +34 -0
- app/api/middleware/exception_handler.py +126 -0
- app/api/middleware/request_logging.py +97 -0
- app/api/middleware/token_auth.py +215 -0
- app/api/models/__init__.py +151 -0
- app/api/models/connection_rules.py +98 -0
- app/api/models/files.py +99 -0
- app/api/models/full_validation.py +554 -0
- app/api/models/project.py +117 -0
- app/api/models/projects.py +80 -0
- app/api/models/schema.py +121 -0
- app/api/models/v2_responses.py +46 -0
- app/api/models/validation.py +303 -0
- app/api/models/workspace.py +118 -0
- app/api/routers/__init__.py +56 -0
- app/api/routers/ai/__init__.py +36 -0
- app/api/routers/ai/chat.py +243 -0
- app/api/routers/ai/generate.py +156 -0
- app/api/routers/ai/hardware.py +199 -0
- app/api/routers/ai/jobs.py +544 -0
- app/api/routers/ai/migrate.py +471 -0
- app/api/routers/ai/models.py +352 -0
- app/api/routers/ai/ollama.py +108 -0
- app/api/routers/ai/providers.py +491 -0
- app/api/routers/ai/router.py +44 -0
- app/api/routers/ai/stream.py +281 -0
- app/api/routers/ai/utils.py +182 -0
- app/api/routers/core/__init__.py +34 -0
- app/api/routers/core/connection_rules.py +188 -0
- app/api/routers/core/data_sources.py +374 -0
- app/api/routers/core/regex.py +333 -0
- app/api/routers/core/reporting.py +263 -0
- app/api/routers/files/__init__.py +24 -0
- app/api/routers/files/ops.py +179 -0
- app/api/routers/files/transfer.py +101 -0
- app/api/routers/preview/__init__.py +35 -0
- app/api/routers/preview/content_mode.py +264 -0
- app/api/routers/preview/header_row.py +154 -0
- app/api/routers/preview/models.py +81 -0
- app/api/routers/preview/path_mode.py +440 -0
- app/api/routers/preview/router.py +30 -0
- app/api/routers/project/__init__.py +61 -0
- app/api/routers/project/base.py +111 -0
- app/api/routers/project/constraint.py +345 -0
- app/api/routers/project/full_config.py +411 -0
- app/api/routers/project/full_config_writer.py +327 -0
- app/api/routers/project/helpers.py +167 -0
- app/api/routers/project/inspection_fix.py +330 -0
- app/api/routers/project/manifest.py +605 -0
- app/api/routers/project/models.py +177 -0
- app/api/routers/project/pattern.py +268 -0
- app/api/routers/project/regex.py +358 -0
- app/api/routers/project/scanner.py +157 -0
- app/api/routers/project/schema.py +567 -0
- app/api/routers/project/schema_helpers.py +149 -0
- app/api/routers/project/settings.py +399 -0
- app/api/routers/project/template.py +325 -0
- app/api/routers/project/validation.py +227 -0
- app/api/routers/project/view.py +177 -0
- app/api/routers/project/workspaces.py +167 -0
- app/api/routers/projects/__init__.py +25 -0
- app/api/routers/projects/create.py +121 -0
- app/api/routers/projects/open.py +103 -0
- app/api/routers/projects/scan.py +100 -0
- app/api/routers/validation/__init__.py +39 -0
- app/api/routers/validation/common.py +263 -0
- app/api/routers/validation/content_mode.py +401 -0
- app/api/routers/validation/history.py +235 -0
- app/api/routers/validation/inline_mode.py +196 -0
- app/api/routers/validation/path_mode.py +314 -0
- app/api/routers/validation/router.py +51 -0
- app/api/services/full_validation_response_builder.py +412 -0
- app/api/services/io_error_messages.py +41 -0
- app/api/services/preview_service.py +248 -0
- app/cli/__init__.py +18 -0
- app/cli/__main__.py +30 -0
- app/cli/shared_services/__init__.py +42 -0
- app/cli/shared_services/config_ops.py +581 -0
- app/cli/shared_services/generation_ops.py +160 -0
- app/cli/shared_services/project_ops.py +288 -0
- app/cli/shell/__init__.py +48 -0
- app/cli/shell/commands/__init__.py +63 -0
- app/cli/shell/commands/ai/__init__.py +260 -0
- app/cli/shell/commands/ai/chat.py +331 -0
- app/cli/shell/commands/ai/delete.py +165 -0
- app/cli/shell/commands/ai/diff.py +70 -0
- app/cli/shell/commands/ai/display.py +428 -0
- app/cli/shell/commands/ai/executor.py +225 -0
- app/cli/shell/commands/ai/executor_utils.py +271 -0
- app/cli/shell/commands/ai/generate.py +285 -0
- app/cli/shell/commands/ai/interaction.py +262 -0
- app/cli/shell/commands/ai/migrate.py +281 -0
- app/cli/shell/commands/ai/resolver.py +224 -0
- app/cli/shell/commands/ai/status.py +101 -0
- app/cli/shell/commands/ai/switch.py +158 -0
- app/cli/shell/commands/ai/utils.py +52 -0
- app/cli/shell/commands/base.py +343 -0
- app/cli/shell/commands/config/__init__.py +131 -0
- app/cli/shell/commands/config/base.py +80 -0
- app/cli/shell/commands/config/check.py +246 -0
- app/cli/shell/commands/config/edit.py +148 -0
- app/cli/shell/commands/config/get.py +111 -0
- app/cli/shell/commands/config/init.py +123 -0
- app/cli/shell/commands/config/inspect.py +164 -0
- app/cli/shell/commands/config/list.py +104 -0
- app/cli/shell/commands/config/set.py +119 -0
- app/cli/shell/commands/config/show.py +150 -0
- app/cli/shell/commands/exit.py +67 -0
- app/cli/shell/commands/help.py +104 -0
- app/cli/shell/commands/infer_schema.py +156 -0
- app/cli/shell/commands/open.py +201 -0
- app/cli/shell/commands/project.py +215 -0
- app/cli/shell/commands/provider.py +619 -0
- app/cli/shell/commands/system.py +170 -0
- app/cli/shell/commands/validate.py +455 -0
- app/cli/shell/completer.py +231 -0
- app/cli/shell/config_storage.py +191 -0
- app/cli/shell/exceptions.py +90 -0
- app/cli/shell/formatter.py +371 -0
- app/cli/shell/interactive_menu.py +348 -0
- app/cli/shell/main.py +286 -0
- app/cli/shell/parser.py +266 -0
- app/cli/start.py +180 -0
- app/cli_main.py +52 -0
- app/mcp_server.py +311 -0
- app/shared/core/__init__.py +18 -0
- app/shared/core/app_version.py +59 -0
- app/shared/core/config/__init__.py +119 -0
- app/shared/core/config/server.py +193 -0
- app/shared/core/data_source/__init__.py +106 -0
- app/shared/core/data_source/loader.py +348 -0
- app/shared/core/data_source/loaders/__init__.py +213 -0
- app/shared/core/data_source/loaders/base.py +215 -0
- app/shared/core/data_source/loaders/converter.py +337 -0
- app/shared/core/data_source/loaders/csv_loader.py +252 -0
- app/shared/core/data_source/loaders/excel_loader.py +666 -0
- app/shared/core/data_source/loaders/extractor.py +268 -0
- app/shared/core/data_source/loaders/json_loader.py +354 -0
- app/shared/core/data_source/loaders/registry.py +88 -0
- app/shared/core/data_source/loaders/sql_loader.py +313 -0
- app/shared/core/data_source/loaders/strategies/__init__.py +100 -0
- app/shared/core/data_source/loaders/strategies/array_parser.py +277 -0
- app/shared/core/data_source/loaders/strategies/lines_parser.py +355 -0
- app/shared/core/data_source/loaders/strategies/object_parser.py +278 -0
- app/shared/core/data_source/schema_info.py +50 -0
- app/shared/core/data_source/specs/__init__.py +128 -0
- app/shared/core/data_source/specs/base.py +258 -0
- app/shared/core/data_source/specs/csv_source.py +137 -0
- app/shared/core/data_source/specs/excel_source.py +141 -0
- app/shared/core/data_source/specs/file_base.py +252 -0
- app/shared/core/data_source/specs/json_source.py +226 -0
- app/shared/core/data_source/specs/sql_source.py +161 -0
- app/shared/core/encoding.py +36 -0
- app/shared/core/io/__init__.py +24 -0
- app/shared/core/io/yaml.py +379 -0
- app/shared/core/manifest_schema/__init__.py +52 -0
- app/shared/core/manifest_schema/types.py +99 -0
- app/shared/core/manifest_schema/version.py +175 -0
- app/shared/core/patterns/__init__.py +24 -0
- app/shared/core/patterns/loader.py +134 -0
- app/shared/core/patterns/writer.py +244 -0
- app/shared/core/project/__init__.py +47 -0
- app/shared/core/project/constraint/builders/__init__.py +50 -0
- app/shared/core/project/constraint/builders/base.py +100 -0
- app/shared/core/project/constraint/builders/composite.py +77 -0
- app/shared/core/project/constraint/builders/conditional.py +67 -0
- app/shared/core/project/constraint/builders/foreign_key.py +53 -0
- app/shared/core/project/constraint/builders/registry.py +64 -0
- app/shared/core/project/constraint/builders/scripted.py +51 -0
- app/shared/core/project/constraint/builders/single_column.py +86 -0
- app/shared/core/project/constraint/builders/unique.py +53 -0
- app/shared/core/project/constraint/factory.py +170 -0
- app/shared/core/project/constraint/reader.py +214 -0
- app/shared/core/project/constraint/registry.py +233 -0
- app/shared/core/project/constraint/types/__init__.py +63 -0
- app/shared/core/project/constraint/types/constraint_file.py +261 -0
- app/shared/core/project/constraint/types/refs.py +460 -0
- app/shared/core/project/constraint/types.py +28 -0
- app/shared/core/project/constraint/writer.py +181 -0
- app/shared/core/project/loader/__init__.py +56 -0
- app/shared/core/project/loader/loader.py +30 -0
- app/shared/core/project/loader/loader_parts/config_inspector.py +137 -0
- app/shared/core/project/loader/loader_parts/embedded_constraints.py +224 -0
- app/shared/core/project/loader/loader_parts/file_loaders.py +58 -0
- app/shared/core/project/loader/loader_parts/inspection_ids.py +85 -0
- app/shared/core/project/loader/loader_parts/inspector_helpers.py +312 -0
- app/shared/core/project/loader/loader_parts/inspector_id_checks.py +274 -0
- app/shared/core/project/loader/loader_parts/inspector_reference_checks.py +690 -0
- app/shared/core/project/loader/loader_parts/inspector_uniqueness_checks.py +286 -0
- app/shared/core/project/loader/loader_parts/loading_error_messages.py +298 -0
- app/shared/core/project/loader/loader_parts/main.py +432 -0
- app/shared/core/project/loader/loader_parts/path_validation.py +117 -0
- app/shared/core/project/loader/loader_parts/runtime.py +125 -0
- app/shared/core/project/loader/types.py +246 -0
- app/shared/core/project/manifest/coverage.py +391 -0
- app/shared/core/project/manifest/reader.py +260 -0
- app/shared/core/project/manifest/types.py +91 -0
- app/shared/core/project/manifest/types_parts/__init__.py +60 -0
- app/shared/core/project/manifest/types_parts/constants.py +40 -0
- app/shared/core/project/manifest/types_parts/data_source.py +68 -0
- app/shared/core/project/manifest/types_parts/info.py +59 -0
- app/shared/core/project/manifest/types_parts/manifest.py +294 -0
- app/shared/core/project/manifest/types_parts/refs.py +153 -0
- app/shared/core/project/manifest/types_parts/settings.py +92 -0
- app/shared/core/project/manifest/types_parts/settings_file_processing.py +64 -0
- app/shared/core/project/manifest/types_parts/settings_script_security.py +78 -0
- app/shared/core/project/manifest/types_parts/settings_validation.py +83 -0
- app/shared/core/project/manifest/types_parts/template.py +66 -0
- app/shared/core/project/manifest/writer.py +262 -0
- app/shared/core/project/manual_data/__init__.py +27 -0
- app/shared/core/project/manual_data/types.py +75 -0
- app/shared/core/project/regex/reader.py +197 -0
- app/shared/core/project/regex/types.py +405 -0
- app/shared/core/project/regex/writer.py +123 -0
- app/shared/core/project/schema/reader.py +170 -0
- app/shared/core/project/schema/types.py +47 -0
- app/shared/core/project/schema/types_parts/__init__.py +36 -0
- app/shared/core/project/schema/types_parts/column.py +174 -0
- app/shared/core/project/schema/types_parts/column_utils.py +72 -0
- app/shared/core/project/schema/types_parts/constraint.py +165 -0
- app/shared/core/project/schema/types_parts/schema_id.py +66 -0
- app/shared/core/project/schema/types_parts/source.py +255 -0
- app/shared/core/project/schema/types_parts/source_options.py +347 -0
- app/shared/core/project/schema/types_parts/table.py +230 -0
- app/shared/core/project/schema/writer.py +139 -0
- app/shared/core/project/schema_ref_check.py +95 -0
- app/shared/core/project/template/__init__.py +27 -0
- app/shared/core/project/template/expander.py +263 -0
- app/shared/core/project/template/reader.py +120 -0
- app/shared/core/project/template/types.py +114 -0
- app/shared/core/project/transform/reader.py +76 -0
- app/shared/core/project/transform/types.py +116 -0
- app/shared/core/project/transform/writer.py +84 -0
- app/shared/core/pydantic_messages.py +59 -0
- app/shared/core/reporter/__init__.py +46 -0
- app/shared/core/reporter/reporter.py +220 -0
- app/shared/core/reporter/reporters/__init__.py +65 -0
- app/shared/core/reporter/reporters/base.py +188 -0
- app/shared/core/reporter/reporters/dingtalk_app_reporter.py +274 -0
- app/shared/core/reporter/reporters/email_reporter.py +271 -0
- app/shared/core/reporter/reporters/feishu_app_reporter.py +467 -0
- app/shared/core/reporter/reporters/local_file_reporter.py +208 -0
- app/shared/core/reporter/reporters/wecom_app_reporter.py +268 -0
- app/shared/core/utils/__init__.py +18 -0
- app/shared/core/utils/path_utils.py +60 -0
- app/shared/core/utils/regex_extract.py +113 -0
- app/shared/domain/__init__.py +114 -0
- app/shared/domain/constraints/__init__.py +76 -0
- app/shared/domain/constraints/allowed_values.py +211 -0
- app/shared/domain/constraints/base.py +141 -0
- app/shared/domain/constraints/charset.py +337 -0
- app/shared/domain/constraints/composite.py +174 -0
- app/shared/domain/constraints/condition_registry.py +130 -0
- app/shared/domain/constraints/conditional.py +629 -0
- app/shared/domain/constraints/date_logic.py +731 -0
- app/shared/domain/constraints/foreign_key.py +261 -0
- app/shared/domain/constraints/key_normalization.py +56 -0
- app/shared/domain/constraints/not_null.py +185 -0
- app/shared/domain/constraints/range.py +360 -0
- app/shared/domain/constraints/regex.py +218 -0
- app/shared/domain/constraints/scripted.py +276 -0
- app/shared/domain/constraints/unique.py +233 -0
- app/shared/domain/data_engine.py +384 -0
- app/shared/domain/data_types.py +113 -0
- app/shared/domain/data_types_parts/__init__.py +67 -0
- app/shared/domain/data_types_parts/base.py +204 -0
- app/shared/domain/data_types_parts/composite.py +250 -0
- app/shared/domain/data_types_parts/expression.py +202 -0
- app/shared/domain/data_types_parts/extracted.py +106 -0
- app/shared/domain/data_types_parts/json_types.py +234 -0
- app/shared/domain/data_types_parts/scalars.py +719 -0
- app/shared/domain/data_types_parts/sequence.py +129 -0
- app/shared/domain/dataset_schema.py +48 -0
- app/shared/domain/eval_sandbox.py +63 -0
- app/shared/domain/expression_system.py +366 -0
- app/shared/domain/regex_flags.py +53 -0
- app/shared/domain/schema/builder.py +223 -0
- app/shared/domain/schema/models.py +339 -0
- app/shared/domain/transforms/__init__.py +28 -0
- app/shared/domain/transforms/aggregate.py +141 -0
- app/shared/domain/transforms/base.py +171 -0
- app/shared/domain/transforms/cast_type.py +120 -0
- app/shared/domain/transforms/concat.py +103 -0
- app/shared/domain/transforms/conditional_assign.py +114 -0
- app/shared/domain/transforms/date_format.py +81 -0
- app/shared/domain/transforms/digits.py +77 -0
- app/shared/domain/transforms/drop_duplicates.py +94 -0
- app/shared/domain/transforms/fill_na.py +94 -0
- app/shared/domain/transforms/filter_rows.py +97 -0
- app/shared/domain/transforms/lookup.py +78 -0
- app/shared/domain/transforms/lower_case.py +70 -0
- app/shared/domain/transforms/map_value.py +90 -0
- app/shared/domain/transforms/math_expr.py +112 -0
- app/shared/domain/transforms/modulo.py +82 -0
- app/shared/domain/transforms/regex_extract.py +103 -0
- app/shared/domain/transforms/registry.py +95 -0
- app/shared/domain/transforms/replace.py +88 -0
- app/shared/domain/transforms/sort_rows.py +94 -0
- app/shared/domain/transforms/string_split.py +84 -0
- app/shared/domain/transforms/strip.py +73 -0
- app/shared/domain/transforms/substring.py +92 -0
- app/shared/domain/transforms/upper_case.py +70 -0
- app/shared/domain/transforms/weighted_sum.py +104 -0
- app/shared/domain/validation_constraints.py +69 -0
- app/shared/services/__init__.py +62 -0
- app/shared/services/ai/__init__.py +48 -0
- app/shared/services/ai/agent/__init__.py +40 -0
- app/shared/services/ai/agent/chat_tools/__init__.py +47 -0
- app/shared/services/ai/agent/chat_tools/apply_actions.py +592 -0
- app/shared/services/ai/agent/chat_tools/ask_user.py +237 -0
- app/shared/services/ai/agent/chat_tools/read_canvas.py +140 -0
- app/shared/services/ai/agent/chat_tools/read_project.py +160 -0
- app/shared/services/ai/agent/chat_tools/read_table.py +301 -0
- app/shared/services/ai/agent/chat_tools/schemas.py +105 -0
- app/shared/services/ai/agent/chat_tools/validate_table.py +131 -0
- app/shared/services/ai/agent/executor.py +554 -0
- app/shared/services/ai/agent/memory.py +256 -0
- app/shared/services/ai/agent/planner.py +282 -0
- app/shared/services/ai/agent/tool_registry.py +296 -0
- app/shared/services/ai/agent/tools/__init__.py +32 -0
- app/shared/services/ai/agent/tools/config_generate.py +133 -0
- app/shared/services/ai/agent/tools/config_refine.py +109 -0
- app/shared/services/ai/agent/tools/config_validate.py +403 -0
- app/shared/services/ai/agent/tools/merge_results.py +136 -0
- app/shared/services/ai/agent/tools/plan_chunks.py +79 -0
- app/shared/services/ai/agent/tools/script_parse.py +359 -0
- app/shared/services/ai/agent/types.py +138 -0
- app/shared/services/ai/chat_agent_runner.py +596 -0
- app/shared/services/ai/chat_orchestrator.py +725 -0
- app/shared/services/ai/failure_messages.py +65 -0
- app/shared/services/ai/job_storage.py +274 -0
- app/shared/services/ai/migrate_service.py +550 -0
- app/shared/services/ai/streaming/__init__.py +65 -0
- app/shared/services/ai/streaming/event_journal.py +197 -0
- app/shared/services/ai/streaming/orchestrator.py +262 -0
- app/shared/services/ai/streaming/pending_interaction_store.py +227 -0
- app/shared/services/ai/streaming/sse_response.py +148 -0
- app/shared/services/ai/streaming/types.py +73 -0
- app/shared/services/ai/types.py +213 -0
- app/shared/services/ai/utils.py +322 -0
- app/shared/services/diff/config_diff.py +320 -0
- app/shared/services/hardware.py +237 -0
- app/shared/services/llm/__init__.py +62 -0
- app/shared/services/llm/actions/__init__.py +42 -0
- app/shared/services/llm/actions/_canvas_validator.py +147 -0
- app/shared/services/llm/actions/_constraint_validator.py +401 -0
- app/shared/services/llm/actions/_regex_validator.py +85 -0
- app/shared/services/llm/actions/_schema_validator.py +82 -0
- app/shared/services/llm/actions/_settings_validator.py +76 -0
- app/shared/services/llm/actions/_transform_validator.py +78 -0
- app/shared/services/llm/actions/action_handlers.py +400 -0
- app/shared/services/llm/actions/action_parser.py +63 -0
- app/shared/services/llm/actions/action_processor.py +468 -0
- app/shared/services/llm/actions/action_validator.py +311 -0
- app/shared/services/llm/actions/diff_compute.py +195 -0
- app/shared/services/llm/actions/regex_handlers.py +284 -0
- app/shared/services/llm/actions/registry.py +336 -0
- app/shared/services/llm/actions/schema_handlers.py +335 -0
- app/shared/services/llm/actions/settings_handlers.py +180 -0
- app/shared/services/llm/actions/specs.py +259 -0
- app/shared/services/llm/actions/transform_handlers.py +279 -0
- app/shared/services/llm/actions/validation_types.py +147 -0
- app/shared/services/llm/cache/__init__.py +18 -0
- app/shared/services/llm/cache/response_cache.py +80 -0
- app/shared/services/llm/chat/__init__.py +35 -0
- app/shared/services/llm/chat/chat_system_prompt.py +561 -0
- app/shared/services/llm/chat/response_parser.py +296 -0
- app/shared/services/llm/config/__init__.py +45 -0
- app/shared/services/llm/config/crypto.py +135 -0
- app/shared/services/llm/config/loader.py +258 -0
- app/shared/services/llm/config/models.py +202 -0
- app/shared/services/llm/config/presets.py +128 -0
- app/shared/services/llm/config_generator.py +163 -0
- app/shared/services/llm/constraints/__init__.py +41 -0
- app/shared/services/llm/constraints/constraint_builder.py +177 -0
- app/shared/services/llm/constraints/constraint_deletion.py +84 -0
- app/shared/services/llm/constraints/constraint_id.py +174 -0
- app/shared/services/llm/constraints/frontend_instructions.py +333 -0
- app/shared/services/llm/constraints/inline_batch.py +279 -0
- app/shared/services/llm/discovery/__init__.py +35 -0
- app/shared/services/llm/discovery/scanner.py +181 -0
- app/shared/services/llm/generation/__init__.py +42 -0
- app/shared/services/llm/generation/agent_wiring.py +156 -0
- app/shared/services/llm/generation/config_builder.py +386 -0
- app/shared/services/llm/generation/errors.py +36 -0
- app/shared/services/llm/generation/existing_config.py +99 -0
- app/shared/services/llm/generation/profiler.py +279 -0
- app/shared/services/llm/generation/prompt_builder.py +199 -0
- app/shared/services/llm/generation/response_parser.py +144 -0
- app/shared/services/llm/generation/service.py +622 -0
- app/shared/services/llm/models.py +104 -0
- app/shared/services/llm/providers/__init__.py +48 -0
- app/shared/services/llm/providers/base.py +310 -0
- app/shared/services/llm/providers/cached_provider.py +90 -0
- app/shared/services/llm/providers/ollama.py +498 -0
- app/shared/services/llm/providers/openai.py +334 -0
- app/shared/services/llm/providers/registry.py +108 -0
- app/shared/services/llm/schema_resolver.py +141 -0
- app/shared/services/llm/suggestion_utils.py +219 -0
- app/shared/services/llm/validate_executor.py +163 -0
- app/shared/services/llm/yaml_io.py +406 -0
- app/shared/services/preview/__init__.py +18 -0
- app/shared/services/preview/loader.py +145 -0
- app/shared/services/preview/path_validation.py +138 -0
- app/shared/services/project_loader.py +57 -0
- app/shared/services/schema_inference.py +260 -0
- app/shared/services/schema_runtime_builder.py +120 -0
- app/shared/services/validation/__init__.py +52 -0
- app/shared/services/validation/chunked_loader.py +509 -0
- app/shared/services/validation/dag/__init__.py +35 -0
- app/shared/services/validation/dag/builder.py +141 -0
- app/shared/services/validation/dag/executor.py +225 -0
- app/shared/services/validation/dag/sorter.py +77 -0
- app/shared/services/validation/data_loader.py +231 -0
- app/shared/services/validation/engine.py +446 -0
- app/shared/services/validation/executor.py +1030 -0
- app/shared/services/validation/extractors.py +266 -0
- app/shared/services/validation/history.py +209 -0
- app/shared/services/validation/json_payload.py +136 -0
- app/shared/services/validation/loader.py +152 -0
- app/shared/services/validation/memory_monitor.py +179 -0
- app/shared/services/validation/postprocess.py +208 -0
- app/shared/services/validation/progress.py +64 -0
- app/shared/services/validation/report_export.py +222 -0
- app/shared/services/validation/resolver.py +180 -0
- app/shared/services/validation/service.py +418 -0
- app/shared/services/validation/types.py +238 -0
- app/shared/services/validation/validators/__init__.py +36 -0
- app/shared/services/validation/validators/adapter.py +182 -0
- app/shared/services/validation/validators/base.py +332 -0
- app/shared/services/validation/validators/composite.py +233 -0
- app/shared/services/validation/validators/date_logic.py +276 -0
- app/start_server.py +133 -0
- precis_cli-0.1.3.dist-info/METADATA +180 -0
- precis_cli-0.1.3.dist-info/RECORD +442 -0
- precis_cli-0.1.3.dist-info/WHEEL +5 -0
- precis_cli-0.1.3.dist-info/entry_points.txt +4 -0
- precis_cli-0.1.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 项目加载器类型定义
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 定义项目加载过程中的核心数据类型
|
|
21
|
+
- 提供 LoadingError、LoadedProject、LoadedRegexNode 等数据类
|
|
22
|
+
|
|
23
|
+
架构设计:
|
|
24
|
+
- 不可变数据类: LoadedProject、LoadedRegexNode 使用 frozen dataclass
|
|
25
|
+
- 延迟导入: 通过 TYPE_CHECKING 避免循环依赖
|
|
26
|
+
|
|
27
|
+
输入示例:
|
|
28
|
+
LoadedProject(
|
|
29
|
+
manifest_path=Path("project.precis.yaml"),
|
|
30
|
+
manifest=manifest,
|
|
31
|
+
schema_files={"users": schema_file},
|
|
32
|
+
...
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
输出示例:
|
|
36
|
+
loading_errors = project.loading_errors # List[LoadingError]
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
import re
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import TYPE_CHECKING, Any, Protocol
|
|
43
|
+
|
|
44
|
+
# 直接导入 RegexNodeFile(类型明确,不会引起循环依赖)
|
|
45
|
+
from ..regex.types import RegexNodeFile
|
|
46
|
+
from ..transform.types import TransformFile
|
|
47
|
+
|
|
48
|
+
# 仅在类型检查阶段导入,避免运行时循环导入问题
|
|
49
|
+
if TYPE_CHECKING:
|
|
50
|
+
from ..constraint.types import ConstraintFile
|
|
51
|
+
from ..manifest.types import ProjectManifest
|
|
52
|
+
from ..manual_data.types import ManualDataFile
|
|
53
|
+
from ..schema.types import TableSchemaFile
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class LoadingError:
|
|
58
|
+
"""@classdesc 加载错误详情
|
|
59
|
+
|
|
60
|
+
当项目加载过程中出现文件不存在、解析失败、路径错误等问题时,
|
|
61
|
+
会创建该对象记录错误的详细信息,便于后续排查和展示。
|
|
62
|
+
|
|
63
|
+
字段说明:
|
|
64
|
+
- error_type: 错误类型标识(如 SchemaNotFound、SchemaParseError 等)
|
|
65
|
+
- file_path: 发生错误的文件绝对路径
|
|
66
|
+
- ref_id: 对应 manifest 中引用的 ID(可选)
|
|
67
|
+
- message: 人类可读的错误描述
|
|
68
|
+
- suggestion: 修复建议
|
|
69
|
+
|
|
70
|
+
=== 面向 UI 的扩展字段(前后端契约) ===
|
|
71
|
+
- id: 稳定唯一 id,用于"忽略"持久化(localStorage 等)
|
|
72
|
+
- severity: 严重度 — blocker(阻塞) | warning(警告) | info(提示)
|
|
73
|
+
- title: 人类可读的短标题(中文 fallback,前端有 title_key 时优先用 i18n 渲染)
|
|
74
|
+
- description: 根因说明(中文 fallback,前端有 description_key 时优先用 i18n 渲染)
|
|
75
|
+
- fix_hint: 高亮显示的修复建议(中文 fallback,前端有 fix_hint_key 时优先用 i18n 渲染)
|
|
76
|
+
- actions: 可执行动作列表(open_file / copy / dismiss / auto_fix / navigate)
|
|
77
|
+
每个 action 可携带 label_key 用于前端 i18n 渲染
|
|
78
|
+
- fix_api: 一键修复的 API 描述(method/path/body)— 仅在"安全操作"时填充
|
|
79
|
+
- context: 上下文数据(如可用表列表、当前值与期望值),用于前端渲染对比表
|
|
80
|
+
|
|
81
|
+
=== i18n 字段(可选) ===
|
|
82
|
+
- title_key / description_key / fix_hint_key: i18n 键名,前端有该 key 时优先渲染
|
|
83
|
+
- message_params: 模板参数字典,用于 i18n 字符串插值(如 {"constraintId": "fk_x"})
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
error_type: str # SchemaNotFound, SchemaParseError, ConstraintNotFound, etc.
|
|
87
|
+
file_path: str # 出错的文件路径
|
|
88
|
+
ref_id: str | None = None # 引用ID
|
|
89
|
+
message: str = "" # 错误信息
|
|
90
|
+
suggestion: str = "" # 修复建议
|
|
91
|
+
|
|
92
|
+
# === 面向 UI 的扩展字段 ===
|
|
93
|
+
id: str = "" # 稳定 id,用于"忽略"持久化
|
|
94
|
+
severity: str = "warning" # blocker | warning | info
|
|
95
|
+
title: str = "" # 人类可读短标题
|
|
96
|
+
description: str = "" # 根因说明
|
|
97
|
+
fix_hint: str = "" # 修复建议(突出显示)
|
|
98
|
+
actions: list[dict] = field(default_factory=list) # 可执行动作列表
|
|
99
|
+
fix_api: dict | None = None # 一键修复 API 描述
|
|
100
|
+
context: dict = field(default_factory=dict) # 上下文数据(用于渲染对比表等)
|
|
101
|
+
|
|
102
|
+
# === i18n 渲染字段(可选,前端有 key 时优先使用) ===
|
|
103
|
+
title_key: str = "" # i18n key,对应 title
|
|
104
|
+
description_key: str = "" # i18n key,对应 description
|
|
105
|
+
fix_hint_key: str = "" # i18n key,对应 fix_hint
|
|
106
|
+
message_params: dict = field(default_factory=dict) # i18n 插值参数
|
|
107
|
+
|
|
108
|
+
def to_dict(self) -> dict:
|
|
109
|
+
"""@methoddesc 将错误对象序列化为字典格式
|
|
110
|
+
|
|
111
|
+
便于转换为 JSON 返回给前端或写入日志。
|
|
112
|
+
|
|
113
|
+
返回值:
|
|
114
|
+
{
|
|
115
|
+
"error_type": ...,
|
|
116
|
+
"file_path": ...,
|
|
117
|
+
"ref_id": ...,
|
|
118
|
+
"message": ...,
|
|
119
|
+
"suggestion": ...,
|
|
120
|
+
"id": ...,
|
|
121
|
+
"severity": ...,
|
|
122
|
+
"title": ...,
|
|
123
|
+
"description": ...,
|
|
124
|
+
"fix_hint": ...,
|
|
125
|
+
"actions": ...,
|
|
126
|
+
"fix_api": ...,
|
|
127
|
+
"context": ...,
|
|
128
|
+
"title_key": ...,
|
|
129
|
+
"description_key": ...,
|
|
130
|
+
"fix_hint_key": ...,
|
|
131
|
+
"message_params": ...,
|
|
132
|
+
}
|
|
133
|
+
"""
|
|
134
|
+
return {
|
|
135
|
+
"error_type": self.error_type,
|
|
136
|
+
"file_path": self.file_path,
|
|
137
|
+
"ref_id": self.ref_id,
|
|
138
|
+
"message": self.message,
|
|
139
|
+
"suggestion": self.suggestion,
|
|
140
|
+
"id": self.id,
|
|
141
|
+
"severity": self.severity,
|
|
142
|
+
"title": self.title,
|
|
143
|
+
"description": self.description,
|
|
144
|
+
"fix_hint": self.fix_hint,
|
|
145
|
+
"actions": self.actions,
|
|
146
|
+
"fix_api": self.fix_api,
|
|
147
|
+
"context": self.context,
|
|
148
|
+
"title_key": self.title_key,
|
|
149
|
+
"description_key": self.description_key,
|
|
150
|
+
"fix_hint_key": self.fix_hint_key,
|
|
151
|
+
"message_params": self.message_params,
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class SchemaBuilder(Protocol):
|
|
156
|
+
"""@classdesc 运行时数据集 Schema 构建器协议(依赖注入用)
|
|
157
|
+
|
|
158
|
+
core 层的 load_project 不应直接依赖 services 层。通过此协议,
|
|
159
|
+
load_project 接收一个外部注入的构建器回调来完成
|
|
160
|
+
"core 文件对象 → domain 运行时对象" 的转换,
|
|
161
|
+
从而把层依赖方向保持为 services → core(而非 core → services)。
|
|
162
|
+
|
|
163
|
+
dataset_schema 的返回类型用 Any,使 core 层不静态依赖 domain。
|
|
164
|
+
具体实现由 services 层提供(见 services/project_loader.build_dataset_schema)。
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
def __call__(
|
|
168
|
+
self,
|
|
169
|
+
schema_files: dict[str, "TableSchemaFile"],
|
|
170
|
+
constraint_files: dict[str, "ConstraintFile"],
|
|
171
|
+
registries: dict[str, Any],
|
|
172
|
+
) -> tuple[Any, list[str]]:
|
|
173
|
+
"""构建运行时数据集 Schema。
|
|
174
|
+
|
|
175
|
+
:param schema_files: schema 配置文件字典
|
|
176
|
+
:param constraint_files: constraint 配置文件字典
|
|
177
|
+
:param registries: 表达式注册表等辅助对象
|
|
178
|
+
:return: (dataset_schema, 警告列表)
|
|
179
|
+
"""
|
|
180
|
+
...
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
@dataclass(frozen=True)
|
|
184
|
+
class LoadedProject:
|
|
185
|
+
"""@classdesc 已加载项目的完整数据结构
|
|
186
|
+
|
|
187
|
+
该对象包含从 manifest 文件加载后的所有配置和运行时对象,
|
|
188
|
+
是项目加载流程的最终产物。
|
|
189
|
+
|
|
190
|
+
字段说明:
|
|
191
|
+
- manifest_path: manifest 文件路径
|
|
192
|
+
- manifest: manifest 原始数据对象
|
|
193
|
+
- schema_files: 所有 schema 配置文件的映射(table_id -> TableSchemaFile)
|
|
194
|
+
- constraint_files: 所有约束配置文件的映射(constraint_id -> ConstraintFile)
|
|
195
|
+
- regex_node_files: 所有正则节点配置文件的映射(regex_id -> RegexNodeFile)
|
|
196
|
+
- dataset_schema: 运行时的数据集 Schema(包含 TableSchema 和约束实例)
|
|
197
|
+
- warnings: 加载过程中的警告信息列表
|
|
198
|
+
- loading_errors: 加载过程中的错误信息列表
|
|
199
|
+
- constraint_source_files: 约束 ID -> 来源文件路径(相对 manifest 目录)映射。
|
|
200
|
+
独立约束文件为 constraint YAML 路径,schema 内嵌约束为 schema 文件路径。
|
|
201
|
+
供校验错误回溯 constraint_file(P0-3),模板展开产物不在映射中。
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
manifest_path: Path
|
|
205
|
+
manifest: "ProjectManifest"
|
|
206
|
+
schema_files: dict[str, "TableSchemaFile"]
|
|
207
|
+
constraint_files: dict[str, "ConstraintFile"]
|
|
208
|
+
regex_node_files: dict[str, RegexNodeFile]
|
|
209
|
+
dataset_schema: Any
|
|
210
|
+
transform_files: dict[str, TransformFile] | None = None
|
|
211
|
+
manual_data_files: dict[str, "ManualDataFile"] | None = None
|
|
212
|
+
warnings: list[str] | None = None
|
|
213
|
+
loading_errors: list[LoadingError] | None = None
|
|
214
|
+
constraint_source_files: dict[str, str] | None = None
|
|
215
|
+
|
|
216
|
+
def __post_init__(self) -> None:
|
|
217
|
+
# frozen=True 表示实例创建后不可变,因此必须通过 object.__setattr__ 来修改属性
|
|
218
|
+
if self.warnings is None:
|
|
219
|
+
object.__setattr__(self, "warnings", [])
|
|
220
|
+
if self.loading_errors is None:
|
|
221
|
+
object.__setattr__(self, "loading_errors", [])
|
|
222
|
+
if self.transform_files is None:
|
|
223
|
+
object.__setattr__(self, "transform_files", {})
|
|
224
|
+
if self.manual_data_files is None:
|
|
225
|
+
object.__setattr__(self, "manual_data_files", {})
|
|
226
|
+
if self.constraint_source_files is None:
|
|
227
|
+
object.__setattr__(self, "constraint_source_files", {})
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@dataclass(frozen=True)
|
|
231
|
+
class LoadedRegexNode:
|
|
232
|
+
"""@classdesc 已加载的正则表达式节点数据
|
|
233
|
+
|
|
234
|
+
将 RegexNodeFile 配置与编译后的正则模式、源表/列引用封装在一起。
|
|
235
|
+
|
|
236
|
+
字段说明:
|
|
237
|
+
- config: 原始的 RegexNodeFile 配置对象
|
|
238
|
+
- pattern: 编译后的正则表达式模式(re.Pattern)
|
|
239
|
+
- source_table: 该正则节点引用的源表 ID(可选)
|
|
240
|
+
- source_column: 该正则节点引用的源列 ID(可选)
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
config: RegexNodeFile
|
|
244
|
+
pattern: re.Pattern
|
|
245
|
+
source_table: str | None
|
|
246
|
+
source_column: str | None
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 项目清单覆盖度检查模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 计算 project.precis.yaml 清单与实际磁盘文件的覆盖关系
|
|
21
|
+
- 检测未入清单的资源(unlisted)和清单引用但缺失的资源(dangling)
|
|
22
|
+
- 为前端提供清单一致性诊断数据
|
|
23
|
+
|
|
24
|
+
架构设计:
|
|
25
|
+
- 纯函数设计: compute_manifest_coverage 无副作用,仅读取文件系统
|
|
26
|
+
- 数据类: CoverageRef、ManifestCoverage 使用 frozen dataclass 保证不可变性
|
|
27
|
+
- 扫描策略: 分别扫描 schemas/、constraints/、regex/ 目录与清单比对
|
|
28
|
+
|
|
29
|
+
输入示例:
|
|
30
|
+
manifest = ProjectManifestV2(
|
|
31
|
+
schemas=[SchemaRef(id="users", path="schemas/users.schema.yaml")],
|
|
32
|
+
constraints=[],
|
|
33
|
+
regex_nodes=[],
|
|
34
|
+
)
|
|
35
|
+
coverage = compute_manifest_coverage("/path/to/project", manifest)
|
|
36
|
+
|
|
37
|
+
输出示例:
|
|
38
|
+
ManifestCoverage(
|
|
39
|
+
unlisted_schemas=[CoverageRef(id="orders", path="schemas/orders.schema.yaml")],
|
|
40
|
+
unlisted_constraints=[],
|
|
41
|
+
unlisted_regex_nodes=[],
|
|
42
|
+
dangling_schemas=[CoverageRef(id="old_users", path="schemas/old_users.schema.yaml")],
|
|
43
|
+
dangling_constraints=[],
|
|
44
|
+
dangling_regex_nodes=[],
|
|
45
|
+
)
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
from __future__ import annotations
|
|
49
|
+
|
|
50
|
+
import logging
|
|
51
|
+
import os
|
|
52
|
+
from dataclasses import dataclass
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
|
|
55
|
+
from app.shared.core.io.yaml import read_yaml
|
|
56
|
+
from app.shared.core.project.manifest.types import ProjectManifestV2
|
|
57
|
+
from app.shared.core.utils.path_utils import normalize_to_posix
|
|
58
|
+
|
|
59
|
+
logger = logging.getLogger(__name__)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class CoverageRef:
|
|
64
|
+
"""@classdesc 单个资源的覆盖度引用
|
|
65
|
+
|
|
66
|
+
用于标识一个具体的配置文件,包含资源 ID 和文件路径。
|
|
67
|
+
|
|
68
|
+
字段说明:
|
|
69
|
+
- id: 资源的唯一标识符(从文件名或文件内容中提取)
|
|
70
|
+
- path: 文件相对于项目根目录的路径
|
|
71
|
+
|
|
72
|
+
示例:
|
|
73
|
+
CoverageRef(id="users", path="schemas/users.schema.yaml")
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
id: str # 资源唯一标识符
|
|
77
|
+
path: str # 文件相对路径
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class ManifestCoverage:
|
|
82
|
+
"""@classdesc 清单覆盖度结果
|
|
83
|
+
|
|
84
|
+
保存清单与实际磁盘文件的对比结果,分为两类:
|
|
85
|
+
- unlisted: 磁盘上有但清单中未引用的文件(遗漏配置)
|
|
86
|
+
- dangling: 清单中引用了但磁盘上缺失的文件(无效引用)
|
|
87
|
+
|
|
88
|
+
字段说明:
|
|
89
|
+
- unlisted_schemas: 未入清单的 schema 文件
|
|
90
|
+
- unlisted_constraints: 未入清单的 constraint 文件
|
|
91
|
+
- unlisted_regex_nodes: 未入清单的 regex 文件
|
|
92
|
+
- dangling_schemas: 引用但缺失的 schema 文件
|
|
93
|
+
- dangling_constraints: 引用但缺失的 constraint 文件
|
|
94
|
+
- dangling_regex_nodes: 引用但缺失的 regex 文件
|
|
95
|
+
|
|
96
|
+
示例:
|
|
97
|
+
ManifestCoverage(
|
|
98
|
+
unlisted_schemas=[CoverageRef(id="orders", path="schemas/orders.schema.yaml")],
|
|
99
|
+
unlisted_constraints=[],
|
|
100
|
+
unlisted_regex_nodes=[],
|
|
101
|
+
dangling_schemas=[CoverageRef(id="old_users", path="schemas/old_users.schema.yaml")],
|
|
102
|
+
dangling_constraints=[],
|
|
103
|
+
dangling_regex_nodes=[],
|
|
104
|
+
)
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
unlisted_schemas: list[CoverageRef] # 磁盘存在但清单未引用的 schema
|
|
108
|
+
unlisted_constraints: list[CoverageRef] # 磁盘存在但清单未引用的 constraint
|
|
109
|
+
unlisted_regex_nodes: list[CoverageRef] # 磁盘存在但清单未引用的 regex
|
|
110
|
+
unlisted_transforms: list[CoverageRef] # 磁盘存在但清单未引用的 transform
|
|
111
|
+
unlisted_manual_data: list[CoverageRef] # 磁盘存在但清单未引用的 manual_data
|
|
112
|
+
dangling_schemas: list[CoverageRef] # 清单引用但磁盘缺失的 schema
|
|
113
|
+
dangling_constraints: list[CoverageRef] # 清单引用但磁盘缺失的 constraint
|
|
114
|
+
dangling_regex_nodes: list[CoverageRef] # 清单引用但磁盘缺失的 regex
|
|
115
|
+
dangling_transforms: list[CoverageRef] # 清单引用但磁盘缺失的 transform
|
|
116
|
+
dangling_manual_data: list[CoverageRef] # 清单引用但磁盘缺失的 manual_data
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def compute_manifest_coverage(config_path: str, manifest: ProjectManifestV2) -> ManifestCoverage:
|
|
120
|
+
"""@methoddesc 计算清单与实际磁盘文件的覆盖关系
|
|
121
|
+
|
|
122
|
+
本函数通过对比 manifest 中记录的文件引用与磁盘上实际存在的文件,
|
|
123
|
+
发现两类问题:
|
|
124
|
+
1. unlisted(未入清单):磁盘上有配置文件,但 manifest 里没有引用它
|
|
125
|
+
2. dangling(悬空引用):manifest 里引用了某个文件,但磁盘上找不到它
|
|
126
|
+
|
|
127
|
+
============================================================================
|
|
128
|
+
处理流程
|
|
129
|
+
============================================================================
|
|
130
|
+
1. 构建三个目录路径:schemas/、constraints/、regex/
|
|
131
|
+
2. 收集 manifest 中已引用的文件路径和 ID
|
|
132
|
+
3. 扫描磁盘目录,找出未入清单的文件(unlisted)
|
|
133
|
+
4. 遍历 manifest 引用,找出缺失的文件(dangling)
|
|
134
|
+
5. 组装并返回 ManifestCoverage 结果对象
|
|
135
|
+
|
|
136
|
+
============================================================================
|
|
137
|
+
参数说明
|
|
138
|
+
============================================================================
|
|
139
|
+
:param config_path: 项目根目录路径(manifest 所在目录)
|
|
140
|
+
:param manifest: 已加载的项目清单对象
|
|
141
|
+
:return: ManifestCoverage 覆盖度结果对象
|
|
142
|
+
|
|
143
|
+
============================================================================
|
|
144
|
+
业务场景
|
|
145
|
+
============================================================================
|
|
146
|
+
- 场景1: 项目健康检查
|
|
147
|
+
用户执行项目诊断时,系统调用此函数发现配置不一致问题。
|
|
148
|
+
- 场景2: 前端项目概览页面
|
|
149
|
+
前端展示项目配置完整性时,调用此接口获取统计数据。
|
|
150
|
+
"""
|
|
151
|
+
# ============================================================================
|
|
152
|
+
# 第一步:构建三个资源目录的绝对路径
|
|
153
|
+
# ============================================================================
|
|
154
|
+
# 分别在项目根目录下查找 schemas、constraints、regex 子目录
|
|
155
|
+
schemas_dir = os.path.join(config_path, "schemas")
|
|
156
|
+
constraints_dir = os.path.join(config_path, "constraints")
|
|
157
|
+
regex_dir = os.path.join(config_path, "regex")
|
|
158
|
+
transforms_dir = os.path.join(config_path, "transforms")
|
|
159
|
+
manual_data_dir = os.path.join(config_path, "manual_data")
|
|
160
|
+
|
|
161
|
+
# ============================================================================
|
|
162
|
+
# 第二步:收集 manifest 中已引用的路径和 ID
|
|
163
|
+
# ============================================================================
|
|
164
|
+
# schema_paths 使用小写路径集合,用于不区分大小写的路径匹配
|
|
165
|
+
# 注意:schemas 使用 path 字段匹配,constraints 和 regex 使用 id 字段匹配
|
|
166
|
+
schema_paths = {normalize_to_posix(r.path or "").lower() for r in (manifest.schemas or [])}
|
|
167
|
+
constraint_ids = {r.id for r in (manifest.constraints or [])}
|
|
168
|
+
regex_ids = {r.id for r in (manifest.regex_nodes or [])}
|
|
169
|
+
transform_ids = {r.id for r in (manifest.transforms or [])}
|
|
170
|
+
manual_data_ids = {r.id for r in (manifest.manual_data or [])}
|
|
171
|
+
|
|
172
|
+
# ============================================================================
|
|
173
|
+
# 第三步:扫描 schemas 目录,找出未入清单的 schema 文件
|
|
174
|
+
# ============================================================================
|
|
175
|
+
# 遍历 schemas/ 目录下所有 .schema.yaml 结尾的文件
|
|
176
|
+
# 对于每个文件:
|
|
177
|
+
# - 如果 manifest 中已引用(按路径匹配),跳过
|
|
178
|
+
# - 否则,尝试读取文件内容获取真实的 id 字段
|
|
179
|
+
# - 如果读取失败,使用文件名(去掉 .schema.yaml 后缀)作为 id
|
|
180
|
+
unlisted_schemas: list[CoverageRef] = []
|
|
181
|
+
if os.path.isdir(schemas_dir):
|
|
182
|
+
for filename in os.listdir(schemas_dir):
|
|
183
|
+
# 只处理 .schema.yaml 后缀的文件
|
|
184
|
+
if filename.lower().endswith(".schema.yaml"):
|
|
185
|
+
# 构建相对路径,统一使用正斜杠
|
|
186
|
+
rel_path = f"schemas/{filename}"
|
|
187
|
+
# 检查 manifest 中是否已引用该路径(不区分大小写)
|
|
188
|
+
if normalize_to_posix(rel_path).lower() in schema_paths:
|
|
189
|
+
continue # 已入清单,跳过
|
|
190
|
+
# 默认使用文件名(去掉 .schema.yaml)作为 id
|
|
191
|
+
rid = filename[:-12]
|
|
192
|
+
try:
|
|
193
|
+
# 尝试读取文件内容,获取真实的 id 字段
|
|
194
|
+
raw = read_yaml(Path(os.path.join(schemas_dir, filename)))
|
|
195
|
+
file_id = raw.get("id") if isinstance(raw, dict) else None
|
|
196
|
+
# 如果文件内 id 是有效的字符串,使用文件内的 id
|
|
197
|
+
if isinstance(file_id, str) and file_id.strip():
|
|
198
|
+
rid = file_id.strip()
|
|
199
|
+
except Exception as e:
|
|
200
|
+
# 读取失败时记录警告,但仍使用文件名作为 id
|
|
201
|
+
logger.warning(f"读取schema文件失败: {filename}, 错误: {e}")
|
|
202
|
+
# 将未入清单的 schema 添加到结果列表
|
|
203
|
+
unlisted_schemas.append(CoverageRef(id=rid, path=rel_path))
|
|
204
|
+
# 按 id 排序,保证输出结果稳定可预测
|
|
205
|
+
unlisted_schemas.sort(key=lambda r: r.id)
|
|
206
|
+
|
|
207
|
+
# ============================================================================
|
|
208
|
+
# 第四步:扫描 constraints 目录,找出未入清单的约束文件
|
|
209
|
+
# ============================================================================
|
|
210
|
+
# constraints 使用文件名(去掉 .constraint.yaml)作为 id,与 manifest 中的 id 匹配
|
|
211
|
+
unlisted_constraints: list[CoverageRef] = []
|
|
212
|
+
if os.path.isdir(constraints_dir):
|
|
213
|
+
for filename in os.listdir(constraints_dir):
|
|
214
|
+
if filename.lower().endswith(".constraint.yaml"):
|
|
215
|
+
# 从文件名中提取约束 id(去掉 .constraint.yaml 后缀)
|
|
216
|
+
rid = filename[:-16]
|
|
217
|
+
# 如果 manifest 中未引用该 id,则标记为未入清单
|
|
218
|
+
if rid not in constraint_ids:
|
|
219
|
+
unlisted_constraints.append(CoverageRef(id=rid, path=f"constraints/{filename}"))
|
|
220
|
+
unlisted_constraints.sort(key=lambda r: r.id)
|
|
221
|
+
|
|
222
|
+
# ============================================================================
|
|
223
|
+
# 第五步:扫描 regex 目录,找出未入清单的正则文件
|
|
224
|
+
# ============================================================================
|
|
225
|
+
# regex 使用文件名(去掉 .regex.yaml)作为 id,与 manifest 中的 id 匹配
|
|
226
|
+
unlisted_regex_nodes: list[CoverageRef] = []
|
|
227
|
+
if os.path.isdir(regex_dir):
|
|
228
|
+
for filename in os.listdir(regex_dir):
|
|
229
|
+
if filename.lower().endswith(".regex.yaml"):
|
|
230
|
+
# 从文件名中提取 regex id(去掉 .regex.yaml 后缀)
|
|
231
|
+
rid = filename[:-11]
|
|
232
|
+
# 如果 manifest 中未引用该 id,则标记为未入清单
|
|
233
|
+
if rid not in regex_ids:
|
|
234
|
+
unlisted_regex_nodes.append(CoverageRef(id=rid, path=f"regex/{filename}"))
|
|
235
|
+
unlisted_regex_nodes.sort(key=lambda r: r.id)
|
|
236
|
+
|
|
237
|
+
# ============================================================================
|
|
238
|
+
# 第五步扩展:扫描 transforms 目录,找出未入清单的 transform 文件
|
|
239
|
+
# ============================================================================
|
|
240
|
+
unlisted_transforms: list[CoverageRef] = []
|
|
241
|
+
if os.path.isdir(transforms_dir):
|
|
242
|
+
for filename in os.listdir(transforms_dir):
|
|
243
|
+
if filename.lower().endswith(".transform.yaml"):
|
|
244
|
+
rid = filename[: -len(".transform.yaml")]
|
|
245
|
+
if rid not in transform_ids:
|
|
246
|
+
unlisted_transforms.append(CoverageRef(id=rid, path=f"transforms/{filename}"))
|
|
247
|
+
unlisted_transforms.sort(key=lambda r: r.id)
|
|
248
|
+
|
|
249
|
+
# ============================================================================
|
|
250
|
+
# 第五步扩展:扫描 manual_data 目录,找出未入清单的 manual_data 文件
|
|
251
|
+
# ============================================================================
|
|
252
|
+
unlisted_manual_data: list[CoverageRef] = []
|
|
253
|
+
if os.path.isdir(manual_data_dir):
|
|
254
|
+
for filename in os.listdir(manual_data_dir):
|
|
255
|
+
if filename.lower().endswith(".manual_data.yaml"):
|
|
256
|
+
rid = filename[: -len(".manual_data.yaml")]
|
|
257
|
+
if rid not in manual_data_ids:
|
|
258
|
+
unlisted_manual_data.append(CoverageRef(id=rid, path=f"manual_data/{filename}"))
|
|
259
|
+
unlisted_manual_data.sort(key=lambda r: r.id)
|
|
260
|
+
|
|
261
|
+
# ============================================================================
|
|
262
|
+
# 第六步:检查 dangling schemas(清单引用但磁盘缺失)
|
|
263
|
+
# ============================================================================
|
|
264
|
+
# 遍历 manifest 中所有 schema 引用,检查对应的文件是否存在于磁盘
|
|
265
|
+
dangling_schemas: list[CoverageRef] = []
|
|
266
|
+
for schema_ref in manifest.schemas or []:
|
|
267
|
+
# 如果文件不存在,标记为悬空引用
|
|
268
|
+
if not os.path.isfile(os.path.join(config_path, schema_ref.path)):
|
|
269
|
+
dangling_schemas.append(CoverageRef(id=schema_ref.id, path=schema_ref.path))
|
|
270
|
+
dangling_schemas.sort(key=lambda r: r.id)
|
|
271
|
+
|
|
272
|
+
# ============================================================================
|
|
273
|
+
# 第七步:检查 dangling constraints(清单引用但磁盘缺失)
|
|
274
|
+
# ============================================================================
|
|
275
|
+
dangling_constraints: list[CoverageRef] = []
|
|
276
|
+
for constraint_ref in manifest.constraints or []:
|
|
277
|
+
if not os.path.isfile(os.path.join(config_path, constraint_ref.path)):
|
|
278
|
+
dangling_constraints.append(CoverageRef(id=constraint_ref.id, path=constraint_ref.path))
|
|
279
|
+
dangling_constraints.sort(key=lambda r: r.id)
|
|
280
|
+
|
|
281
|
+
# ============================================================================
|
|
282
|
+
# 第八步:检查 dangling regex nodes(清单引用但磁盘缺失)
|
|
283
|
+
# ============================================================================
|
|
284
|
+
dangling_regex_nodes: list[CoverageRef] = []
|
|
285
|
+
for regex_ref in manifest.regex_nodes or []:
|
|
286
|
+
if not os.path.isfile(os.path.join(config_path, regex_ref.path)):
|
|
287
|
+
dangling_regex_nodes.append(CoverageRef(id=regex_ref.id, path=regex_ref.path))
|
|
288
|
+
dangling_regex_nodes.sort(key=lambda r: r.id)
|
|
289
|
+
|
|
290
|
+
# ============================================================================
|
|
291
|
+
# 第八步扩展:检查 dangling transforms / manual_data(清单引用但磁盘缺失)
|
|
292
|
+
# ============================================================================
|
|
293
|
+
dangling_transforms: list[CoverageRef] = []
|
|
294
|
+
for transform_ref in manifest.transforms or []:
|
|
295
|
+
if not os.path.isfile(os.path.join(config_path, transform_ref.path)):
|
|
296
|
+
dangling_transforms.append(CoverageRef(id=transform_ref.id, path=transform_ref.path))
|
|
297
|
+
dangling_transforms.sort(key=lambda r: r.id)
|
|
298
|
+
|
|
299
|
+
dangling_manual_data: list[CoverageRef] = []
|
|
300
|
+
for manual_ref in manifest.manual_data or []:
|
|
301
|
+
if not os.path.isfile(os.path.join(config_path, manual_ref.path)):
|
|
302
|
+
dangling_manual_data.append(CoverageRef(id=manual_ref.id, path=manual_ref.path))
|
|
303
|
+
dangling_manual_data.sort(key=lambda r: r.id)
|
|
304
|
+
|
|
305
|
+
# ============================================================================
|
|
306
|
+
# 第九步:组装并返回覆盖度结果
|
|
307
|
+
# ============================================================================
|
|
308
|
+
return ManifestCoverage(
|
|
309
|
+
unlisted_schemas=unlisted_schemas,
|
|
310
|
+
unlisted_constraints=unlisted_constraints,
|
|
311
|
+
unlisted_regex_nodes=unlisted_regex_nodes,
|
|
312
|
+
unlisted_transforms=unlisted_transforms,
|
|
313
|
+
unlisted_manual_data=unlisted_manual_data,
|
|
314
|
+
dangling_schemas=dangling_schemas,
|
|
315
|
+
dangling_constraints=dangling_constraints,
|
|
316
|
+
dangling_regex_nodes=dangling_regex_nodes,
|
|
317
|
+
dangling_transforms=dangling_transforms,
|
|
318
|
+
dangling_manual_data=dangling_manual_data,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def coverage_to_api_dict(coverage: ManifestCoverage) -> dict:
|
|
323
|
+
"""@methoddesc 将 ManifestCoverage 对象转换为 API 响应字典
|
|
324
|
+
|
|
325
|
+
本函数将覆盖度检查结果转换为前端易用的字典格式,
|
|
326
|
+
包含一个 is_complete 标志和按类型分组的 unlisted/dangling 列表。
|
|
327
|
+
|
|
328
|
+
参数说明:
|
|
329
|
+
:param coverage: ManifestCoverage 覆盖度结果对象
|
|
330
|
+
:return: API 响应字典,结构如下:
|
|
331
|
+
{
|
|
332
|
+
"is_complete": bool, # 是否完全覆盖(无任何问题)
|
|
333
|
+
"unlisted": {
|
|
334
|
+
"schemas": [{"id": str, "path": str}, ...],
|
|
335
|
+
"constraints": [...],
|
|
336
|
+
"regex_nodes": [...],
|
|
337
|
+
},
|
|
338
|
+
"dangling": {
|
|
339
|
+
"schemas": [...],
|
|
340
|
+
"constraints": [...],
|
|
341
|
+
"regex_nodes": [...],
|
|
342
|
+
},
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
示例:
|
|
346
|
+
>>> coverage = ManifestCoverage(
|
|
347
|
+
... unlisted_schemas=[CoverageRef(id="orders", path="schemas/orders.schema.yaml")],
|
|
348
|
+
... unlisted_constraints=[],
|
|
349
|
+
... unlisted_regex_nodes=[],
|
|
350
|
+
... dangling_schemas=[],
|
|
351
|
+
... dangling_constraints=[],
|
|
352
|
+
... dangling_regex_nodes=[],
|
|
353
|
+
... )
|
|
354
|
+
>>> result = coverage_to_api_dict(coverage)
|
|
355
|
+
>>> result["is_complete"]
|
|
356
|
+
False
|
|
357
|
+
>>> len(result["unlisted"]["schemas"])
|
|
358
|
+
1
|
|
359
|
+
"""
|
|
360
|
+
# is_complete 为 True 的条件:所有 unlisted 和 dangling 列表都为空
|
|
361
|
+
# 即没有任何遗漏配置,也没有任何无效引用
|
|
362
|
+
return {
|
|
363
|
+
"is_complete": not (
|
|
364
|
+
coverage.unlisted_schemas
|
|
365
|
+
or coverage.unlisted_constraints
|
|
366
|
+
or coverage.unlisted_regex_nodes
|
|
367
|
+
or coverage.unlisted_transforms
|
|
368
|
+
or coverage.unlisted_manual_data
|
|
369
|
+
or coverage.dangling_schemas
|
|
370
|
+
or coverage.dangling_constraints
|
|
371
|
+
or coverage.dangling_regex_nodes
|
|
372
|
+
or coverage.dangling_transforms
|
|
373
|
+
or coverage.dangling_manual_data
|
|
374
|
+
),
|
|
375
|
+
# 未入清单的资源(磁盘有但清单未引用)
|
|
376
|
+
"unlisted": {
|
|
377
|
+
"schemas": [r.__dict__ for r in coverage.unlisted_schemas],
|
|
378
|
+
"constraints": [r.__dict__ for r in coverage.unlisted_constraints],
|
|
379
|
+
"regex_nodes": [r.__dict__ for r in coverage.unlisted_regex_nodes],
|
|
380
|
+
"transforms": [r.__dict__ for r in coverage.unlisted_transforms],
|
|
381
|
+
"manual_data": [r.__dict__ for r in coverage.unlisted_manual_data],
|
|
382
|
+
},
|
|
383
|
+
# 悬空引用的资源(清单引用但磁盘缺失)
|
|
384
|
+
"dangling": {
|
|
385
|
+
"schemas": [r.__dict__ for r in coverage.dangling_schemas],
|
|
386
|
+
"constraints": [r.__dict__ for r in coverage.dangling_constraints],
|
|
387
|
+
"regex_nodes": [r.__dict__ for r in coverage.dangling_regex_nodes],
|
|
388
|
+
"transforms": [r.__dict__ for r in coverage.dangling_transforms],
|
|
389
|
+
"manual_data": [r.__dict__ for r in coverage.dangling_manual_data],
|
|
390
|
+
},
|
|
391
|
+
}
|