precis-cli 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +16 -0
- app/api/__init__.py +32 -0
- app/api/dependencies.py +185 -0
- app/api/main.py +301 -0
- app/api/middleware/__init__.py +34 -0
- app/api/middleware/exception_handler.py +126 -0
- app/api/middleware/request_logging.py +97 -0
- app/api/middleware/token_auth.py +215 -0
- app/api/models/__init__.py +151 -0
- app/api/models/connection_rules.py +98 -0
- app/api/models/files.py +99 -0
- app/api/models/full_validation.py +554 -0
- app/api/models/project.py +117 -0
- app/api/models/projects.py +80 -0
- app/api/models/schema.py +121 -0
- app/api/models/v2_responses.py +46 -0
- app/api/models/validation.py +303 -0
- app/api/models/workspace.py +118 -0
- app/api/routers/__init__.py +56 -0
- app/api/routers/ai/__init__.py +36 -0
- app/api/routers/ai/chat.py +243 -0
- app/api/routers/ai/generate.py +156 -0
- app/api/routers/ai/hardware.py +199 -0
- app/api/routers/ai/jobs.py +544 -0
- app/api/routers/ai/migrate.py +471 -0
- app/api/routers/ai/models.py +352 -0
- app/api/routers/ai/ollama.py +108 -0
- app/api/routers/ai/providers.py +491 -0
- app/api/routers/ai/router.py +44 -0
- app/api/routers/ai/stream.py +281 -0
- app/api/routers/ai/utils.py +182 -0
- app/api/routers/core/__init__.py +34 -0
- app/api/routers/core/connection_rules.py +188 -0
- app/api/routers/core/data_sources.py +374 -0
- app/api/routers/core/regex.py +333 -0
- app/api/routers/core/reporting.py +263 -0
- app/api/routers/files/__init__.py +24 -0
- app/api/routers/files/ops.py +179 -0
- app/api/routers/files/transfer.py +101 -0
- app/api/routers/preview/__init__.py +35 -0
- app/api/routers/preview/content_mode.py +264 -0
- app/api/routers/preview/header_row.py +154 -0
- app/api/routers/preview/models.py +81 -0
- app/api/routers/preview/path_mode.py +440 -0
- app/api/routers/preview/router.py +30 -0
- app/api/routers/project/__init__.py +61 -0
- app/api/routers/project/base.py +111 -0
- app/api/routers/project/constraint.py +345 -0
- app/api/routers/project/full_config.py +411 -0
- app/api/routers/project/full_config_writer.py +327 -0
- app/api/routers/project/helpers.py +167 -0
- app/api/routers/project/inspection_fix.py +330 -0
- app/api/routers/project/manifest.py +605 -0
- app/api/routers/project/models.py +177 -0
- app/api/routers/project/pattern.py +268 -0
- app/api/routers/project/regex.py +358 -0
- app/api/routers/project/scanner.py +157 -0
- app/api/routers/project/schema.py +567 -0
- app/api/routers/project/schema_helpers.py +149 -0
- app/api/routers/project/settings.py +399 -0
- app/api/routers/project/template.py +325 -0
- app/api/routers/project/validation.py +227 -0
- app/api/routers/project/view.py +177 -0
- app/api/routers/project/workspaces.py +167 -0
- app/api/routers/projects/__init__.py +25 -0
- app/api/routers/projects/create.py +121 -0
- app/api/routers/projects/open.py +103 -0
- app/api/routers/projects/scan.py +100 -0
- app/api/routers/validation/__init__.py +39 -0
- app/api/routers/validation/common.py +263 -0
- app/api/routers/validation/content_mode.py +401 -0
- app/api/routers/validation/history.py +235 -0
- app/api/routers/validation/inline_mode.py +196 -0
- app/api/routers/validation/path_mode.py +314 -0
- app/api/routers/validation/router.py +51 -0
- app/api/services/full_validation_response_builder.py +412 -0
- app/api/services/io_error_messages.py +41 -0
- app/api/services/preview_service.py +248 -0
- app/cli/__init__.py +18 -0
- app/cli/__main__.py +30 -0
- app/cli/shared_services/__init__.py +42 -0
- app/cli/shared_services/config_ops.py +581 -0
- app/cli/shared_services/generation_ops.py +160 -0
- app/cli/shared_services/project_ops.py +288 -0
- app/cli/shell/__init__.py +48 -0
- app/cli/shell/commands/__init__.py +63 -0
- app/cli/shell/commands/ai/__init__.py +260 -0
- app/cli/shell/commands/ai/chat.py +331 -0
- app/cli/shell/commands/ai/delete.py +165 -0
- app/cli/shell/commands/ai/diff.py +70 -0
- app/cli/shell/commands/ai/display.py +428 -0
- app/cli/shell/commands/ai/executor.py +225 -0
- app/cli/shell/commands/ai/executor_utils.py +271 -0
- app/cli/shell/commands/ai/generate.py +285 -0
- app/cli/shell/commands/ai/interaction.py +262 -0
- app/cli/shell/commands/ai/migrate.py +281 -0
- app/cli/shell/commands/ai/resolver.py +224 -0
- app/cli/shell/commands/ai/status.py +101 -0
- app/cli/shell/commands/ai/switch.py +158 -0
- app/cli/shell/commands/ai/utils.py +52 -0
- app/cli/shell/commands/base.py +343 -0
- app/cli/shell/commands/config/__init__.py +131 -0
- app/cli/shell/commands/config/base.py +80 -0
- app/cli/shell/commands/config/check.py +246 -0
- app/cli/shell/commands/config/edit.py +148 -0
- app/cli/shell/commands/config/get.py +111 -0
- app/cli/shell/commands/config/init.py +123 -0
- app/cli/shell/commands/config/inspect.py +164 -0
- app/cli/shell/commands/config/list.py +104 -0
- app/cli/shell/commands/config/set.py +119 -0
- app/cli/shell/commands/config/show.py +150 -0
- app/cli/shell/commands/exit.py +67 -0
- app/cli/shell/commands/help.py +104 -0
- app/cli/shell/commands/infer_schema.py +156 -0
- app/cli/shell/commands/open.py +201 -0
- app/cli/shell/commands/project.py +215 -0
- app/cli/shell/commands/provider.py +619 -0
- app/cli/shell/commands/system.py +170 -0
- app/cli/shell/commands/validate.py +455 -0
- app/cli/shell/completer.py +231 -0
- app/cli/shell/config_storage.py +191 -0
- app/cli/shell/exceptions.py +90 -0
- app/cli/shell/formatter.py +371 -0
- app/cli/shell/interactive_menu.py +348 -0
- app/cli/shell/main.py +286 -0
- app/cli/shell/parser.py +266 -0
- app/cli/start.py +180 -0
- app/cli_main.py +52 -0
- app/mcp_server.py +311 -0
- app/shared/core/__init__.py +18 -0
- app/shared/core/app_version.py +59 -0
- app/shared/core/config/__init__.py +119 -0
- app/shared/core/config/server.py +193 -0
- app/shared/core/data_source/__init__.py +106 -0
- app/shared/core/data_source/loader.py +348 -0
- app/shared/core/data_source/loaders/__init__.py +213 -0
- app/shared/core/data_source/loaders/base.py +215 -0
- app/shared/core/data_source/loaders/converter.py +337 -0
- app/shared/core/data_source/loaders/csv_loader.py +252 -0
- app/shared/core/data_source/loaders/excel_loader.py +666 -0
- app/shared/core/data_source/loaders/extractor.py +268 -0
- app/shared/core/data_source/loaders/json_loader.py +354 -0
- app/shared/core/data_source/loaders/registry.py +88 -0
- app/shared/core/data_source/loaders/sql_loader.py +313 -0
- app/shared/core/data_source/loaders/strategies/__init__.py +100 -0
- app/shared/core/data_source/loaders/strategies/array_parser.py +277 -0
- app/shared/core/data_source/loaders/strategies/lines_parser.py +355 -0
- app/shared/core/data_source/loaders/strategies/object_parser.py +278 -0
- app/shared/core/data_source/schema_info.py +50 -0
- app/shared/core/data_source/specs/__init__.py +128 -0
- app/shared/core/data_source/specs/base.py +258 -0
- app/shared/core/data_source/specs/csv_source.py +137 -0
- app/shared/core/data_source/specs/excel_source.py +141 -0
- app/shared/core/data_source/specs/file_base.py +252 -0
- app/shared/core/data_source/specs/json_source.py +226 -0
- app/shared/core/data_source/specs/sql_source.py +161 -0
- app/shared/core/encoding.py +36 -0
- app/shared/core/io/__init__.py +24 -0
- app/shared/core/io/yaml.py +379 -0
- app/shared/core/manifest_schema/__init__.py +52 -0
- app/shared/core/manifest_schema/types.py +99 -0
- app/shared/core/manifest_schema/version.py +175 -0
- app/shared/core/patterns/__init__.py +24 -0
- app/shared/core/patterns/loader.py +134 -0
- app/shared/core/patterns/writer.py +244 -0
- app/shared/core/project/__init__.py +47 -0
- app/shared/core/project/constraint/builders/__init__.py +50 -0
- app/shared/core/project/constraint/builders/base.py +100 -0
- app/shared/core/project/constraint/builders/composite.py +77 -0
- app/shared/core/project/constraint/builders/conditional.py +67 -0
- app/shared/core/project/constraint/builders/foreign_key.py +53 -0
- app/shared/core/project/constraint/builders/registry.py +64 -0
- app/shared/core/project/constraint/builders/scripted.py +51 -0
- app/shared/core/project/constraint/builders/single_column.py +86 -0
- app/shared/core/project/constraint/builders/unique.py +53 -0
- app/shared/core/project/constraint/factory.py +170 -0
- app/shared/core/project/constraint/reader.py +214 -0
- app/shared/core/project/constraint/registry.py +233 -0
- app/shared/core/project/constraint/types/__init__.py +63 -0
- app/shared/core/project/constraint/types/constraint_file.py +261 -0
- app/shared/core/project/constraint/types/refs.py +460 -0
- app/shared/core/project/constraint/types.py +28 -0
- app/shared/core/project/constraint/writer.py +181 -0
- app/shared/core/project/loader/__init__.py +56 -0
- app/shared/core/project/loader/loader.py +30 -0
- app/shared/core/project/loader/loader_parts/config_inspector.py +137 -0
- app/shared/core/project/loader/loader_parts/embedded_constraints.py +224 -0
- app/shared/core/project/loader/loader_parts/file_loaders.py +58 -0
- app/shared/core/project/loader/loader_parts/inspection_ids.py +85 -0
- app/shared/core/project/loader/loader_parts/inspector_helpers.py +312 -0
- app/shared/core/project/loader/loader_parts/inspector_id_checks.py +274 -0
- app/shared/core/project/loader/loader_parts/inspector_reference_checks.py +690 -0
- app/shared/core/project/loader/loader_parts/inspector_uniqueness_checks.py +286 -0
- app/shared/core/project/loader/loader_parts/loading_error_messages.py +298 -0
- app/shared/core/project/loader/loader_parts/main.py +432 -0
- app/shared/core/project/loader/loader_parts/path_validation.py +117 -0
- app/shared/core/project/loader/loader_parts/runtime.py +125 -0
- app/shared/core/project/loader/types.py +246 -0
- app/shared/core/project/manifest/coverage.py +391 -0
- app/shared/core/project/manifest/reader.py +260 -0
- app/shared/core/project/manifest/types.py +91 -0
- app/shared/core/project/manifest/types_parts/__init__.py +60 -0
- app/shared/core/project/manifest/types_parts/constants.py +40 -0
- app/shared/core/project/manifest/types_parts/data_source.py +68 -0
- app/shared/core/project/manifest/types_parts/info.py +59 -0
- app/shared/core/project/manifest/types_parts/manifest.py +294 -0
- app/shared/core/project/manifest/types_parts/refs.py +153 -0
- app/shared/core/project/manifest/types_parts/settings.py +92 -0
- app/shared/core/project/manifest/types_parts/settings_file_processing.py +64 -0
- app/shared/core/project/manifest/types_parts/settings_script_security.py +78 -0
- app/shared/core/project/manifest/types_parts/settings_validation.py +83 -0
- app/shared/core/project/manifest/types_parts/template.py +66 -0
- app/shared/core/project/manifest/writer.py +262 -0
- app/shared/core/project/manual_data/__init__.py +27 -0
- app/shared/core/project/manual_data/types.py +75 -0
- app/shared/core/project/regex/reader.py +197 -0
- app/shared/core/project/regex/types.py +405 -0
- app/shared/core/project/regex/writer.py +123 -0
- app/shared/core/project/schema/reader.py +170 -0
- app/shared/core/project/schema/types.py +47 -0
- app/shared/core/project/schema/types_parts/__init__.py +36 -0
- app/shared/core/project/schema/types_parts/column.py +174 -0
- app/shared/core/project/schema/types_parts/column_utils.py +72 -0
- app/shared/core/project/schema/types_parts/constraint.py +165 -0
- app/shared/core/project/schema/types_parts/schema_id.py +66 -0
- app/shared/core/project/schema/types_parts/source.py +255 -0
- app/shared/core/project/schema/types_parts/source_options.py +347 -0
- app/shared/core/project/schema/types_parts/table.py +230 -0
- app/shared/core/project/schema/writer.py +139 -0
- app/shared/core/project/schema_ref_check.py +95 -0
- app/shared/core/project/template/__init__.py +27 -0
- app/shared/core/project/template/expander.py +263 -0
- app/shared/core/project/template/reader.py +120 -0
- app/shared/core/project/template/types.py +114 -0
- app/shared/core/project/transform/reader.py +76 -0
- app/shared/core/project/transform/types.py +116 -0
- app/shared/core/project/transform/writer.py +84 -0
- app/shared/core/pydantic_messages.py +59 -0
- app/shared/core/reporter/__init__.py +46 -0
- app/shared/core/reporter/reporter.py +220 -0
- app/shared/core/reporter/reporters/__init__.py +65 -0
- app/shared/core/reporter/reporters/base.py +188 -0
- app/shared/core/reporter/reporters/dingtalk_app_reporter.py +274 -0
- app/shared/core/reporter/reporters/email_reporter.py +271 -0
- app/shared/core/reporter/reporters/feishu_app_reporter.py +467 -0
- app/shared/core/reporter/reporters/local_file_reporter.py +208 -0
- app/shared/core/reporter/reporters/wecom_app_reporter.py +268 -0
- app/shared/core/utils/__init__.py +18 -0
- app/shared/core/utils/path_utils.py +60 -0
- app/shared/core/utils/regex_extract.py +113 -0
- app/shared/domain/__init__.py +114 -0
- app/shared/domain/constraints/__init__.py +76 -0
- app/shared/domain/constraints/allowed_values.py +211 -0
- app/shared/domain/constraints/base.py +141 -0
- app/shared/domain/constraints/charset.py +337 -0
- app/shared/domain/constraints/composite.py +174 -0
- app/shared/domain/constraints/condition_registry.py +130 -0
- app/shared/domain/constraints/conditional.py +629 -0
- app/shared/domain/constraints/date_logic.py +731 -0
- app/shared/domain/constraints/foreign_key.py +261 -0
- app/shared/domain/constraints/key_normalization.py +56 -0
- app/shared/domain/constraints/not_null.py +185 -0
- app/shared/domain/constraints/range.py +360 -0
- app/shared/domain/constraints/regex.py +218 -0
- app/shared/domain/constraints/scripted.py +276 -0
- app/shared/domain/constraints/unique.py +233 -0
- app/shared/domain/data_engine.py +384 -0
- app/shared/domain/data_types.py +113 -0
- app/shared/domain/data_types_parts/__init__.py +67 -0
- app/shared/domain/data_types_parts/base.py +204 -0
- app/shared/domain/data_types_parts/composite.py +250 -0
- app/shared/domain/data_types_parts/expression.py +202 -0
- app/shared/domain/data_types_parts/extracted.py +106 -0
- app/shared/domain/data_types_parts/json_types.py +234 -0
- app/shared/domain/data_types_parts/scalars.py +719 -0
- app/shared/domain/data_types_parts/sequence.py +129 -0
- app/shared/domain/dataset_schema.py +48 -0
- app/shared/domain/eval_sandbox.py +63 -0
- app/shared/domain/expression_system.py +366 -0
- app/shared/domain/regex_flags.py +53 -0
- app/shared/domain/schema/builder.py +223 -0
- app/shared/domain/schema/models.py +339 -0
- app/shared/domain/transforms/__init__.py +28 -0
- app/shared/domain/transforms/aggregate.py +141 -0
- app/shared/domain/transforms/base.py +171 -0
- app/shared/domain/transforms/cast_type.py +120 -0
- app/shared/domain/transforms/concat.py +103 -0
- app/shared/domain/transforms/conditional_assign.py +114 -0
- app/shared/domain/transforms/date_format.py +81 -0
- app/shared/domain/transforms/digits.py +77 -0
- app/shared/domain/transforms/drop_duplicates.py +94 -0
- app/shared/domain/transforms/fill_na.py +94 -0
- app/shared/domain/transforms/filter_rows.py +97 -0
- app/shared/domain/transforms/lookup.py +78 -0
- app/shared/domain/transforms/lower_case.py +70 -0
- app/shared/domain/transforms/map_value.py +90 -0
- app/shared/domain/transforms/math_expr.py +112 -0
- app/shared/domain/transforms/modulo.py +82 -0
- app/shared/domain/transforms/regex_extract.py +103 -0
- app/shared/domain/transforms/registry.py +95 -0
- app/shared/domain/transforms/replace.py +88 -0
- app/shared/domain/transforms/sort_rows.py +94 -0
- app/shared/domain/transforms/string_split.py +84 -0
- app/shared/domain/transforms/strip.py +73 -0
- app/shared/domain/transforms/substring.py +92 -0
- app/shared/domain/transforms/upper_case.py +70 -0
- app/shared/domain/transforms/weighted_sum.py +104 -0
- app/shared/domain/validation_constraints.py +69 -0
- app/shared/services/__init__.py +62 -0
- app/shared/services/ai/__init__.py +48 -0
- app/shared/services/ai/agent/__init__.py +40 -0
- app/shared/services/ai/agent/chat_tools/__init__.py +47 -0
- app/shared/services/ai/agent/chat_tools/apply_actions.py +592 -0
- app/shared/services/ai/agent/chat_tools/ask_user.py +237 -0
- app/shared/services/ai/agent/chat_tools/read_canvas.py +140 -0
- app/shared/services/ai/agent/chat_tools/read_project.py +160 -0
- app/shared/services/ai/agent/chat_tools/read_table.py +301 -0
- app/shared/services/ai/agent/chat_tools/schemas.py +105 -0
- app/shared/services/ai/agent/chat_tools/validate_table.py +131 -0
- app/shared/services/ai/agent/executor.py +554 -0
- app/shared/services/ai/agent/memory.py +256 -0
- app/shared/services/ai/agent/planner.py +282 -0
- app/shared/services/ai/agent/tool_registry.py +296 -0
- app/shared/services/ai/agent/tools/__init__.py +32 -0
- app/shared/services/ai/agent/tools/config_generate.py +133 -0
- app/shared/services/ai/agent/tools/config_refine.py +109 -0
- app/shared/services/ai/agent/tools/config_validate.py +403 -0
- app/shared/services/ai/agent/tools/merge_results.py +136 -0
- app/shared/services/ai/agent/tools/plan_chunks.py +79 -0
- app/shared/services/ai/agent/tools/script_parse.py +359 -0
- app/shared/services/ai/agent/types.py +138 -0
- app/shared/services/ai/chat_agent_runner.py +596 -0
- app/shared/services/ai/chat_orchestrator.py +725 -0
- app/shared/services/ai/failure_messages.py +65 -0
- app/shared/services/ai/job_storage.py +274 -0
- app/shared/services/ai/migrate_service.py +550 -0
- app/shared/services/ai/streaming/__init__.py +65 -0
- app/shared/services/ai/streaming/event_journal.py +197 -0
- app/shared/services/ai/streaming/orchestrator.py +262 -0
- app/shared/services/ai/streaming/pending_interaction_store.py +227 -0
- app/shared/services/ai/streaming/sse_response.py +148 -0
- app/shared/services/ai/streaming/types.py +73 -0
- app/shared/services/ai/types.py +213 -0
- app/shared/services/ai/utils.py +322 -0
- app/shared/services/diff/config_diff.py +320 -0
- app/shared/services/hardware.py +237 -0
- app/shared/services/llm/__init__.py +62 -0
- app/shared/services/llm/actions/__init__.py +42 -0
- app/shared/services/llm/actions/_canvas_validator.py +147 -0
- app/shared/services/llm/actions/_constraint_validator.py +401 -0
- app/shared/services/llm/actions/_regex_validator.py +85 -0
- app/shared/services/llm/actions/_schema_validator.py +82 -0
- app/shared/services/llm/actions/_settings_validator.py +76 -0
- app/shared/services/llm/actions/_transform_validator.py +78 -0
- app/shared/services/llm/actions/action_handlers.py +400 -0
- app/shared/services/llm/actions/action_parser.py +63 -0
- app/shared/services/llm/actions/action_processor.py +468 -0
- app/shared/services/llm/actions/action_validator.py +311 -0
- app/shared/services/llm/actions/diff_compute.py +195 -0
- app/shared/services/llm/actions/regex_handlers.py +284 -0
- app/shared/services/llm/actions/registry.py +336 -0
- app/shared/services/llm/actions/schema_handlers.py +335 -0
- app/shared/services/llm/actions/settings_handlers.py +180 -0
- app/shared/services/llm/actions/specs.py +259 -0
- app/shared/services/llm/actions/transform_handlers.py +279 -0
- app/shared/services/llm/actions/validation_types.py +147 -0
- app/shared/services/llm/cache/__init__.py +18 -0
- app/shared/services/llm/cache/response_cache.py +80 -0
- app/shared/services/llm/chat/__init__.py +35 -0
- app/shared/services/llm/chat/chat_system_prompt.py +561 -0
- app/shared/services/llm/chat/response_parser.py +296 -0
- app/shared/services/llm/config/__init__.py +45 -0
- app/shared/services/llm/config/crypto.py +135 -0
- app/shared/services/llm/config/loader.py +258 -0
- app/shared/services/llm/config/models.py +202 -0
- app/shared/services/llm/config/presets.py +128 -0
- app/shared/services/llm/config_generator.py +163 -0
- app/shared/services/llm/constraints/__init__.py +41 -0
- app/shared/services/llm/constraints/constraint_builder.py +177 -0
- app/shared/services/llm/constraints/constraint_deletion.py +84 -0
- app/shared/services/llm/constraints/constraint_id.py +174 -0
- app/shared/services/llm/constraints/frontend_instructions.py +333 -0
- app/shared/services/llm/constraints/inline_batch.py +279 -0
- app/shared/services/llm/discovery/__init__.py +35 -0
- app/shared/services/llm/discovery/scanner.py +181 -0
- app/shared/services/llm/generation/__init__.py +42 -0
- app/shared/services/llm/generation/agent_wiring.py +156 -0
- app/shared/services/llm/generation/config_builder.py +386 -0
- app/shared/services/llm/generation/errors.py +36 -0
- app/shared/services/llm/generation/existing_config.py +99 -0
- app/shared/services/llm/generation/profiler.py +279 -0
- app/shared/services/llm/generation/prompt_builder.py +199 -0
- app/shared/services/llm/generation/response_parser.py +144 -0
- app/shared/services/llm/generation/service.py +622 -0
- app/shared/services/llm/models.py +104 -0
- app/shared/services/llm/providers/__init__.py +48 -0
- app/shared/services/llm/providers/base.py +310 -0
- app/shared/services/llm/providers/cached_provider.py +90 -0
- app/shared/services/llm/providers/ollama.py +498 -0
- app/shared/services/llm/providers/openai.py +334 -0
- app/shared/services/llm/providers/registry.py +108 -0
- app/shared/services/llm/schema_resolver.py +141 -0
- app/shared/services/llm/suggestion_utils.py +219 -0
- app/shared/services/llm/validate_executor.py +163 -0
- app/shared/services/llm/yaml_io.py +406 -0
- app/shared/services/preview/__init__.py +18 -0
- app/shared/services/preview/loader.py +145 -0
- app/shared/services/preview/path_validation.py +138 -0
- app/shared/services/project_loader.py +57 -0
- app/shared/services/schema_inference.py +260 -0
- app/shared/services/schema_runtime_builder.py +120 -0
- app/shared/services/validation/__init__.py +52 -0
- app/shared/services/validation/chunked_loader.py +509 -0
- app/shared/services/validation/dag/__init__.py +35 -0
- app/shared/services/validation/dag/builder.py +141 -0
- app/shared/services/validation/dag/executor.py +225 -0
- app/shared/services/validation/dag/sorter.py +77 -0
- app/shared/services/validation/data_loader.py +231 -0
- app/shared/services/validation/engine.py +446 -0
- app/shared/services/validation/executor.py +1030 -0
- app/shared/services/validation/extractors.py +266 -0
- app/shared/services/validation/history.py +209 -0
- app/shared/services/validation/json_payload.py +136 -0
- app/shared/services/validation/loader.py +152 -0
- app/shared/services/validation/memory_monitor.py +179 -0
- app/shared/services/validation/postprocess.py +208 -0
- app/shared/services/validation/progress.py +64 -0
- app/shared/services/validation/report_export.py +222 -0
- app/shared/services/validation/resolver.py +180 -0
- app/shared/services/validation/service.py +418 -0
- app/shared/services/validation/types.py +238 -0
- app/shared/services/validation/validators/__init__.py +36 -0
- app/shared/services/validation/validators/adapter.py +182 -0
- app/shared/services/validation/validators/base.py +332 -0
- app/shared/services/validation/validators/composite.py +233 -0
- app/shared/services/validation/validators/date_logic.py +276 -0
- app/start_server.py +133 -0
- precis_cli-0.1.3.dist-info/METADATA +180 -0
- precis_cli-0.1.3.dist-info/RECORD +442 -0
- precis_cli-0.1.3.dist-info/WHEEL +5 -0
- precis_cli-0.1.3.dist-info/entry_points.txt +4 -0
- precis_cli-0.1.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview Manifest 主类型定义模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 定义项目清单 (ProjectManifest) 的核心数据结构
|
|
21
|
+
- 整合项目信息、设置、Schema/Constraint/Regex 引用、数据源配置
|
|
22
|
+
|
|
23
|
+
架构设计:
|
|
24
|
+
- 主入口模型: ProjectManifest 是 manifest.yaml 解析后的根对象
|
|
25
|
+
- ID 唯一性校验: 自动检测并处理重复的 ID
|
|
26
|
+
- 默认值处理: schemas/constraints/regex_nodes/data_sources 默认为空列表
|
|
27
|
+
|
|
28
|
+
输入示例 (manifest.yaml):
|
|
29
|
+
version: 2
|
|
30
|
+
|
|
31
|
+
project:
|
|
32
|
+
name: 我的数据项目
|
|
33
|
+
description: 这是一个测试项目
|
|
34
|
+
|
|
35
|
+
settings:
|
|
36
|
+
file_processing:
|
|
37
|
+
default_encoding: utf-8
|
|
38
|
+
csv_delimiter: ","
|
|
39
|
+
validation:
|
|
40
|
+
parallel_workers: 4
|
|
41
|
+
|
|
42
|
+
schemas:
|
|
43
|
+
- id: users
|
|
44
|
+
path: schemas/users.yaml
|
|
45
|
+
- id: orders
|
|
46
|
+
path: schemas/orders.yaml
|
|
47
|
+
|
|
48
|
+
constraints:
|
|
49
|
+
- id: users_not_null
|
|
50
|
+
path: constraints/users_not_null.yaml
|
|
51
|
+
|
|
52
|
+
regex_nodes:
|
|
53
|
+
- id: email_pattern
|
|
54
|
+
path: regex/email.yaml
|
|
55
|
+
|
|
56
|
+
data_sources:
|
|
57
|
+
- id: primary
|
|
58
|
+
path: data
|
|
59
|
+
mode: relative
|
|
60
|
+
description: 主数据目录
|
|
61
|
+
|
|
62
|
+
patterns_dir: patterns
|
|
63
|
+
|
|
64
|
+
输出示例 (Python 对象):
|
|
65
|
+
ProjectManifest(
|
|
66
|
+
version=2,
|
|
67
|
+
project=ProjectInfo(name="我的数据项目", description="这是一个测试项目"),
|
|
68
|
+
settings=ProjectSettings(...),
|
|
69
|
+
schemas=[SchemaRef(...), ...],
|
|
70
|
+
constraints=[ConstraintRef(...), ...],
|
|
71
|
+
regex_nodes=[RegexRef(...), ...],
|
|
72
|
+
data_sources=[DataSourceRef(id="primary", path="data", mode="relative")],
|
|
73
|
+
patterns_dir="patterns",
|
|
74
|
+
warnings=[]
|
|
75
|
+
)
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
from __future__ import annotations
|
|
79
|
+
|
|
80
|
+
from typing import Any
|
|
81
|
+
|
|
82
|
+
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
83
|
+
|
|
84
|
+
from app.shared.core.project.manifest.types_parts.constants import V2_VERSION
|
|
85
|
+
from app.shared.core.project.manifest.types_parts.data_source import DataSourceRef
|
|
86
|
+
from app.shared.core.project.manifest.types_parts.info import ProjectInfo
|
|
87
|
+
from app.shared.core.project.manifest.types_parts.refs import (
|
|
88
|
+
ConstraintRef,
|
|
89
|
+
ManualDataRef,
|
|
90
|
+
RegexRef,
|
|
91
|
+
SchemaRef,
|
|
92
|
+
TransformRef,
|
|
93
|
+
)
|
|
94
|
+
from app.shared.core.project.manifest.types_parts.settings import ProjectSettings
|
|
95
|
+
from app.shared.core.project.manifest.types_parts.template import TemplateInstanceRef, TemplateRef
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class ProjectManifest(BaseModel):
|
|
99
|
+
"""@classdesc 项目清单主类型
|
|
100
|
+
|
|
101
|
+
这是 manifest.yaml 文件解析后的根对象,包含了项目的所有配置引用。
|
|
102
|
+
|
|
103
|
+
字段说明:
|
|
104
|
+
- version: 配置版本号,当前固定为 2
|
|
105
|
+
- project: 项目基本信息 (名称、描述等)
|
|
106
|
+
- settings: 项目设置 (文件处理、验证规则等)
|
|
107
|
+
- schemas: Schema 文件引用列表
|
|
108
|
+
- constraints: Constraint 文件引用列表
|
|
109
|
+
- regex_nodes: Regex 节点文件引用列表
|
|
110
|
+
- data_sources: 数据源目录引用列表(用于全量校验)
|
|
111
|
+
- patterns_dir: patterns 目录路径
|
|
112
|
+
- warnings: 加载过程中产生的警告信息
|
|
113
|
+
|
|
114
|
+
数据校验:
|
|
115
|
+
- ID 唯一性: 自动检测重复的 schema/constraint/regex/data_source ID,保留第一个
|
|
116
|
+
- None 转换: schemas/constraints/regex_nodes/data_sources 如果为 None,自动转为空列表
|
|
117
|
+
|
|
118
|
+
示例:
|
|
119
|
+
# YAML 文件内容
|
|
120
|
+
version: 2
|
|
121
|
+
project:
|
|
122
|
+
name: 用户管理系统
|
|
123
|
+
schemas:
|
|
124
|
+
- id: users
|
|
125
|
+
path: schemas/users.yaml
|
|
126
|
+
- id: users # 重复 ID
|
|
127
|
+
path: schemas/users_dup.yaml
|
|
128
|
+
data_sources:
|
|
129
|
+
- id: primary
|
|
130
|
+
path: data
|
|
131
|
+
mode: relative
|
|
132
|
+
|
|
133
|
+
# 解析后的对象 (重复的会被自动处理)
|
|
134
|
+
ProjectManifest(
|
|
135
|
+
version=2,
|
|
136
|
+
project=ProjectInfo(name="用户管理系统", ...),
|
|
137
|
+
schemas=[SchemaRef(id="users", path="schemas/users.yaml")],
|
|
138
|
+
data_sources=[DataSourceRef(id="primary", path="data", mode="relative")],
|
|
139
|
+
warnings=["Schema ID 'users' 重复,已跳过 'schemas/users_dup.yaml'"]
|
|
140
|
+
)
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
version: int = Field(V2_VERSION, description="配置版本号(固定为 2)")
|
|
144
|
+
project: ProjectInfo
|
|
145
|
+
settings: ProjectSettings = Field(default_factory=ProjectSettings, description="项目设置")
|
|
146
|
+
schemas: list[SchemaRef] = Field(default_factory=list, description="Schema 文件索引")
|
|
147
|
+
constraints: list[ConstraintRef] = Field(default_factory=list, description="Constraint 文件索引")
|
|
148
|
+
regex_nodes: list[RegexRef] = Field(default_factory=list, description="Regex 节点文件索引")
|
|
149
|
+
transforms: list[TransformRef] = Field(default_factory=list, description="Transform 功能节点文件索引")
|
|
150
|
+
manual_data: list[ManualDataRef] = Field(default_factory=list, description="ManualData 内联数据节点文件索引")
|
|
151
|
+
data_sources: list[DataSourceRef] = Field(default_factory=list, description="数据源目录索引")
|
|
152
|
+
templates: list[TemplateRef] = Field(default_factory=list, description="模板定义文件索引")
|
|
153
|
+
template_instances: list[TemplateInstanceRef] = Field(default_factory=list, description="模板实例索引")
|
|
154
|
+
patterns_dir: str = Field("patterns", description="patterns 目录(相对路径)")
|
|
155
|
+
warnings: list[str] = Field(default_factory=list, description="加载时的警告信息")
|
|
156
|
+
|
|
157
|
+
@field_validator(
|
|
158
|
+
"schemas",
|
|
159
|
+
"constraints",
|
|
160
|
+
"regex_nodes",
|
|
161
|
+
"transforms",
|
|
162
|
+
"manual_data",
|
|
163
|
+
"data_sources",
|
|
164
|
+
"templates",
|
|
165
|
+
"template_instances",
|
|
166
|
+
mode="before",
|
|
167
|
+
)
|
|
168
|
+
@classmethod
|
|
169
|
+
def _coerce_none_to_list(cls, v: Any) -> Any:
|
|
170
|
+
if v is None:
|
|
171
|
+
return []
|
|
172
|
+
return v
|
|
173
|
+
|
|
174
|
+
@model_validator(mode="after")
|
|
175
|
+
def _validate_unique_ids(self) -> ProjectManifest:
|
|
176
|
+
warnings: list[str] = []
|
|
177
|
+
|
|
178
|
+
seen_schema_ids: dict[str, SchemaRef] = {}
|
|
179
|
+
unique_schemas: list[SchemaRef] = []
|
|
180
|
+
for schema in self.schemas:
|
|
181
|
+
if schema.id in seen_schema_ids:
|
|
182
|
+
existing_schema: SchemaRef = seen_schema_ids[schema.id]
|
|
183
|
+
warnings.append(
|
|
184
|
+
f"两个表结构定义使用了相同的 ID「{schema.id}」:已跳过 {schema.path},保留 {existing_schema.path}"
|
|
185
|
+
)
|
|
186
|
+
else:
|
|
187
|
+
seen_schema_ids[schema.id] = schema
|
|
188
|
+
unique_schemas.append(schema)
|
|
189
|
+
# 就地重赋值用 object.__setattr__ 绕开 fields_set 标记(回归: 普通赋值会把
|
|
190
|
+
# 字段加入 model_fields_set,令 PUT /config/full 的"未显式设置则保留磁盘值"
|
|
191
|
+
# 合并防线对全部列表字段失效——见 full_config_writer._should_merge 与
|
|
192
|
+
# test_full_config_reliability 的 fields_set 纯净性守卫)
|
|
193
|
+
object.__setattr__(self, "schemas", unique_schemas)
|
|
194
|
+
|
|
195
|
+
seen_constraint_ids: dict[str, ConstraintRef] = {}
|
|
196
|
+
unique_constraints: list[ConstraintRef] = []
|
|
197
|
+
for constraint in self.constraints:
|
|
198
|
+
if constraint.id in seen_constraint_ids:
|
|
199
|
+
existing_constraint: ConstraintRef = seen_constraint_ids[constraint.id]
|
|
200
|
+
warnings.append(
|
|
201
|
+
f"两条约束使用了相同的 ID「{constraint.id}」:已跳过 {constraint.path},保留 {existing_constraint.path}"
|
|
202
|
+
)
|
|
203
|
+
else:
|
|
204
|
+
seen_constraint_ids[constraint.id] = constraint
|
|
205
|
+
unique_constraints.append(constraint)
|
|
206
|
+
object.__setattr__(self, "constraints", unique_constraints)
|
|
207
|
+
|
|
208
|
+
seen_regex_ids: dict[str, RegexRef] = {}
|
|
209
|
+
unique_regex_nodes: list[RegexRef] = []
|
|
210
|
+
for regex in self.regex_nodes:
|
|
211
|
+
if regex.id in seen_regex_ids:
|
|
212
|
+
existing_regex: RegexRef = seen_regex_ids[regex.id]
|
|
213
|
+
warnings.append(
|
|
214
|
+
f"两个正则定义使用了相同的 ID「{regex.id}」:已跳过 {regex.path},保留 {existing_regex.path}"
|
|
215
|
+
)
|
|
216
|
+
else:
|
|
217
|
+
seen_regex_ids[regex.id] = regex
|
|
218
|
+
unique_regex_nodes.append(regex)
|
|
219
|
+
object.__setattr__(self, "regex_nodes", unique_regex_nodes)
|
|
220
|
+
|
|
221
|
+
seen_transform_ids: dict[str, TransformRef] = {}
|
|
222
|
+
unique_transforms: list[TransformRef] = []
|
|
223
|
+
for transform in self.transforms:
|
|
224
|
+
if transform.id in seen_transform_ids:
|
|
225
|
+
existing_transform: TransformRef = seen_transform_ids[transform.id]
|
|
226
|
+
warnings.append(
|
|
227
|
+
f"两个数据转换使用了相同的 ID「{transform.id}」:已跳过 {transform.path},保留 {existing_transform.path}"
|
|
228
|
+
)
|
|
229
|
+
else:
|
|
230
|
+
seen_transform_ids[transform.id] = transform
|
|
231
|
+
unique_transforms.append(transform)
|
|
232
|
+
object.__setattr__(self, "transforms", unique_transforms)
|
|
233
|
+
|
|
234
|
+
# ManualData ID 去重
|
|
235
|
+
seen_manual_data_ids: dict[str, ManualDataRef] = {}
|
|
236
|
+
unique_manual_data: list[ManualDataRef] = []
|
|
237
|
+
for md in self.manual_data:
|
|
238
|
+
if md.id in seen_manual_data_ids:
|
|
239
|
+
existing_manual_data: ManualDataRef = seen_manual_data_ids[md.id]
|
|
240
|
+
warnings.append(
|
|
241
|
+
f"两个手动数据定义使用了相同的 ID「{md.id}」:已跳过 {md.path},保留 {existing_manual_data.path}"
|
|
242
|
+
)
|
|
243
|
+
else:
|
|
244
|
+
seen_manual_data_ids[md.id] = md
|
|
245
|
+
unique_manual_data.append(md)
|
|
246
|
+
object.__setattr__(self, "manual_data", unique_manual_data)
|
|
247
|
+
|
|
248
|
+
seen_data_source_ids: dict[str, DataSourceRef] = {}
|
|
249
|
+
unique_data_sources: list[DataSourceRef] = []
|
|
250
|
+
for ds in self.data_sources:
|
|
251
|
+
if ds.id in seen_data_source_ids:
|
|
252
|
+
existing_data_source: DataSourceRef = seen_data_source_ids[ds.id]
|
|
253
|
+
warnings.append(
|
|
254
|
+
f"两个数据源目录使用了相同的 ID「{ds.id}」:已跳过 {ds.path},保留 {existing_data_source.path}"
|
|
255
|
+
)
|
|
256
|
+
else:
|
|
257
|
+
seen_data_source_ids[ds.id] = ds
|
|
258
|
+
unique_data_sources.append(ds)
|
|
259
|
+
object.__setattr__(self, "data_sources", unique_data_sources)
|
|
260
|
+
|
|
261
|
+
# 模板定义 ID 去重
|
|
262
|
+
seen_template_ids: dict[str, TemplateRef] = {}
|
|
263
|
+
unique_templates: list[TemplateRef] = []
|
|
264
|
+
for tmpl in self.templates:
|
|
265
|
+
if tmpl.id in seen_template_ids:
|
|
266
|
+
existing_template: TemplateRef = seen_template_ids[tmpl.id]
|
|
267
|
+
warnings.append(f"Template ID '{tmpl.id}' 重复,已跳过 '{tmpl.path}',保留 '{existing_template.path}'")
|
|
268
|
+
else:
|
|
269
|
+
seen_template_ids[tmpl.id] = tmpl
|
|
270
|
+
unique_templates.append(tmpl)
|
|
271
|
+
object.__setattr__(self, "templates", unique_templates)
|
|
272
|
+
|
|
273
|
+
# 模板实例 ID 去重
|
|
274
|
+
seen_instance_ids: dict[str, TemplateInstanceRef] = {}
|
|
275
|
+
unique_instances: list[TemplateInstanceRef] = []
|
|
276
|
+
for inst in self.template_instances:
|
|
277
|
+
if inst.id in seen_instance_ids:
|
|
278
|
+
existing_instance: TemplateInstanceRef = seen_instance_ids[inst.id]
|
|
279
|
+
warnings.append(
|
|
280
|
+
f"TemplateInstance ID '{inst.id}' 重复,"
|
|
281
|
+
f"已跳过 (template_id={inst.template_id}),"
|
|
282
|
+
f"保留 (template_id={existing_instance.template_id})"
|
|
283
|
+
)
|
|
284
|
+
else:
|
|
285
|
+
seen_instance_ids[inst.id] = inst
|
|
286
|
+
unique_instances.append(inst)
|
|
287
|
+
object.__setattr__(self, "template_instances", unique_instances)
|
|
288
|
+
|
|
289
|
+
# 回归: 追加而非整体覆盖——文件里手写的 warnings 是用户的磁盘内容,
|
|
290
|
+
# 覆盖赋值会在加载→保存 roundtrip 中把它从磁盘上抹掉。
|
|
291
|
+
# 去重校验每次加载只在确有重复时产生新告警(重复引用已在上方被剔除,
|
|
292
|
+
# 下次加载不再触发),因此追加不会随 roundtrip 无限增长。
|
|
293
|
+
object.__setattr__(self, "warnings", list(self.warnings or []) + warnings)
|
|
294
|
+
return self
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview Manifest 引用类型定义模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 定义 manifest 文件中对其他配置文件的引用结构
|
|
21
|
+
- 包括 Schema、Constraint、Regex 三种引用类型
|
|
22
|
+
|
|
23
|
+
架构设计:
|
|
24
|
+
- 轻量级引用: 仅包含 ID 和路径,不包含完整内容
|
|
25
|
+
- 延迟加载: 实际内容在后续步骤加载
|
|
26
|
+
|
|
27
|
+
输入示例 (manifest.yaml):
|
|
28
|
+
version: 2
|
|
29
|
+
schemas:
|
|
30
|
+
- id: users # SchemaRef.id
|
|
31
|
+
path: schemas/users.yaml # SchemaRef.path
|
|
32
|
+
- id: orders
|
|
33
|
+
path: schemas/orders.yaml
|
|
34
|
+
constraints:
|
|
35
|
+
- id: users_not_null
|
|
36
|
+
path: constraints/users_not_null.yaml
|
|
37
|
+
regex_nodes:
|
|
38
|
+
- id: email_pattern
|
|
39
|
+
path: regex/email.yaml
|
|
40
|
+
|
|
41
|
+
输出示例 (Python 对象):
|
|
42
|
+
# SchemaRef 对象
|
|
43
|
+
SchemaRef(id="users", path="schemas/users.yaml")
|
|
44
|
+
|
|
45
|
+
# ConstraintRef 对象
|
|
46
|
+
ConstraintRef(id="users_not_null", path="constraints/users_not_null.yaml")
|
|
47
|
+
|
|
48
|
+
# RegexRef 对象
|
|
49
|
+
RegexRef(id="email_pattern", path="regex/email.yaml")
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
from pydantic import BaseModel, Field
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class SchemaRef(BaseModel):
|
|
58
|
+
"""@classdesc Schema 文件引用
|
|
59
|
+
|
|
60
|
+
用于在 manifest.yaml 中引用一个 Schema 配置文件。
|
|
61
|
+
|
|
62
|
+
字段说明:
|
|
63
|
+
- id: Schema 的唯一标识符,应与 schema 文件内部的 id 字段一致
|
|
64
|
+
- path: 相对于 manifest.yaml 所在目录的路径
|
|
65
|
+
|
|
66
|
+
示例:
|
|
67
|
+
# manifest.yaml 中的定义
|
|
68
|
+
schemas:
|
|
69
|
+
- id: users
|
|
70
|
+
path: schemas/users.yaml
|
|
71
|
+
|
|
72
|
+
# 对应的 Python 对象
|
|
73
|
+
SchemaRef(id="users", path="schemas/users.yaml")
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
id: str = Field(..., description="表 ID(与 schema 文件内 id 一致)")
|
|
77
|
+
path: str = Field(..., description="schema 文件相对路径(相对于 manifest 所在目录)")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ConstraintRef(BaseModel):
|
|
81
|
+
"""@classdesc Constraint 文件引用
|
|
82
|
+
|
|
83
|
+
用于在 manifest.yaml 中引用一个 Constraint 配置文件。
|
|
84
|
+
|
|
85
|
+
示例:
|
|
86
|
+
# manifest.yaml 中的定义
|
|
87
|
+
constraints:
|
|
88
|
+
- id: users_not_null
|
|
89
|
+
path: constraints/users_not_null.yaml
|
|
90
|
+
|
|
91
|
+
# 对应的 Python 对象
|
|
92
|
+
ConstraintRef(id="users_not_null", path="constraints/users_not_null.yaml")
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
id: str = Field(..., description="约束 ID(与 constraint 文件内 id 一致)")
|
|
96
|
+
path: str = Field(..., description="constraint 文件相对路径(相对于 manifest 所在目录)")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class RegexRef(BaseModel):
|
|
100
|
+
"""@classdesc Regex 节点文件引用
|
|
101
|
+
|
|
102
|
+
用于在 manifest.yaml 中引用一个 Regex 节点配置文件。
|
|
103
|
+
|
|
104
|
+
示例:
|
|
105
|
+
# manifest.yaml 中的定义
|
|
106
|
+
regex_nodes:
|
|
107
|
+
- id: email_pattern
|
|
108
|
+
path: regex/email.yaml
|
|
109
|
+
|
|
110
|
+
# 对应的 Python 对象
|
|
111
|
+
RegexRef(id="email_pattern", path="regex/email.yaml")
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
id: str = Field(..., description="Regex 节点 ID(与 regex 文件内 id 一致)")
|
|
115
|
+
path: str = Field(..., description="regex 文件相对路径(相对于 manifest 所在目录)")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class TransformRef(BaseModel):
|
|
119
|
+
"""@classdesc Transform 节点文件引用
|
|
120
|
+
|
|
121
|
+
用于在 manifest.yaml 中引用一个 Transform 功能节点配置文件。
|
|
122
|
+
|
|
123
|
+
示例:
|
|
124
|
+
# manifest.yaml 中的定义
|
|
125
|
+
transforms:
|
|
126
|
+
- id: split_id_card
|
|
127
|
+
path: transforms/split_id_card.transform.yaml
|
|
128
|
+
|
|
129
|
+
# 对应的 Python 对象
|
|
130
|
+
TransformRef(id="split_id_card", path="transforms/split_id_card.transform.yaml")
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
id: str = Field(..., description="Transform 节点 ID(与 transform 文件内 id 一致)")
|
|
134
|
+
path: str = Field(..., description="transform 文件相对路径(相对于 manifest 所在目录)")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class ManualDataRef(BaseModel):
|
|
138
|
+
"""@classdesc ManualData 节点文件引用
|
|
139
|
+
|
|
140
|
+
用于在 manifest.yaml 中引用一个 ManualData 内联数据节点配置文件。
|
|
141
|
+
|
|
142
|
+
示例:
|
|
143
|
+
# manifest.yaml 中的定义
|
|
144
|
+
manual_data:
|
|
145
|
+
- id: ti1__md1
|
|
146
|
+
path: manual_data/ti1__md1.yaml
|
|
147
|
+
|
|
148
|
+
# 对应的 Python 对象
|
|
149
|
+
ManualDataRef(id="ti1__md1", path="manual_data/ti1__md1.yaml")
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
id: str = Field(..., description="ManualData 节点 ID(与 manual_data 文件内 id 一致)")
|
|
153
|
+
path: str = Field(..., description="manual_data 文件相对路径(相对于 manifest 所在目录)")
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 项目设置类型定义模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 整合项目各类设置的结构
|
|
21
|
+
- 包括验证设置、文件处理设置、脚本安全设置
|
|
22
|
+
|
|
23
|
+
架构设计:
|
|
24
|
+
- 组合模式: ProjectSettings 组合多个子设置
|
|
25
|
+
- 默认工厂: 每个子设置都有默认值
|
|
26
|
+
|
|
27
|
+
输入示例 (manifest.yaml):
|
|
28
|
+
settings:
|
|
29
|
+
validation:
|
|
30
|
+
parallel_workers: 4
|
|
31
|
+
strict_mode: true
|
|
32
|
+
file_processing:
|
|
33
|
+
default_encoding: utf-8
|
|
34
|
+
csv_delimiter: ","
|
|
35
|
+
script_security:
|
|
36
|
+
allowed_imports:
|
|
37
|
+
- math
|
|
38
|
+
- datetime
|
|
39
|
+
max_execution_time: 30
|
|
40
|
+
|
|
41
|
+
输出示例:
|
|
42
|
+
ProjectSettings(
|
|
43
|
+
validation=ValidationSettings(parallel_workers=4, strict_mode=True),
|
|
44
|
+
file_processing=FileProcessingSettings(
|
|
45
|
+
default_encoding="utf-8",
|
|
46
|
+
csv_delimiter=","
|
|
47
|
+
),
|
|
48
|
+
script_security=ScriptSecuritySettings(
|
|
49
|
+
allowed_imports=["math", "datetime"],
|
|
50
|
+
max_execution_time=30
|
|
51
|
+
)
|
|
52
|
+
)
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
from __future__ import annotations
|
|
56
|
+
|
|
57
|
+
from pydantic import BaseModel, Field
|
|
58
|
+
|
|
59
|
+
from app.shared.core.project.manifest.types_parts.settings_file_processing import FileProcessingSettings
|
|
60
|
+
from app.shared.core.project.manifest.types_parts.settings_script_security import ScriptSecuritySettings
|
|
61
|
+
from app.shared.core.project.manifest.types_parts.settings_validation import ValidationSettings
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class ProjectSettings(BaseModel):
|
|
65
|
+
"""
|
|
66
|
+
@classdesc 项目设置
|
|
67
|
+
|
|
68
|
+
包含项目运行时的各类配置选项。
|
|
69
|
+
|
|
70
|
+
字段说明:
|
|
71
|
+
- validation: 验证行为设置 (并行数、严格模式等)
|
|
72
|
+
- file_processing: 文件处理设置 (编码、日期格式、分隔符等)
|
|
73
|
+
- script_security: 脚本安全设置 (允许的导入、最大执行时间等)
|
|
74
|
+
|
|
75
|
+
默认值:
|
|
76
|
+
所有子设置都有默认值,可以不填写使用默认配置
|
|
77
|
+
|
|
78
|
+
示例:
|
|
79
|
+
# 最小配置 (使用默认值)
|
|
80
|
+
settings: {}
|
|
81
|
+
|
|
82
|
+
# 自定义配置
|
|
83
|
+
settings:
|
|
84
|
+
validation:
|
|
85
|
+
parallel_workers: 8
|
|
86
|
+
file_processing:
|
|
87
|
+
encoding: gbk
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
validation: ValidationSettings = Field(default_factory=ValidationSettings, description="校验行为设置")
|
|
91
|
+
file_processing: FileProcessingSettings = Field(default_factory=FileProcessingSettings, description="文件处理设置")
|
|
92
|
+
script_security: ScriptSecuritySettings = Field(default_factory=ScriptSecuritySettings, description="脚本安全设置")
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 文件处理设置类型定义模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 定义文件处理相关的配置选项
|
|
21
|
+
- 包括编码、分隔符
|
|
22
|
+
|
|
23
|
+
架构设计:
|
|
24
|
+
- 类型安全: 使用 Literal 限制可选值
|
|
25
|
+
- 默认值: 提供合理的默认配置
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from typing import Literal
|
|
31
|
+
|
|
32
|
+
from pydantic import BaseModel, Field
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class FileProcessingSettings(BaseModel):
|
|
36
|
+
"""@classdesc 文件处理设置
|
|
37
|
+
|
|
38
|
+
定义如何读取数据文件。
|
|
39
|
+
|
|
40
|
+
字段说明:
|
|
41
|
+
- default_encoding: 文件字符编码,auto 表示自动检测
|
|
42
|
+
- csv_delimiter: CSV 文件字段分隔符
|
|
43
|
+
|
|
44
|
+
输入示例 (manifest.yaml):
|
|
45
|
+
settings:
|
|
46
|
+
file_processing:
|
|
47
|
+
default_encoding: utf-8
|
|
48
|
+
csv_delimiter: ";"
|
|
49
|
+
|
|
50
|
+
输出示例:
|
|
51
|
+
FileProcessingSettings(
|
|
52
|
+
default_encoding="utf-8",
|
|
53
|
+
csv_delimiter=";",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
默认值:
|
|
57
|
+
FileProcessingSettings(
|
|
58
|
+
default_encoding="utf-8",
|
|
59
|
+
csv_delimiter=",",
|
|
60
|
+
)
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
default_encoding: Literal["utf-8", "gbk", "auto"] = Field("utf-8", description="读取文件时使用的字符编码")
|
|
64
|
+
csv_delimiter: str = Field(",", description="CSV文件的字段分隔符")
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#
|
|
3
|
+
# Copyright 2026 Precis Team
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
"""
|
|
17
|
+
@fileoverview 脚本安全设置类型定义模块
|
|
18
|
+
|
|
19
|
+
功能概述:
|
|
20
|
+
- 定义脚本执行的安全策略
|
|
21
|
+
- 控制 eval/exec 的使用、沙箱模式、超时等
|
|
22
|
+
|
|
23
|
+
架构设计:
|
|
24
|
+
- 安全优先: 默认禁用危险函数
|
|
25
|
+
- 范围限制: timeout_seconds 有取值范围
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
from pydantic import BaseModel, Field
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ScriptSecuritySettings(BaseModel):
|
|
34
|
+
"""@classdesc 脚本安全设置
|
|
35
|
+
|
|
36
|
+
定义脚本执行的权限和安全策略。
|
|
37
|
+
|
|
38
|
+
字段说明:
|
|
39
|
+
- allow_eval: 是否允许使用 eval() 函数 (危险! 可能执行任意代码)
|
|
40
|
+
- allow_exec: 是否允许使用 exec() 函数 (危险! 可能执行任意代码)
|
|
41
|
+
- sandbox_mode: 是否在沙箱环境中执行 (限制文件系统访问等)
|
|
42
|
+
- timeout_seconds: 脚本最大执行时间 (1-60秒)
|
|
43
|
+
|
|
44
|
+
原理说明:
|
|
45
|
+
为什么默认禁用 eval/exec?
|
|
46
|
+
- eval() 和 exec() 可以执行任意 Python 代码
|
|
47
|
+
- 如果用户输入被拼接到这些函数中,可能导致代码注入攻击
|
|
48
|
+
- 建议使用安全的表达式解析器代替
|
|
49
|
+
|
|
50
|
+
输入示例 (manifest.yaml):
|
|
51
|
+
settings:
|
|
52
|
+
script_security:
|
|
53
|
+
allow_eval: false
|
|
54
|
+
allow_exec: false
|
|
55
|
+
sandbox_mode: true
|
|
56
|
+
timeout_seconds: 30
|
|
57
|
+
|
|
58
|
+
输出示例:
|
|
59
|
+
ScriptSecuritySettings(
|
|
60
|
+
allow_eval=False,
|
|
61
|
+
allow_exec=False,
|
|
62
|
+
sandbox_mode=True,
|
|
63
|
+
timeout_seconds=30
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
默认值 (最安全配置):
|
|
67
|
+
ScriptSecuritySettings(
|
|
68
|
+
allow_eval=False,
|
|
69
|
+
allow_exec=False,
|
|
70
|
+
sandbox_mode=True,
|
|
71
|
+
timeout_seconds=10
|
|
72
|
+
)
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
allow_eval: bool = Field(False, description="是否允许执行 Python eval() 函数")
|
|
76
|
+
allow_exec: bool = Field(False, description="是否允许执行 Python exec() 函数")
|
|
77
|
+
sandbox_mode: bool = Field(True, description="是否在受限环境中执行脚本")
|
|
78
|
+
timeout_seconds: int = Field(10, ge=1, le=60, description="脚本单次执行的最大时间(秒)")
|