precis-cli 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (442) hide show
  1. app/__init__.py +16 -0
  2. app/api/__init__.py +32 -0
  3. app/api/dependencies.py +185 -0
  4. app/api/main.py +301 -0
  5. app/api/middleware/__init__.py +34 -0
  6. app/api/middleware/exception_handler.py +126 -0
  7. app/api/middleware/request_logging.py +97 -0
  8. app/api/middleware/token_auth.py +215 -0
  9. app/api/models/__init__.py +151 -0
  10. app/api/models/connection_rules.py +98 -0
  11. app/api/models/files.py +99 -0
  12. app/api/models/full_validation.py +554 -0
  13. app/api/models/project.py +117 -0
  14. app/api/models/projects.py +80 -0
  15. app/api/models/schema.py +121 -0
  16. app/api/models/v2_responses.py +46 -0
  17. app/api/models/validation.py +303 -0
  18. app/api/models/workspace.py +118 -0
  19. app/api/routers/__init__.py +56 -0
  20. app/api/routers/ai/__init__.py +36 -0
  21. app/api/routers/ai/chat.py +243 -0
  22. app/api/routers/ai/generate.py +156 -0
  23. app/api/routers/ai/hardware.py +199 -0
  24. app/api/routers/ai/jobs.py +544 -0
  25. app/api/routers/ai/migrate.py +471 -0
  26. app/api/routers/ai/models.py +352 -0
  27. app/api/routers/ai/ollama.py +108 -0
  28. app/api/routers/ai/providers.py +491 -0
  29. app/api/routers/ai/router.py +44 -0
  30. app/api/routers/ai/stream.py +281 -0
  31. app/api/routers/ai/utils.py +182 -0
  32. app/api/routers/core/__init__.py +34 -0
  33. app/api/routers/core/connection_rules.py +188 -0
  34. app/api/routers/core/data_sources.py +374 -0
  35. app/api/routers/core/regex.py +333 -0
  36. app/api/routers/core/reporting.py +263 -0
  37. app/api/routers/files/__init__.py +24 -0
  38. app/api/routers/files/ops.py +179 -0
  39. app/api/routers/files/transfer.py +101 -0
  40. app/api/routers/preview/__init__.py +35 -0
  41. app/api/routers/preview/content_mode.py +264 -0
  42. app/api/routers/preview/header_row.py +154 -0
  43. app/api/routers/preview/models.py +81 -0
  44. app/api/routers/preview/path_mode.py +440 -0
  45. app/api/routers/preview/router.py +30 -0
  46. app/api/routers/project/__init__.py +61 -0
  47. app/api/routers/project/base.py +111 -0
  48. app/api/routers/project/constraint.py +345 -0
  49. app/api/routers/project/full_config.py +411 -0
  50. app/api/routers/project/full_config_writer.py +327 -0
  51. app/api/routers/project/helpers.py +167 -0
  52. app/api/routers/project/inspection_fix.py +330 -0
  53. app/api/routers/project/manifest.py +605 -0
  54. app/api/routers/project/models.py +177 -0
  55. app/api/routers/project/pattern.py +268 -0
  56. app/api/routers/project/regex.py +358 -0
  57. app/api/routers/project/scanner.py +157 -0
  58. app/api/routers/project/schema.py +567 -0
  59. app/api/routers/project/schema_helpers.py +149 -0
  60. app/api/routers/project/settings.py +399 -0
  61. app/api/routers/project/template.py +325 -0
  62. app/api/routers/project/validation.py +227 -0
  63. app/api/routers/project/view.py +177 -0
  64. app/api/routers/project/workspaces.py +167 -0
  65. app/api/routers/projects/__init__.py +25 -0
  66. app/api/routers/projects/create.py +121 -0
  67. app/api/routers/projects/open.py +103 -0
  68. app/api/routers/projects/scan.py +100 -0
  69. app/api/routers/validation/__init__.py +39 -0
  70. app/api/routers/validation/common.py +263 -0
  71. app/api/routers/validation/content_mode.py +401 -0
  72. app/api/routers/validation/history.py +235 -0
  73. app/api/routers/validation/inline_mode.py +196 -0
  74. app/api/routers/validation/path_mode.py +314 -0
  75. app/api/routers/validation/router.py +51 -0
  76. app/api/services/full_validation_response_builder.py +412 -0
  77. app/api/services/io_error_messages.py +41 -0
  78. app/api/services/preview_service.py +248 -0
  79. app/cli/__init__.py +18 -0
  80. app/cli/__main__.py +30 -0
  81. app/cli/shared_services/__init__.py +42 -0
  82. app/cli/shared_services/config_ops.py +581 -0
  83. app/cli/shared_services/generation_ops.py +160 -0
  84. app/cli/shared_services/project_ops.py +288 -0
  85. app/cli/shell/__init__.py +48 -0
  86. app/cli/shell/commands/__init__.py +63 -0
  87. app/cli/shell/commands/ai/__init__.py +260 -0
  88. app/cli/shell/commands/ai/chat.py +331 -0
  89. app/cli/shell/commands/ai/delete.py +165 -0
  90. app/cli/shell/commands/ai/diff.py +70 -0
  91. app/cli/shell/commands/ai/display.py +428 -0
  92. app/cli/shell/commands/ai/executor.py +225 -0
  93. app/cli/shell/commands/ai/executor_utils.py +271 -0
  94. app/cli/shell/commands/ai/generate.py +285 -0
  95. app/cli/shell/commands/ai/interaction.py +262 -0
  96. app/cli/shell/commands/ai/migrate.py +281 -0
  97. app/cli/shell/commands/ai/resolver.py +224 -0
  98. app/cli/shell/commands/ai/status.py +101 -0
  99. app/cli/shell/commands/ai/switch.py +158 -0
  100. app/cli/shell/commands/ai/utils.py +52 -0
  101. app/cli/shell/commands/base.py +343 -0
  102. app/cli/shell/commands/config/__init__.py +131 -0
  103. app/cli/shell/commands/config/base.py +80 -0
  104. app/cli/shell/commands/config/check.py +246 -0
  105. app/cli/shell/commands/config/edit.py +148 -0
  106. app/cli/shell/commands/config/get.py +111 -0
  107. app/cli/shell/commands/config/init.py +123 -0
  108. app/cli/shell/commands/config/inspect.py +164 -0
  109. app/cli/shell/commands/config/list.py +104 -0
  110. app/cli/shell/commands/config/set.py +119 -0
  111. app/cli/shell/commands/config/show.py +150 -0
  112. app/cli/shell/commands/exit.py +67 -0
  113. app/cli/shell/commands/help.py +104 -0
  114. app/cli/shell/commands/infer_schema.py +156 -0
  115. app/cli/shell/commands/open.py +201 -0
  116. app/cli/shell/commands/project.py +215 -0
  117. app/cli/shell/commands/provider.py +619 -0
  118. app/cli/shell/commands/system.py +170 -0
  119. app/cli/shell/commands/validate.py +455 -0
  120. app/cli/shell/completer.py +231 -0
  121. app/cli/shell/config_storage.py +191 -0
  122. app/cli/shell/exceptions.py +90 -0
  123. app/cli/shell/formatter.py +371 -0
  124. app/cli/shell/interactive_menu.py +348 -0
  125. app/cli/shell/main.py +286 -0
  126. app/cli/shell/parser.py +266 -0
  127. app/cli/start.py +180 -0
  128. app/cli_main.py +52 -0
  129. app/mcp_server.py +311 -0
  130. app/shared/core/__init__.py +18 -0
  131. app/shared/core/app_version.py +59 -0
  132. app/shared/core/config/__init__.py +119 -0
  133. app/shared/core/config/server.py +193 -0
  134. app/shared/core/data_source/__init__.py +106 -0
  135. app/shared/core/data_source/loader.py +348 -0
  136. app/shared/core/data_source/loaders/__init__.py +213 -0
  137. app/shared/core/data_source/loaders/base.py +215 -0
  138. app/shared/core/data_source/loaders/converter.py +337 -0
  139. app/shared/core/data_source/loaders/csv_loader.py +252 -0
  140. app/shared/core/data_source/loaders/excel_loader.py +666 -0
  141. app/shared/core/data_source/loaders/extractor.py +268 -0
  142. app/shared/core/data_source/loaders/json_loader.py +354 -0
  143. app/shared/core/data_source/loaders/registry.py +88 -0
  144. app/shared/core/data_source/loaders/sql_loader.py +313 -0
  145. app/shared/core/data_source/loaders/strategies/__init__.py +100 -0
  146. app/shared/core/data_source/loaders/strategies/array_parser.py +277 -0
  147. app/shared/core/data_source/loaders/strategies/lines_parser.py +355 -0
  148. app/shared/core/data_source/loaders/strategies/object_parser.py +278 -0
  149. app/shared/core/data_source/schema_info.py +50 -0
  150. app/shared/core/data_source/specs/__init__.py +128 -0
  151. app/shared/core/data_source/specs/base.py +258 -0
  152. app/shared/core/data_source/specs/csv_source.py +137 -0
  153. app/shared/core/data_source/specs/excel_source.py +141 -0
  154. app/shared/core/data_source/specs/file_base.py +252 -0
  155. app/shared/core/data_source/specs/json_source.py +226 -0
  156. app/shared/core/data_source/specs/sql_source.py +161 -0
  157. app/shared/core/encoding.py +36 -0
  158. app/shared/core/io/__init__.py +24 -0
  159. app/shared/core/io/yaml.py +379 -0
  160. app/shared/core/manifest_schema/__init__.py +52 -0
  161. app/shared/core/manifest_schema/types.py +99 -0
  162. app/shared/core/manifest_schema/version.py +175 -0
  163. app/shared/core/patterns/__init__.py +24 -0
  164. app/shared/core/patterns/loader.py +134 -0
  165. app/shared/core/patterns/writer.py +244 -0
  166. app/shared/core/project/__init__.py +47 -0
  167. app/shared/core/project/constraint/builders/__init__.py +50 -0
  168. app/shared/core/project/constraint/builders/base.py +100 -0
  169. app/shared/core/project/constraint/builders/composite.py +77 -0
  170. app/shared/core/project/constraint/builders/conditional.py +67 -0
  171. app/shared/core/project/constraint/builders/foreign_key.py +53 -0
  172. app/shared/core/project/constraint/builders/registry.py +64 -0
  173. app/shared/core/project/constraint/builders/scripted.py +51 -0
  174. app/shared/core/project/constraint/builders/single_column.py +86 -0
  175. app/shared/core/project/constraint/builders/unique.py +53 -0
  176. app/shared/core/project/constraint/factory.py +170 -0
  177. app/shared/core/project/constraint/reader.py +214 -0
  178. app/shared/core/project/constraint/registry.py +233 -0
  179. app/shared/core/project/constraint/types/__init__.py +63 -0
  180. app/shared/core/project/constraint/types/constraint_file.py +261 -0
  181. app/shared/core/project/constraint/types/refs.py +460 -0
  182. app/shared/core/project/constraint/types.py +28 -0
  183. app/shared/core/project/constraint/writer.py +181 -0
  184. app/shared/core/project/loader/__init__.py +56 -0
  185. app/shared/core/project/loader/loader.py +30 -0
  186. app/shared/core/project/loader/loader_parts/config_inspector.py +137 -0
  187. app/shared/core/project/loader/loader_parts/embedded_constraints.py +224 -0
  188. app/shared/core/project/loader/loader_parts/file_loaders.py +58 -0
  189. app/shared/core/project/loader/loader_parts/inspection_ids.py +85 -0
  190. app/shared/core/project/loader/loader_parts/inspector_helpers.py +312 -0
  191. app/shared/core/project/loader/loader_parts/inspector_id_checks.py +274 -0
  192. app/shared/core/project/loader/loader_parts/inspector_reference_checks.py +690 -0
  193. app/shared/core/project/loader/loader_parts/inspector_uniqueness_checks.py +286 -0
  194. app/shared/core/project/loader/loader_parts/loading_error_messages.py +298 -0
  195. app/shared/core/project/loader/loader_parts/main.py +432 -0
  196. app/shared/core/project/loader/loader_parts/path_validation.py +117 -0
  197. app/shared/core/project/loader/loader_parts/runtime.py +125 -0
  198. app/shared/core/project/loader/types.py +246 -0
  199. app/shared/core/project/manifest/coverage.py +391 -0
  200. app/shared/core/project/manifest/reader.py +260 -0
  201. app/shared/core/project/manifest/types.py +91 -0
  202. app/shared/core/project/manifest/types_parts/__init__.py +60 -0
  203. app/shared/core/project/manifest/types_parts/constants.py +40 -0
  204. app/shared/core/project/manifest/types_parts/data_source.py +68 -0
  205. app/shared/core/project/manifest/types_parts/info.py +59 -0
  206. app/shared/core/project/manifest/types_parts/manifest.py +294 -0
  207. app/shared/core/project/manifest/types_parts/refs.py +153 -0
  208. app/shared/core/project/manifest/types_parts/settings.py +92 -0
  209. app/shared/core/project/manifest/types_parts/settings_file_processing.py +64 -0
  210. app/shared/core/project/manifest/types_parts/settings_script_security.py +78 -0
  211. app/shared/core/project/manifest/types_parts/settings_validation.py +83 -0
  212. app/shared/core/project/manifest/types_parts/template.py +66 -0
  213. app/shared/core/project/manifest/writer.py +262 -0
  214. app/shared/core/project/manual_data/__init__.py +27 -0
  215. app/shared/core/project/manual_data/types.py +75 -0
  216. app/shared/core/project/regex/reader.py +197 -0
  217. app/shared/core/project/regex/types.py +405 -0
  218. app/shared/core/project/regex/writer.py +123 -0
  219. app/shared/core/project/schema/reader.py +170 -0
  220. app/shared/core/project/schema/types.py +47 -0
  221. app/shared/core/project/schema/types_parts/__init__.py +36 -0
  222. app/shared/core/project/schema/types_parts/column.py +174 -0
  223. app/shared/core/project/schema/types_parts/column_utils.py +72 -0
  224. app/shared/core/project/schema/types_parts/constraint.py +165 -0
  225. app/shared/core/project/schema/types_parts/schema_id.py +66 -0
  226. app/shared/core/project/schema/types_parts/source.py +255 -0
  227. app/shared/core/project/schema/types_parts/source_options.py +347 -0
  228. app/shared/core/project/schema/types_parts/table.py +230 -0
  229. app/shared/core/project/schema/writer.py +139 -0
  230. app/shared/core/project/schema_ref_check.py +95 -0
  231. app/shared/core/project/template/__init__.py +27 -0
  232. app/shared/core/project/template/expander.py +263 -0
  233. app/shared/core/project/template/reader.py +120 -0
  234. app/shared/core/project/template/types.py +114 -0
  235. app/shared/core/project/transform/reader.py +76 -0
  236. app/shared/core/project/transform/types.py +116 -0
  237. app/shared/core/project/transform/writer.py +84 -0
  238. app/shared/core/pydantic_messages.py +59 -0
  239. app/shared/core/reporter/__init__.py +46 -0
  240. app/shared/core/reporter/reporter.py +220 -0
  241. app/shared/core/reporter/reporters/__init__.py +65 -0
  242. app/shared/core/reporter/reporters/base.py +188 -0
  243. app/shared/core/reporter/reporters/dingtalk_app_reporter.py +274 -0
  244. app/shared/core/reporter/reporters/email_reporter.py +271 -0
  245. app/shared/core/reporter/reporters/feishu_app_reporter.py +467 -0
  246. app/shared/core/reporter/reporters/local_file_reporter.py +208 -0
  247. app/shared/core/reporter/reporters/wecom_app_reporter.py +268 -0
  248. app/shared/core/utils/__init__.py +18 -0
  249. app/shared/core/utils/path_utils.py +60 -0
  250. app/shared/core/utils/regex_extract.py +113 -0
  251. app/shared/domain/__init__.py +114 -0
  252. app/shared/domain/constraints/__init__.py +76 -0
  253. app/shared/domain/constraints/allowed_values.py +211 -0
  254. app/shared/domain/constraints/base.py +141 -0
  255. app/shared/domain/constraints/charset.py +337 -0
  256. app/shared/domain/constraints/composite.py +174 -0
  257. app/shared/domain/constraints/condition_registry.py +130 -0
  258. app/shared/domain/constraints/conditional.py +629 -0
  259. app/shared/domain/constraints/date_logic.py +731 -0
  260. app/shared/domain/constraints/foreign_key.py +261 -0
  261. app/shared/domain/constraints/key_normalization.py +56 -0
  262. app/shared/domain/constraints/not_null.py +185 -0
  263. app/shared/domain/constraints/range.py +360 -0
  264. app/shared/domain/constraints/regex.py +218 -0
  265. app/shared/domain/constraints/scripted.py +276 -0
  266. app/shared/domain/constraints/unique.py +233 -0
  267. app/shared/domain/data_engine.py +384 -0
  268. app/shared/domain/data_types.py +113 -0
  269. app/shared/domain/data_types_parts/__init__.py +67 -0
  270. app/shared/domain/data_types_parts/base.py +204 -0
  271. app/shared/domain/data_types_parts/composite.py +250 -0
  272. app/shared/domain/data_types_parts/expression.py +202 -0
  273. app/shared/domain/data_types_parts/extracted.py +106 -0
  274. app/shared/domain/data_types_parts/json_types.py +234 -0
  275. app/shared/domain/data_types_parts/scalars.py +719 -0
  276. app/shared/domain/data_types_parts/sequence.py +129 -0
  277. app/shared/domain/dataset_schema.py +48 -0
  278. app/shared/domain/eval_sandbox.py +63 -0
  279. app/shared/domain/expression_system.py +366 -0
  280. app/shared/domain/regex_flags.py +53 -0
  281. app/shared/domain/schema/builder.py +223 -0
  282. app/shared/domain/schema/models.py +339 -0
  283. app/shared/domain/transforms/__init__.py +28 -0
  284. app/shared/domain/transforms/aggregate.py +141 -0
  285. app/shared/domain/transforms/base.py +171 -0
  286. app/shared/domain/transforms/cast_type.py +120 -0
  287. app/shared/domain/transforms/concat.py +103 -0
  288. app/shared/domain/transforms/conditional_assign.py +114 -0
  289. app/shared/domain/transforms/date_format.py +81 -0
  290. app/shared/domain/transforms/digits.py +77 -0
  291. app/shared/domain/transforms/drop_duplicates.py +94 -0
  292. app/shared/domain/transforms/fill_na.py +94 -0
  293. app/shared/domain/transforms/filter_rows.py +97 -0
  294. app/shared/domain/transforms/lookup.py +78 -0
  295. app/shared/domain/transforms/lower_case.py +70 -0
  296. app/shared/domain/transforms/map_value.py +90 -0
  297. app/shared/domain/transforms/math_expr.py +112 -0
  298. app/shared/domain/transforms/modulo.py +82 -0
  299. app/shared/domain/transforms/regex_extract.py +103 -0
  300. app/shared/domain/transforms/registry.py +95 -0
  301. app/shared/domain/transforms/replace.py +88 -0
  302. app/shared/domain/transforms/sort_rows.py +94 -0
  303. app/shared/domain/transforms/string_split.py +84 -0
  304. app/shared/domain/transforms/strip.py +73 -0
  305. app/shared/domain/transforms/substring.py +92 -0
  306. app/shared/domain/transforms/upper_case.py +70 -0
  307. app/shared/domain/transforms/weighted_sum.py +104 -0
  308. app/shared/domain/validation_constraints.py +69 -0
  309. app/shared/services/__init__.py +62 -0
  310. app/shared/services/ai/__init__.py +48 -0
  311. app/shared/services/ai/agent/__init__.py +40 -0
  312. app/shared/services/ai/agent/chat_tools/__init__.py +47 -0
  313. app/shared/services/ai/agent/chat_tools/apply_actions.py +592 -0
  314. app/shared/services/ai/agent/chat_tools/ask_user.py +237 -0
  315. app/shared/services/ai/agent/chat_tools/read_canvas.py +140 -0
  316. app/shared/services/ai/agent/chat_tools/read_project.py +160 -0
  317. app/shared/services/ai/agent/chat_tools/read_table.py +301 -0
  318. app/shared/services/ai/agent/chat_tools/schemas.py +105 -0
  319. app/shared/services/ai/agent/chat_tools/validate_table.py +131 -0
  320. app/shared/services/ai/agent/executor.py +554 -0
  321. app/shared/services/ai/agent/memory.py +256 -0
  322. app/shared/services/ai/agent/planner.py +282 -0
  323. app/shared/services/ai/agent/tool_registry.py +296 -0
  324. app/shared/services/ai/agent/tools/__init__.py +32 -0
  325. app/shared/services/ai/agent/tools/config_generate.py +133 -0
  326. app/shared/services/ai/agent/tools/config_refine.py +109 -0
  327. app/shared/services/ai/agent/tools/config_validate.py +403 -0
  328. app/shared/services/ai/agent/tools/merge_results.py +136 -0
  329. app/shared/services/ai/agent/tools/plan_chunks.py +79 -0
  330. app/shared/services/ai/agent/tools/script_parse.py +359 -0
  331. app/shared/services/ai/agent/types.py +138 -0
  332. app/shared/services/ai/chat_agent_runner.py +596 -0
  333. app/shared/services/ai/chat_orchestrator.py +725 -0
  334. app/shared/services/ai/failure_messages.py +65 -0
  335. app/shared/services/ai/job_storage.py +274 -0
  336. app/shared/services/ai/migrate_service.py +550 -0
  337. app/shared/services/ai/streaming/__init__.py +65 -0
  338. app/shared/services/ai/streaming/event_journal.py +197 -0
  339. app/shared/services/ai/streaming/orchestrator.py +262 -0
  340. app/shared/services/ai/streaming/pending_interaction_store.py +227 -0
  341. app/shared/services/ai/streaming/sse_response.py +148 -0
  342. app/shared/services/ai/streaming/types.py +73 -0
  343. app/shared/services/ai/types.py +213 -0
  344. app/shared/services/ai/utils.py +322 -0
  345. app/shared/services/diff/config_diff.py +320 -0
  346. app/shared/services/hardware.py +237 -0
  347. app/shared/services/llm/__init__.py +62 -0
  348. app/shared/services/llm/actions/__init__.py +42 -0
  349. app/shared/services/llm/actions/_canvas_validator.py +147 -0
  350. app/shared/services/llm/actions/_constraint_validator.py +401 -0
  351. app/shared/services/llm/actions/_regex_validator.py +85 -0
  352. app/shared/services/llm/actions/_schema_validator.py +82 -0
  353. app/shared/services/llm/actions/_settings_validator.py +76 -0
  354. app/shared/services/llm/actions/_transform_validator.py +78 -0
  355. app/shared/services/llm/actions/action_handlers.py +400 -0
  356. app/shared/services/llm/actions/action_parser.py +63 -0
  357. app/shared/services/llm/actions/action_processor.py +468 -0
  358. app/shared/services/llm/actions/action_validator.py +311 -0
  359. app/shared/services/llm/actions/diff_compute.py +195 -0
  360. app/shared/services/llm/actions/regex_handlers.py +284 -0
  361. app/shared/services/llm/actions/registry.py +336 -0
  362. app/shared/services/llm/actions/schema_handlers.py +335 -0
  363. app/shared/services/llm/actions/settings_handlers.py +180 -0
  364. app/shared/services/llm/actions/specs.py +259 -0
  365. app/shared/services/llm/actions/transform_handlers.py +279 -0
  366. app/shared/services/llm/actions/validation_types.py +147 -0
  367. app/shared/services/llm/cache/__init__.py +18 -0
  368. app/shared/services/llm/cache/response_cache.py +80 -0
  369. app/shared/services/llm/chat/__init__.py +35 -0
  370. app/shared/services/llm/chat/chat_system_prompt.py +561 -0
  371. app/shared/services/llm/chat/response_parser.py +296 -0
  372. app/shared/services/llm/config/__init__.py +45 -0
  373. app/shared/services/llm/config/crypto.py +135 -0
  374. app/shared/services/llm/config/loader.py +258 -0
  375. app/shared/services/llm/config/models.py +202 -0
  376. app/shared/services/llm/config/presets.py +128 -0
  377. app/shared/services/llm/config_generator.py +163 -0
  378. app/shared/services/llm/constraints/__init__.py +41 -0
  379. app/shared/services/llm/constraints/constraint_builder.py +177 -0
  380. app/shared/services/llm/constraints/constraint_deletion.py +84 -0
  381. app/shared/services/llm/constraints/constraint_id.py +174 -0
  382. app/shared/services/llm/constraints/frontend_instructions.py +333 -0
  383. app/shared/services/llm/constraints/inline_batch.py +279 -0
  384. app/shared/services/llm/discovery/__init__.py +35 -0
  385. app/shared/services/llm/discovery/scanner.py +181 -0
  386. app/shared/services/llm/generation/__init__.py +42 -0
  387. app/shared/services/llm/generation/agent_wiring.py +156 -0
  388. app/shared/services/llm/generation/config_builder.py +386 -0
  389. app/shared/services/llm/generation/errors.py +36 -0
  390. app/shared/services/llm/generation/existing_config.py +99 -0
  391. app/shared/services/llm/generation/profiler.py +279 -0
  392. app/shared/services/llm/generation/prompt_builder.py +199 -0
  393. app/shared/services/llm/generation/response_parser.py +144 -0
  394. app/shared/services/llm/generation/service.py +622 -0
  395. app/shared/services/llm/models.py +104 -0
  396. app/shared/services/llm/providers/__init__.py +48 -0
  397. app/shared/services/llm/providers/base.py +310 -0
  398. app/shared/services/llm/providers/cached_provider.py +90 -0
  399. app/shared/services/llm/providers/ollama.py +498 -0
  400. app/shared/services/llm/providers/openai.py +334 -0
  401. app/shared/services/llm/providers/registry.py +108 -0
  402. app/shared/services/llm/schema_resolver.py +141 -0
  403. app/shared/services/llm/suggestion_utils.py +219 -0
  404. app/shared/services/llm/validate_executor.py +163 -0
  405. app/shared/services/llm/yaml_io.py +406 -0
  406. app/shared/services/preview/__init__.py +18 -0
  407. app/shared/services/preview/loader.py +145 -0
  408. app/shared/services/preview/path_validation.py +138 -0
  409. app/shared/services/project_loader.py +57 -0
  410. app/shared/services/schema_inference.py +260 -0
  411. app/shared/services/schema_runtime_builder.py +120 -0
  412. app/shared/services/validation/__init__.py +52 -0
  413. app/shared/services/validation/chunked_loader.py +509 -0
  414. app/shared/services/validation/dag/__init__.py +35 -0
  415. app/shared/services/validation/dag/builder.py +141 -0
  416. app/shared/services/validation/dag/executor.py +225 -0
  417. app/shared/services/validation/dag/sorter.py +77 -0
  418. app/shared/services/validation/data_loader.py +231 -0
  419. app/shared/services/validation/engine.py +446 -0
  420. app/shared/services/validation/executor.py +1030 -0
  421. app/shared/services/validation/extractors.py +266 -0
  422. app/shared/services/validation/history.py +209 -0
  423. app/shared/services/validation/json_payload.py +136 -0
  424. app/shared/services/validation/loader.py +152 -0
  425. app/shared/services/validation/memory_monitor.py +179 -0
  426. app/shared/services/validation/postprocess.py +208 -0
  427. app/shared/services/validation/progress.py +64 -0
  428. app/shared/services/validation/report_export.py +222 -0
  429. app/shared/services/validation/resolver.py +180 -0
  430. app/shared/services/validation/service.py +418 -0
  431. app/shared/services/validation/types.py +238 -0
  432. app/shared/services/validation/validators/__init__.py +36 -0
  433. app/shared/services/validation/validators/adapter.py +182 -0
  434. app/shared/services/validation/validators/base.py +332 -0
  435. app/shared/services/validation/validators/composite.py +233 -0
  436. app/shared/services/validation/validators/date_logic.py +276 -0
  437. app/start_server.py +133 -0
  438. precis_cli-0.1.3.dist-info/METADATA +180 -0
  439. precis_cli-0.1.3.dist-info/RECORD +442 -0
  440. precis_cli-0.1.3.dist-info/WHEEL +5 -0
  441. precis_cli-0.1.3.dist-info/entry_points.txt +4 -0
  442. precis_cli-0.1.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,193 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # Copyright 2026 Precis Team
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """@fileoverview 后端服务启动配置单一事实源
17
+
18
+ 功能概述:
19
+ - 集中管理后端端口、主机、端口文件协议等启动相关配置
20
+ - 提供 OS 原子端口分配(避免固定端口的冲突面与 TOCTOU 竞态)
21
+ - 通过 ``.backend-port`` 文件向外部(Vite 代理 / Electron 主进程)发布实际端口
22
+
23
+ 设计原则:
24
+ - 本模块位于 ``core/`` 层(基础设施),含 socket + 文件 I/O,不违反三层分离
25
+ - 所有启动入口(``start_server.py`` / ``cli/start.py``)统一从此模块取配置
26
+ - ``VITE_BACKEND_PORT`` 环境变量作为"固定端口逃生舱"保留向后兼容;
27
+ 未设置时使用 ``port=0`` 由 OS 动态分配
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import os
33
+ import socket
34
+ from pathlib import Path
35
+
36
+ # ============================================================================
37
+ # 常量(单一事实源)
38
+ # ============================================================================
39
+
40
+ #: 后端默认端口。0 = 由 OS 动态分配(推荐);正数 = 固定端口
41
+ DEFAULT_BACKEND_PORT = 0
42
+
43
+ #: 后端默认监听地址(loopback,不暴露到网络)
44
+ DEFAULT_BACKEND_HOST = "127.0.0.1"
45
+
46
+ #: 端口文件名,写在后端 cwd(即 ``backend/``)下,内容为纯端口号
47
+ BACKEND_PORT_FILE = ".backend-port"
48
+
49
+
50
+ # ============================================================================
51
+ # 端口解析
52
+ # ============================================================================
53
+
54
+
55
+ def resolve_port(cli_port: int | None) -> int:
56
+ """确定后端监听端口。
57
+
58
+ 优先级(高 → 低):
59
+ 1. CLI 参数 ``--port``(显式指定)
60
+ 2. 环境变量 ``VITE_BACKEND_PORT``(向后兼容的固定端口逃生舱)
61
+ 3. 默认值 ``DEFAULT_BACKEND_PORT``(=0,OS 动态分配)
62
+
63
+ Args:
64
+ cli_port: 命令行传入的端口号;None 表示未指定
65
+
66
+ Returns:
67
+ 最终使用的端口号。0 表示交由 OS 动态分配。
68
+ """
69
+ if cli_port is not None:
70
+ return cli_port
71
+
72
+ env_port = os.environ.get("VITE_BACKEND_PORT")
73
+ if env_port:
74
+ try:
75
+ return int(env_port)
76
+ except ValueError:
77
+ pass
78
+
79
+ return DEFAULT_BACKEND_PORT
80
+
81
+
82
+ # ============================================================================
83
+ # OS 原子端口分配
84
+ # ============================================================================
85
+
86
+
87
+ def acquire_port(preferred: int = 0) -> int:
88
+ """获取一个可监听的端口。
89
+
90
+ - ``preferred == 0``:通过 ``socket.bind(('127.0.0.1', 0))`` 让 OS 原子分配一个
91
+ 临时端口。这是 loopback 单机场景下最安全的方式——无递归、无竞态、永不冲突。
92
+ - ``preferred > 0``:直接返回该端口号,由调用方(uvicorn)负责绑定;若已被占用
93
+ 会在 uvicorn 启动时报错(固定端口是用户显式选择的,可接受)。
94
+
95
+ Note:
96
+ 对于 ``preferred == 0``,bind 后立即 close 再把端口交给 uvicorn,存在极小的
97
+ TOCTOU 窗口(另一进程抢占)。但 loopback 单机场景下概率极低且可接受——
98
+ 这与 Electron 旧 ``findAvailablePort`` 同理,但更简单(无递归)。
99
+
100
+ Args:
101
+ preferred: 期望端口。0 = OS 分配。
102
+
103
+ Returns:
104
+ 可用的端口号。
105
+ """
106
+ if preferred > 0:
107
+ return preferred
108
+
109
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
110
+ s.bind((DEFAULT_BACKEND_HOST, 0))
111
+ # getsockname() 返回 tuple,取 [1] 为端口;mypy 将其推断为 Any,用 int() 收窄类型
112
+ return int(s.getsockname()[1])
113
+
114
+
115
+ # ============================================================================
116
+ # 端口文件协议
117
+ # ============================================================================
118
+
119
+
120
+ def _port_file_path() -> Path:
121
+ """返回端口文件路径(后端 cwd 下)。
122
+
123
+ 后端进程的 cwd 是 ``backend/``,故端口文件落在 ``backend/.backend-port``。
124
+ """
125
+ return Path.cwd() / BACKEND_PORT_FILE
126
+
127
+
128
+ def write_port_file(port: int) -> None:
129
+ """将实际端口写入 ``.backend-port`` 文件,供 Vite 代理 / Electron 主进程发现。
130
+
131
+ 写入前若文件已存在(上次异常退出残留),先覆盖。
132
+ """
133
+ _port_file_path().write_text(str(port), encoding="utf-8")
134
+
135
+
136
+ def clear_port_file() -> None:
137
+ """删除端口文件(后端退出时清理)。文件不存在时静默忽略。"""
138
+ try:
139
+ _port_file_path().unlink(missing_ok=True)
140
+ except OSError:
141
+ # 权限或并发问题:端口文件残留不影响下次启动(会被覆盖),忽略
142
+ pass
143
+
144
+
145
+ def register_port_file_cleanup() -> None:
146
+ """注册端口文件清理的兜底机制(atexit + signal)。
147
+
148
+ Windows 下 uvicorn 收到 SIGINT/异常后,调用方的 finally 不一定执行
149
+ (信号处理链路与 Unix 不同)。此函数注册多重清理兜底,多次调用安全。
150
+
151
+ 信号处理语义:handler 内先清理端口文件,再链回默认行为——
152
+ - SIGINT: 交回 ``signal.default_int_handler`` 抛 KeyboardInterrupt,
153
+ 让 uvicorn/调用方走正常关闭路径;
154
+ - SIGTERM: Python 无默认 handler,清理后以 ``sys.exit(128 + signum)`` 终止。
155
+ 若只清理不退出,信号会被 handler 吞掉,进程永远无法用 Ctrl+C 停止。
156
+
157
+ 应在 write_port_file() 之后、uvicorn.run() 之前调用。
158
+ (注意:uvicorn.run() 默认会安装自己的 SIGINT/SIGTERM handler 并覆盖本注册,
159
+ 此时清理由 atexit 兜底;本 handler 服务于非 uvicorn 入口与提前到达的信号。)
160
+ """
161
+ import atexit
162
+ import signal
163
+ import sys
164
+ from types import FrameType
165
+
166
+ atexit.register(clear_port_file)
167
+
168
+ def _cleanup_and_exit(signum: int, frame: FrameType | None) -> None:
169
+ clear_port_file()
170
+ if signum == signal.SIGINT:
171
+ # 链回默认 INT 行为:抛 KeyboardInterrupt,不吞信号
172
+ signal.default_int_handler(signum, frame)
173
+ # SIGTERM 无 Python 默认 handler:清理后按约定退出码终止
174
+ sys.exit(128 + signum)
175
+
176
+ signal.signal(signal.SIGTERM, _cleanup_and_exit)
177
+ try:
178
+ signal.signal(signal.SIGINT, _cleanup_and_exit)
179
+ except (OSError, ValueError):
180
+ # 某些环境(如嵌入运行时)无法注册 SIGINT,忽略
181
+ pass
182
+
183
+
184
+ __all__ = [
185
+ "DEFAULT_BACKEND_PORT",
186
+ "DEFAULT_BACKEND_HOST",
187
+ "BACKEND_PORT_FILE",
188
+ "resolve_port",
189
+ "acquire_port",
190
+ "write_port_file",
191
+ "clear_port_file",
192
+ "register_port_file_cleanup",
193
+ ]
@@ -0,0 +1,106 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # Copyright 2026 Precis Team
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """
17
+ @fileoverview 数据源模块统一入口
18
+
19
+ 功能概述:
20
+ - 提供统一的数据加载接口,支持多种数据源类型(Excel、CSV、JSON、SQL)
21
+ - 导出所有数据源规格类,供上层服务和 API 路由构建数据源配置
22
+ - 导出加载器工具和缓存管理函数,实现数据的高效加载与复用
23
+ - 作为 app.shared.core.data_source 包的门面(Facade),封装内部子模块细节
24
+
25
+ 架构设计:
26
+ - 门面模式: 通过 __init__.py 聚合 specs、loader、loaders 等子模块的公共接口,
27
+ 调用方只需导入本模块即可获得完整的数据源操作能力。
28
+ - 规格类继承体系: 所有具体规格类(ExcelSourceSpec、CSVSourceSpec 等)
29
+ 均继承自 DataSourceSpec 基类,支持多态反序列化和统一校验。
30
+ - 加载器注册表: LOADER_REGISTRY 维护数据源类型到加载器实现的映射,
31
+ load_grouped_sources 根据规格自动分派到对应的加载器,实现开闭原则。
32
+
33
+ 输入示例:
34
+ >>> from app.shared.core.data_source import JSONSourceSpec, load_grouped_sources
35
+ >>> spec = JSONSourceSpec(
36
+ ... path="data.json",
37
+ ... format="auto",
38
+ ... json_path="$.data.items"
39
+ ... )
40
+
41
+ 输出示例:
42
+ >>> datasets, errors = load_grouped_sources({"data.json": [spec]})
43
+ >>> # datasets: dict[str, pd.DataFrame] — 文件名到 DataFrame 的映射
44
+ >>> # errors: list[DataLoadError] — 加载过程中收集的错误列表
45
+
46
+ 注意事项:
47
+ - 新增数据源类型时,需在 specs 中定义规格类、在 loaders 中实现加载器,
48
+ 并在本模块重导出,同时更新 LOADER_REGISTRY 的映射关系。
49
+ - __all__ 显式控制公开 API,避免内部辅助函数被意外暴露。
50
+ """
51
+
52
+ # 从 loader 子模块导入加载器注册表及工具函数
53
+ # LOADER_REGISTRY: 数据源类型到加载器类的映射表,支持运行时扩展
54
+ # load_grouped_sources: 批量加载入口,按文件分组并行处理
55
+ # can_load: 预检函数,判断给定规格是否可被当前注册表支持
56
+ from .loader import (
57
+ LOADER_REGISTRY,
58
+ can_load,
59
+ load_grouped_sources,
60
+ )
61
+
62
+ # 从 loaders 子模块导入加载异常类型
63
+ # DataLoadError: 统一的数据加载异常,包含文件路径、错误原因等上下文信息
64
+ from .loaders import DataLoadError
65
+
66
+ # 数据源 Schema 信息 DTO,用于 core 层与 domain 解耦
67
+ from .schema_info import DataSourceInfo
68
+
69
+ # 从 specs 子模块导入所有数据源规格类
70
+ # DataSourceSpec / FileSourceSpec: 抽象基类,定义规格的通用接口
71
+ # ExcelSpec / CSVSpec / JSONSpec: 向后兼容的别名导出
72
+ # ExcelSourceSpec / CSVSourceSpec / JSONSourceSpec / SQLSourceSpec: 具体规格实现
73
+ from .specs import (
74
+ CSVSourceSpec,
75
+ CSVSpec,
76
+ DataSourceSpec,
77
+ ExcelSourceSpec,
78
+ ExcelSpec,
79
+ FileSourceSpec,
80
+ JSONSourceSpec,
81
+ JSONSpec,
82
+ SQLSourceSpec,
83
+ )
84
+
85
+ # 控制 `from module import *` 时导出的公开名称
86
+ # 按类别分组:基础类型 → 具体规格 → 加载器工具,便于阅读和维护
87
+ __all__ = [
88
+ # 基础类型
89
+ "DataSourceSpec",
90
+ "FileSourceSpec",
91
+ # 数据源信息 DTO
92
+ "DataSourceInfo",
93
+ # 具体规格(向后兼容别名 + 标准名称)
94
+ "ExcelSpec",
95
+ "CSVSpec",
96
+ "JSONSpec",
97
+ "ExcelSourceSpec",
98
+ "CSVSourceSpec",
99
+ "JSONSourceSpec",
100
+ "SQLSourceSpec",
101
+ # 加载器工具
102
+ "DataLoadError",
103
+ "load_grouped_sources",
104
+ "can_load",
105
+ "LOADER_REGISTRY",
106
+ ]
@@ -0,0 +1,348 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # Copyright 2026 Precis Team
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """
17
+ @fileoverview 数据源加载模块
18
+
19
+ 功能概述:
20
+ - 提供 Excel/CSV/JSON 文件的统一加载入口
21
+ - 支持按文件分组批量加载多个数据源规格
22
+ - 提供加载能力检测工具函数
23
+
24
+ 架构设计:
25
+ - 使用类式加载器(CSVLoader, ExcelLoader, JSONLoader)
26
+ - 每次调用均从磁盘读取最新数据,不使用缓存
27
+ - core 层只依赖 DataSourceInfo DTO,不直接依赖 domain.TableSchema
28
+
29
+ 输入示例:
30
+ file_to_schemas = {
31
+ "data/users.xlsx": [DataSourceInfo(schema_id="users", sheet_name="Sheet1", header_row=0)],
32
+ "data/products.csv": [DataSourceInfo(schema_id="products", header_row=0)]
33
+ }
34
+
35
+ 输出示例:
36
+ datasets, errors = load_grouped_sources(file_to_schemas)
37
+ # datasets: {"users": pd.DataFrame, ...}
38
+ # errors: ["data/products.csv: 文件不存在", ...]
39
+ """
40
+
41
+ from __future__ import annotations
42
+
43
+ import logging
44
+ import os
45
+ from pathlib import Path
46
+ from typing import Any, Literal
47
+
48
+ import pandas as pd
49
+
50
+ from app.shared.core.data_source.loaders.base import DataLoadError
51
+ from app.shared.core.data_source.loaders.csv_loader import CSVLoader
52
+ from app.shared.core.data_source.loaders.excel_loader import ExcelLoader
53
+ from app.shared.core.data_source.loaders.json_loader import JSONLoader
54
+ from app.shared.core.data_source.schema_info import DataSourceInfo
55
+ from app.shared.core.data_source.specs.csv_source import CSVSourceSpec
56
+ from app.shared.core.data_source.specs.excel_source import ExcelSourceSpec
57
+ from app.shared.core.data_source.specs.json_source import JSONSourceSpec
58
+
59
+ logger = logging.getLogger(__name__)
60
+
61
+
62
+ def _load_excel_with_new_loader(
63
+ filepath: str,
64
+ schemas: list[DataSourceInfo],
65
+ *,
66
+ file_to_sheet_names: dict[str, str] | None = None,
67
+ errors: list[dict[str, Any]] | None = None,
68
+ **kwargs: Any,
69
+ ) -> dict[str, pd.DataFrame]:
70
+ """
71
+ @methoddesc 使用新版 ExcelLoader 加载 Excel 文件(支持多 sheet)。
72
+
73
+ 将 DataSourceInfo 列表转换为 sheet 配置字典,
74
+ 调用 ExcelLoader.load_multi_sheet() 实现一次性加载多个 sheet。
75
+
76
+ Args:
77
+ filepath: Excel 文件的完整路径
78
+ schemas: 引用该文件的 schema 列表,每个 schema 包含 sheet 名称和表头行号等配置
79
+ file_to_sheet_names: 可选,按文件路径提供默认 sheet 名称;当 schema 未指定 sheet_name 时作为回退
80
+
81
+ Returns:
82
+ 字典,键为 schema_id,值为对应的 DataFrame
83
+ 如果 schemas 中没有有效的 sheet 配置,返回空字典
84
+
85
+ 示例:
86
+ >>> schemas = [DataSourceInfo(schema_id="users", sheet_name="Sheet1", header_row=0)]
87
+ >>> result = _load_excel_with_new_loader("data.xlsx", schemas)
88
+ >>> # result: {"users": DataFrame}
89
+ """
90
+ # engine 是文件级读取参数,从第一个 schema 的 source_config 读取;
91
+ # 仅接受合法枚举值(与 ExcelSourceSpec.engine 的 Literal 约束一致),非法值回退默认 openpyxl
92
+ first_config: dict[str, Any] = (schemas[0].source_config or {}) if schemas else {}
93
+ engine: Literal["openpyxl", "xlrd"] = "openpyxl"
94
+ if first_config.get("engine") in ("openpyxl", "xlrd"):
95
+ engine = first_config["engine"] # type: ignore[assignment] # 白名单校验后收窄
96
+ spec = ExcelSourceSpec.model_construct(path=filepath, engine=engine)
97
+ loader = ExcelLoader(spec)
98
+ default_sheet = (file_to_sheet_names or {}).get(filepath)
99
+ sheet_configs = {}
100
+ # 未配 sheet 且无回退映射的 schema 逐个上报 loading error(不再静默丢表——
101
+ # 无约束的表消失后校验报告给出"全部通过"假阳性),与同函数族"文件不存在"
102
+ # 的错误条目结构一致(error_type/message/source_path)
103
+ for s in schemas:
104
+ if s.sheet_name or default_sheet:
105
+ sheet_configs[s.schema_id] = {
106
+ "sheet_name": s.sheet_name or default_sheet,
107
+ "header_row": s.header_row,
108
+ "dtype_inference": s.source_config.get("dtype_inference", True),
109
+ "skip_rows": s.source_config.get("skip_rows", 0),
110
+ "nrows": s.source_config.get("nrows"),
111
+ }
112
+ elif errors is not None:
113
+ errors.append(
114
+ {
115
+ "error_type": "SheetNotConfigured",
116
+ "message": (f"Schema '{s.schema_id}' 未指定 sheet 且文件 '{filepath}' 无默认 sheet 回退,表被跳过"),
117
+ "source_path": filepath,
118
+ }
119
+ )
120
+ if not sheet_configs:
121
+ return {}
122
+ return loader.load_multi_sheet(sheet_configs)
123
+
124
+
125
+ def _load_csv_with_new_loader(
126
+ filepath: str,
127
+ schemas: list[DataSourceInfo],
128
+ *,
129
+ default_encoding: str = "utf-8",
130
+ csv_delimiter: str = ",",
131
+ errors: list[dict[str, Any]] | None = None,
132
+ **kwargs: Any,
133
+ ) -> dict[str, pd.DataFrame]:
134
+ """
135
+ @methoddesc 使用新版 CSVLoader 加载 CSV 文件。
136
+
137
+ CSV 文件通常只对应一个 schema。如果多个 schema 引用同一个 CSV 文件,
138
+ 只处理 schemas 列表中的第一个 schema,其余 schema 记入 errors 上报。
139
+
140
+ Args:
141
+ filepath: CSV 文件的完整路径
142
+ schemas: 引用该文件的 schema 列表(期望长度为 1)
143
+ default_encoding: CSV 文件默认编码
144
+ csv_delimiter: CSV 字段分隔符
145
+ errors: 可选,加载期错误收集列表(多 schema 引用时逐个追加丢失条目)
146
+
147
+ Returns:
148
+ 字典,键为 schema_id,值为对应的 DataFrame
149
+ 如果 schemas 为空,返回空字典
150
+
151
+ 示例:
152
+ >>> schemas = [DataSourceInfo(schema_id="orders", header_row=0)]
153
+ >>> result = _load_csv_with_new_loader("data.csv", schemas)
154
+ >>> # result: {"orders": DataFrame}
155
+ """
156
+ if len(schemas) != 1:
157
+ # 多 schema 引用同一 CSV 不被支持:仅处理第一个(历史行为保留),但把
158
+ # 其余 schema 的丢失上报 loading error(不再只有 logger.warning 不可见)
159
+ logger.warning(f"CSV 文件 '{os.path.basename(filepath)}' 被多个 Schema 引用,这不被支持。跳过。")
160
+ if errors is not None:
161
+ for s in schemas[1:]:
162
+ errors.append(
163
+ {
164
+ "error_type": "DuplicateCsvReference",
165
+ "message": (
166
+ f"CSV 文件 '{filepath}' 被多个 Schema 引用(不支持),"
167
+ f"仅加载第一个 Schema '{schemas[0].schema_id}',Schema '{s.schema_id}' 的表被跳过"
168
+ ),
169
+ "source_path": filepath,
170
+ }
171
+ )
172
+ if not schemas:
173
+ return {}
174
+
175
+ info = schemas[0]
176
+ # 回归修复: schema 的 source_config 中的 CSV 读取参数(分隔符/编码/跳行等)过去被丢弃,
177
+ # 一律使用 manifest 级默认值,导致分号分隔、GBK 编码等文件解析错位。现优先读
178
+ # source_config,manifest 默认值仅作兜底(与 preview/loader.py 的读取方式保持一致)。
179
+ source_config = info.source_config or {}
180
+
181
+ # on_bad_lines 是 Literal 枚举(model_construct 不校验),非法值兜底为 "error"
182
+ # (与 CSVSourceSpec/CSVOptions 默认同口径:坏行静默跳过=校验工具漏报红线),
183
+ # 避免把任意字符串透传给 pandas.read_csv 导致加载失败
184
+ on_bad_lines: Literal["error", "warn", "skip"] = "error"
185
+ if source_config.get("on_bad_lines") in ("error", "warn", "skip"):
186
+ on_bad_lines = source_config["on_bad_lines"] # type: ignore[assignment] # 白名单校验后收窄
187
+
188
+ spec = CSVSourceSpec.model_construct(
189
+ path=filepath,
190
+ header_row=info.header_row,
191
+ encoding=source_config.get("encoding", default_encoding),
192
+ delimiter=source_config.get("delimiter", csv_delimiter),
193
+ skip_rows=int(source_config.get("skip_rows", 0) or 0),
194
+ quotechar=source_config.get("quotechar", '"'),
195
+ on_bad_lines=on_bad_lines,
196
+ )
197
+ loader = CSVLoader(spec)
198
+ df = loader.load()
199
+ return {info.schema_id: df}
200
+
201
+
202
+ def _load_json_with_new_loader(
203
+ filepath: str,
204
+ schemas: list[DataSourceInfo],
205
+ **kwargs: Any,
206
+ ) -> dict[str, pd.DataFrame]:
207
+ """
208
+ @methoddesc 使用新版 JSONLoader 加载 JSON 文件。
209
+
210
+ JSON 文件支持多个 schema 引用,但只处理第一个 schema 的配置。
211
+ 从 schema 的 source_config 中读取 format、json_path、flatten 等参数。
212
+
213
+ Args:
214
+ filepath: JSON 文件的完整路径
215
+ schemas: 引用该文件的 schema 列表
216
+
217
+ Returns:
218
+ 字典,键为 schema_id,值为对应的 DataFrame
219
+ 如果 schemas 为空,会抛出 IndexError(调用方应确保非空)
220
+
221
+ 示例:
222
+ >>> schemas = [DataSourceInfo(schema_id="users", source_config={"format": "auto", "json_path": "$.data"})]
223
+ >>> result = _load_json_with_new_loader("data.json", schemas)
224
+ >>> # result: {"users": DataFrame}
225
+ """
226
+ if len(schemas) > 1:
227
+ logger.warning(f"JSON 文件 '{os.path.basename(filepath)}' 被多个 Schema 引用,只处理第一个。")
228
+
229
+ info = schemas[0]
230
+ source_config = info.source_config or {}
231
+
232
+ spec = JSONSourceSpec.model_construct(
233
+ path=filepath,
234
+ format=source_config.get("format", "auto"),
235
+ json_path=source_config.get("json_path"),
236
+ record_path=source_config.get("record_path"),
237
+ meta_prefix=source_config.get("meta_prefix", "meta."),
238
+ sep=source_config.get("sep", "."),
239
+ dtype=source_config.get("dtype"),
240
+ flatten=source_config.get("flatten", False),
241
+ )
242
+ loader = JSONLoader(spec)
243
+ df = loader.load()
244
+ return {info.schema_id: df}
245
+
246
+
247
+ # 文件扩展名到加载函数的映射
248
+ # 每个扩展名对应一个私有加载函数,负责将文件路径和 schema 列表转换为 {schema_id: DataFrame}
249
+ _LOADER_FNS = {
250
+ ".xlsx": _load_excel_with_new_loader,
251
+ ".xls": _load_excel_with_new_loader,
252
+ ".csv": _load_csv_with_new_loader,
253
+ ".json": _load_json_with_new_loader,
254
+ ".jsonl": _load_json_with_new_loader,
255
+ }
256
+
257
+ # 公开的加载器注册表,与 _LOADER_FNS 内容相同
258
+ # 供 can_load() 等函数查询支持的文件类型
259
+ LOADER_REGISTRY = dict(_LOADER_FNS)
260
+
261
+
262
+ def can_load(path: str) -> bool:
263
+ """
264
+ @methoddesc 检查指定路径的文件是否可以被加载。
265
+
266
+ :param path: 文件路径(可以是完整路径或文件名)
267
+ :return: 如果文件扩展名在 LOADER_REGISTRY 中返回 True,否则 False
268
+ """
269
+ ext = Path(path).suffix.lower()
270
+ return ext in LOADER_REGISTRY
271
+
272
+
273
+ def load_grouped_sources(
274
+ file_to_schemas: dict[str, list[DataSourceInfo]],
275
+ *,
276
+ default_encoding: str = "utf-8",
277
+ csv_delimiter: str = ",",
278
+ file_to_sheet_names: dict[str, str] | None = None,
279
+ ) -> tuple[dict[str, pd.DataFrame], list[dict[str, Any]]]:
280
+ """
281
+ @methoddesc 批量加载多个数据源文件。
282
+
283
+ 使用新版类式加载器(CSVLoader, ExcelLoader, JSONLoader)按文件类型分发加载。
284
+
285
+ :param file_to_schemas: 字典,键为文件路径,值为对应的 DataSourceInfo 列表
286
+ :param default_encoding: CSV 文件的默认编码,默认为 "utf-8"
287
+ :param csv_delimiter: CSV 文件的分隔符,默认为 ","
288
+ :param file_to_sheet_names: 可选,字典,键为文件路径,值为 sheet 名称;Excel 加载时若 schema 未指定 sheet_name 则以此作为回退
289
+ :return: 元组 (datasets, errors)
290
+ """
291
+ datasets: dict[str, pd.DataFrame] = {}
292
+ errors: list[dict[str, Any]] = []
293
+
294
+ for full_path, schemas_for_file in file_to_schemas.items():
295
+ file_ext = os.path.splitext(full_path)[1].lower()
296
+
297
+ if file_ext not in _LOADER_FNS:
298
+ errors.append(
299
+ {
300
+ "error_type": "UnsupportedFileType",
301
+ "message": f"源文件格式不被支持: {file_ext}。支持格式: {list(_LOADER_FNS.keys())}",
302
+ "source_path": full_path,
303
+ }
304
+ )
305
+ continue
306
+
307
+ if not os.path.exists(full_path):
308
+ errors.append(
309
+ {
310
+ "error_type": "FileNotFound",
311
+ "message": "源文件不存在",
312
+ "source_path": full_path,
313
+ }
314
+ )
315
+ continue
316
+
317
+ try:
318
+ loader_fn = _LOADER_FNS[file_ext]
319
+ logger.debug(f"[Load] 从磁盘加载 '{os.path.basename(full_path)}'...")
320
+ file_datasets = loader_fn(
321
+ full_path,
322
+ schemas_for_file,
323
+ default_encoding=default_encoding,
324
+ csv_delimiter=csv_delimiter,
325
+ file_to_sheet_names=file_to_sheet_names,
326
+ errors=errors,
327
+ )
328
+ datasets.update(file_datasets)
329
+
330
+ except DataLoadError as e:
331
+ errors.append(
332
+ {
333
+ "error_type": "LoadFailed",
334
+ "message": str(e),
335
+ "source_path": full_path,
336
+ }
337
+ )
338
+ except Exception as e:
339
+ logger.exception("加载源文件时发生未预期错误: %s", full_path)
340
+ errors.append(
341
+ {
342
+ "error_type": "LoadFailed",
343
+ "message": str(e),
344
+ "source_path": full_path,
345
+ }
346
+ )
347
+
348
+ return datasets, errors