precis-cli 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (442) hide show
  1. app/__init__.py +16 -0
  2. app/api/__init__.py +32 -0
  3. app/api/dependencies.py +185 -0
  4. app/api/main.py +301 -0
  5. app/api/middleware/__init__.py +34 -0
  6. app/api/middleware/exception_handler.py +126 -0
  7. app/api/middleware/request_logging.py +97 -0
  8. app/api/middleware/token_auth.py +215 -0
  9. app/api/models/__init__.py +151 -0
  10. app/api/models/connection_rules.py +98 -0
  11. app/api/models/files.py +99 -0
  12. app/api/models/full_validation.py +554 -0
  13. app/api/models/project.py +117 -0
  14. app/api/models/projects.py +80 -0
  15. app/api/models/schema.py +121 -0
  16. app/api/models/v2_responses.py +46 -0
  17. app/api/models/validation.py +303 -0
  18. app/api/models/workspace.py +118 -0
  19. app/api/routers/__init__.py +56 -0
  20. app/api/routers/ai/__init__.py +36 -0
  21. app/api/routers/ai/chat.py +243 -0
  22. app/api/routers/ai/generate.py +156 -0
  23. app/api/routers/ai/hardware.py +199 -0
  24. app/api/routers/ai/jobs.py +544 -0
  25. app/api/routers/ai/migrate.py +471 -0
  26. app/api/routers/ai/models.py +352 -0
  27. app/api/routers/ai/ollama.py +108 -0
  28. app/api/routers/ai/providers.py +491 -0
  29. app/api/routers/ai/router.py +44 -0
  30. app/api/routers/ai/stream.py +281 -0
  31. app/api/routers/ai/utils.py +182 -0
  32. app/api/routers/core/__init__.py +34 -0
  33. app/api/routers/core/connection_rules.py +188 -0
  34. app/api/routers/core/data_sources.py +374 -0
  35. app/api/routers/core/regex.py +333 -0
  36. app/api/routers/core/reporting.py +263 -0
  37. app/api/routers/files/__init__.py +24 -0
  38. app/api/routers/files/ops.py +179 -0
  39. app/api/routers/files/transfer.py +101 -0
  40. app/api/routers/preview/__init__.py +35 -0
  41. app/api/routers/preview/content_mode.py +264 -0
  42. app/api/routers/preview/header_row.py +154 -0
  43. app/api/routers/preview/models.py +81 -0
  44. app/api/routers/preview/path_mode.py +440 -0
  45. app/api/routers/preview/router.py +30 -0
  46. app/api/routers/project/__init__.py +61 -0
  47. app/api/routers/project/base.py +111 -0
  48. app/api/routers/project/constraint.py +345 -0
  49. app/api/routers/project/full_config.py +411 -0
  50. app/api/routers/project/full_config_writer.py +327 -0
  51. app/api/routers/project/helpers.py +167 -0
  52. app/api/routers/project/inspection_fix.py +330 -0
  53. app/api/routers/project/manifest.py +605 -0
  54. app/api/routers/project/models.py +177 -0
  55. app/api/routers/project/pattern.py +268 -0
  56. app/api/routers/project/regex.py +358 -0
  57. app/api/routers/project/scanner.py +157 -0
  58. app/api/routers/project/schema.py +567 -0
  59. app/api/routers/project/schema_helpers.py +149 -0
  60. app/api/routers/project/settings.py +399 -0
  61. app/api/routers/project/template.py +325 -0
  62. app/api/routers/project/validation.py +227 -0
  63. app/api/routers/project/view.py +177 -0
  64. app/api/routers/project/workspaces.py +167 -0
  65. app/api/routers/projects/__init__.py +25 -0
  66. app/api/routers/projects/create.py +121 -0
  67. app/api/routers/projects/open.py +103 -0
  68. app/api/routers/projects/scan.py +100 -0
  69. app/api/routers/validation/__init__.py +39 -0
  70. app/api/routers/validation/common.py +263 -0
  71. app/api/routers/validation/content_mode.py +401 -0
  72. app/api/routers/validation/history.py +235 -0
  73. app/api/routers/validation/inline_mode.py +196 -0
  74. app/api/routers/validation/path_mode.py +314 -0
  75. app/api/routers/validation/router.py +51 -0
  76. app/api/services/full_validation_response_builder.py +412 -0
  77. app/api/services/io_error_messages.py +41 -0
  78. app/api/services/preview_service.py +248 -0
  79. app/cli/__init__.py +18 -0
  80. app/cli/__main__.py +30 -0
  81. app/cli/shared_services/__init__.py +42 -0
  82. app/cli/shared_services/config_ops.py +581 -0
  83. app/cli/shared_services/generation_ops.py +160 -0
  84. app/cli/shared_services/project_ops.py +288 -0
  85. app/cli/shell/__init__.py +48 -0
  86. app/cli/shell/commands/__init__.py +63 -0
  87. app/cli/shell/commands/ai/__init__.py +260 -0
  88. app/cli/shell/commands/ai/chat.py +331 -0
  89. app/cli/shell/commands/ai/delete.py +165 -0
  90. app/cli/shell/commands/ai/diff.py +70 -0
  91. app/cli/shell/commands/ai/display.py +428 -0
  92. app/cli/shell/commands/ai/executor.py +225 -0
  93. app/cli/shell/commands/ai/executor_utils.py +271 -0
  94. app/cli/shell/commands/ai/generate.py +285 -0
  95. app/cli/shell/commands/ai/interaction.py +262 -0
  96. app/cli/shell/commands/ai/migrate.py +281 -0
  97. app/cli/shell/commands/ai/resolver.py +224 -0
  98. app/cli/shell/commands/ai/status.py +101 -0
  99. app/cli/shell/commands/ai/switch.py +158 -0
  100. app/cli/shell/commands/ai/utils.py +52 -0
  101. app/cli/shell/commands/base.py +343 -0
  102. app/cli/shell/commands/config/__init__.py +131 -0
  103. app/cli/shell/commands/config/base.py +80 -0
  104. app/cli/shell/commands/config/check.py +246 -0
  105. app/cli/shell/commands/config/edit.py +148 -0
  106. app/cli/shell/commands/config/get.py +111 -0
  107. app/cli/shell/commands/config/init.py +123 -0
  108. app/cli/shell/commands/config/inspect.py +164 -0
  109. app/cli/shell/commands/config/list.py +104 -0
  110. app/cli/shell/commands/config/set.py +119 -0
  111. app/cli/shell/commands/config/show.py +150 -0
  112. app/cli/shell/commands/exit.py +67 -0
  113. app/cli/shell/commands/help.py +104 -0
  114. app/cli/shell/commands/infer_schema.py +156 -0
  115. app/cli/shell/commands/open.py +201 -0
  116. app/cli/shell/commands/project.py +215 -0
  117. app/cli/shell/commands/provider.py +619 -0
  118. app/cli/shell/commands/system.py +170 -0
  119. app/cli/shell/commands/validate.py +455 -0
  120. app/cli/shell/completer.py +231 -0
  121. app/cli/shell/config_storage.py +191 -0
  122. app/cli/shell/exceptions.py +90 -0
  123. app/cli/shell/formatter.py +371 -0
  124. app/cli/shell/interactive_menu.py +348 -0
  125. app/cli/shell/main.py +286 -0
  126. app/cli/shell/parser.py +266 -0
  127. app/cli/start.py +180 -0
  128. app/cli_main.py +52 -0
  129. app/mcp_server.py +311 -0
  130. app/shared/core/__init__.py +18 -0
  131. app/shared/core/app_version.py +59 -0
  132. app/shared/core/config/__init__.py +119 -0
  133. app/shared/core/config/server.py +193 -0
  134. app/shared/core/data_source/__init__.py +106 -0
  135. app/shared/core/data_source/loader.py +348 -0
  136. app/shared/core/data_source/loaders/__init__.py +213 -0
  137. app/shared/core/data_source/loaders/base.py +215 -0
  138. app/shared/core/data_source/loaders/converter.py +337 -0
  139. app/shared/core/data_source/loaders/csv_loader.py +252 -0
  140. app/shared/core/data_source/loaders/excel_loader.py +666 -0
  141. app/shared/core/data_source/loaders/extractor.py +268 -0
  142. app/shared/core/data_source/loaders/json_loader.py +354 -0
  143. app/shared/core/data_source/loaders/registry.py +88 -0
  144. app/shared/core/data_source/loaders/sql_loader.py +313 -0
  145. app/shared/core/data_source/loaders/strategies/__init__.py +100 -0
  146. app/shared/core/data_source/loaders/strategies/array_parser.py +277 -0
  147. app/shared/core/data_source/loaders/strategies/lines_parser.py +355 -0
  148. app/shared/core/data_source/loaders/strategies/object_parser.py +278 -0
  149. app/shared/core/data_source/schema_info.py +50 -0
  150. app/shared/core/data_source/specs/__init__.py +128 -0
  151. app/shared/core/data_source/specs/base.py +258 -0
  152. app/shared/core/data_source/specs/csv_source.py +137 -0
  153. app/shared/core/data_source/specs/excel_source.py +141 -0
  154. app/shared/core/data_source/specs/file_base.py +252 -0
  155. app/shared/core/data_source/specs/json_source.py +226 -0
  156. app/shared/core/data_source/specs/sql_source.py +161 -0
  157. app/shared/core/encoding.py +36 -0
  158. app/shared/core/io/__init__.py +24 -0
  159. app/shared/core/io/yaml.py +379 -0
  160. app/shared/core/manifest_schema/__init__.py +52 -0
  161. app/shared/core/manifest_schema/types.py +99 -0
  162. app/shared/core/manifest_schema/version.py +175 -0
  163. app/shared/core/patterns/__init__.py +24 -0
  164. app/shared/core/patterns/loader.py +134 -0
  165. app/shared/core/patterns/writer.py +244 -0
  166. app/shared/core/project/__init__.py +47 -0
  167. app/shared/core/project/constraint/builders/__init__.py +50 -0
  168. app/shared/core/project/constraint/builders/base.py +100 -0
  169. app/shared/core/project/constraint/builders/composite.py +77 -0
  170. app/shared/core/project/constraint/builders/conditional.py +67 -0
  171. app/shared/core/project/constraint/builders/foreign_key.py +53 -0
  172. app/shared/core/project/constraint/builders/registry.py +64 -0
  173. app/shared/core/project/constraint/builders/scripted.py +51 -0
  174. app/shared/core/project/constraint/builders/single_column.py +86 -0
  175. app/shared/core/project/constraint/builders/unique.py +53 -0
  176. app/shared/core/project/constraint/factory.py +170 -0
  177. app/shared/core/project/constraint/reader.py +214 -0
  178. app/shared/core/project/constraint/registry.py +233 -0
  179. app/shared/core/project/constraint/types/__init__.py +63 -0
  180. app/shared/core/project/constraint/types/constraint_file.py +261 -0
  181. app/shared/core/project/constraint/types/refs.py +460 -0
  182. app/shared/core/project/constraint/types.py +28 -0
  183. app/shared/core/project/constraint/writer.py +181 -0
  184. app/shared/core/project/loader/__init__.py +56 -0
  185. app/shared/core/project/loader/loader.py +30 -0
  186. app/shared/core/project/loader/loader_parts/config_inspector.py +137 -0
  187. app/shared/core/project/loader/loader_parts/embedded_constraints.py +224 -0
  188. app/shared/core/project/loader/loader_parts/file_loaders.py +58 -0
  189. app/shared/core/project/loader/loader_parts/inspection_ids.py +85 -0
  190. app/shared/core/project/loader/loader_parts/inspector_helpers.py +312 -0
  191. app/shared/core/project/loader/loader_parts/inspector_id_checks.py +274 -0
  192. app/shared/core/project/loader/loader_parts/inspector_reference_checks.py +690 -0
  193. app/shared/core/project/loader/loader_parts/inspector_uniqueness_checks.py +286 -0
  194. app/shared/core/project/loader/loader_parts/loading_error_messages.py +298 -0
  195. app/shared/core/project/loader/loader_parts/main.py +432 -0
  196. app/shared/core/project/loader/loader_parts/path_validation.py +117 -0
  197. app/shared/core/project/loader/loader_parts/runtime.py +125 -0
  198. app/shared/core/project/loader/types.py +246 -0
  199. app/shared/core/project/manifest/coverage.py +391 -0
  200. app/shared/core/project/manifest/reader.py +260 -0
  201. app/shared/core/project/manifest/types.py +91 -0
  202. app/shared/core/project/manifest/types_parts/__init__.py +60 -0
  203. app/shared/core/project/manifest/types_parts/constants.py +40 -0
  204. app/shared/core/project/manifest/types_parts/data_source.py +68 -0
  205. app/shared/core/project/manifest/types_parts/info.py +59 -0
  206. app/shared/core/project/manifest/types_parts/manifest.py +294 -0
  207. app/shared/core/project/manifest/types_parts/refs.py +153 -0
  208. app/shared/core/project/manifest/types_parts/settings.py +92 -0
  209. app/shared/core/project/manifest/types_parts/settings_file_processing.py +64 -0
  210. app/shared/core/project/manifest/types_parts/settings_script_security.py +78 -0
  211. app/shared/core/project/manifest/types_parts/settings_validation.py +83 -0
  212. app/shared/core/project/manifest/types_parts/template.py +66 -0
  213. app/shared/core/project/manifest/writer.py +262 -0
  214. app/shared/core/project/manual_data/__init__.py +27 -0
  215. app/shared/core/project/manual_data/types.py +75 -0
  216. app/shared/core/project/regex/reader.py +197 -0
  217. app/shared/core/project/regex/types.py +405 -0
  218. app/shared/core/project/regex/writer.py +123 -0
  219. app/shared/core/project/schema/reader.py +170 -0
  220. app/shared/core/project/schema/types.py +47 -0
  221. app/shared/core/project/schema/types_parts/__init__.py +36 -0
  222. app/shared/core/project/schema/types_parts/column.py +174 -0
  223. app/shared/core/project/schema/types_parts/column_utils.py +72 -0
  224. app/shared/core/project/schema/types_parts/constraint.py +165 -0
  225. app/shared/core/project/schema/types_parts/schema_id.py +66 -0
  226. app/shared/core/project/schema/types_parts/source.py +255 -0
  227. app/shared/core/project/schema/types_parts/source_options.py +347 -0
  228. app/shared/core/project/schema/types_parts/table.py +230 -0
  229. app/shared/core/project/schema/writer.py +139 -0
  230. app/shared/core/project/schema_ref_check.py +95 -0
  231. app/shared/core/project/template/__init__.py +27 -0
  232. app/shared/core/project/template/expander.py +263 -0
  233. app/shared/core/project/template/reader.py +120 -0
  234. app/shared/core/project/template/types.py +114 -0
  235. app/shared/core/project/transform/reader.py +76 -0
  236. app/shared/core/project/transform/types.py +116 -0
  237. app/shared/core/project/transform/writer.py +84 -0
  238. app/shared/core/pydantic_messages.py +59 -0
  239. app/shared/core/reporter/__init__.py +46 -0
  240. app/shared/core/reporter/reporter.py +220 -0
  241. app/shared/core/reporter/reporters/__init__.py +65 -0
  242. app/shared/core/reporter/reporters/base.py +188 -0
  243. app/shared/core/reporter/reporters/dingtalk_app_reporter.py +274 -0
  244. app/shared/core/reporter/reporters/email_reporter.py +271 -0
  245. app/shared/core/reporter/reporters/feishu_app_reporter.py +467 -0
  246. app/shared/core/reporter/reporters/local_file_reporter.py +208 -0
  247. app/shared/core/reporter/reporters/wecom_app_reporter.py +268 -0
  248. app/shared/core/utils/__init__.py +18 -0
  249. app/shared/core/utils/path_utils.py +60 -0
  250. app/shared/core/utils/regex_extract.py +113 -0
  251. app/shared/domain/__init__.py +114 -0
  252. app/shared/domain/constraints/__init__.py +76 -0
  253. app/shared/domain/constraints/allowed_values.py +211 -0
  254. app/shared/domain/constraints/base.py +141 -0
  255. app/shared/domain/constraints/charset.py +337 -0
  256. app/shared/domain/constraints/composite.py +174 -0
  257. app/shared/domain/constraints/condition_registry.py +130 -0
  258. app/shared/domain/constraints/conditional.py +629 -0
  259. app/shared/domain/constraints/date_logic.py +731 -0
  260. app/shared/domain/constraints/foreign_key.py +261 -0
  261. app/shared/domain/constraints/key_normalization.py +56 -0
  262. app/shared/domain/constraints/not_null.py +185 -0
  263. app/shared/domain/constraints/range.py +360 -0
  264. app/shared/domain/constraints/regex.py +218 -0
  265. app/shared/domain/constraints/scripted.py +276 -0
  266. app/shared/domain/constraints/unique.py +233 -0
  267. app/shared/domain/data_engine.py +384 -0
  268. app/shared/domain/data_types.py +113 -0
  269. app/shared/domain/data_types_parts/__init__.py +67 -0
  270. app/shared/domain/data_types_parts/base.py +204 -0
  271. app/shared/domain/data_types_parts/composite.py +250 -0
  272. app/shared/domain/data_types_parts/expression.py +202 -0
  273. app/shared/domain/data_types_parts/extracted.py +106 -0
  274. app/shared/domain/data_types_parts/json_types.py +234 -0
  275. app/shared/domain/data_types_parts/scalars.py +719 -0
  276. app/shared/domain/data_types_parts/sequence.py +129 -0
  277. app/shared/domain/dataset_schema.py +48 -0
  278. app/shared/domain/eval_sandbox.py +63 -0
  279. app/shared/domain/expression_system.py +366 -0
  280. app/shared/domain/regex_flags.py +53 -0
  281. app/shared/domain/schema/builder.py +223 -0
  282. app/shared/domain/schema/models.py +339 -0
  283. app/shared/domain/transforms/__init__.py +28 -0
  284. app/shared/domain/transforms/aggregate.py +141 -0
  285. app/shared/domain/transforms/base.py +171 -0
  286. app/shared/domain/transforms/cast_type.py +120 -0
  287. app/shared/domain/transforms/concat.py +103 -0
  288. app/shared/domain/transforms/conditional_assign.py +114 -0
  289. app/shared/domain/transforms/date_format.py +81 -0
  290. app/shared/domain/transforms/digits.py +77 -0
  291. app/shared/domain/transforms/drop_duplicates.py +94 -0
  292. app/shared/domain/transforms/fill_na.py +94 -0
  293. app/shared/domain/transforms/filter_rows.py +97 -0
  294. app/shared/domain/transforms/lookup.py +78 -0
  295. app/shared/domain/transforms/lower_case.py +70 -0
  296. app/shared/domain/transforms/map_value.py +90 -0
  297. app/shared/domain/transforms/math_expr.py +112 -0
  298. app/shared/domain/transforms/modulo.py +82 -0
  299. app/shared/domain/transforms/regex_extract.py +103 -0
  300. app/shared/domain/transforms/registry.py +95 -0
  301. app/shared/domain/transforms/replace.py +88 -0
  302. app/shared/domain/transforms/sort_rows.py +94 -0
  303. app/shared/domain/transforms/string_split.py +84 -0
  304. app/shared/domain/transforms/strip.py +73 -0
  305. app/shared/domain/transforms/substring.py +92 -0
  306. app/shared/domain/transforms/upper_case.py +70 -0
  307. app/shared/domain/transforms/weighted_sum.py +104 -0
  308. app/shared/domain/validation_constraints.py +69 -0
  309. app/shared/services/__init__.py +62 -0
  310. app/shared/services/ai/__init__.py +48 -0
  311. app/shared/services/ai/agent/__init__.py +40 -0
  312. app/shared/services/ai/agent/chat_tools/__init__.py +47 -0
  313. app/shared/services/ai/agent/chat_tools/apply_actions.py +592 -0
  314. app/shared/services/ai/agent/chat_tools/ask_user.py +237 -0
  315. app/shared/services/ai/agent/chat_tools/read_canvas.py +140 -0
  316. app/shared/services/ai/agent/chat_tools/read_project.py +160 -0
  317. app/shared/services/ai/agent/chat_tools/read_table.py +301 -0
  318. app/shared/services/ai/agent/chat_tools/schemas.py +105 -0
  319. app/shared/services/ai/agent/chat_tools/validate_table.py +131 -0
  320. app/shared/services/ai/agent/executor.py +554 -0
  321. app/shared/services/ai/agent/memory.py +256 -0
  322. app/shared/services/ai/agent/planner.py +282 -0
  323. app/shared/services/ai/agent/tool_registry.py +296 -0
  324. app/shared/services/ai/agent/tools/__init__.py +32 -0
  325. app/shared/services/ai/agent/tools/config_generate.py +133 -0
  326. app/shared/services/ai/agent/tools/config_refine.py +109 -0
  327. app/shared/services/ai/agent/tools/config_validate.py +403 -0
  328. app/shared/services/ai/agent/tools/merge_results.py +136 -0
  329. app/shared/services/ai/agent/tools/plan_chunks.py +79 -0
  330. app/shared/services/ai/agent/tools/script_parse.py +359 -0
  331. app/shared/services/ai/agent/types.py +138 -0
  332. app/shared/services/ai/chat_agent_runner.py +596 -0
  333. app/shared/services/ai/chat_orchestrator.py +725 -0
  334. app/shared/services/ai/failure_messages.py +65 -0
  335. app/shared/services/ai/job_storage.py +274 -0
  336. app/shared/services/ai/migrate_service.py +550 -0
  337. app/shared/services/ai/streaming/__init__.py +65 -0
  338. app/shared/services/ai/streaming/event_journal.py +197 -0
  339. app/shared/services/ai/streaming/orchestrator.py +262 -0
  340. app/shared/services/ai/streaming/pending_interaction_store.py +227 -0
  341. app/shared/services/ai/streaming/sse_response.py +148 -0
  342. app/shared/services/ai/streaming/types.py +73 -0
  343. app/shared/services/ai/types.py +213 -0
  344. app/shared/services/ai/utils.py +322 -0
  345. app/shared/services/diff/config_diff.py +320 -0
  346. app/shared/services/hardware.py +237 -0
  347. app/shared/services/llm/__init__.py +62 -0
  348. app/shared/services/llm/actions/__init__.py +42 -0
  349. app/shared/services/llm/actions/_canvas_validator.py +147 -0
  350. app/shared/services/llm/actions/_constraint_validator.py +401 -0
  351. app/shared/services/llm/actions/_regex_validator.py +85 -0
  352. app/shared/services/llm/actions/_schema_validator.py +82 -0
  353. app/shared/services/llm/actions/_settings_validator.py +76 -0
  354. app/shared/services/llm/actions/_transform_validator.py +78 -0
  355. app/shared/services/llm/actions/action_handlers.py +400 -0
  356. app/shared/services/llm/actions/action_parser.py +63 -0
  357. app/shared/services/llm/actions/action_processor.py +468 -0
  358. app/shared/services/llm/actions/action_validator.py +311 -0
  359. app/shared/services/llm/actions/diff_compute.py +195 -0
  360. app/shared/services/llm/actions/regex_handlers.py +284 -0
  361. app/shared/services/llm/actions/registry.py +336 -0
  362. app/shared/services/llm/actions/schema_handlers.py +335 -0
  363. app/shared/services/llm/actions/settings_handlers.py +180 -0
  364. app/shared/services/llm/actions/specs.py +259 -0
  365. app/shared/services/llm/actions/transform_handlers.py +279 -0
  366. app/shared/services/llm/actions/validation_types.py +147 -0
  367. app/shared/services/llm/cache/__init__.py +18 -0
  368. app/shared/services/llm/cache/response_cache.py +80 -0
  369. app/shared/services/llm/chat/__init__.py +35 -0
  370. app/shared/services/llm/chat/chat_system_prompt.py +561 -0
  371. app/shared/services/llm/chat/response_parser.py +296 -0
  372. app/shared/services/llm/config/__init__.py +45 -0
  373. app/shared/services/llm/config/crypto.py +135 -0
  374. app/shared/services/llm/config/loader.py +258 -0
  375. app/shared/services/llm/config/models.py +202 -0
  376. app/shared/services/llm/config/presets.py +128 -0
  377. app/shared/services/llm/config_generator.py +163 -0
  378. app/shared/services/llm/constraints/__init__.py +41 -0
  379. app/shared/services/llm/constraints/constraint_builder.py +177 -0
  380. app/shared/services/llm/constraints/constraint_deletion.py +84 -0
  381. app/shared/services/llm/constraints/constraint_id.py +174 -0
  382. app/shared/services/llm/constraints/frontend_instructions.py +333 -0
  383. app/shared/services/llm/constraints/inline_batch.py +279 -0
  384. app/shared/services/llm/discovery/__init__.py +35 -0
  385. app/shared/services/llm/discovery/scanner.py +181 -0
  386. app/shared/services/llm/generation/__init__.py +42 -0
  387. app/shared/services/llm/generation/agent_wiring.py +156 -0
  388. app/shared/services/llm/generation/config_builder.py +386 -0
  389. app/shared/services/llm/generation/errors.py +36 -0
  390. app/shared/services/llm/generation/existing_config.py +99 -0
  391. app/shared/services/llm/generation/profiler.py +279 -0
  392. app/shared/services/llm/generation/prompt_builder.py +199 -0
  393. app/shared/services/llm/generation/response_parser.py +144 -0
  394. app/shared/services/llm/generation/service.py +622 -0
  395. app/shared/services/llm/models.py +104 -0
  396. app/shared/services/llm/providers/__init__.py +48 -0
  397. app/shared/services/llm/providers/base.py +310 -0
  398. app/shared/services/llm/providers/cached_provider.py +90 -0
  399. app/shared/services/llm/providers/ollama.py +498 -0
  400. app/shared/services/llm/providers/openai.py +334 -0
  401. app/shared/services/llm/providers/registry.py +108 -0
  402. app/shared/services/llm/schema_resolver.py +141 -0
  403. app/shared/services/llm/suggestion_utils.py +219 -0
  404. app/shared/services/llm/validate_executor.py +163 -0
  405. app/shared/services/llm/yaml_io.py +406 -0
  406. app/shared/services/preview/__init__.py +18 -0
  407. app/shared/services/preview/loader.py +145 -0
  408. app/shared/services/preview/path_validation.py +138 -0
  409. app/shared/services/project_loader.py +57 -0
  410. app/shared/services/schema_inference.py +260 -0
  411. app/shared/services/schema_runtime_builder.py +120 -0
  412. app/shared/services/validation/__init__.py +52 -0
  413. app/shared/services/validation/chunked_loader.py +509 -0
  414. app/shared/services/validation/dag/__init__.py +35 -0
  415. app/shared/services/validation/dag/builder.py +141 -0
  416. app/shared/services/validation/dag/executor.py +225 -0
  417. app/shared/services/validation/dag/sorter.py +77 -0
  418. app/shared/services/validation/data_loader.py +231 -0
  419. app/shared/services/validation/engine.py +446 -0
  420. app/shared/services/validation/executor.py +1030 -0
  421. app/shared/services/validation/extractors.py +266 -0
  422. app/shared/services/validation/history.py +209 -0
  423. app/shared/services/validation/json_payload.py +136 -0
  424. app/shared/services/validation/loader.py +152 -0
  425. app/shared/services/validation/memory_monitor.py +179 -0
  426. app/shared/services/validation/postprocess.py +208 -0
  427. app/shared/services/validation/progress.py +64 -0
  428. app/shared/services/validation/report_export.py +222 -0
  429. app/shared/services/validation/resolver.py +180 -0
  430. app/shared/services/validation/service.py +418 -0
  431. app/shared/services/validation/types.py +238 -0
  432. app/shared/services/validation/validators/__init__.py +36 -0
  433. app/shared/services/validation/validators/adapter.py +182 -0
  434. app/shared/services/validation/validators/base.py +332 -0
  435. app/shared/services/validation/validators/composite.py +233 -0
  436. app/shared/services/validation/validators/date_logic.py +276 -0
  437. app/start_server.py +133 -0
  438. precis_cli-0.1.3.dist-info/METADATA +180 -0
  439. precis_cli-0.1.3.dist-info/RECORD +442 -0
  440. precis_cli-0.1.3.dist-info/WHEEL +5 -0
  441. precis_cli-0.1.3.dist-info/entry_points.txt +4 -0
  442. precis_cli-0.1.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,412 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # Copyright 2026 Precis Team
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """
17
+ @fileoverview 全量校验响应构建器
18
+
19
+ 功能概述:
20
+ - 将 ValidationExecutor 的输出转换为 FullValidationResponse
21
+ - 封装错误分类、ID -> Name 映射、通过项构造、统计聚合、覆盖率组装
22
+ - 使 API 路由层仅负责参数校验与执行器调用
23
+
24
+ 输入示例:
25
+ builder = FullValidationResponseBuilder(executor, started=time.monotonic(), coverage=coverage)
26
+ response = builder.build_from_result(result)
27
+
28
+ 输出示例:
29
+ FullValidationResponse 实例,包含 success/summary/errors/passed_items/statistics 等字段
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import logging
35
+ import time
36
+ from typing import Any
37
+
38
+ from app.api.models import (
39
+ FullValidationErrorItem,
40
+ FullValidationResponse,
41
+ FullValidationSummary,
42
+ ValidationPassedItem,
43
+ ValidationStatistics,
44
+ )
45
+ from app.shared.services.validation.executor import ValidationExecutor
46
+
47
+ logger = logging.getLogger(__name__)
48
+
49
+
50
+ class FullValidationResponseBuilder:
51
+ """
52
+ 全量校验响应构建器。
53
+
54
+ 负责把执行器返回的原始结果字典转换为前端需要的 Pydantic 响应模型,
55
+ 同时支持执行器初始化失败或执行异常时的降级响应构建。
56
+ """
57
+
58
+ # 回归 #10: 单次响应返回的错误条目上限。超过此数时截断 errors 列表(保留真实总数),
59
+ # 防止百万级错误导致响应数百 MB、前端卡死。5000 条足以覆盖典型排查需求。
60
+ MAX_ERRORS = 5000
61
+
62
+ def __init__(
63
+ self,
64
+ executor: ValidationExecutor | None,
65
+ started: float,
66
+ coverage: Any | None = None,
67
+ ):
68
+ """
69
+ 初始化响应构建器。
70
+
71
+ 参数:
72
+ executor: 校验执行器实例;初始化失败时可为 None
73
+ started: 路由层记录的起始时间(time.monotonic())
74
+ coverage: 预计算的覆盖率 payload(可选)
75
+ """
76
+ self.executor = executor
77
+ self.started = started
78
+ self.coverage = coverage
79
+
80
+ def _duration_ms(self) -> int:
81
+ """计算从 started 到当前的耗时(毫秒)。"""
82
+ return int((time.monotonic() - self.started) * 1000)
83
+
84
+ def _build_id_to_name(self) -> dict[str, str]:
85
+ """构建 Schema ID -> 显示名称 的映射。"""
86
+ id_to_name: dict[str, str] = {}
87
+ if not self.executor or not self.executor.dataset_schema:
88
+ return id_to_name
89
+
90
+ for tid, schema in self.executor.dataset_schema.tables.items():
91
+ target_name = schema.name or tid
92
+ if schema.id:
93
+ id_to_name[schema.id] = target_name
94
+ id_to_name[tid] = target_name
95
+
96
+ return id_to_name
97
+
98
+ def _build_errors(self, result: dict[str, Any]) -> list[FullValidationErrorItem]:
99
+ """将 executor 结果中的 loading_errors 与 errors 转换为响应错误项。"""
100
+ errors: list[FullValidationErrorItem] = []
101
+
102
+ # 加载阶段错误
103
+ for le in result.get("loading_errors", []):
104
+ error_type = str(le.get("error_type") or "LoadError")
105
+ # message 为空的加载错误(如 SchemaIdDuplicate 只填 title/description)回退到
106
+ # 描述/标题,避免用户只看到一句孤零零的"建议:"而不知道问题本身是什么
107
+ message = str(le.get("message") or le.get("description") or le.get("title") or "")
108
+ suggestion = le.get("suggestion", "")
109
+ if suggestion:
110
+ message = f"{message}\n建议: {suggestion}"
111
+
112
+ errors.append(
113
+ FullValidationErrorItem(
114
+ stage="loading",
115
+ error_type=error_type,
116
+ check_type="DataLoad",
117
+ message=message,
118
+ table=le.get("table"),
119
+ source_path=le.get("file_path"),
120
+ source_file=le.get("source_file"),
121
+ source_sheet=le.get("source_sheet"),
122
+ )
123
+ )
124
+
125
+ # 校验错误
126
+ for err in result.get("errors", []):
127
+ # 跳过重复的超时错误
128
+ if err.get("error_type") == "Timeout" and any(e.error_type == "Timeout" for e in errors):
129
+ continue
130
+
131
+ stage = str(err.get("stage") or "constraint")
132
+ message = str(err.get("message") or err.get("error_message") or "")
133
+ row_index_raw = err.get("row_index")
134
+ row_index = None if row_index_raw is None or row_index_raw != row_index_raw else row_index_raw
135
+
136
+ errors.append(
137
+ FullValidationErrorItem(
138
+ stage=stage,
139
+ error_type=str(err.get("error_type") or "ValidationError"),
140
+ check_type=err.get("check_type"),
141
+ message=message,
142
+ table=err.get("table"),
143
+ table_id=err.get("table_id"),
144
+ column=err.get("column"),
145
+ column_id=err.get("column_id"),
146
+ row_index=row_index,
147
+ # 回归 C3: 用 is not None 而非 falsy 判断,避免 value=0/False/"" 被吞成 None,
148
+ # 妨碍用户定位(如 Range 对值 0 的违规)。
149
+ value=str(err.get("value")) if err.get("value") is not None else None,
150
+ source_path=err.get("source_path"),
151
+ source_file=err.get("source_file"),
152
+ source_sheet=err.get("source_sheet"),
153
+ )
154
+ )
155
+
156
+ return errors
157
+
158
+ def _build_passed_items(self, result: dict[str, Any], id_to_name: dict[str, str]) -> list[ValidationPassedItem]:
159
+ """从 executor 结果中构造所有通过项。"""
160
+ passed_items: list[ValidationPassedItem] = []
161
+ raw_datasets = result.get("raw_datasets", {})
162
+
163
+ # 1. 成功加载的数据表
164
+ for table_id in raw_datasets.keys():
165
+ table_name = id_to_name.get(table_id, table_id)
166
+ dataset_info = raw_datasets.get(table_id, {})
167
+ source_file = None
168
+ source_sheet = None
169
+ if isinstance(dataset_info, dict):
170
+ source_file = dataset_info.get("source_file")
171
+ source_sheet = dataset_info.get("source_sheet")
172
+
173
+ passed_items.append(
174
+ ValidationPassedItem(
175
+ stage="loading",
176
+ check_type="DataLoad",
177
+ message=f"数据表 '{table_name}' 加载成功",
178
+ table=table_name,
179
+ source_file=source_file,
180
+ source_sheet=source_sheet,
181
+ )
182
+ )
183
+
184
+ validation_details = result.get("validation_details", {})
185
+
186
+ # 2. 格式校验通过项
187
+ for format_check in validation_details.get("format_checks", []):
188
+ if not format_check.get("passed", False):
189
+ continue
190
+ passed_items.append(
191
+ ValidationPassedItem(
192
+ stage="format",
193
+ check_type="FormatValidation",
194
+ message=f"数据表 '{format_check['table']}' 格式校验通过",
195
+ table=format_check["table"],
196
+ source_file=format_check.get("source_file"),
197
+ source_sheet=format_check.get("source_sheet"),
198
+ )
199
+ )
200
+
201
+ # 3. 约束校验通过项
202
+ for constraint_check in validation_details.get("constraint_checks", []):
203
+ if not constraint_check.get("passed", False):
204
+ continue
205
+ constraint_type = constraint_check.get("constraint_type", "Unknown")
206
+ passed_items.append(
207
+ ValidationPassedItem(
208
+ stage="constraint",
209
+ check_type=constraint_type,
210
+ message=constraint_check.get("description", "") or f"{constraint_type} 约束校验通过",
211
+ table=constraint_check.get("table"),
212
+ source_file=constraint_check.get("source_file"),
213
+ source_sheet=constraint_check.get("source_sheet"),
214
+ )
215
+ )
216
+
217
+ return passed_items
218
+
219
+ def _build_statistics(
220
+ self,
221
+ errors: list[FullValidationErrorItem],
222
+ passed_items: list[ValidationPassedItem],
223
+ ) -> ValidationStatistics:
224
+ """按类型和表聚合统计信息。
225
+
226
+ 回归 C1: 顶层 total_checks/passed_count/failed_count/pass_rate 必须按"检查项"粒度
227
+ 计算(与 passed_items 同粒度),不能分子按项、分母按错误行混算。原实现
228
+ failed_count=len(errors)(行数),导致 1 个约束在 N 行违规 → 通过率≈1/N,即使
229
+ 其余检查全过也显示灾难性低通过率,污染历史趋势。
230
+ - 顶层计数:每个不同的 (stage, check_type, table) 失败检查记 1(与通过的检查项同粒度)。
231
+ - by_type/by_table 明细:保留按错误条目计数(便于用户看每类/每表的具体错误规模)。
232
+ """
233
+ by_type: dict[str, dict[str, int]] = {}
234
+ by_table: dict[str, dict[str, int]] = {}
235
+
236
+ def _ensure_bucket(container: dict[str, dict[str, int]], key: Any) -> dict[str, int]:
237
+ target_key = str(key or "Unknown")
238
+ if target_key not in container:
239
+ container[target_key] = {"total": 0, "passed": 0, "failed": 0}
240
+ return container[target_key]
241
+
242
+ for item in passed_items:
243
+ type_bucket = _ensure_bucket(by_type, item.check_type)
244
+ type_bucket["total"] += 1
245
+ type_bucket["passed"] += 1
246
+ if item.table:
247
+ table_bucket = _ensure_bucket(by_table, item.table)
248
+ table_bucket["total"] += 1
249
+ table_bucket["passed"] += 1
250
+
251
+ # 收集"失败检查项"的唯一标识(检查项粒度),与 passed_items 对齐。
252
+ # 标识 = (stage, check_type/error_type, table);同一检查在多行违规只算 1 个失败检查。
253
+ failed_check_keys: set[tuple[str, str, str]] = set()
254
+ for error_item in errors:
255
+ check_key = (
256
+ error_item.stage,
257
+ str(error_item.check_type or error_item.error_type or "Unknown"),
258
+ str(error_item.table or ""),
259
+ )
260
+ failed_check_keys.add(check_key)
261
+
262
+ type_bucket = _ensure_bucket(by_type, error_item.check_type or error_item.error_type)
263
+ type_bucket["total"] += 1
264
+ type_bucket["failed"] += 1
265
+ if error_item.table:
266
+ table_bucket = _ensure_bucket(by_table, error_item.table)
267
+ table_bucket["total"] += 1
268
+ table_bucket["failed"] += 1
269
+
270
+ passed_count = len(passed_items)
271
+ failed_count = len(failed_check_keys)
272
+ total_checks = passed_count + failed_count
273
+ pass_rate = 100.0 if total_checks == 0 else (passed_count / total_checks * 100)
274
+
275
+ return ValidationStatistics(
276
+ total_checks=total_checks,
277
+ passed_count=passed_count,
278
+ failed_count=failed_count,
279
+ pass_rate=pass_rate,
280
+ by_type=by_type,
281
+ by_table=by_table,
282
+ )
283
+
284
+ def _build_summary(
285
+ self,
286
+ result: dict[str, Any],
287
+ errors: list[FullValidationErrorItem],
288
+ *,
289
+ total_error_count_real: int | None = None,
290
+ ) -> FullValidationSummary:
291
+ """构造摘要统计。
292
+
293
+ 参数:
294
+ total_error_count_real: 真实错误总数(截断前)。回归 #10: 当 errors 被截断时,
295
+ len(errors) < 真实总数,total_error_count 必须用真实值让用户了解问题规模。
296
+ """
297
+ loading_error_count = len([e for e in errors if e.stage == "loading"])
298
+ format_error_count = len([e for e in errors if e.stage == "format"])
299
+ constraint_error_count = len([e for e in errors if e.stage == "constraint"])
300
+
301
+ raw_datasets = result.get("raw_datasets", {})
302
+ files_loaded = len(set(raw_datasets.keys()))
303
+
304
+ files_total = 0
305
+ if self.executor and self.executor.dataset_schema:
306
+ files_total = len(self.executor.dataset_schema.tables)
307
+
308
+ # total_error_count 用真实总数(截断前),供用户了解问题规模与前端显示"X/Y 条"。
309
+ total_for_summary = total_error_count_real if total_error_count_real is not None else len(errors)
310
+
311
+ return FullValidationSummary(
312
+ files_total=files_total,
313
+ files_loaded=files_loaded,
314
+ tables_loaded=len(raw_datasets),
315
+ loading_error_count=loading_error_count,
316
+ format_error_count=format_error_count,
317
+ constraint_error_count=constraint_error_count,
318
+ total_error_count=total_for_summary,
319
+ duration_ms=result.get("duration_ms", self._duration_ms()),
320
+ # C6: 透出遇错即停的中断状态(executor 在 result["interrupted"] 标记)
321
+ interrupted=bool(result.get("interrupted", False)),
322
+ # D6: 透出超时中断状态(executor 在 result["timeout_occurred"] 标记,原为死字段)
323
+ timed_out=bool(result.get("timeout_occurred", False)),
324
+ )
325
+
326
+ def build_from_result(self, result: dict[str, Any]) -> FullValidationResponse:
327
+ """
328
+ 基于 executor 成功返回的结果字典构建完整响应。
329
+
330
+ 参数:
331
+ result: ValidationExecutor.execute 返回的字典
332
+
333
+ 返回:
334
+ FullValidationResponse 实例
335
+ """
336
+ errors = self._build_errors(result)
337
+ id_to_name = self._build_id_to_name()
338
+ passed_items = self._build_passed_items(result, id_to_name)
339
+
340
+ # 统计必须基于完整错误列表(by_type/by_table 聚合 + failed_count 检查项粒度),
341
+ # 在截断之前计算。
342
+ statistics = self._build_statistics(errors, passed_items)
343
+
344
+ # 回归 #10: 错误列表截断保护。整列格式错/大规模约束违规时,百万级错误 dict 会让
345
+ # 响应数百 MB、前端卡死。这里在响应层截断 errors 到上限,但 total_error_count 保留
346
+ # 真实总数(供用户了解问题规模)、errors_truncated 标记供前端显示"显示 X/Y 条"。
347
+ total_error_count_real = len(errors)
348
+ truncated = total_error_count_real > self.MAX_ERRORS
349
+ if truncated:
350
+ errors = errors[: self.MAX_ERRORS]
351
+
352
+ summary = self._build_summary(result, errors, total_error_count_real=total_error_count_real)
353
+ summary.errors_truncated = truncated
354
+
355
+ return FullValidationResponse(
356
+ success=(total_error_count_real == 0),
357
+ summary=summary,
358
+ errors=errors,
359
+ passed_items=passed_items,
360
+ statistics=statistics,
361
+ error=None,
362
+ warnings=result.get("warnings", []),
363
+ coverage=self.coverage,
364
+ )
365
+
366
+ def build_error(
367
+ self,
368
+ error: Exception,
369
+ *,
370
+ files_total: int = 0,
371
+ ) -> FullValidationResponse:
372
+ """
373
+ 构建执行器初始化失败或执行异常时的降级响应。
374
+
375
+ 参数:
376
+ error: 异常对象
377
+ files_total: 扫描到的源文件总数
378
+
379
+ 返回:
380
+ FullValidationResponse 实例(success=False)
381
+ """
382
+ summary = FullValidationSummary(
383
+ files_total=files_total,
384
+ files_loaded=0,
385
+ tables_loaded=0,
386
+ loading_error_count=0,
387
+ format_error_count=0,
388
+ constraint_error_count=0,
389
+ total_error_count=0,
390
+ duration_ms=self._duration_ms(),
391
+ )
392
+
393
+ return FullValidationResponse(
394
+ success=False,
395
+ summary=summary,
396
+ errors=[],
397
+ passed_items=[],
398
+ statistics=ValidationStatistics(
399
+ total_checks=0,
400
+ passed_count=0,
401
+ failed_count=0,
402
+ # 回归 #9: 执行失败时 pass_rate 必须为 0.0 而非 100.0。原实现返回 100.0,
403
+ # 与 success=False 矛盾,前端会把崩溃的运行以"满分通过"写入历史趋势,
404
+ # 污染纵向对比(通过率趋势)。失败即 0% 通过。
405
+ pass_rate=0.0,
406
+ by_type={},
407
+ by_table={},
408
+ ),
409
+ error=str(error),
410
+ warnings=[],
411
+ coverage=self.coverage,
412
+ )
@@ -0,0 +1,41 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # Copyright 2026 Precis Team
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """
17
+ @fileoverview 文件读写异常的用户可读描述
18
+
19
+ 视图/工作区/报告配置等 JSON 文件的读写异常原文(OSError 英文)
20
+ 直接拼进 HTTP detail 会成为用户看不懂的提示。本模块把常见 OS
21
+ 错误翻译为中文原因短语,未识别的异常保留原文。
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+
27
+ def describe_io_error(exc: BaseException) -> str:
28
+ """把文件读写异常翻译成用户可读的原因短语。
29
+
30
+ Args:
31
+ exc: 读写文件时捕获的异常
32
+
33
+ Returns:
34
+ 中文原因短语;未识别的异常返回原文
35
+ """
36
+ if isinstance(exc, PermissionError):
37
+ return "文件正被其他程序占用,或没有读写权限"
38
+ if isinstance(exc, FileNotFoundError):
39
+ return "文件不存在,可能已被移动或删除"
40
+ text = str(exc).strip()
41
+ return text or "未知原因"
@@ -0,0 +1,248 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ #
3
+ # Copyright 2026 Precis Team
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ """
17
+ @fileoverview 预览数据处理服务模块
18
+
19
+ 功能概述:
20
+ - 提取 DataFrame 到二维列表的公共转换逻辑
21
+ - 封装文件类型检测
22
+ - 封装临时文件清理
23
+ - 提供基于路径和基于内容(上传)的统一预览接口
24
+
25
+ 输入示例:
26
+ data, sheets = preview_from_path("/path/to/file.xlsx", max_rows=100, max_cols=20)
27
+ data, sheets = preview_from_content(content, ".xlsx", max_rows=100, max_cols=20, sheet_name="Sheet2")
28
+
29
+ 输出示例:
30
+ ([["col1", "col2"], ["a", "b"]], ["Sheet1", "Sheet2"])
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import logging
36
+ import os
37
+ import tempfile
38
+ import time
39
+
40
+ import pandas as pd
41
+ from fastapi import HTTPException
42
+
43
+ from app.shared.core.data_source.loaders.excel_loader import get_excel_sheet_names
44
+ from app.shared.services.preview.loader import load_preview_data
45
+
46
+ logger = logging.getLogger(__name__)
47
+
48
+
49
+ def detect_file_type(file_ext: str) -> str:
50
+ """根据文件扩展名判断文件类型。
51
+
52
+ Args:
53
+ file_ext: 文件扩展名(包含点号,如 .xlsx)
54
+
55
+ Returns:
56
+ "excel" | "csv" | "json"
57
+
58
+ Raises:
59
+ HTTPException: 不支持的文件类型
60
+ """
61
+ if file_ext in [".xlsx", ".xls"]:
62
+ return "excel"
63
+ if file_ext == ".csv":
64
+ return "csv"
65
+ if file_ext in [".json", ".jsonl", ".ndjson"]:
66
+ return "json"
67
+ raise HTTPException(status_code=400, detail=f"不支持的文件类型: {file_ext}")
68
+
69
+
70
+ def df_to_list(df: pd.DataFrame, max_cols: int) -> list[list[str]]:
71
+ """将 DataFrame 转为二维字符串列表,限制列数。
72
+
73
+ Args:
74
+ df: pandas DataFrame
75
+ max_cols: 最大列数
76
+
77
+ Returns:
78
+ 二维字符串列表
79
+ """
80
+ data: list[list[str]] = []
81
+ for _, row in df.iterrows():
82
+ row_data = [str(cell) for cell in row.tolist()[:max_cols]]
83
+ data.append(row_data)
84
+ return data
85
+
86
+
87
+ def cleanup_temp_file(tmp_path: str) -> None:
88
+ """清理临时文件,最多重试 3 次。
89
+
90
+ Args:
91
+ tmp_path: 临时文件路径
92
+ """
93
+ if not os.path.exists(tmp_path):
94
+ return
95
+ for _attempt in range(3):
96
+ try:
97
+ os.unlink(tmp_path)
98
+ logger.info(f"[PREVIEW] 已删除临时文件: {tmp_path}")
99
+ break
100
+ except PermissionError:
101
+ time.sleep(0.1)
102
+ except Exception as e:
103
+ logger.info(f"[PREVIEW] 无法删除临时文件: {e}")
104
+ break
105
+
106
+
107
+ def preview_from_path(
108
+ file_path: str,
109
+ max_rows: int,
110
+ max_cols: int,
111
+ sheet_name: str | None = None,
112
+ ) -> tuple[list[list[str]], str, int, list[str] | None, str | None]:
113
+ """基于文件路径加载预览数据。
114
+
115
+ Args:
116
+ file_path: 文件路径
117
+ max_rows: 最大行数
118
+ max_cols: 最大列数
119
+ sheet_name: 指定工作表名称(仅 Excel)
120
+
121
+ Returns:
122
+ (data, file_type, total_rows, sheets, current_sheet)
123
+
124
+ Raises:
125
+ HTTPException: 文件不存在或读取失败
126
+ """
127
+ if not os.path.exists(file_path):
128
+ raise HTTPException(status_code=404, detail=f"文件未找到: {file_path}")
129
+
130
+ file_ext = os.path.splitext(file_path)[1].lower()
131
+ file_type = detect_file_type(file_ext)
132
+
133
+ if file_type == "excel":
134
+ try:
135
+ if sheet_name is None:
136
+ # 引擎按扩展名自适应(.xls→xlrd)、句柄在助手内确保关闭
137
+ sheets = get_excel_sheet_names(file_path)
138
+ sheet_name = sheets[0] if sheets else "Sheet1"
139
+
140
+ df, sheet_names = load_preview_data(
141
+ file_path=file_path,
142
+ file_type="excel",
143
+ max_rows=max_rows,
144
+ sheet_name=sheet_name,
145
+ )
146
+ data = df_to_list(df, max_cols)
147
+ return data, file_type, len(df), sheet_names, sheet_name
148
+ except ValueError as e:
149
+ if "Worksheet" in str(e):
150
+ raise HTTPException(status_code=404, detail=f"工作表 '{sheet_name}' 不存在") from e
151
+ raise HTTPException(status_code=500, detail=f"读取工作表失败: {str(e)}") from e
152
+ except Exception as e:
153
+ raise HTTPException(status_code=500, detail=f"读取Excel文件失败: {str(e)}") from e
154
+
155
+ # JSON
156
+ if file_type == "json":
157
+ try:
158
+ df, _ = load_preview_data(file_path=file_path, file_type="json", max_rows=max_rows)
159
+ data = df_to_list(df, max_cols)
160
+ return data, file_type, len(df), None, None
161
+ except Exception as e:
162
+ raise HTTPException(status_code=500, detail=f"读取JSON文件失败: {str(e)}") from e
163
+
164
+ # CSV
165
+ try:
166
+ df, _ = load_preview_data(file_path=file_path, file_type="csv", max_rows=max_rows)
167
+ data = df_to_list(df, max_cols)
168
+ return data, file_type, len(df), None, None
169
+ except Exception as e:
170
+ raise HTTPException(status_code=500, detail=f"读取CSV文件失败: {str(e)}") from e
171
+
172
+
173
+ def preview_from_content(
174
+ content: bytes,
175
+ file_ext: str,
176
+ max_rows: int,
177
+ max_cols: int,
178
+ sheet_name: str | None = None,
179
+ ) -> tuple[list[list[str]], str, int, list[str] | None, str | None]:
180
+ """基于文件内容(上传)加载预览数据。
181
+
182
+ 将内容写入临时文件后解析,处理完成后自动清理。
183
+
184
+ Args:
185
+ content: 文件二进制内容
186
+ file_ext: 文件扩展名
187
+ max_rows: 最大行数
188
+ max_cols: 最大列数
189
+ sheet_name: 指定工作表名称(仅 Excel)
190
+
191
+ Returns:
192
+ (data, file_type, total_rows, sheets, current_sheet)
193
+
194
+ Raises:
195
+ HTTPException: 内容为空或读取失败
196
+ """
197
+ if len(content) == 0:
198
+ raise HTTPException(status_code=400, detail="文件内容为空")
199
+
200
+ file_type = detect_file_type(file_ext)
201
+ tmp_path = ""
202
+
203
+ try:
204
+ with tempfile.NamedTemporaryFile(suffix=file_ext, delete=False) as tmp:
205
+ tmp.write(content)
206
+ tmp_path = tmp.name
207
+
208
+ if file_type == "excel":
209
+ try:
210
+ # 引擎按扩展名自适应(.xls→xlrd)、句柄在助手内确保关闭
211
+ sheets = get_excel_sheet_names(tmp_path)
212
+ actual_sheet = sheet_name or (sheets[0] if sheets else "Sheet1")
213
+
214
+ df, sheet_names = load_preview_data(
215
+ file_path=tmp_path,
216
+ file_type="excel",
217
+ max_rows=max_rows,
218
+ sheet_name=actual_sheet,
219
+ )
220
+ data = df_to_list(df, max_cols)
221
+ return data, file_type, len(df), sheet_names, actual_sheet
222
+ except Exception as e:
223
+ raise HTTPException(status_code=500, detail=f"读取Excel文件失败: {str(e)}") from e
224
+ finally:
225
+ import gc
226
+
227
+ gc.collect()
228
+
229
+ # JSON
230
+ if file_type == "json":
231
+ try:
232
+ df, _ = load_preview_data(file_path=tmp_path, file_type="json", max_rows=max_rows)
233
+ data = df_to_list(df, max_cols)
234
+ return data, file_type, len(df), None, None
235
+ except Exception as e:
236
+ raise HTTPException(status_code=500, detail=f"读取JSON文件失败: {str(e)}") from e
237
+
238
+ # CSV
239
+ try:
240
+ df, _ = load_preview_data(file_path=tmp_path, file_type="csv", max_rows=max_rows)
241
+ data = df_to_list(df, max_cols)
242
+ return data, file_type, len(df), None, None
243
+ except Exception as e:
244
+ raise HTTPException(status_code=500, detail=f"读取CSV文件失败: {str(e)}") from e
245
+
246
+ finally:
247
+ if tmp_path:
248
+ cleanup_temp_file(tmp_path)