spectra-cli 4.2.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (365) hide show
  1. package/README.md +12 -5
  2. package/dist/.spectra-build-meta.json +9 -0
  3. package/dist/adapters/language-adapter.d.ts +7 -12
  4. package/dist/adapters/language-adapter.d.ts.map +1 -1
  5. package/dist/adapters/python-adapter.d.ts.map +1 -1
  6. package/dist/adapters/python-adapter.js +11 -2
  7. package/dist/adapters/python-adapter.js.map +1 -1
  8. package/dist/batch/batch-orchestrator.d.ts +44 -1
  9. package/dist/batch/batch-orchestrator.d.ts.map +1 -1
  10. package/dist/batch/batch-orchestrator.js +312 -76
  11. package/dist/batch/batch-orchestrator.js.map +1 -1
  12. package/dist/batch/delta-regenerator.d.ts.map +1 -1
  13. package/dist/batch/delta-regenerator.js +21 -28
  14. package/dist/batch/delta-regenerator.js.map +1 -1
  15. package/dist/batch/module-grouper.d.ts +7 -0
  16. package/dist/batch/module-grouper.d.ts.map +1 -1
  17. package/dist/batch/module-grouper.js +2 -0
  18. package/dist/batch/module-grouper.js.map +1 -1
  19. package/dist/batch/regen-plan.d.ts +78 -0
  20. package/dist/batch/regen-plan.d.ts.map +1 -0
  21. package/dist/batch/regen-plan.js +93 -0
  22. package/dist/batch/regen-plan.js.map +1 -0
  23. package/dist/cli/commands/batch.d.ts.map +1 -1
  24. package/dist/cli/commands/batch.js +36 -8
  25. package/dist/cli/commands/batch.js.map +1 -1
  26. package/dist/cli/commands/scaffold-kb.d.ts +9 -0
  27. package/dist/cli/commands/scaffold-kb.d.ts.map +1 -0
  28. package/dist/cli/commands/scaffold-kb.js +179 -0
  29. package/dist/cli/commands/scaffold-kb.js.map +1 -0
  30. package/dist/cli/index.js +24 -5
  31. package/dist/cli/index.js.map +1 -1
  32. package/dist/cli/utils/error-handler.d.ts +0 -6
  33. package/dist/cli/utils/error-handler.d.ts.map +1 -1
  34. package/dist/cli/utils/error-handler.js +0 -13
  35. package/dist/cli/utils/error-handler.js.map +1 -1
  36. package/dist/cli/utils/parse-args.d.ts +43 -3
  37. package/dist/cli/utils/parse-args.d.ts.map +1 -1
  38. package/dist/cli/utils/parse-args.js +98 -12
  39. package/dist/cli/utils/parse-args.js.map +1 -1
  40. package/dist/cli/version-meta.d.ts +16 -0
  41. package/dist/cli/version-meta.d.ts.map +1 -0
  42. package/dist/cli/version-meta.js +28 -0
  43. package/dist/cli/version-meta.js.map +1 -0
  44. package/dist/core/ast-analyzer.d.ts.map +1 -1
  45. package/dist/core/ast-analyzer.js +7 -11
  46. package/dist/core/ast-analyzer.js.map +1 -1
  47. package/dist/core/import-resolver.d.ts +70 -67
  48. package/dist/core/import-resolver.d.ts.map +1 -1
  49. package/dist/core/import-resolver.js +341 -200
  50. package/dist/core/import-resolver.js.map +1 -1
  51. package/dist/core/single-spec-orchestrator.d.ts +21 -0
  52. package/dist/core/single-spec-orchestrator.d.ts.map +1 -1
  53. package/dist/core/single-spec-orchestrator.js +41 -14
  54. package/dist/core/single-spec-orchestrator.js.map +1 -1
  55. package/dist/core/skeleton-hash.d.ts +33 -0
  56. package/dist/core/skeleton-hash.d.ts.map +1 -0
  57. package/dist/core/skeleton-hash.js +76 -0
  58. package/dist/core/skeleton-hash.js.map +1 -0
  59. package/dist/core/tree-sitter-analyzer.d.ts +3 -2
  60. package/dist/core/tree-sitter-analyzer.d.ts.map +1 -1
  61. package/dist/core/tree-sitter-analyzer.js +5 -6
  62. package/dist/core/tree-sitter-analyzer.js.map +1 -1
  63. package/dist/core/tree-sitter-fallback.d.ts +7 -2
  64. package/dist/core/tree-sitter-fallback.d.ts.map +1 -1
  65. package/dist/core/tree-sitter-fallback.js +18 -15
  66. package/dist/core/tree-sitter-fallback.js.map +1 -1
  67. package/dist/extraction/image-extractor.d.ts.map +1 -1
  68. package/dist/extraction/image-extractor.js +3 -1
  69. package/dist/extraction/image-extractor.js.map +1 -1
  70. package/dist/extraction/markdown-extractor.d.ts.map +1 -1
  71. package/dist/extraction/markdown-extractor.js +3 -1
  72. package/dist/extraction/markdown-extractor.js.map +1 -1
  73. package/dist/generator/frontmatter.d.ts +5 -0
  74. package/dist/generator/frontmatter.d.ts.map +1 -1
  75. package/dist/generator/frontmatter.js +4 -0
  76. package/dist/generator/frontmatter.js.map +1 -1
  77. package/dist/kb-mcp/index.d.ts +14 -0
  78. package/dist/kb-mcp/index.d.ts.map +1 -0
  79. package/dist/kb-mcp/index.js +24 -0
  80. package/dist/kb-mcp/index.js.map +1 -0
  81. package/dist/kb-mcp/lib/kb-error.d.ts +21 -0
  82. package/dist/kb-mcp/lib/kb-error.d.ts.map +1 -0
  83. package/dist/kb-mcp/lib/kb-error.js +28 -0
  84. package/dist/kb-mcp/lib/kb-error.js.map +1 -0
  85. package/dist/kb-mcp/lib/kb-locator.d.ts +58 -0
  86. package/dist/kb-mcp/lib/kb-locator.d.ts.map +1 -0
  87. package/dist/kb-mcp/lib/kb-locator.js +85 -0
  88. package/dist/kb-mcp/lib/kb-locator.js.map +1 -0
  89. package/dist/kb-mcp/lib/result-merger.d.ts +35 -0
  90. package/dist/kb-mcp/lib/result-merger.d.ts.map +1 -0
  91. package/dist/kb-mcp/lib/result-merger.js +64 -0
  92. package/dist/kb-mcp/lib/result-merger.js.map +1 -0
  93. package/dist/kb-mcp/server.d.ts +12 -0
  94. package/dist/kb-mcp/server.d.ts.map +1 -0
  95. package/dist/kb-mcp/server.js +29 -0
  96. package/dist/kb-mcp/server.js.map +1 -0
  97. package/dist/kb-mcp/tools/kb-api-lookup.d.ts +28 -0
  98. package/dist/kb-mcp/tools/kb-api-lookup.d.ts.map +1 -0
  99. package/dist/kb-mcp/tools/kb-api-lookup.js +237 -0
  100. package/dist/kb-mcp/tools/kb-api-lookup.js.map +1 -0
  101. package/dist/kb-mcp/tools/kb-doc-lookup.d.ts +20 -0
  102. package/dist/kb-mcp/tools/kb-doc-lookup.d.ts.map +1 -0
  103. package/dist/kb-mcp/tools/kb-doc-lookup.js +92 -0
  104. package/dist/kb-mcp/tools/kb-doc-lookup.js.map +1 -0
  105. package/dist/kb-mcp/tools/kb-search.d.ts +23 -0
  106. package/dist/kb-mcp/tools/kb-search.d.ts.map +1 -0
  107. package/dist/kb-mcp/tools/kb-search.js +126 -0
  108. package/dist/kb-mcp/tools/kb-search.js.map +1 -0
  109. package/dist/knowledge-graph/import-resolver.d.ts +11 -86
  110. package/dist/knowledge-graph/import-resolver.d.ts.map +1 -1
  111. package/dist/knowledge-graph/import-resolver.js +10 -333
  112. package/dist/knowledge-graph/import-resolver.js.map +1 -1
  113. package/dist/knowledge-graph/incremental.d.ts +25 -4
  114. package/dist/knowledge-graph/incremental.d.ts.map +1 -1
  115. package/dist/knowledge-graph/incremental.js +37 -15
  116. package/dist/knowledge-graph/incremental.js.map +1 -1
  117. package/dist/knowledge-graph/index.d.ts +11 -0
  118. package/dist/knowledge-graph/index.d.ts.map +1 -1
  119. package/dist/knowledge-graph/index.js +43 -1
  120. package/dist/knowledge-graph/index.js.map +1 -1
  121. package/dist/knowledge-graph/module-derivation.d.ts +14 -0
  122. package/dist/knowledge-graph/module-derivation.d.ts.map +1 -1
  123. package/dist/knowledge-graph/module-derivation.js +44 -44
  124. package/dist/knowledge-graph/module-derivation.js.map +1 -1
  125. package/dist/knowledge-graph/persistence.d.ts +56 -16
  126. package/dist/knowledge-graph/persistence.d.ts.map +1 -1
  127. package/dist/knowledge-graph/persistence.js +86 -32
  128. package/dist/knowledge-graph/persistence.js.map +1 -1
  129. package/dist/knowledge-graph/query-helpers.d.ts +39 -12
  130. package/dist/knowledge-graph/query-helpers.d.ts.map +1 -1
  131. package/dist/knowledge-graph/query-helpers.js +199 -34
  132. package/dist/knowledge-graph/query-helpers.js.map +1 -1
  133. package/dist/knowledge-graph/relativize.d.ts +51 -0
  134. package/dist/knowledge-graph/relativize.d.ts.map +1 -0
  135. package/dist/knowledge-graph/relativize.js +104 -0
  136. package/dist/knowledge-graph/relativize.js.map +1 -0
  137. package/dist/mcp/agent-context-tools.d.ts +3 -54
  138. package/dist/mcp/agent-context-tools.d.ts.map +1 -1
  139. package/dist/mcp/agent-context-tools.js +141 -189
  140. package/dist/mcp/agent-context-tools.js.map +1 -1
  141. package/dist/mcp/file-nav-tools.d.ts +42 -0
  142. package/dist/mcp/file-nav-tools.d.ts.map +1 -0
  143. package/dist/mcp/file-nav-tools.js +318 -0
  144. package/dist/mcp/file-nav-tools.js.map +1 -0
  145. package/dist/mcp/graph-tools.d.ts +2 -0
  146. package/dist/mcp/graph-tools.d.ts.map +1 -1
  147. package/dist/mcp/graph-tools.js +134 -123
  148. package/dist/mcp/graph-tools.js.map +1 -1
  149. package/dist/mcp/lib/file-nav-helpers.d.ts +128 -0
  150. package/dist/mcp/lib/file-nav-helpers.d.ts.map +1 -0
  151. package/dist/mcp/lib/file-nav-helpers.js +305 -0
  152. package/dist/mcp/lib/file-nav-helpers.js.map +1 -0
  153. package/dist/mcp/lib/telemetry.d.ts +69 -0
  154. package/dist/mcp/lib/telemetry.d.ts.map +1 -0
  155. package/dist/mcp/lib/telemetry.js +113 -0
  156. package/dist/mcp/lib/telemetry.js.map +1 -0
  157. package/dist/mcp/lib/tool-response.d.ts +46 -0
  158. package/dist/mcp/lib/tool-response.d.ts.map +1 -0
  159. package/dist/mcp/lib/tool-response.js +83 -0
  160. package/dist/mcp/lib/tool-response.js.map +1 -0
  161. package/dist/mcp/server.d.ts +13 -2
  162. package/dist/mcp/server.d.ts.map +1 -1
  163. package/dist/mcp/server.js +209 -150
  164. package/dist/mcp/server.js.map +1 -1
  165. package/dist/models/code-skeleton.d.ts +4 -2
  166. package/dist/models/code-skeleton.d.ts.map +1 -1
  167. package/dist/models/code-skeleton.js +2 -1
  168. package/dist/models/code-skeleton.js.map +1 -1
  169. package/dist/models/module-spec.d.ts +18 -0
  170. package/dist/models/module-spec.d.ts.map +1 -1
  171. package/dist/models/module-spec.js +6 -0
  172. package/dist/models/module-spec.js.map +1 -1
  173. package/dist/panoramic/builders/doc-graph-builder.d.ts +13 -0
  174. package/dist/panoramic/builders/doc-graph-builder.d.ts.map +1 -1
  175. package/dist/panoramic/builders/doc-graph-builder.js +20 -0
  176. package/dist/panoramic/builders/doc-graph-builder.js.map +1 -1
  177. package/dist/panoramic/graph/graph-builder.d.ts +72 -2
  178. package/dist/panoramic/graph/graph-builder.d.ts.map +1 -1
  179. package/dist/panoramic/graph/graph-builder.js +257 -51
  180. package/dist/panoramic/graph/graph-builder.js.map +1 -1
  181. package/dist/panoramic/graph/graph-query.d.ts +16 -0
  182. package/dist/panoramic/graph/graph-query.d.ts.map +1 -1
  183. package/dist/panoramic/graph/graph-query.js +49 -1
  184. package/dist/panoramic/graph/graph-query.js.map +1 -1
  185. package/dist/panoramic/graph/index.d.ts +2 -1
  186. package/dist/panoramic/graph/index.d.ts.map +1 -1
  187. package/dist/panoramic/graph/index.js +1 -1
  188. package/dist/panoramic/graph/index.js.map +1 -1
  189. package/dist/panoramic/pipelines/adr-evidence-verifier.d.ts.map +1 -1
  190. package/dist/panoramic/pipelines/adr-evidence-verifier.js +1 -29
  191. package/dist/panoramic/pipelines/adr-evidence-verifier.js.map +1 -1
  192. package/dist/panoramic/query.d.ts +1 -0
  193. package/dist/panoramic/query.d.ts.map +1 -1
  194. package/dist/panoramic/query.js +5 -0
  195. package/dist/panoramic/query.js.map +1 -1
  196. package/dist/scaffold-kb/api-entities-serializer.d.ts +12 -0
  197. package/dist/scaffold-kb/api-entities-serializer.d.ts.map +1 -0
  198. package/dist/scaffold-kb/api-entities-serializer.js +144 -0
  199. package/dist/scaffold-kb/api-entities-serializer.js.map +1 -0
  200. package/dist/scaffold-kb/arbitration.d.ts +44 -0
  201. package/dist/scaffold-kb/arbitration.d.ts.map +1 -0
  202. package/dist/scaffold-kb/arbitration.js +144 -0
  203. package/dist/scaffold-kb/arbitration.js.map +1 -0
  204. package/dist/scaffold-kb/chunk-splitter.d.ts +20 -0
  205. package/dist/scaffold-kb/chunk-splitter.d.ts.map +1 -0
  206. package/dist/scaffold-kb/chunk-splitter.js +309 -0
  207. package/dist/scaffold-kb/chunk-splitter.js.map +1 -0
  208. package/dist/scaffold-kb/doc-graph-builder.d.ts +26 -0
  209. package/dist/scaffold-kb/doc-graph-builder.d.ts.map +1 -0
  210. package/dist/scaffold-kb/doc-graph-builder.js +72 -0
  211. package/dist/scaffold-kb/doc-graph-builder.js.map +1 -0
  212. package/dist/scaffold-kb/entity-extractor.d.ts +40 -0
  213. package/dist/scaffold-kb/entity-extractor.d.ts.map +1 -0
  214. package/dist/scaffold-kb/entity-extractor.js +0 -0
  215. package/dist/scaffold-kb/entity-extractor.js.map +1 -0
  216. package/dist/scaffold-kb/entity-heuristic.d.ts +14 -0
  217. package/dist/scaffold-kb/entity-heuristic.d.ts.map +1 -0
  218. package/dist/scaffold-kb/entity-heuristic.js +107 -0
  219. package/dist/scaffold-kb/entity-heuristic.js.map +1 -0
  220. package/dist/scaffold-kb/entity-matcher.d.ts +20 -0
  221. package/dist/scaffold-kb/entity-matcher.d.ts.map +1 -0
  222. package/dist/scaffold-kb/entity-matcher.js +53 -0
  223. package/dist/scaffold-kb/entity-matcher.js.map +1 -0
  224. package/dist/scaffold-kb/entity-util.d.ts +11 -0
  225. package/dist/scaffold-kb/entity-util.d.ts.map +1 -0
  226. package/dist/scaffold-kb/entity-util.js +23 -0
  227. package/dist/scaffold-kb/entity-util.js.map +1 -0
  228. package/dist/scaffold-kb/evidence-envelope.d.ts +17 -0
  229. package/dist/scaffold-kb/evidence-envelope.d.ts.map +1 -0
  230. package/dist/scaffold-kb/evidence-envelope.js +32 -0
  231. package/dist/scaffold-kb/evidence-envelope.js.map +1 -0
  232. package/dist/scaffold-kb/index.d.ts +20 -0
  233. package/dist/scaffold-kb/index.d.ts.map +1 -0
  234. package/dist/scaffold-kb/index.js +102 -0
  235. package/dist/scaffold-kb/index.js.map +1 -0
  236. package/dist/scaffold-kb/ingest/ingest-core.d.ts +47 -0
  237. package/dist/scaffold-kb/ingest/ingest-core.d.ts.map +1 -0
  238. package/dist/scaffold-kb/ingest/ingest-core.js +231 -0
  239. package/dist/scaffold-kb/ingest/ingest-core.js.map +1 -0
  240. package/dist/scaffold-kb/ingest/office-parser.d.ts +33 -0
  241. package/dist/scaffold-kb/ingest/office-parser.d.ts.map +1 -0
  242. package/dist/scaffold-kb/ingest/office-parser.js +187 -0
  243. package/dist/scaffold-kb/ingest/office-parser.js.map +1 -0
  244. package/dist/scaffold-kb/ingest/url-fetcher.d.ts +43 -0
  245. package/dist/scaffold-kb/ingest/url-fetcher.d.ts.map +1 -0
  246. package/dist/scaffold-kb/ingest/url-fetcher.js +245 -0
  247. package/dist/scaffold-kb/ingest/url-fetcher.js.map +1 -0
  248. package/dist/scaffold-kb/ingester.d.ts +38 -0
  249. package/dist/scaffold-kb/ingester.d.ts.map +1 -0
  250. package/dist/scaffold-kb/ingester.js +261 -0
  251. package/dist/scaffold-kb/ingester.js.map +1 -0
  252. package/dist/scaffold-kb/injection-format.d.ts +25 -0
  253. package/dist/scaffold-kb/injection-format.d.ts.map +1 -0
  254. package/dist/scaffold-kb/injection-format.js +55 -0
  255. package/dist/scaffold-kb/injection-format.js.map +1 -0
  256. package/dist/scaffold-kb/kb-writer.d.ts +13 -0
  257. package/dist/scaffold-kb/kb-writer.d.ts.map +1 -0
  258. package/dist/scaffold-kb/kb-writer.js +69 -0
  259. package/dist/scaffold-kb/kb-writer.js.map +1 -0
  260. package/dist/scaffold-kb/keyword-extract.d.ts +16 -0
  261. package/dist/scaffold-kb/keyword-extract.d.ts.map +1 -0
  262. package/dist/scaffold-kb/keyword-extract.js +49 -0
  263. package/dist/scaffold-kb/keyword-extract.js.map +1 -0
  264. package/dist/scaffold-kb/query-sanitizer.d.ts +28 -0
  265. package/dist/scaffold-kb/query-sanitizer.d.ts.map +1 -0
  266. package/dist/scaffold-kb/query-sanitizer.js +38 -0
  267. package/dist/scaffold-kb/query-sanitizer.js.map +1 -0
  268. package/dist/scaffold-kb/recall-eval.d.ts +48 -0
  269. package/dist/scaffold-kb/recall-eval.d.ts.map +1 -0
  270. package/dist/scaffold-kb/recall-eval.js +55 -0
  271. package/dist/scaffold-kb/recall-eval.js.map +1 -0
  272. package/dist/scaffold-kb/schema-compat.d.ts +22 -0
  273. package/dist/scaffold-kb/schema-compat.d.ts.map +1 -0
  274. package/dist/scaffold-kb/schema-compat.js +32 -0
  275. package/dist/scaffold-kb/schema-compat.js.map +1 -0
  276. package/dist/scaffold-kb/search-core.d.ts +42 -0
  277. package/dist/scaffold-kb/search-core.d.ts.map +1 -0
  278. package/dist/scaffold-kb/search-core.js +89 -0
  279. package/dist/scaffold-kb/search-core.js.map +1 -0
  280. package/dist/scaffold-kb/sqlite-engine.d.ts +58 -0
  281. package/dist/scaffold-kb/sqlite-engine.d.ts.map +1 -0
  282. package/dist/scaffold-kb/sqlite-engine.js +61 -0
  283. package/dist/scaffold-kb/sqlite-engine.js.map +1 -0
  284. package/dist/scaffold-kb/sqlite-writer.d.ts +18 -0
  285. package/dist/scaffold-kb/sqlite-writer.d.ts.map +1 -0
  286. package/dist/scaffold-kb/sqlite-writer.js +76 -0
  287. package/dist/scaffold-kb/sqlite-writer.js.map +1 -0
  288. package/dist/scaffold-kb/tokenizer.d.ts +26 -0
  289. package/dist/scaffold-kb/tokenizer.d.ts.map +1 -0
  290. package/dist/scaffold-kb/tokenizer.js +97 -0
  291. package/dist/scaffold-kb/tokenizer.js.map +1 -0
  292. package/dist/scaffold-kb/types.d.ts +157 -0
  293. package/dist/scaffold-kb/types.d.ts.map +1 -0
  294. package/dist/scaffold-kb/types.js +5 -0
  295. package/dist/scaffold-kb/types.js.map +1 -0
  296. package/dist/utils/file-scanner.d.ts +15 -0
  297. package/dist/utils/file-scanner.d.ts.map +1 -1
  298. package/dist/utils/file-scanner.js +17 -0
  299. package/dist/utils/file-scanner.js.map +1 -1
  300. package/dist/utils/string-distance.d.ts +20 -0
  301. package/dist/utils/string-distance.d.ts.map +1 -0
  302. package/dist/utils/string-distance.js +38 -0
  303. package/dist/utils/string-distance.js.map +1 -0
  304. package/package.json +11 -2
  305. package/plugins/demo-kb-en/.claude-plugin/plugin.json +11 -0
  306. package/plugins/demo-kb-en/.mcp.json +8 -0
  307. package/plugins/demo-kb-en/FIXTURE.json +23 -0
  308. package/plugins/demo-kb-en/FIXTURE.md +53 -0
  309. package/plugins/demo-kb-en/ingest-samples/meeting-notes.md +13 -0
  310. package/plugins/demo-kb-en/kb/api-entities.json +3445 -0
  311. package/plugins/demo-kb-en/kb/chunks.sqlite +0 -0
  312. package/plugins/demo-kb-en/kb/doc-graph.json +113 -0
  313. package/plugins/demo-kb-zh/.claude-plugin/plugin.json +11 -0
  314. package/plugins/demo-kb-zh/.mcp.json +8 -0
  315. package/plugins/demo-kb-zh/FIXTURE.json +23 -0
  316. package/plugins/demo-kb-zh/FIXTURE.md +63 -0
  317. package/plugins/demo-kb-zh/ingest-samples/meeting-notes.md +13 -0
  318. package/plugins/demo-kb-zh/kb/api-entities.json +990 -0
  319. package/plugins/demo-kb-zh/kb/chunks.sqlite +0 -0
  320. package/plugins/demo-kb-zh/kb/doc-graph.json +113 -0
  321. package/plugins/spec-driver/.claude-plugin/plugin.json +1 -1
  322. package/plugins/spec-driver/README.md +33 -1
  323. package/plugins/spec-driver/agents/verify.md +88 -0
  324. package/plugins/spec-driver/config/orchestration.yaml +39 -31
  325. package/plugins/spec-driver/contracts/orchestration-overrides-contract.yaml +27 -0
  326. package/plugins/spec-driver/contracts/orchestration-schema.mjs +272 -227
  327. package/plugins/spec-driver/contracts/wrapper-source-of-truth.yaml +3 -0
  328. package/plugins/spec-driver/hooks/hooks.json +9 -0
  329. package/plugins/spec-driver/hooks/stop-fix-compliance-check.sh +36 -0
  330. package/plugins/spec-driver/lib/delegation-contract.mjs +84 -0
  331. package/plugins/spec-driver/lib/orchestration-resolver.mjs +49 -0
  332. package/plugins/spec-driver/lib/orchestrator.mjs +29 -1
  333. package/plugins/spec-driver/scripts/codex-skills.sh +28 -28
  334. package/plugins/spec-driver/scripts/dev/spike-fix-compliance-e2e.mjs +130 -0
  335. package/plugins/spec-driver/scripts/fix-compliance-judge.mjs +415 -0
  336. package/plugins/spec-driver/scripts/generate-adoption-insights.mjs +48 -1
  337. package/plugins/spec-driver/scripts/goal-loop-cli.mjs +307 -0
  338. package/plugins/spec-driver/scripts/init-project.sh +56 -0
  339. package/plugins/spec-driver/scripts/kb-prequery.mjs +125 -0
  340. package/plugins/spec-driver/scripts/lib/config-schema.mjs +200 -94
  341. package/plugins/spec-driver/scripts/lib/ensure-gitignore.sh +277 -0
  342. package/plugins/spec-driver/scripts/lib/extract-wrapper-body.mjs +136 -0
  343. package/plugins/spec-driver/scripts/lib/fix-compliance-core.mjs +434 -0
  344. package/plugins/spec-driver/scripts/lib/fix-compliance-io.mjs +335 -0
  345. package/plugins/spec-driver/scripts/lib/goal-loop-core.mjs +783 -0
  346. package/plugins/spec-driver/scripts/lib/init-project-output.sh +28 -0
  347. package/plugins/spec-driver/scripts/lib/load-zod.mjs +62 -0
  348. package/plugins/spec-driver/scripts/lib/project-profile-resolver.mjs +141 -49
  349. package/plugins/spec-driver/scripts/lib/project-profile-schema.mjs +88 -54
  350. package/plugins/spec-driver/scripts/postinstall.sh +13 -1
  351. package/plugins/spec-driver/scripts/record-workflow-run.mjs +75 -0
  352. package/plugins/spec-driver/scripts/sync-delegation-contract.mjs +187 -0
  353. package/plugins/spec-driver/scripts/validate-orchestrator-models.mjs +156 -0
  354. package/plugins/spec-driver/scripts/validate-wrapper-sources.mjs +46 -1
  355. package/plugins/spec-driver/skills/spec-driver-feature/SKILL.md +292 -0
  356. package/plugins/spec-driver/skills/spec-driver-fix/SKILL.md +87 -8
  357. package/plugins/spec-driver/skills/spec-driver-implement/SKILL.md +8 -0
  358. package/plugins/spec-driver/skills/spec-driver-resume/SKILL.md +12 -1
  359. package/plugins/spec-driver/skills/spec-driver-story/SKILL.md +20 -0
  360. package/plugins/spec-driver/templates/delegation-contract.md +27 -0
  361. package/plugins/spec-driver/templates/goal-loop-override-template.yaml +284 -0
  362. package/plugins/spec-driver/templates/specify-base/project-context-template.yaml +10 -0
  363. package/plugins/spectra/.claude-plugin/plugin.json +1 -1
  364. package/plugins/spectra/README.md +1 -1
  365. package/templates/module-spec.hbs +1 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ingest-core.js","sourceRoot":"","sources":["../../../src/scaffold-kb/ingest/ingest-core.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAI3C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC5E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAC7F,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAEzE,MAAM,OAAO,WAAY,SAAQ,KAAK;IACpC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF;AA+BD,MAAM,GAAG,GAAG,iBAAiB,CAAC;AAC9B,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACtC,CAAC;AACD,SAAS,OAAO,CAAC,EAAU;IACzB,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AACD,SAAS,OAAO,CAAC,GAAW;IAC1B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,WAAW,CACxB,MAAoB,EACpB,QAA6B;IAE7B,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7B,OAAO;YACL,GAAG,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE;YAC/G,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,QAAQ;SACjB,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,eAAe,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACxF,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,IAAI,GAAqB,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAE,UAAU,GAAG,EAAuB,CAAC;QACrG,OAAO;YACL,GAAG,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE;YACvG,IAAI;YACJ,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IACD,wBAAwB;IACxB,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO;QACL,GAAG,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE;QACvG,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,EAAE;KACX,CAAC;AACJ,CAAC;AAUD;;;;GAIG;AACH,KAAK,UAAU,YAAY,CAAC,aAAqB;IAC/C,MAAM,KAAK,GAAe,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACvF,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IACxD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,OAAO;IAElD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,eAAe,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,wBAAwB,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CACpB,EAAE,EACF;+FACyF,IAAI;4EACvB,EACtE,EAAE,CACH,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,EAAE,GAAG,CAAC,CAAU,EAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzF,IAAI,CAAC,IAAI,CAAC;gBACR,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1F,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtC,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aACrG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,0DAA0D;QAC1D,MAAM,IAAI,WAAW,CACnB,mDAAoD,CAAW,CAAC,OAAO,EAAE,CAC1E,CAAC;IACJ,CAAC;IAED,oCAAoC;IACpC,IAAI,QAAQ,GAAgB,EAAE,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAC;QACzD,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7E,IAAI,CAAC;gBAAE,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,GAAG,EAAE,CAAC;IAChB,CAAC;IACD,kDAAkD;IAClD,IAAI,KAAK,GAAc,EAAE,CAAC;IAC1B,IAAI,KAAK,GAAc,EAAE,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;QACxD,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAGpD,CAAC;YACF,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YAChI,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,KAAK,GAAG,EAAE,CAAC;QACX,KAAK,GAAG,EAAE,CAAC;IACb,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAClD,CAAC;AAED,6CAA6C;AAC7C,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAuB,EACvB,aAAqB,EACrB,OAAsB,EAAE;IAExB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC;IAC/C,MAAM,OAAO,GAAyB,EAAE,CAAC;IACzC,MAAM,OAAO,GAAsE,EAAE,CAAC;IAEtF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5F,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAG,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,aAAa,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAExD,+FAA+F;IAC/F,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;IACpF,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrE,wCAAwC;IACxC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAC1E,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEvE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IACjE,MAAM,SAAS,GAAY,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,KAAK,GAAc,CAAC,GAAG,SAAS,CAAC,CAAC;IAExC,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACvF,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE;gBAAE,SAAS,CAAC,SAAS;YACxD,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;gBAAE,SAAS,CAAC,4BAA4B;YACzE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAC9B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM;gBACnG,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO;aAC7F,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;gBAAE,aAAa,EAAE,CAAC,CAAC,eAAe;QAC1E,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,MAAM,WAAW,GAA0C,EAAE,CAAC;IAC9D,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;QAAE,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAG,IAAI,GAAG,EAAqB,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAAE,cAAc,EAAE,CAAC;QACpD,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,SAAS,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACjE,MAAM,UAAU,GAAkB;QAChC,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW;KAC9F,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAEhR,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,SAAS,EAAE,aAAa;QACxB,WAAW,EAAE,cAAc;QAC3B,WAAW,EAAE,SAAS,CAAC,MAAM;QAC7B,aAAa,EAAE,WAAW,CAAC,MAAM;QACjC,OAAO,EAAE,OAAO;QAChB,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;KAC/G,CAAC;AACJ,CAAC;AAED,mBAAmB;AACnB,MAAM,UAAU,YAAY,CAAC,aAAqB,EAAE,IAAgB;IAClE,sBAAsB,CAAC,aAAa,EAAE;QACpC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;QAC1D,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;QAC3D,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;KACjE,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,33 @@
1
+ /**
2
+ * F192 T015 — 办公文档文本抽取(FR-011 攻击矩阵)
3
+ *
4
+ * docx/pptx:fflate 解压(仅取需要的 OOXML entry,过滤 path traversal + declared/实际
5
+ * size 上限防 zip bomb)→ regex 抽 <w:t>/<a:t>(零 XML 解析器攻击面)+ DOCTYPE/ENTITY
6
+ * 预拒(XXE)+ 手动实体解码。不读 .rels → 天然忽略 external relationships。
7
+ * pdf:unpdf text-layer 抽取(serverless pdfjs,无 canvas、不渲染、不执行动作)+ 字节上限
8
+ * + /Launch、/OpenAction+/JavaScript 主动内容预扫描拒绝。
9
+ * md:直接 UTF-8 解码。
10
+ * 加密/损坏/超大 → OfficeParseError(安全拒绝,不崩)。
11
+ */
12
+ export type OfficeFormat = 'docx' | 'pptx' | 'pdf' | 'md';
13
+ export interface OfficeParseOptions {
14
+ maxFileBytes?: number;
15
+ maxEntryBytes?: number;
16
+ /** 全部抽取 entry 的累计解压字节上限(zip bomb 防护) */
17
+ maxTotalBytes?: number;
18
+ }
19
+ export declare class OfficeParseError extends Error {
20
+ constructor(message: string);
21
+ }
22
+ /** zip entry 名安全:拒 `..`、绝对路径、盘符(path traversal) */
23
+ export declare function isSafeEntryName(name: string): boolean;
24
+ /** regex 抽取 OOXML `<tag>…</tag>` 文本(DTD 预拒 + 实体解码) */
25
+ export declare function extractOoxmlText(xml: string, tag: string): string;
26
+ /** 抽取办公文档文本(按 format 分派,全程安全拒绝、不崩、不外联) */
27
+ export declare function parseOfficeFile(bytes: Uint8Array, format: OfficeFormat, options?: OfficeParseOptions): Promise<{
28
+ text: string;
29
+ format: OfficeFormat;
30
+ }>;
31
+ /** 从文件扩展名推断 format(caller 用);未知返回 null */
32
+ export declare function detectOfficeFormat(filename: string): OfficeFormat | null;
33
+ //# sourceMappingURL=office-parser.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"office-parser.d.ts","sourceRoot":"","sources":["../../../src/scaffold-kb/ingest/office-parser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAE1D,MAAM,WAAW,kBAAkB;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAID,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAI5B;AAED,mDAAmD;AACnD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAKrD;AA0BD,sDAAsD;AACtD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAOjE;AA+FD,0CAA0C;AAC1C,wBAAsB,eAAe,CACnC,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,YAAY,EACpB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAA;CAAE,CAAC,CAuBjD;AAED,0CAA0C;AAC1C,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAIxE"}
@@ -0,0 +1,187 @@
1
+ /**
2
+ * F192 T015 — 办公文档文本抽取(FR-011 攻击矩阵)
3
+ *
4
+ * docx/pptx:fflate 解压(仅取需要的 OOXML entry,过滤 path traversal + declared/实际
5
+ * size 上限防 zip bomb)→ regex 抽 <w:t>/<a:t>(零 XML 解析器攻击面)+ DOCTYPE/ENTITY
6
+ * 预拒(XXE)+ 手动实体解码。不读 .rels → 天然忽略 external relationships。
7
+ * pdf:unpdf text-layer 抽取(serverless pdfjs,无 canvas、不渲染、不执行动作)+ 字节上限
8
+ * + /Launch、/OpenAction+/JavaScript 主动内容预扫描拒绝。
9
+ * md:直接 UTF-8 解码。
10
+ * 加密/损坏/超大 → OfficeParseError(安全拒绝,不崩)。
11
+ */
12
+ import { Unzip, UnzipInflate } from 'fflate';
13
+ import { extractText, getDocumentProxy } from 'unpdf';
14
+ const DEFAULTS = { maxFileBytes: 50_000_000, maxEntryBytes: 30_000_000, maxTotalBytes: 60_000_000 };
15
+ export class OfficeParseError extends Error {
16
+ constructor(message) {
17
+ super(message);
18
+ this.name = 'OfficeParseError';
19
+ }
20
+ }
21
+ /** zip entry 名安全:拒 `..`、绝对路径、盘符(path traversal) */
22
+ export function isSafeEntryName(name) {
23
+ if (name.includes('..'))
24
+ return false;
25
+ if (name.startsWith('/') || name.startsWith('\\'))
26
+ return false;
27
+ if (/^[a-zA-Z]:/.test(name))
28
+ return false;
29
+ return true;
30
+ }
31
+ /** XXE 防护:解析前拒绝含 DOCTYPE/ENTITY 的 XML(不止禁实体替换) */
32
+ function rejectDtd(xml) {
33
+ if (/<!DOCTYPE/i.test(xml) || /<!ENTITY/i.test(xml)) {
34
+ throw new OfficeParseError('OOXML 含 DOCTYPE/ENTITY(XXE 防护拒绝)');
35
+ }
36
+ }
37
+ const NAMED_ENTITIES = {
38
+ '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'",
39
+ };
40
+ function decodeXmlEntities(s) {
41
+ return s
42
+ .replace(/&(?:amp|lt|gt|quot|apos);/g, (m) => NAMED_ENTITIES[m] ?? m)
43
+ .replace(/&#(\d+);/g, (_, d) => safeCodePoint(Number(d)))
44
+ .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => safeCodePoint(parseInt(h, 16)));
45
+ }
46
+ function safeCodePoint(n) {
47
+ try {
48
+ return n >= 0 && n <= 0x10ffff ? String.fromCodePoint(n) : '';
49
+ }
50
+ catch {
51
+ return '';
52
+ }
53
+ }
54
+ /** regex 抽取 OOXML `<tag>…</tag>` 文本(DTD 预拒 + 实体解码) */
55
+ export function extractOoxmlText(xml, tag) {
56
+ rejectDtd(xml);
57
+ const re = new RegExp(`<${tag}\\b[^>]*>([^<]*)</${tag}>`, 'g');
58
+ const parts = [];
59
+ let m;
60
+ while ((m = re.exec(xml)) !== null)
61
+ parts.push(decodeXmlEntities(m[1] ?? ''));
62
+ return parts.join('');
63
+ }
64
+ function concatChunks(chunks, total) {
65
+ const out = new Uint8Array(total);
66
+ let off = 0;
67
+ for (const c of chunks) {
68
+ out.set(c, off);
69
+ off += c.length;
70
+ }
71
+ return out;
72
+ }
73
+ /**
74
+ * 流式解压取匹配 wantedRe 的 entry(C-1):仅 start 需要的 entry(不解压其它);
75
+ * 逐 chunk 累加 entry 实际字节 + 全局累计字节,任一超限立即终止(streaming,不整包 materialize)。
76
+ */
77
+ function unzipWanted(bytes, wantedRe, maxEntryBytes, maxTotalBytes) {
78
+ const result = {};
79
+ let totalBytes = 0;
80
+ let fatal = null;
81
+ const unzipper = new Unzip((file) => {
82
+ if (fatal)
83
+ return;
84
+ if (!isSafeEntryName(file.name) || !wantedRe.test(file.name))
85
+ return; // 不 start → 不解压
86
+ // declared size 早拦(已知时)
87
+ if (typeof file.originalSize === 'number' && file.originalSize > maxEntryBytes) {
88
+ fatal = new OfficeParseError(`entry ${file.name} declared size 超 ${maxEntryBytes}(zip bomb)`);
89
+ return;
90
+ }
91
+ const chunks = [];
92
+ let entryBytes = 0;
93
+ file.ondata = (err, chunk, final) => {
94
+ if (fatal)
95
+ return;
96
+ if (err) {
97
+ fatal = new OfficeParseError(`解压 ${file.name} 失败: ${err.message}`);
98
+ return;
99
+ }
100
+ entryBytes += chunk.length;
101
+ totalBytes += chunk.length;
102
+ if (entryBytes > maxEntryBytes || totalBytes > maxTotalBytes) {
103
+ fatal = new OfficeParseError(`解压超字节上限(zip bomb 防护): ${file.name}`);
104
+ return;
105
+ }
106
+ chunks.push(chunk);
107
+ if (final)
108
+ result[file.name] = concatChunks(chunks, entryBytes);
109
+ };
110
+ file.start();
111
+ });
112
+ unzipper.register(UnzipInflate);
113
+ try {
114
+ unzipper.push(bytes, true);
115
+ }
116
+ catch (e) {
117
+ throw new OfficeParseError(`zip 解析失败(损坏/加密/非法): ${e.message}`);
118
+ }
119
+ if (fatal)
120
+ throw fatal;
121
+ return result;
122
+ }
123
+ function parseDocx(bytes, maxEntryBytes, maxTotalBytes) {
124
+ const entries = unzipWanted(bytes, /^word\/document\.xml$/, maxEntryBytes, maxTotalBytes);
125
+ const xml = entries['word/document.xml'];
126
+ if (!xml)
127
+ throw new OfficeParseError('docx 缺 word/document.xml(损坏或非 docx)');
128
+ return extractOoxmlText(new TextDecoder().decode(xml), 'w:t');
129
+ }
130
+ function parsePptx(bytes, maxEntryBytes, maxTotalBytes) {
131
+ const entries = unzipWanted(bytes, /^ppt\/slides\/slide\d+\.xml$/, maxEntryBytes, maxTotalBytes);
132
+ const names = Object.keys(entries).sort();
133
+ if (names.length === 0)
134
+ throw new OfficeParseError('pptx 无幻灯片(损坏或非 pptx)');
135
+ return names.map((k) => extractOoxmlText(new TextDecoder().decode(entries[k]), 'a:t')).join('\n');
136
+ }
137
+ async function parsePdf(bytes) {
138
+ // 主动内容预扫描(不执行;text-only 抽取本就不触发动作,此为 defense-in-depth)。
139
+ // 先解码 PDF name `#xx` 转义(防 /L#61unch 绕过),再匹配;扫描窗口放宽到 10MB。
140
+ const raw = new TextDecoder('latin1').decode(bytes.subarray(0, Math.min(bytes.length, 10_000_000)));
141
+ const decoded = raw.replace(/#([0-9a-fA-F]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
142
+ const hasJs = /\/JavaScript\b|\/JS\b/.test(decoded);
143
+ if (/\/Launch\b/.test(decoded) || ((/\/OpenAction\b/.test(decoded) || /\/AA\b/.test(decoded)) && hasJs)) {
144
+ throw new OfficeParseError('PDF 含主动内容(/Launch 或 OpenAction|AA + JavaScript)—— 拒绝');
145
+ }
146
+ try {
147
+ const pdf = await getDocumentProxy(new Uint8Array(bytes));
148
+ const res = await extractText(pdf, { mergePages: true });
149
+ return Array.isArray(res.text) ? res.text.join('\n') : String(res.text);
150
+ }
151
+ catch (e) {
152
+ throw new OfficeParseError(`PDF 解析失败(加密/损坏/无文本层): ${e.message}`);
153
+ }
154
+ }
155
+ /** 抽取办公文档文本(按 format 分派,全程安全拒绝、不崩、不外联) */
156
+ export async function parseOfficeFile(bytes, format, options = {}) {
157
+ const maxFileBytes = options.maxFileBytes ?? DEFAULTS.maxFileBytes;
158
+ const maxEntryBytes = options.maxEntryBytes ?? DEFAULTS.maxEntryBytes;
159
+ const maxTotalBytes = options.maxTotalBytes ?? DEFAULTS.maxTotalBytes;
160
+ if (bytes.length > maxFileBytes) {
161
+ throw new OfficeParseError(`文件超 ${maxFileBytes} 字节上限`);
162
+ }
163
+ let text;
164
+ switch (format) {
165
+ case 'docx':
166
+ text = parseDocx(bytes, maxEntryBytes, maxTotalBytes);
167
+ break;
168
+ case 'pptx':
169
+ text = parsePptx(bytes, maxEntryBytes, maxTotalBytes);
170
+ break;
171
+ case 'pdf':
172
+ text = await parsePdf(bytes);
173
+ break;
174
+ case 'md':
175
+ text = new TextDecoder().decode(bytes);
176
+ break;
177
+ }
178
+ return { text, format };
179
+ }
180
+ /** 从文件扩展名推断 format(caller 用);未知返回 null */
181
+ export function detectOfficeFormat(filename) {
182
+ const ext = filename.toLowerCase().split('.').pop() ?? '';
183
+ if (ext === 'docx' || ext === 'pptx' || ext === 'pdf' || ext === 'md')
184
+ return ext;
185
+ return null;
186
+ }
187
+ //# sourceMappingURL=office-parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"office-parser.js","sourceRoot":"","sources":["../../../src/scaffold-kb/ingest/office-parser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAC;AAWtD,MAAM,QAAQ,GAAG,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;AAEpG,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,mDAAmD;AACnD,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,kDAAkD;AAClD,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,kCAAkC,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAA2B;IAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG;CACrE,CAAC;AACF,SAAS,iBAAiB,CAAC,CAAS;IAClC,OAAO,CAAC;SACL,OAAO,CAAC,4BAA4B,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACpE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;SACxD,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AAC9E,CAAC;AACD,SAAS,aAAa,CAAC,CAAS;IAC9B,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,GAAW;IACvD,SAAS,CAAC,GAAG,CAAC,CAAC;IACf,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,qBAAqB,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,YAAY,CAAC,MAAoB,EAAE,KAAa;IACvD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAChB,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAClB,KAAiB,EACjB,QAAgB,EAChB,aAAqB,EACrB,aAAqB;IAErB,MAAM,MAAM,GAA+B,EAAE,CAAC;IAC9C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAiB,IAAI,CAAC;IAE/B,MAAM,QAAQ,GAAG,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;QAClC,IAAI,KAAK;YAAE,OAAO;QAClB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,gBAAgB;QACtF,wBAAwB;QACxB,IAAI,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ,IAAI,IAAI,CAAC,YAAY,GAAG,aAAa,EAAE,CAAC;YAC/E,KAAK,GAAG,IAAI,gBAAgB,CAAC,SAAS,IAAI,CAAC,IAAI,oBAAoB,aAAa,YAAY,CAAC,CAAC;YAC9F,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAClC,IAAI,KAAK;gBAAE,OAAO;YAClB,IAAI,GAAG,EAAE,CAAC;gBACR,KAAK,GAAG,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,IAAI,QAAQ,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YACD,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,IAAI,UAAU,GAAG,aAAa,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;gBAC7D,KAAK,GAAG,IAAI,gBAAgB,CAAC,yBAAyB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAClE,CAAC,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,gBAAgB,CAAC,uBAAwB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,KAAK;QAAE,MAAM,KAAK,CAAC;IACvB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,KAAiB,EAAE,aAAqB,EAAE,aAAqB;IAChF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,uBAAuB,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IAC1F,MAAM,GAAG,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACzC,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,gBAAgB,CAAC,qCAAqC,CAAC,CAAC;IAC5E,OAAO,gBAAgB,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,SAAS,CAAC,KAAiB,EAAE,aAAqB,EAAE,aAAqB;IAChF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,8BAA8B,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IACjG,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,gBAAgB,CAAC,sBAAsB,CAAC,CAAC;IAC3E,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrG,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAiB;IACvC,wDAAwD;IACxD,0DAA0D;IAC1D,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACpG,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxG,MAAM,IAAI,gBAAgB,CAAC,sDAAsD,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,gBAAgB,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,0CAA0C;AAC1C,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAiB,EACjB,MAAoB,EACpB,UAA8B,EAAE;IAEhC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY,CAAC;IACnE,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa,CAAC;IACtE,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa,CAAC;IACtE,IAAI,KAAK,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;QAChC,MAAM,IAAI,gBAAgB,CAAC,OAAO,YAAY,OAAO,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,IAAY,CAAC;IACjB,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,MAAM;YACT,IAAI,GAAG,SAAS,CAAC,KAAK,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YACtD,MAAM;QACR,KAAK,MAAM;YACT,IAAI,GAAG,SAAS,CAAC,KAAK,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YACtD,MAAM;QACR,KAAK,KAAK;YACR,IAAI,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC7B,MAAM;QACR,KAAK,IAAI;YACP,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM;IACV,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IAC1D,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAClF,OAAO,IAAI,CAAC;AACd,CAAC"}
@@ -0,0 +1,43 @@
1
+ /**
2
+ * F192 T014 — SSRF 安全 URL 抓取(FR-010,唯一联网点)
3
+ *
4
+ * 用 node:http/https + 自定义 `lookup`:DNS 解析后校验 IP,连接用校验后的 IP
5
+ * (node 直接用 lookup 结果连,无 rebinding 重解析窗口,C-5)。
6
+ * 协议白名单 + 内网/loopback/link-local/ULA/mapped 封锁 + 手写逐跳 redirect(限跳数 +
7
+ * 跨协议拒绝)+ 连接/读超时 + 流式字节上限 + content-type 校验。
8
+ *
9
+ * isBlockedIp / makeSsrfLookup 为纯/可注入函数,SSRF 核心逻辑可确定性单测。
10
+ */
11
+ import { lookup as nodeDnsLookup } from 'node:dns';
12
+ export interface SafeFetchOptions {
13
+ /** 单次请求 idle 超时(连接/读取静默) */
14
+ timeoutMs?: number;
15
+ /** 总 wall-clock 超时(覆盖 DNS+连接+读取+全部 redirect,防慢速攻击) */
16
+ totalTimeoutMs?: number;
17
+ maxBytes?: number;
18
+ maxRedirects?: number;
19
+ /** 注入 DNS lookup(测试用),缺省 node:dns.lookup */
20
+ dnsLookup?: typeof nodeDnsLookup;
21
+ }
22
+ export interface SafeFetchResult {
23
+ finalUrl: string;
24
+ contentType: string;
25
+ markdown: string;
26
+ }
27
+ export declare class SsrfError extends Error {
28
+ constructor(message: string);
29
+ }
30
+ /** 判断 IP 是否在封锁段(loopback/私网/link-local/ULA/未指定/IPv4-mapped IPv6) */
31
+ export declare function isBlockedIp(ip: string): boolean;
32
+ type LookupCallback = (err: NodeJS.ErrnoException | null, address: string, family: number) => void;
33
+ /** 构造校验 IP 的 lookup(DNS 解析后封锁非法 IP,连接用校验结果,防 rebinding) */
34
+ export declare function makeSsrfLookup(dnsLookup?: typeof nodeDnsLookup): (hostname: string, options: unknown, callback: LookupCallback) => void;
35
+ /** 从 HTML 抽主内容 → Markdown(readability + turndown;linkedom 不执行脚本) */
36
+ export declare function htmlToMarkdown(html: string, url: string): {
37
+ title: string;
38
+ markdown: string;
39
+ };
40
+ /** SSRF 安全抓取 URL → 主内容 Markdown(手写逐跳 redirect,每跳重校验) */
41
+ export declare function safeFetchUrl(url: string, options?: SafeFetchOptions): Promise<SafeFetchResult>;
42
+ export {};
43
+ //# sourceMappingURL=url-fetcher.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url-fetcher.d.ts","sourceRoot":"","sources":["../../../src/scaffold-kb/ingest/url-fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,UAAU,CAAC;AAMnD,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,OAAO,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,SAAU,SAAQ,KAAK;gBACtB,OAAO,EAAE,MAAM;CAI5B;AAKD,oEAAoE;AACpE,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAK/C;AA+DD,KAAK,cAAc,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;AAEnG,2DAA2D;AAC3D,wBAAgB,cAAc,CAAC,SAAS,GAAE,OAAO,aAA6B,IACjD,UAAU,MAAM,EAAE,SAAS,OAAO,EAAE,UAAU,cAAc,KAAG,IAAI,CAe/F;AA6DD,oEAAoE;AACpE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAY7F;AAED,wDAAwD;AACxD,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAkDxG"}
@@ -0,0 +1,245 @@
1
+ /**
2
+ * F192 T014 — SSRF 安全 URL 抓取(FR-010,唯一联网点)
3
+ *
4
+ * 用 node:http/https + 自定义 `lookup`:DNS 解析后校验 IP,连接用校验后的 IP
5
+ * (node 直接用 lookup 结果连,无 rebinding 重解析窗口,C-5)。
6
+ * 协议白名单 + 内网/loopback/link-local/ULA/mapped 封锁 + 手写逐跳 redirect(限跳数 +
7
+ * 跨协议拒绝)+ 连接/读超时 + 流式字节上限 + content-type 校验。
8
+ *
9
+ * isBlockedIp / makeSsrfLookup 为纯/可注入函数,SSRF 核心逻辑可确定性单测。
10
+ */
11
+ import { request as httpsRequest } from 'node:https';
12
+ import { request as httpRequest } from 'node:http';
13
+ import { lookup as nodeDnsLookup } from 'node:dns';
14
+ import { isIP } from 'node:net';
15
+ import { Readability } from '@mozilla/readability';
16
+ import { parseHTML } from 'linkedom';
17
+ import TurndownService from 'turndown';
18
+ export class SsrfError extends Error {
19
+ constructor(message) {
20
+ super(message);
21
+ this.name = 'SsrfError';
22
+ }
23
+ }
24
+ const DEFAULTS = { timeoutMs: 10_000, totalTimeoutMs: 30_000, maxBytes: 5_000_000, maxRedirects: 5 };
25
+ const ALLOWED_MIME = new Set(['text/html', 'application/xhtml+xml', 'text/plain', 'text/markdown']);
26
+ /** 判断 IP 是否在封锁段(loopback/私网/link-local/ULA/未指定/IPv4-mapped IPv6) */
27
+ export function isBlockedIp(ip) {
28
+ const kind = isIP(ip);
29
+ if (kind === 0)
30
+ return true; // 非法 IP 一律封锁(保守)
31
+ if (kind === 4)
32
+ return isBlockedV4(ip);
33
+ return isBlockedV6(ip);
34
+ }
35
+ function isBlockedV4(ip) {
36
+ const parts = ip.split('.').map((n) => parseInt(n, 10));
37
+ if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255))
38
+ return true;
39
+ const [a, b] = parts;
40
+ if (a === 0)
41
+ return true; // 0.0.0.0/8 未指定
42
+ if (a === 127)
43
+ return true; // loopback
44
+ if (a === 10)
45
+ return true; // 私网
46
+ if (a === 172 && b >= 16 && b <= 31)
47
+ return true; // 私网
48
+ if (a === 192 && b === 168)
49
+ return true; // 私网
50
+ if (a === 169 && b === 254)
51
+ return true; // link-local
52
+ if (a === 100 && b >= 64 && b <= 127)
53
+ return true; // CGNAT 100.64/10
54
+ return false;
55
+ }
56
+ /** 展开 IPv6 为 8 个 hextet(含内嵌 IPv4 转换);非法返回 null */
57
+ function expandV6(ipRaw) {
58
+ let s = (ipRaw.toLowerCase().split('%')[0] ?? ipRaw).trim();
59
+ // 内嵌 IPv4(::ffff:1.2.3.4 / ::1.2.3.4)→ 转两个 hextet
60
+ const v4Match = /^(.*:)((?:\d{1,3}\.){3}\d{1,3})$/.exec(s);
61
+ if (v4Match) {
62
+ const v4 = v4Match[2].split('.').map((n) => parseInt(n, 10));
63
+ if (v4.length !== 4 || v4.some((n) => !Number.isInteger(n) || n < 0 || n > 255))
64
+ return null;
65
+ const g7 = (v4[0] << 8) | v4[1];
66
+ const g8 = (v4[2] << 8) | v4[3];
67
+ s = `${v4Match[1]}${g7.toString(16)}:${g8.toString(16)}`;
68
+ }
69
+ const parts = s.split('::');
70
+ if (parts.length > 2)
71
+ return null;
72
+ const head = parts[0] ? parts[0].split(':') : [];
73
+ const tail = parts.length === 2 && parts[1] ? parts[1].split(':') : [];
74
+ if (parts.length === 1 && head.length !== 8)
75
+ return null;
76
+ const fill = 8 - head.length - tail.length;
77
+ if (fill < 0 || (parts.length === 2 && fill < 1))
78
+ return null;
79
+ const groups = parts.length === 2 ? [...head, ...Array(fill).fill('0'), ...tail] : head;
80
+ if (groups.length !== 8)
81
+ return null;
82
+ const nums = groups.map((g) => (g === '' ? 0 : parseInt(g, 16)));
83
+ if (nums.some((n) => !Number.isInteger(n) || n < 0 || n > 0xffff))
84
+ return null;
85
+ return nums;
86
+ }
87
+ function isBlockedV6(ipRaw) {
88
+ const g = expandV6(ipRaw);
89
+ if (!g)
90
+ return true; // 解析失败保守封锁
91
+ const [g0, g1, g2, g3, g4, g5, g6, g7] = g;
92
+ // :: 未指定 / ::1 loopback
93
+ if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0 && g6 === 0 && (g7 === 0 || g7 === 1))
94
+ return true;
95
+ // fe80::/10 link-local(首 hextet 0xfe80–0xfebf)
96
+ if (g0 >= 0xfe80 && g0 <= 0xfebf)
97
+ return true;
98
+ // fc00::/7 ULA(首 hextet 0xfc00–0xfdff)
99
+ if (g0 >= 0xfc00 && g0 <= 0xfdff)
100
+ return true;
101
+ // ::ffff:0:0/96 IPv4-mapped → 校验内嵌 v4
102
+ if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0xffff) {
103
+ return isBlockedV4(`${(g6 >> 8) & 0xff}.${g6 & 0xff}.${(g7 >> 8) & 0xff}.${g7 & 0xff}`);
104
+ }
105
+ // ::/96 IPv4-compatible(已废弃)非 ::/::1 → 校验内嵌 v4
106
+ if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0 && (g6 !== 0 || g7 > 1)) {
107
+ return isBlockedV4(`${(g6 >> 8) & 0xff}.${g6 & 0xff}.${(g7 >> 8) & 0xff}.${g7 & 0xff}`);
108
+ }
109
+ return false;
110
+ }
111
+ /** 构造校验 IP 的 lookup(DNS 解析后封锁非法 IP,连接用校验结果,防 rebinding) */
112
+ export function makeSsrfLookup(dnsLookup = nodeDnsLookup) {
113
+ return function ssrfLookup(hostname, options, callback) {
114
+ dnsLookup(hostname, { all: true }, (err, addresses) => {
115
+ if (err) {
116
+ callback(err, '', 4);
117
+ return;
118
+ }
119
+ const list = Array.isArray(addresses) ? addresses : [];
120
+ const ok = list.find((a) => !isBlockedIp(a.address));
121
+ if (!ok) {
122
+ callback(new SsrfError(`目标 ${hostname} 解析到被封锁地址(内网/loopback/link-local)`), '', 4);
123
+ return;
124
+ }
125
+ callback(null, ok.address, ok.family);
126
+ });
127
+ };
128
+ }
129
+ /** 单次请求(含协议白名单 + IP literal 校验 + lookup + 超时 + 流式字节上限 + 总 deadline signal);不跟随 redirect */
130
+ function fetchRaw(url, opts, signal) {
131
+ return new Promise((resolve, reject) => {
132
+ let parsed;
133
+ try {
134
+ parsed = new URL(url);
135
+ }
136
+ catch {
137
+ reject(new SsrfError(`非法 URL: ${url}`));
138
+ return;
139
+ }
140
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
141
+ reject(new SsrfError(`协议不在白名单(仅 http/https): ${parsed.protocol}`));
142
+ return;
143
+ }
144
+ // C-1:IP literal(十进制/十六进制/八进制已被 URL 规范化为 dotted;及 IPv6 字面量)
145
+ // node http 对 IP literal **不触发 lookup**,必须在此显式校验,否则 SSRF 防线被完全绕过。
146
+ const host = parsed.hostname.replace(/^\[/, '').replace(/\]$/, '');
147
+ if (isIP(host) !== 0 && isBlockedIp(host)) {
148
+ reject(new SsrfError(`目标 IP 在封锁段: ${host}`));
149
+ return;
150
+ }
151
+ const requestFn = parsed.protocol === 'https:' ? httpsRequest : httpRequest;
152
+ const req = requestFn(url, { lookup: makeSsrfLookup(opts.dnsLookup), timeout: opts.timeoutMs, method: 'GET', signal }, (res) => {
153
+ const chunks = [];
154
+ let total = 0;
155
+ res.on('data', (c) => {
156
+ total += c.length;
157
+ if (total > opts.maxBytes) {
158
+ req.destroy(new SsrfError(`响应体超过 ${opts.maxBytes} 字节上限`));
159
+ return;
160
+ }
161
+ chunks.push(c);
162
+ });
163
+ res.on('end', () => {
164
+ resolve({ status: res.statusCode ?? 0, headers: res.headers, body: Buffer.concat(chunks).toString('utf-8') });
165
+ });
166
+ });
167
+ req.on('timeout', () => req.destroy(new SsrfError('请求超时')));
168
+ req.on('error', (e) => reject(e));
169
+ req.end();
170
+ });
171
+ }
172
+ const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
173
+ /** 从 HTML 抽主内容 → Markdown(readability + turndown;linkedom 不执行脚本) */
174
+ export function htmlToMarkdown(html, url) {
175
+ try {
176
+ const { document } = parseHTML(html);
177
+ const reader = new Readability(document);
178
+ const article = reader.parse();
179
+ const contentHtml = article?.content ?? html;
180
+ const title = article?.title ?? '';
181
+ return { title, markdown: turndown.turndown(contentHtml) };
182
+ }
183
+ catch {
184
+ // readability 失败 → 整页转换兜底
185
+ return { title: '', markdown: turndown.turndown(html) };
186
+ }
187
+ }
188
+ /** SSRF 安全抓取 URL → 主内容 Markdown(手写逐跳 redirect,每跳重校验) */
189
+ export async function safeFetchUrl(url, options = {}) {
190
+ const opts = {
191
+ timeoutMs: options.timeoutMs ?? DEFAULTS.timeoutMs,
192
+ totalTimeoutMs: options.totalTimeoutMs ?? DEFAULTS.totalTimeoutMs,
193
+ maxBytes: options.maxBytes ?? DEFAULTS.maxBytes,
194
+ maxRedirects: options.maxRedirects ?? DEFAULTS.maxRedirects,
195
+ dnsLookup: options.dnsLookup ?? nodeDnsLookup,
196
+ };
197
+ // 总 wall-clock deadline(覆盖全流程,防慢速攻击;W)
198
+ const controller = new AbortController();
199
+ const wallTimer = setTimeout(() => controller.abort(), opts.totalTimeoutMs);
200
+ try {
201
+ let current = url;
202
+ let prevProtocol = null;
203
+ for (let hop = 0; hop <= opts.maxRedirects; hop++) {
204
+ let proto;
205
+ try {
206
+ proto = new URL(current).protocol;
207
+ }
208
+ catch {
209
+ throw new SsrfError(`非法 URL: ${current}`);
210
+ }
211
+ // 跨协议降级(https→http)拒绝
212
+ if (prevProtocol === 'https:' && proto === 'http:') {
213
+ throw new SsrfError('重定向跨协议降级(https→http)被拒绝');
214
+ }
215
+ prevProtocol = proto;
216
+ const res = await fetchRaw(current, opts, controller.signal);
217
+ if (res.status >= 300 && res.status < 400) {
218
+ const loc = res.headers['location'];
219
+ const locStr = Array.isArray(loc) ? loc[0] : loc;
220
+ if (!locStr)
221
+ throw new SsrfError('重定向无 Location');
222
+ if (hop === opts.maxRedirects)
223
+ throw new SsrfError(`重定向超过最大跳数 ${opts.maxRedirects}`);
224
+ current = new URL(locStr, current).toString(); // 解析相对/protocol-relative,下一跳重新校验
225
+ continue;
226
+ }
227
+ if (res.status >= 400)
228
+ throw new SsrfError(`HTTP ${res.status}`);
229
+ // content-type 精确校验(主 MIME 精确匹配,拒绝空、拒绝子串绕过;W)
230
+ const ctRaw = res.headers['content-type'];
231
+ const ctFull = (Array.isArray(ctRaw) ? (ctRaw[0] ?? '') : (ctRaw ?? '')).toLowerCase();
232
+ const mime = ctFull.split(';')[0]?.trim() ?? '';
233
+ if (!ALLOWED_MIME.has(mime)) {
234
+ throw new SsrfError(`content-type 不接受(仅 HTML/text,拒绝空): ${ctFull || '(空)'}`);
235
+ }
236
+ const { markdown } = htmlToMarkdown(res.body, current);
237
+ return { finalUrl: current, contentType: mime, markdown };
238
+ }
239
+ throw new SsrfError(`重定向超过最大跳数 ${opts.maxRedirects}`);
240
+ }
241
+ finally {
242
+ clearTimeout(wallTimer);
243
+ }
244
+ }
245
+ //# sourceMappingURL=url-fetcher.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"url-fetcher.js","sourceRoot":"","sources":["../../../src/scaffold-kb/ingest/url-fetcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAmBvC,MAAM,OAAO,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;AACrG,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,uBAAuB,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC,CAAC;AAEpG,oEAAoE;AACpE,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,iBAAiB;IAC9C,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC,EAAE,CAAC,CAAC;IACvC,OAAO,WAAW,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAAC,EAAU;IAC7B,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACnG,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAyC,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,gBAAgB;IAC1C,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,WAAW;IACvC,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,KAAK;IAChC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,KAAK;IACvD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,KAAK;IAC9C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,aAAa;IACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,kBAAkB;IACrE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kDAAkD;AAClD,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,kDAAkD;IAClD,MAAM,OAAO,GAAG,kCAAkC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7F,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;QAClC,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IAC3D,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACjD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzD,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/E,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,WAAW;IAChC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,CAAqE,CAAC;IAC/G,wBAAwB;IACxB,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5H,+CAA+C;IAC/C,IAAI,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC9C,uCAAuC;IACvC,IAAI,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC9C,sCAAsC;IACtC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;QAC9E,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,+CAA+C;IAC/C,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;QACjG,OAAO,WAAW,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAID,2DAA2D;AAC3D,MAAM,UAAU,cAAc,CAAC,YAAkC,aAAa;IAC5E,OAAO,SAAS,UAAU,CAAC,QAAgB,EAAE,OAAgB,EAAE,QAAwB;QACrF,SAAS,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;YACpD,IAAI,GAAG,EAAE,CAAC;gBACR,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;gBACrB,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACrD,IAAI,CAAC,EAAE,EAAE,CAAC;gBACR,QAAQ,CAAC,IAAI,SAAS,CAAC,MAAM,QAAQ,mCAAmC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;gBAClF,OAAO;YACT,CAAC;YACD,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAQD,2FAA2F;AAC3F,SAAS,QAAQ,CACf,GAAW,EACX,IAAyF,EACzF,MAAmB;IAEnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,MAAW,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,SAAS,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAChE,MAAM,CAAC,IAAI,SAAS,CAAC,0BAA0B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QACD,4DAA4D;QAC5D,kEAAkE;QAClE,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,SAAS,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;QAC5E,MAAM,GAAG,GAAG,SAAS,CACnB,GAAG,EACH,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAC1F,CAAC,GAAG,EAAE,EAAE;YACN,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;gBAC3B,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;gBAClB,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAC1B,GAAG,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,SAAS,IAAI,CAAC,QAAQ,OAAO,CAAC,CAAC,CAAC;oBAC1D,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACjB,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,UAAU,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAChH,CAAC,CAAC,CAAC;QACL,CAAC,CACF,CAAC;QACF,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5D,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAExF,oEAAoE;AACpE,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAmE,CAAC,CAAC;QACpG,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,WAAW,GAAG,OAAO,EAAE,OAAO,IAAI,IAAI,CAAC;QAC7C,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;QAC1B,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IAC1D,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,UAA4B,EAAE;IAC5E,MAAM,IAAI,GAAG;QACX,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAClD,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC,cAAc;QACjE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC/C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,QAAQ,CAAC,YAAY;QAC3D,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,aAAa;KAC9C,CAAC;IACF,uCAAuC;IACvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAC5E,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;YAClD,IAAI,KAAa,CAAC;YAClB,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,SAAS,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,sBAAsB;YACtB,IAAI,YAAY,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;gBACnD,MAAM,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;YACjD,CAAC;YACD,YAAY,GAAG,KAAK,CAAC;YACrB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YAC7D,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC1C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACpC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;gBACjD,IAAI,CAAC,MAAM;oBAAE,MAAM,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;gBAClD,IAAI,GAAG,KAAK,IAAI,CAAC,YAAY;oBAAE,MAAM,IAAI,SAAS,CAAC,aAAa,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;gBACrF,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,iCAAiC;gBAChF,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;gBAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjE,8CAA8C;YAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACvF,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAChD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,SAAS,CAAC,sCAAsC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACvD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5D,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,aAAa,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IACxD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,SAAS,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC"}
@@ -0,0 +1,38 @@
1
+ /**
2
+ * F190 scaffold-kb — DocumentIngester
3
+ *
4
+ * 提供两种文档输入模式(FR-001):
5
+ * - `--dir`:递归扫描目录下的 *.md 文件,解析为 ParsedDoc
6
+ * - `--llms-txt`:从远程 URL 取 llms.txt 格式,解析条目后逐一抓取文档
7
+ *
8
+ * 两种模式可同时使用:以 llms-txt 为主,--dir 作补充,按 id 去重(llms-txt 优先保留)。
9
+ * EC-008:llms.txt URL 取回或格式解析失败 → throw(原子性,不返回部分结果)。
10
+ */
11
+ import type { ParsedDoc } from './types.js';
12
+ /**
13
+ * ingestDocuments 的输入选项
14
+ */
15
+ export interface IngestOptions {
16
+ /** 本地 Markdown 文档目录 */
17
+ dirPath?: string;
18
+ /** 远程 llms.txt 索引 URL */
19
+ llmsTxtUrl?: string;
20
+ /** 语言标记(如 'zh' | 'en'),传给所有解析出的文档 */
21
+ lang?: string;
22
+ /**
23
+ * 注入 fetch 实现(用于测试时替换真实网络请求)。
24
+ * 缺省使用 Node.js 内置 `fetch`。
25
+ * 函数签名:(url: string) => Promise<string>,返回响应正文字符串。
26
+ */
27
+ fetchImpl?: (url: string) => Promise<string>;
28
+ }
29
+ /**
30
+ * 从多来源抓取文档,返回去重后的 ParsedDoc 列表。
31
+ *
32
+ * 行为契约:
33
+ * - 两种模式均未提供 → throw(含用法提示)
34
+ * - llms-txt 优先:同 id 以 llms-txt 结果保留
35
+ * - EC-008:llms.txt 任一环节失败 → throw,不返回部分结果
36
+ */
37
+ export declare function ingestDocuments(opts: IngestOptions): Promise<ParsedDoc[]>;
38
+ //# sourceMappingURL=ingester.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ingester.d.ts","sourceRoot":"","sources":["../../src/scaffold-kb/ingester.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uBAAuB;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yBAAyB;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qCAAqC;IACrC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9C;AA4OD;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CA6C/E"}