prts-terrarchive 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GAME_ASSETS.md +26 -0
- package/LICENSE +21 -0
- package/README.en.md +274 -0
- package/README.md +423 -0
- package/THIRD_PARTY_NOTICES.md +61 -0
- package/bin/bake-rhine-details.mjs +34 -0
- package/bin/install.js +273 -0
- package/bin/pack-map-assets.mjs +110 -0
- package/bin/preview-rhine.mjs +71 -0
- package/bin/rhine-detail-baker.ts +84 -0
- package/contracts/cloud_tools_v1.schema.json +243 -0
- package/contracts/corpus_i18n_v1.schema.json +179 -0
- package/contracts/corpus_tools_v1.schema.json +1801 -0
- package/cordis.patch.yml +32 -0
- package/docs/endfield-official-i18n.md +103 -0
- package/lib/client.js +3912 -0
- package/lib/endfield-map/map.js.br +0 -0
- package/lib/endfield-map/map.js.gz +0 -0
- package/lib/endfield-map/resources/map-00b0d0744a1b4404.json.br +0 -0
- package/lib/endfield-map/resources/map-00b0d0744a1b4404.json.gz +0 -0
- package/lib/endfield-map/resources/map-01616f534aaaeca1.png +0 -0
- package/lib/endfield-map/resources/map-1562e0dff4be3d23.png +0 -0
- package/lib/endfield-map/resources/map-160fe040f080f6a3.png +0 -0
- package/lib/endfield-map/resources/map-17b9d8be25dc2b39.json.br +0 -0
- package/lib/endfield-map/resources/map-17b9d8be25dc2b39.json.gz +0 -0
- package/lib/endfield-map/resources/map-1c4245c4e8b80c43.json.br +0 -0
- package/lib/endfield-map/resources/map-1c4245c4e8b80c43.json.gz +0 -0
- package/lib/endfield-map/resources/map-1d30baaaef3e29cc.png +0 -0
- package/lib/endfield-map/resources/map-21a61386af79cd12.json.br +0 -0
- package/lib/endfield-map/resources/map-21a61386af79cd12.json.gz +0 -0
- package/lib/endfield-map/resources/map-24b920b8dab49b44.json.br +0 -0
- package/lib/endfield-map/resources/map-24b920b8dab49b44.json.gz +0 -0
- package/lib/endfield-map/resources/map-26caf8d71247abb7.png +0 -0
- package/lib/endfield-map/resources/map-2729942c89545e73.json.br +0 -0
- package/lib/endfield-map/resources/map-2729942c89545e73.json.gz +0 -0
- package/lib/endfield-map/resources/map-29063a9802f67014.json.br +0 -0
- package/lib/endfield-map/resources/map-29063a9802f67014.json.gz +0 -0
- package/lib/endfield-map/resources/map-2f35d09a3ab60a63.json.br +0 -0
- package/lib/endfield-map/resources/map-2f35d09a3ab60a63.json.gz +0 -0
- package/lib/endfield-map/resources/map-2f98a873da81425e.json.br +0 -0
- package/lib/endfield-map/resources/map-2f98a873da81425e.json.gz +0 -0
- package/lib/endfield-map/resources/map-3ab978ae44190f80.json.br +0 -0
- package/lib/endfield-map/resources/map-3ab978ae44190f80.json.gz +0 -0
- package/lib/endfield-map/resources/map-3b62c80d14208902.json.br +0 -0
- package/lib/endfield-map/resources/map-3b62c80d14208902.json.gz +0 -0
- package/lib/endfield-map/resources/map-3d486e887ae121ac.png +0 -0
- package/lib/endfield-map/resources/map-418dc4865608fce5.png +0 -0
- package/lib/endfield-map/resources/map-4366b84607335f76.json.br +0 -0
- package/lib/endfield-map/resources/map-4366b84607335f76.json.gz +0 -0
- package/lib/endfield-map/resources/map-4371259d3ab7aac7.png +0 -0
- package/lib/endfield-map/resources/map-48dcc0fd55e13f7f.png +0 -0
- package/lib/endfield-map/resources/map-4aabda0e41197570.json.br +0 -0
- package/lib/endfield-map/resources/map-4aabda0e41197570.json.gz +0 -0
- package/lib/endfield-map/resources/map-505f5687b287043c.json.br +0 -0
- package/lib/endfield-map/resources/map-505f5687b287043c.json.gz +0 -0
- package/lib/endfield-map/resources/map-5177e9d1d46f47e2.png +0 -0
- package/lib/endfield-map/resources/map-51baeb59b7f8c28d.png +0 -0
- package/lib/endfield-map/resources/map-549561b46e621874.png +0 -0
- package/lib/endfield-map/resources/map-56c6a0bef799c8dc.json.br +0 -0
- package/lib/endfield-map/resources/map-56c6a0bef799c8dc.json.gz +0 -0
- package/lib/endfield-map/resources/map-5a6963909423b674.png +0 -0
- package/lib/endfield-map/resources/map-5a9c533c0552d7b7.png +0 -0
- package/lib/endfield-map/resources/map-5d20ae9ffa9a02b0.png +0 -0
- package/lib/endfield-map/resources/map-626c8847393f60bf.png +0 -0
- package/lib/endfield-map/resources/map-63d43cbec3bed3b8.png +0 -0
- package/lib/endfield-map/resources/map-661b6ec56e722574.png +0 -0
- package/lib/endfield-map/resources/map-6ddd384eca190828.png +0 -0
- package/lib/endfield-map/resources/map-6fc1aa9144f8156c.json.br +0 -0
- package/lib/endfield-map/resources/map-6fc1aa9144f8156c.json.gz +0 -0
- package/lib/endfield-map/resources/map-738ebaa002c33cfc.png +0 -0
- package/lib/endfield-map/resources/map-758c13be6085ec07.json.br +0 -0
- package/lib/endfield-map/resources/map-758c13be6085ec07.json.gz +0 -0
- package/lib/endfield-map/resources/map-78929f26d6844509.png +0 -0
- package/lib/endfield-map/resources/map-7f09c142950a0f74.png +0 -0
- package/lib/endfield-map/resources/map-8385e11f39e1a51c.json.br +0 -0
- package/lib/endfield-map/resources/map-8385e11f39e1a51c.json.gz +0 -0
- package/lib/endfield-map/resources/map-881343926a568f27.json.br +0 -0
- package/lib/endfield-map/resources/map-881343926a568f27.json.gz +0 -0
- package/lib/endfield-map/resources/map-8d1ade3f01d952b0.png +0 -0
- package/lib/endfield-map/resources/map-8fef3495bb7ed60f.png +0 -0
- package/lib/endfield-map/resources/map-907a7ecf8a0ce352.png +0 -0
- package/lib/endfield-map/resources/map-9113d5951c4db903.json.br +0 -0
- package/lib/endfield-map/resources/map-9113d5951c4db903.json.gz +0 -0
- package/lib/endfield-map/resources/map-94ad6634436224e3.json.br +0 -0
- package/lib/endfield-map/resources/map-94ad6634436224e3.json.gz +0 -0
- package/lib/endfield-map/resources/map-99722e61da5f3e53.png +0 -0
- package/lib/endfield-map/resources/map-9be9596cc059abe3.png +0 -0
- package/lib/endfield-map/resources/map-a4db9a4763e47fbf.png +0 -0
- package/lib/endfield-map/resources/map-a4e8ef07159997e7.json.br +0 -0
- package/lib/endfield-map/resources/map-a4e8ef07159997e7.json.gz +0 -0
- package/lib/endfield-map/resources/map-ab35d06875ee418f.png +0 -0
- package/lib/endfield-map/resources/map-ac51bb3e7623cd8b.png +0 -0
- package/lib/endfield-map/resources/map-ac830a66296bc83a.png +0 -0
- package/lib/endfield-map/resources/map-b3fd04209718e55e.json.br +0 -0
- package/lib/endfield-map/resources/map-b3fd04209718e55e.json.gz +0 -0
- package/lib/endfield-map/resources/map-b8164f1af7c5d895.png +0 -0
- package/lib/endfield-map/resources/map-b8cde93b965384d8.json.br +0 -0
- package/lib/endfield-map/resources/map-b8cde93b965384d8.json.gz +0 -0
- package/lib/endfield-map/resources/map-b8e48511502e0e8d.json.br +0 -0
- package/lib/endfield-map/resources/map-b8e48511502e0e8d.json.gz +0 -0
- package/lib/endfield-map/resources/map-bfc58277d5ba649d.png +0 -0
- package/lib/endfield-map/resources/map-c013f82048c6891a.png +0 -0
- package/lib/endfield-map/resources/map-c49c947841ae4ab5.json.br +0 -0
- package/lib/endfield-map/resources/map-c49c947841ae4ab5.json.gz +0 -0
- package/lib/endfield-map/resources/map-c4d1b1c6d6706c81.json.br +0 -0
- package/lib/endfield-map/resources/map-c4d1b1c6d6706c81.json.gz +0 -0
- package/lib/endfield-map/resources/map-c607f8b4c8ad8967.png +0 -0
- package/lib/endfield-map/resources/map-c6f20276b8781326.json.br +0 -0
- package/lib/endfield-map/resources/map-c6f20276b8781326.json.gz +0 -0
- package/lib/endfield-map/resources/map-d51adf5952af1e8e.png +0 -0
- package/lib/endfield-map/resources/map-d695eface22f8b74.json.br +0 -0
- package/lib/endfield-map/resources/map-d695eface22f8b74.json.gz +0 -0
- package/lib/endfield-map/resources/map-db11d394456a7465.png +0 -0
- package/lib/endfield-map/resources/map-dcc4d218581e3dc6.png +0 -0
- package/lib/endfield-map/resources/map-ddd43dbb8ab8a4e3.json.br +0 -0
- package/lib/endfield-map/resources/map-ddd43dbb8ab8a4e3.json.gz +0 -0
- package/lib/endfield-map/resources/map-defe5db7dabf748a.png +0 -0
- package/lib/endfield-map/resources/map-e1aeb833e22520aa.json.br +0 -0
- package/lib/endfield-map/resources/map-e1aeb833e22520aa.json.gz +0 -0
- package/lib/endfield-map/resources/map-e25ee9af6e493500.png +0 -0
- package/lib/endfield-map/resources/map-e3824082cd6f747d.json.br +0 -0
- package/lib/endfield-map/resources/map-e3824082cd6f747d.json.gz +0 -0
- package/lib/endfield-map/resources/map-e694a91e630b1bd6.png +0 -0
- package/lib/endfield-map/resources/map-eaa93af79d12a87d.png +0 -0
- package/lib/endfield-map/resources/map-ec9575ee65d7766f.json.br +0 -0
- package/lib/endfield-map/resources/map-ec9575ee65d7766f.json.gz +0 -0
- package/lib/endfield-map/resources/map-ede421a329dc78a4.png +0 -0
- package/lib/endfield-map/resources/map-eff3bce8f6888fc5.json.br +0 -0
- package/lib/endfield-map/resources/map-eff3bce8f6888fc5.json.gz +0 -0
- package/lib/endfield-map/resources/map-fb5f4a59a181452c.png +0 -0
- package/lib/endfield-map/resources/map-fdeaa17eeba90d8e.json.br +0 -0
- package/lib/endfield-map/resources/map-fdeaa17eeba90d8e.json.gz +0 -0
- package/lib/rhine/assets/archive-assembly.glb +0 -0
- package/lib/rhine/assets/archive-cassette.glb +0 -0
- package/lib/rhine/fonts/MiSans-Bold.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-Demibold.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-Light.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-Regular.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-license.pdf +0 -0
- package/lib/rhine/fonts/NOTICE.txt +6 -0
- package/lib/rhine/licenses/RhineLabUI-MIT.txt +21 -0
- package/lib/rhine/licenses/entities-BSD-2-Clause.txt +11 -0
- package/lib/rhine/licenses/linkify-it-MIT.txt +22 -0
- package/lib/rhine/licenses/markdown-it-MIT.txt +22 -0
- package/lib/rhine/licenses/mdurl-MIT.txt +45 -0
- package/lib/rhine/licenses/punycode.js-MIT.txt +20 -0
- package/lib/rhine/licenses/rolling-number-MIT.txt +21 -0
- package/lib/rhine/licenses/three-MIT.txt +21 -0
- package/lib/rhine/licenses/uc.micro-MIT.txt +20 -0
- package/lib/rhine/rhine.css +1 -0
- package/lib/rhine/rhine.js +5106 -0
- package/lib/skins/common.css +31 -0
- package/lib/skins/endfield-aic.css +141 -0
- package/lib/skins/prts-agent.css +136 -0
- package/lib/skins/rhine-lab.css +70 -0
- package/package.json +112 -0
- package/presets/definition.js +26 -0
- package/presets/prts/agent.cordis.yml +25 -0
- package/presets/prts/preset.yml +3 -0
- package/presets/register.js +86 -0
- package/resources/entity-routing.json +21 -0
- package/skills/prts-investigation/SKILL.md +121 -0
- package/skills/prts-retrieval/SKILL.md +35 -0
- package/skills/prts-retrieval/references/cloud-tools.md +13 -0
- package/skills/prts-retrieval/references/diagnostics.md +15 -0
- package/skills/prts-retrieval/references/local-tools.md +24 -0
- package/skills/prts-retrieval/references/module-arknights.md +23 -0
- package/skills/prts-retrieval/references/module-dual.md +15 -0
- package/skills/prts-retrieval/references/module-endfield.md +21 -0
- package/skills/prts-retrieval/references/retrieval-process.md +44 -0
- package/skills/prts-retrieval/references/retrieval-recipes-dual.md +34 -0
- package/skills/prts-retrieval/references/retrieval-recipes-endfield.md +37 -0
- package/skills/prts-retrieval/references/retrieval-recipes.md +39 -0
- package/skills/prts-retrieval/references/tools-runtime.md +48 -0
- package/skills/prts-retrieval/references/wiki-schema.md +38 -0
- package/src/archive-presentation.js +287 -0
- package/src/cloud-projection.js +124 -0
- package/src/cloud.js +498 -0
- package/src/entity-recognizer.js +311 -0
- package/src/entity-routing.js +82 -0
- package/src/evidence-state.js +332 -0
- package/src/i18n.js +365 -0
- package/src/index.js +1261 -0
- package/src/installer.js +1362 -0
- package/src/investigation-review.js +87 -0
- package/src/investigation-store.js +579 -0
- package/src/investigation-tools.js +113 -0
- package/src/localization-format.js +23 -0
- package/src/manual-search.js +103 -0
- package/src/read.js +1097 -0
- package/src/release-compatibility.js +73 -0
- package/src/release-watch.js +40 -0
- package/src/search-projection.js +139 -0
- package/src/search.js +1522 -0
- package/src/skill.js +98 -0
- package/src/source-map.js +348 -0
- package/src/state.js +314 -0
- package/src/store.js +1569 -0
- package/src/timeline.js +386 -0
- package/src/ui.js +1353 -0
- package/src/wiki.js +111 -0
- package/tsconfig.rhine.json +15 -0
- package/ui/rhine/archive-navigation-limit.ts +31 -0
- package/ui/rhine/archive-rack.ts +83 -0
- package/ui/rhine/assets/evidence-observatory.webp +0 -0
- package/ui/rhine/board-plane-transform.ts +24 -0
- package/ui/rhine/catalogue.ts +72 -0
- package/ui/rhine/deferred-preparation.ts +146 -0
- package/ui/rhine/evidence-board-controls.css +22 -0
- package/ui/rhine/evidence-board-history.ts +73 -0
- package/ui/rhine/evidence-board-model.ts +189 -0
- package/ui/rhine/evidence-board-panel.ts +762 -0
- package/ui/rhine/evidence-board-scene.ts +548 -0
- package/ui/rhine/evidence-board.css +191 -0
- package/ui/rhine/evidence-card-art.ts +470 -0
- package/ui/rhine/evidence-card-style.ts +17 -0
- package/ui/rhine/evidence-inbox-scene.ts +43 -0
- package/ui/rhine/evidence-inbox.css +29 -0
- package/ui/rhine/evidence-inbox.ts +104 -0
- package/ui/rhine/index.ts +12 -0
- package/ui/rhine/interaction.css +20 -0
- package/ui/rhine/investigation-board-plane.ts +5 -0
- package/ui/rhine/investigation-board.css +34 -0
- package/ui/rhine/investigation-board.ts +348 -0
- package/ui/rhine/investigation-edits.ts +46 -0
- package/ui/rhine/investigation-layout.ts +29 -0
- package/ui/rhine/label-prefetch.ts +34 -0
- package/ui/rhine/looping-title.ts +40 -0
- package/ui/rhine/manual-search.css +41 -0
- package/ui/rhine/manual-search.ts +89 -0
- package/ui/rhine/original/appearance.ts +180 -0
- package/ui/rhine/original/archive-label.ts +84 -0
- package/ui/rhine/original/archive-lighting.ts +39 -0
- package/ui/rhine/original/archive-loop.ts +82 -0
- package/ui/rhine/original/archive-refill.ts +64 -0
- package/ui/rhine/original/array-detail-data.ts +2 -0
- package/ui/rhine/original/array-detail.ts +41 -0
- package/ui/rhine/original/array-interior.ts +411 -0
- package/ui/rhine/original/array-occlusion.ts +166 -0
- package/ui/rhine/original/array-shell.ts +38 -0
- package/ui/rhine/original/array-visibility.ts +309 -0
- package/ui/rhine/original/asset-url.ts +4 -0
- package/ui/rhine/original/brand.ts +12 -0
- package/ui/rhine/original/cassette-lod.ts +81 -0
- package/ui/rhine/original/data.ts +21 -0
- package/ui/rhine/original/decryption.ts +185 -0
- package/ui/rhine/original/detail-signature.ts +18 -0
- package/ui/rhine/original/glass-reveal.ts +45 -0
- package/ui/rhine/original/internal-optics.ts +70 -0
- package/ui/rhine/original/label-renderer.ts +119 -0
- package/ui/rhine/original/label-textures.ts +117 -0
- package/ui/rhine/original/model-viewer.ts +658 -0
- package/ui/rhine/original/motion.ts +116 -0
- package/ui/rhine/original/opaque-backfaces.ts +64 -0
- package/ui/rhine/original/program-preparation.ts +111 -0
- package/ui/rhine/original/quality-renderer.ts +97 -0
- package/ui/rhine/original/quality-settings.ts +111 -0
- package/ui/rhine/original/render-quality.ts +163 -0
- package/ui/rhine/original/scene.ts +2051 -0
- package/ui/rhine/original/shelf-interior.ts +142 -0
- package/ui/rhine/original/ui-transitions.ts +111 -0
- package/ui/rhine/original/viewer-camera.ts +92 -0
- package/ui/rhine/original/viewport-layout.ts +41 -0
- package/ui/rhine/original-ui.css +2334 -0
- package/ui/rhine/readability.css +46 -0
- package/ui/rhine/reading-object.ts +96 -0
- package/ui/rhine/reading-origin.ts +28 -0
- package/ui/rhine/reading-room.css +55 -0
- package/ui/rhine/report-markdown.ts +98 -0
- package/ui/rhine/scene-edge-navigation.ts +84 -0
- package/ui/rhine/scene.ts +1726 -0
- package/ui/rhine/shelf-layout.ts +38 -0
- package/ui/rhine/snapshot-update-gate.ts +22 -0
- package/ui/rhine/source-reading.ts +15 -0
- package/ui/rhine/source-selection.css +164 -0
- package/ui/rhine/temporary-performance-observers.ts +163 -0
- package/ui/rhine/temporary-performance-panel.ts +124 -0
- package/ui/rhine/temporary-performance.css +19 -0
- package/ui/rhine/temporary-performance.ts +552 -0
- package/ui/rhine/temporary-render-diagnostics.ts +105 -0
- package/ui/rhine/tool-activity.ts +69 -0
- package/ui/rhine/types.ts +219 -0
- package/ui/rhine/visible-scene-hit.ts +13 -0
- package/ui/rhine/workbench.css +818 -0
- package/ui/rhine/workbench.ts +2245 -0
- package/ui/rhine/workspace-travel.ts +32 -0
- package/vite.rhine.config.ts +19 -0
package/src/installer.js
ADDED
|
@@ -0,0 +1,1362 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 设置页显式调用的资料包下载器。下载完成并激活 release 后,CorpusStore 才会打开。
|
|
3
|
+
*
|
|
4
|
+
* 下载前先从 PRTS.chat current、release manifest 与 pack manifest 取得受信
|
|
5
|
+
* release ID、data_version 和逐文件摘要;下载源只负责提供与摘要匹配的字节:
|
|
6
|
+
* 1. modelscope —— PRTS.chat 登记的固定 release 镜像;
|
|
7
|
+
* 2. site —— PRTS.chat 站点资源接口(published/preview 匿名可取)。
|
|
8
|
+
*
|
|
9
|
+
* 任一源下载失败即切换下一源;分片只按 PRTS.chat 可信清单的 sha256 校验,
|
|
10
|
+
* 已存在且校验一致的文件跳过(跨源断点续传天然成立:同一构建的分片哈希相同)。
|
|
11
|
+
* 全部通过后才写 current.json 指针——中途失败不产生“半激活”状态。
|
|
12
|
+
*
|
|
13
|
+
* ModelScope 布局(agent/scripts/publish_modelscope_mirrors.py 发布):
|
|
14
|
+
* releases/<release_id>/dataset-manifest.json ← 仅供镜像自检,插件不信任此清单
|
|
15
|
+
* releases/<release_id>/<pack>/{pack-manifest.json, shards/*.jsonl.gz,
|
|
16
|
+
* search-index/*.bin.gz, catalog/documents.jsonl.gz}
|
|
17
|
+
*
|
|
18
|
+
* 站点布局(backend/routers/agent_data.py):
|
|
19
|
+
* /api/agent/data/releases/<id>/release-manifest.json
|
|
20
|
+
* /api/agent/data/releases/<id>/<pack>/pack-manifest.json ← 逐分片 size+sha256
|
|
21
|
+
* /api/agent/data/releases/<id>/<pack>/{shards|search-index}/...
|
|
22
|
+
*/
|
|
23
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
24
|
+
import { createReadStream } from 'node:fs'
|
|
25
|
+
import { lstat, mkdir, open, readFile, readdir, realpath, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
26
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
|
27
|
+
import { AGENT_VERSION, compareSemver, parseSemver } from './release-compatibility.js'
|
|
28
|
+
import { localizationAssets } from './localization-format.js'
|
|
29
|
+
|
|
30
|
+
/** ModelScope 新版按游戏分为两个数据集;旧键仅用于历史 release 兼容。 */
|
|
31
|
+
export const MODELSCOPE_REPOS = Object.freeze({
|
|
32
|
+
arknights: 'HTiantian/prts-agent-corpus-arknights',
|
|
33
|
+
endfield: 'HTiantian/prts-agent-corpus-endfield',
|
|
34
|
+
official: 'HTiantian/prts-agent-corpus-arknights-gamedata',
|
|
35
|
+
community: 'HTiantian/prts-agent-corpus-selfbuilt',
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* 这里仅登记旧三数据集 release 的不可变组合,保证已发布版本继续可下载。
|
|
40
|
+
* 新版本按游戏进入 arknights/endfield 两仓,并以 PRTS.chat 注册的 mirror 为准。
|
|
41
|
+
* 组合关系不能按“各仓最新”猜测,否则并发发布时会拼出未经审核的版本。
|
|
42
|
+
*/
|
|
43
|
+
export const MODELSCOPE_RELEASE_COMPOSITIONS = Object.freeze({
|
|
44
|
+
'agent-corpus-v2-20260903-xuesong-youmeng-v1': Object.freeze({
|
|
45
|
+
layout: 'legacy-three-dataset-v1',
|
|
46
|
+
dataVersion: '77df7c534525256af1dd36b68128cdd878ac2f3bc109636c5051fa85dd3dae09',
|
|
47
|
+
releases: Object.freeze({
|
|
48
|
+
official: 'agent-corpus-v1-20260826-timeline-v1',
|
|
49
|
+
endfield: 'agent-corpus-v2-20260903-xuesong-youmeng-v1',
|
|
50
|
+
community: 'agent-corpus-v1-20260826-timeline-v1',
|
|
51
|
+
}),
|
|
52
|
+
}),
|
|
53
|
+
'agent-corpus-v2-20260905-character-activity-split-v1': Object.freeze({
|
|
54
|
+
layout: 'legacy-three-dataset-v1',
|
|
55
|
+
dataVersion: 'ebf6bec17dc40894c8bc1987197f34bd9800be77baa578de4a04f241c542fba9',
|
|
56
|
+
releases: Object.freeze({
|
|
57
|
+
official: 'agent-corpus-v2-20260904-retraveler-alias-fix-v1',
|
|
58
|
+
endfield: 'agent-corpus-v2-20260904-retraveler-alias-fix-v1',
|
|
59
|
+
community: 'agent-corpus-v2-20260905-character-activity-split-v1',
|
|
60
|
+
}),
|
|
61
|
+
}),
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
/** 默认 pin 的 release(站点已发布;ModelScope 未就绪时自动回退站点源)。 */
|
|
65
|
+
export const DEFAULT_RELEASE_ID = 'agent-corpus-v2-20260903-xuesong-youmeng-v1'
|
|
66
|
+
|
|
67
|
+
export const DEFAULT_SITE_BASE_URL = 'https://prts.chat'
|
|
68
|
+
|
|
69
|
+
/** 字节回退站点必须是 HTTPS;仅本地开发允许环回 HTTP。它不参与选版或摘要签发。 */
|
|
70
|
+
export function normalizeSiteBaseUrl(value = DEFAULT_SITE_BASE_URL) {
|
|
71
|
+
let url
|
|
72
|
+
try { url = new URL(String(value ?? '')) } catch {
|
|
73
|
+
throw new InstallerFault('INVALID_REQUEST', 'siteBaseUrl 不是有效 URL')
|
|
74
|
+
}
|
|
75
|
+
const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname)
|
|
76
|
+
if (url.username || url.password || url.search || url.hash
|
|
77
|
+
|| (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback))) {
|
|
78
|
+
throw new InstallerFault('INVALID_REQUEST',
|
|
79
|
+
'siteBaseUrl 必须使用 HTTPS(本地环回可用 HTTP),且不能包含凭证、查询或片段')
|
|
80
|
+
}
|
|
81
|
+
return url.toString().replace(/\/+$/, '')
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* release id 白名单:必须以字母/数字开头,禁止路径分隔符与纯点号段("."、
|
|
86
|
+
* ".." 会被直接拒绝),避免 releaseId 拼进 releasesDir 后逃逸到上级目录
|
|
87
|
+
* (ui.js 的 delete 路由会对该目录执行递归删除)。
|
|
88
|
+
*/
|
|
89
|
+
export const RELEASE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
|
|
90
|
+
const SHA256_PATTERN = /^[0-9a-f]{64}$/
|
|
91
|
+
const PACK_IDS = ['official_game', 'endfield_official_game', 'endfield_reviewed_knowledge',
|
|
92
|
+
'reviewed_wiki', 'terra_journey', 'entities', 'references']
|
|
93
|
+
const PACK_DESCRIPTOR_FIELDS = Object.freeze([
|
|
94
|
+
'pack_id', 'manifest_path', 'authority', 'data_version', 'document_count',
|
|
95
|
+
'line_count', 'compressed_size', 'uncompressed_size', 'shard_count',
|
|
96
|
+
])
|
|
97
|
+
const PACK_DESCRIPTOR_FIELD_SET = new Set(PACK_DESCRIPTOR_FIELDS)
|
|
98
|
+
const REQUIRED_GAME_PACK = Object.freeze({
|
|
99
|
+
arknights: 'official_game',
|
|
100
|
+
endfield: 'endfield_official_game',
|
|
101
|
+
})
|
|
102
|
+
const ASSET_PATH_PATTERN = /^(?:shards\/[A-Za-z0-9._-]+\.jsonl|search-index\/[A-Za-z0-9._-]+\.bin|(?:catalog|localization)\/[A-Za-z0-9._-]+\.jsonl)\.gz$/
|
|
103
|
+
const wait = (milliseconds) => new Promise((resolve) => { setTimeout(resolve, milliseconds) })
|
|
104
|
+
|
|
105
|
+
/** 清单类请求的显式超时与响应体大小上限(被劫持源不得用超大清单拖垮内存/磁盘)。 */
|
|
106
|
+
const MANIFEST_TIMEOUT_MS = 20_000
|
|
107
|
+
const MAX_MANIFEST_BYTES = 8 * 1024 * 1024
|
|
108
|
+
const MAX_CURRENT_POINTER_BYTES = 64 * 1024
|
|
109
|
+
/** 源未提供 sha256 的文件(站点 pack/release 清单本体)允许的最大体积。 */
|
|
110
|
+
const MAX_UNVERIFIED_BYTES = 4 * 1024 * 1024
|
|
111
|
+
const RELEASE_ALGORITHM = 'prts-browser-corpus-release-v1'
|
|
112
|
+
const PACK_ALGORITHMS = Object.freeze({
|
|
113
|
+
'prts-browser-corpus-pack-v1': 1,
|
|
114
|
+
'prts-browser-corpus-pack-v2': 2,
|
|
115
|
+
})
|
|
116
|
+
export const SUPPORTED_SEARCH_INDEX_ALGORITHMS = Object.freeze({
|
|
117
|
+
'prts-browser-trigram-postings-v1': 1,
|
|
118
|
+
'prts-browser-ngram-postings-v2': 2,
|
|
119
|
+
})
|
|
120
|
+
// 只接受本模块从固定 PRTS.chat origin 解析出来的对象作为跨层快照。WeakMap
|
|
121
|
+
// 保存一份与公开返回值分离的不可变副本;即使内部调用者尝试修改返回对象,
|
|
122
|
+
// 下载器也只读取私有 canonical 值。
|
|
123
|
+
const trustedCurrentSnapshots = new WeakMap()
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* 资料包资源上限是本地解析器的安全不变量,不随部署配置放宽。当前正式包约
|
|
127
|
+
* 621 个资源、155 MiB 压缩 / 617 MiB 解压;这里保留数倍增长空间,同时
|
|
128
|
+
* 阻止恶意清单借文件数、下载总量或 gzip 膨胀耗尽 Host 资源。
|
|
129
|
+
*/
|
|
130
|
+
export const CORPUS_RESOURCE_LIMITS = Object.freeze({
|
|
131
|
+
maxAssets: 4096,
|
|
132
|
+
maxAssetCompressedBytes: 64 * 1024 * 1024,
|
|
133
|
+
maxAssetUncompressedBytes: 128 * 1024 * 1024,
|
|
134
|
+
maxReleaseCompressedBytes: 1024 * 1024 * 1024,
|
|
135
|
+
maxReleaseUncompressedBytes: 4 * 1024 * 1024 * 1024,
|
|
136
|
+
})
|
|
137
|
+
/**
|
|
138
|
+
* 单文件下载整体超时:慢速滴流的源不应长期占用下载槽位。按体积给足带宽
|
|
139
|
+
* 余量(≥64 KiB/s),下限 60s、上限 15 分钟;体积未知时按 60s 计。
|
|
140
|
+
*/
|
|
141
|
+
const downloadTimeoutMs = (size) => (Number.isInteger(size) && size > 0
|
|
142
|
+
? Math.min(15 * 60_000, Math.max(60_000, 60_000 + Math.ceil(size / 65536) * 1000))
|
|
143
|
+
: 60_000)
|
|
144
|
+
|
|
145
|
+
const isTimeoutError = (error) => error?.name === 'TimeoutError'
|
|
146
|
+
|| (error?.name === 'AbortError' && error?.message?.includes('timeout'))
|
|
147
|
+
|
|
148
|
+
async function linuxProcessStart(pid) {
|
|
149
|
+
if (process.platform !== 'linux' || !Number.isInteger(pid) || pid < 1) return null
|
|
150
|
+
try {
|
|
151
|
+
const value = await readFile(`/proc/${pid}/stat`, 'utf8')
|
|
152
|
+
const suffix = value.slice(value.lastIndexOf(') ') + 2).trim().split(/\s+/u)
|
|
153
|
+
return suffix[19] || null // proc_pid_stat(5) field 22, suffix starts at field 3
|
|
154
|
+
} catch {
|
|
155
|
+
return null
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const processGone = async (pid, recordedStart = null) => {
|
|
160
|
+
try {
|
|
161
|
+
process.kill(pid, 0)
|
|
162
|
+
if (recordedStart) {
|
|
163
|
+
const currentStart = await linuxProcessStart(pid)
|
|
164
|
+
if (currentStart && currentStart !== recordedStart) return true
|
|
165
|
+
}
|
|
166
|
+
return false
|
|
167
|
+
} catch (probe) {
|
|
168
|
+
return probe?.code === 'ESRCH'
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const LEASE_NAME_PATTERN = /^lease-(\d+)-(\d+|na)-([0-9a-f]{24})$/u
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* 跨进程锁采用 Lamport bakery 式的唯一 lease 文件,而不是共用一个需要
|
|
176
|
+
* “判断后删除”的 lockfile。每个竞争者只会删除自己的随机文件;崩溃残留
|
|
177
|
+
* 也按该唯一文件回收,因而不存在 reaper 把新主锁误删或自身残留永久
|
|
178
|
+
* 業死锁的竞态。文件初始内容为 choosing,随后写入单调 ticket;同 ticket
|
|
179
|
+
* 用唯一文件名排序。
|
|
180
|
+
*/
|
|
181
|
+
async function readLease(lockDir, name) {
|
|
182
|
+
const match = LEASE_NAME_PATTERN.exec(name)
|
|
183
|
+
if (!match) return null
|
|
184
|
+
const path = join(lockDir, name)
|
|
185
|
+
try {
|
|
186
|
+
const info = await lstat(path)
|
|
187
|
+
if (!info.isFile() || info.isSymbolicLink()) return null
|
|
188
|
+
const content = info.size <= 64 ? String(await readFile(path, 'utf8')).trim() : ''
|
|
189
|
+
const ticket = /^[1-9]\d{0,15}$/u.test(content) ? Number(content) : null
|
|
190
|
+
return {
|
|
191
|
+
name, path, pid: Number(match[1]), processStart: match[2] === 'na' ? null : match[2],
|
|
192
|
+
ticket: Number.isSafeInteger(ticket) ? ticket : null,
|
|
193
|
+
}
|
|
194
|
+
} catch (error) {
|
|
195
|
+
if (error?.code === 'ENOENT') return null
|
|
196
|
+
throw error
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
async function activeLeases(lockDir) {
|
|
201
|
+
const names = await readdir(lockDir)
|
|
202
|
+
const leases = (await Promise.all(names.map((name) => readLease(lockDir, name))))
|
|
203
|
+
.filter(Boolean)
|
|
204
|
+
const active = []
|
|
205
|
+
for (const lease of leases) {
|
|
206
|
+
if (await processGone(lease.pid, lease.processStart)) {
|
|
207
|
+
// 路径含 96-bit 随机 token,新竞争者不会复用;这里精确删除
|
|
208
|
+
// 已消失进程的 lease,不会命中另一个刚创建的锁。
|
|
209
|
+
await rm(lease.path, { force: true }).catch(() => {})
|
|
210
|
+
} else {
|
|
211
|
+
active.push(lease)
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return active
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async function acquireReleaseMutationLock(releasesDir) {
|
|
218
|
+
await mkdir(resolve(releasesDir), { recursive: true, mode: 0o700 })
|
|
219
|
+
const releasesRoot = await realpath(resolve(releasesDir))
|
|
220
|
+
const lockDirPath = join(releasesRoot, '.release-mutation-locks')
|
|
221
|
+
await mkdir(lockDirPath, { mode: 0o700 }).catch((error) => {
|
|
222
|
+
if (error?.code !== 'EEXIST') throw error
|
|
223
|
+
})
|
|
224
|
+
const lockDir = await requireContainedDirectory(lockDirPath, releasesRoot,
|
|
225
|
+
'资料变更锁目录', 'INVALID_RELEASE')
|
|
226
|
+
const processStart = await linuxProcessStart(process.pid)
|
|
227
|
+
const token = randomBytes(12).toString('hex')
|
|
228
|
+
const name = `lease-${process.pid}-${processStart ?? 'na'}-${token}`
|
|
229
|
+
const path = join(lockDir, name)
|
|
230
|
+
const handle = await open(path, 'wx', 0o600)
|
|
231
|
+
const deadline = Date.now() + 30_000
|
|
232
|
+
try {
|
|
233
|
+
await handle.writeFile('choosing\n')
|
|
234
|
+
await handle.sync()
|
|
235
|
+
const existing = await activeLeases(lockDir)
|
|
236
|
+
const maximum = existing.reduce((value, lease) =>
|
|
237
|
+
lease.ticket == null ? value : Math.max(value, lease.ticket), 0)
|
|
238
|
+
const ticket = maximum + 1
|
|
239
|
+
await handle.truncate(0)
|
|
240
|
+
await handle.write(`${ticket}\n`, 0, 'utf8')
|
|
241
|
+
await handle.sync()
|
|
242
|
+
for (;;) {
|
|
243
|
+
const leases = await activeLeases(lockDir)
|
|
244
|
+
const blocked = leases.some((lease) => lease.name !== name
|
|
245
|
+
&& (lease.ticket == null || lease.ticket < ticket
|
|
246
|
+
|| (lease.ticket === ticket && lease.name < name)))
|
|
247
|
+
if (!blocked) {
|
|
248
|
+
return async () => {
|
|
249
|
+
await handle.close()
|
|
250
|
+
await rm(path, { force: true })
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (Date.now() >= deadline) {
|
|
254
|
+
throw new InstallerFault('DOWNLOAD_BUSY', '另一个进程正在下载、激活或删除资料版本')
|
|
255
|
+
}
|
|
256
|
+
await wait(100)
|
|
257
|
+
}
|
|
258
|
+
} catch (error) {
|
|
259
|
+
await handle.close().catch(() => {})
|
|
260
|
+
await rm(path, { force: true }).catch(() => {})
|
|
261
|
+
throw error
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** 跨 Host 进程串行化 release 指针与目录变更。 */
|
|
266
|
+
export async function withReleaseMutationLock(releasesDir, operation) {
|
|
267
|
+
const release = await acquireReleaseMutationLock(releasesDir)
|
|
268
|
+
try {
|
|
269
|
+
return await operation()
|
|
270
|
+
} finally {
|
|
271
|
+
await release()
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async function ensureManagedReleaseDirectory(releasesDir, releaseId) {
|
|
276
|
+
await mkdir(resolve(releasesDir), { recursive: true, mode: 0o700 })
|
|
277
|
+
const releasesRoot = await realpath(resolve(releasesDir))
|
|
278
|
+
const requested = join(releasesRoot, releaseId)
|
|
279
|
+
await mkdir(requested, { recursive: true, mode: 0o700 })
|
|
280
|
+
const releaseDir = await requireContainedDirectory(requested, releasesRoot,
|
|
281
|
+
`release ${releaseId}`, 'INVALID_RELEASE')
|
|
282
|
+
return { releasesRoot, releaseDir }
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
async function prepareAssetTarget(releaseDir, relativePath) {
|
|
286
|
+
const [packId, category, filename, ...extra] = relativePath.split('/')
|
|
287
|
+
if (extra.length || !PACK_IDS.includes(packId)
|
|
288
|
+
|| !['shards', 'search-index', 'catalog', 'localization'].includes(category) || !filename) {
|
|
289
|
+
throw new InstallerFault('INVALID_MANIFEST', `资源路径非法: ${relativePath}`)
|
|
290
|
+
}
|
|
291
|
+
let parent = releaseDir
|
|
292
|
+
for (const segment of [packId, category]) {
|
|
293
|
+
const child = join(parent, segment)
|
|
294
|
+
await mkdir(child, { recursive: false, mode: 0o700 }).catch((error) => {
|
|
295
|
+
if (error?.code !== 'EEXIST') throw error
|
|
296
|
+
})
|
|
297
|
+
const link = await lstat(child)
|
|
298
|
+
if (!link.isDirectory() || link.isSymbolicLink()) {
|
|
299
|
+
throw new InstallerFault('INVALID_RELEASE', `资源父目录不是受管目录: ${relativePath}`)
|
|
300
|
+
}
|
|
301
|
+
const actual = await realpath(child)
|
|
302
|
+
if (!isContainedPath(releaseDir, actual)) {
|
|
303
|
+
throw new InstallerFault('INVALID_RELEASE', `资源父目录越出 release: ${relativePath}`)
|
|
304
|
+
}
|
|
305
|
+
parent = child
|
|
306
|
+
}
|
|
307
|
+
return join(parent, filename)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
async function writeJsonAtomically(path, value) {
|
|
311
|
+
const temporary = `${path}.${randomBytes(6).toString('hex')}.tmp`
|
|
312
|
+
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })
|
|
313
|
+
await rename(temporary, path)
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async function currentReleaseReady(releasesDir, requested, expectedDataVersion = null) {
|
|
317
|
+
try {
|
|
318
|
+
const pointer = await readCurrentReleasePointer(releasesDir)
|
|
319
|
+
const releaseId = pointer.release_id
|
|
320
|
+
// 调用方给出 releaseId 就是显式 pin;绝不能因为 current 指向另一个
|
|
321
|
+
// 完整版本就返回 present,否则用户要的版本实际上从未准备。
|
|
322
|
+
if (releaseId !== requested) return false
|
|
323
|
+
const manifest = await validateLocalRelease(releasesDir, releaseId, { verifyHashes: true })
|
|
324
|
+
return (!pointer.data_version || pointer.data_version === manifest.data_version)
|
|
325
|
+
&& (!expectedDataVersion || manifest.data_version === expectedDataVersion)
|
|
326
|
+
} catch {
|
|
327
|
+
return false
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export class InstallerFault extends Error {
|
|
332
|
+
constructor(code, message) {
|
|
333
|
+
super(message)
|
|
334
|
+
this.code = code
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** 返回当前 release 相对启用游戏所缺的官方资料包。 */
|
|
339
|
+
export function missingEnabledGamePacks(manifest, enabledGames = []) {
|
|
340
|
+
const packs = new Set((manifest?.packs || []).map((pack) => String(pack?.pack_id || '')))
|
|
341
|
+
return [...new Set(enabledGames)].filter((game) => REQUIRED_GAME_PACK[game]
|
|
342
|
+
&& !packs.has(REQUIRED_GAME_PACK[game]))
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function sha256File(path) {
|
|
346
|
+
return new Promise((resolve, reject) => {
|
|
347
|
+
const hash = createHash('sha256')
|
|
348
|
+
createReadStream(path).on('data', (chunk) => hash.update(chunk))
|
|
349
|
+
.on('error', reject)
|
|
350
|
+
.on('end', () => resolve(hash.digest('hex')))
|
|
351
|
+
})
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
const isContainedPath = (root, target) => {
|
|
355
|
+
const child = relative(root, target)
|
|
356
|
+
return child !== '' && child !== '..' && !child.startsWith(`..${sep}`) && !isAbsolute(child)
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** 有界读取受管 current.json,避免符号链接或超大本地指针先于 release 校验生效。 */
|
|
360
|
+
export async function readCurrentReleasePointer(releasesDir) {
|
|
361
|
+
const root = await realpath(resolve(releasesDir))
|
|
362
|
+
const path = join(root, 'current.json')
|
|
363
|
+
const link = await lstat(path)
|
|
364
|
+
if (!link.isFile() || link.isSymbolicLink() || link.size > MAX_CURRENT_POINTER_BYTES) {
|
|
365
|
+
throw new InstallerFault('INVALID_RELEASE', 'current.json 不是有效的受管指针')
|
|
366
|
+
}
|
|
367
|
+
const actual = await realpath(path)
|
|
368
|
+
if (!isContainedPath(root, actual)) {
|
|
369
|
+
throw new InstallerFault('INVALID_RELEASE', 'current.json 越出 releases 目录')
|
|
370
|
+
}
|
|
371
|
+
let pointer
|
|
372
|
+
try { pointer = JSON.parse(await readFile(path, 'utf8')) } catch {
|
|
373
|
+
throw new InstallerFault('INVALID_RELEASE', 'current.json 不是有效 JSON')
|
|
374
|
+
}
|
|
375
|
+
const releaseId = String(pointer?.release_id ?? '')
|
|
376
|
+
if (!pointer || typeof pointer !== 'object' || Array.isArray(pointer)
|
|
377
|
+
|| !RELEASE_ID_PATTERN.test(releaseId)
|
|
378
|
+
|| (pointer.data_version != null
|
|
379
|
+
&& !SHA256_PATTERN.test(String(pointer.data_version)))) {
|
|
380
|
+
throw new InstallerFault('INVALID_RELEASE', 'current.json 内容非法')
|
|
381
|
+
}
|
|
382
|
+
return { ...pointer, release_id: releaseId }
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
const invalid = (code, message) => {
|
|
386
|
+
throw new InstallerFault(code, message)
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function requireInteger(value, label, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER,
|
|
390
|
+
code = 'INVALID_MANIFEST' } = {}) {
|
|
391
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
392
|
+
invalid(code, `${label} 非法`)
|
|
393
|
+
}
|
|
394
|
+
return value
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function normalizePackDescriptor(descriptor, label, code = 'INVALID_MANIFEST') {
|
|
398
|
+
if (!descriptor || typeof descriptor !== 'object' || Array.isArray(descriptor)
|
|
399
|
+
|| Object.keys(descriptor).some((key) => !PACK_DESCRIPTOR_FIELD_SET.has(key))) {
|
|
400
|
+
invalid(code, `${label} pack 描述包含非法字段`)
|
|
401
|
+
}
|
|
402
|
+
const packId = requirePackId(String(descriptor.pack_id ?? ''), code)
|
|
403
|
+
const authority = descriptor.authority == null ? 'official' : String(descriptor.authority)
|
|
404
|
+
if (descriptor.manifest_path !== `${packId}/pack-manifest.json`
|
|
405
|
+
|| !SHA256_PATTERN.test(String(descriptor.data_version ?? ''))
|
|
406
|
+
|| !authority || authority.length > 128 || /[\p{Cc}\p{Cf}]/u.test(authority)) {
|
|
407
|
+
invalid(code, `${label} pack 描述非法: ${packId}`)
|
|
408
|
+
}
|
|
409
|
+
return Object.freeze({
|
|
410
|
+
pack_id: packId,
|
|
411
|
+
manifest_path: descriptor.manifest_path,
|
|
412
|
+
authority,
|
|
413
|
+
data_version: descriptor.data_version,
|
|
414
|
+
document_count: requireInteger(descriptor.document_count, `${label}.${packId}.document_count`, {
|
|
415
|
+
minimum: 1, maximum: 10_000_000, code,
|
|
416
|
+
}),
|
|
417
|
+
line_count: requireInteger(descriptor.line_count, `${label}.${packId}.line_count`, {
|
|
418
|
+
maximum: 100_000_000, code,
|
|
419
|
+
}),
|
|
420
|
+
compressed_size: requireInteger(descriptor.compressed_size, `${label}.${packId}.compressed_size`, {
|
|
421
|
+
minimum: 1, maximum: CORPUS_RESOURCE_LIMITS.maxReleaseCompressedBytes, code,
|
|
422
|
+
}),
|
|
423
|
+
uncompressed_size: requireInteger(descriptor.uncompressed_size, `${label}.${packId}.uncompressed_size`, {
|
|
424
|
+
minimum: 1, maximum: CORPUS_RESOURCE_LIMITS.maxReleaseUncompressedBytes, code,
|
|
425
|
+
}),
|
|
426
|
+
shard_count: requireInteger(descriptor.shard_count, `${label}.${packId}.shard_count`, {
|
|
427
|
+
minimum: 1, maximum: CORPUS_RESOURCE_LIMITS.maxAssets, code,
|
|
428
|
+
}),
|
|
429
|
+
})
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
async function requireContainedDirectory(path, root, label, code = 'INVALID_RELEASE') {
|
|
433
|
+
const link = await lstat(path)
|
|
434
|
+
if (!link.isDirectory() || link.isSymbolicLink()) invalid(code, `${label} 不是受管目录`)
|
|
435
|
+
const actual = await realpath(path)
|
|
436
|
+
if (!isContainedPath(root, actual)) invalid(code, `${label} 越出 releases 目录`)
|
|
437
|
+
return actual
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
async function readContainedJson(path, root, label, code = 'INVALID_RELEASE') {
|
|
441
|
+
const link = await lstat(path)
|
|
442
|
+
if (!link.isFile() || link.isSymbolicLink() || link.size > MAX_MANIFEST_BYTES) {
|
|
443
|
+
invalid(code, `${label} 不是有效的受管清单`)
|
|
444
|
+
}
|
|
445
|
+
const actual = await realpath(path)
|
|
446
|
+
if (!isContainedPath(root, actual)) invalid(code, `${label} 越出 release 目录`)
|
|
447
|
+
let value
|
|
448
|
+
try {
|
|
449
|
+
value = JSON.parse(await readFile(path, 'utf8'))
|
|
450
|
+
} catch (error) {
|
|
451
|
+
if (error instanceof InstallerFault) throw error
|
|
452
|
+
invalid(code, `${label} 不是有效 JSON`)
|
|
453
|
+
}
|
|
454
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
455
|
+
invalid(code, `${label} 顶层必须是对象`)
|
|
456
|
+
}
|
|
457
|
+
return value
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function validateReleaseHeader(releaseId, manifest, code = 'INVALID_MANIFEST') {
|
|
461
|
+
if (manifest?.algorithm !== RELEASE_ALGORITHM || manifest?.schema_version !== 1
|
|
462
|
+
|| manifest?.release_id !== releaseId
|
|
463
|
+
|| !SHA256_PATTERN.test(String(manifest?.data_version ?? ''))
|
|
464
|
+
|| !Array.isArray(manifest?.required_packs) || manifest.required_packs.length === 0
|
|
465
|
+
|| manifest.required_packs.length > PACK_IDS.length
|
|
466
|
+
|| !Array.isArray(manifest?.packs) || manifest.packs.length === 0
|
|
467
|
+
|| manifest.packs.length > PACK_IDS.length) {
|
|
468
|
+
invalid(code, 'release-manifest 内容不完整或与 releaseId 不匹配')
|
|
469
|
+
}
|
|
470
|
+
const minimumAgentVersion = manifest.minimum_agent_version
|
|
471
|
+
if (!parseSemver(minimumAgentVersion)) {
|
|
472
|
+
invalid(code, 'release-manifest minimum_agent_version 缺失或不是有效 SemVer')
|
|
473
|
+
}
|
|
474
|
+
if (!parseSemver(AGENT_VERSION)) {
|
|
475
|
+
invalid('INCOMPATIBLE_RELEASE', `插件自身版本不是有效 SemVer: ${AGENT_VERSION}`)
|
|
476
|
+
}
|
|
477
|
+
if (compareSemver(AGENT_VERSION, minimumAgentVersion) < 0) {
|
|
478
|
+
invalid('INCOMPATIBLE_RELEASE',
|
|
479
|
+
`资料版本至少需要 prts-terrarchive ${minimumAgentVersion},当前为 ${AGENT_VERSION}`)
|
|
480
|
+
}
|
|
481
|
+
const descriptors = new Map()
|
|
482
|
+
for (const rawDescriptor of manifest.packs) {
|
|
483
|
+
const descriptor = normalizePackDescriptor(rawDescriptor, 'release', code)
|
|
484
|
+
const packId = descriptor.pack_id
|
|
485
|
+
if (descriptors.has(packId)) {
|
|
486
|
+
invalid(code, `release pack 描述非法: ${packId}`)
|
|
487
|
+
}
|
|
488
|
+
descriptors.set(packId, descriptor)
|
|
489
|
+
}
|
|
490
|
+
const required = new Set()
|
|
491
|
+
for (const value of manifest.required_packs) {
|
|
492
|
+
const packId = requirePackId(String(value || ''), code)
|
|
493
|
+
if (required.has(packId) || !descriptors.has(packId)) {
|
|
494
|
+
invalid(code, `required pack 描述非法: ${packId}`)
|
|
495
|
+
}
|
|
496
|
+
required.add(packId)
|
|
497
|
+
}
|
|
498
|
+
return descriptors
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
function validatePackManifest(packId, pack, descriptor, totals, code = 'INVALID_MANIFEST') {
|
|
502
|
+
const expectedSchema = PACK_ALGORITHMS[String(pack?.algorithm ?? '')]
|
|
503
|
+
if (!expectedSchema || pack?.schema_version !== expectedSchema
|
|
504
|
+
|| (pack.pack_id != null && pack.pack_id !== packId)
|
|
505
|
+
|| !SHA256_PATTERN.test(String(pack?.data_version ?? ''))
|
|
506
|
+
|| !Array.isArray(pack?.shards) || pack.shards.length === 0) {
|
|
507
|
+
invalid(code, `pack-manifest 内容不完整: ${packId}`)
|
|
508
|
+
}
|
|
509
|
+
if (pack.search_index != null && (!pack.search_index || typeof pack.search_index !== 'object'
|
|
510
|
+
|| Array.isArray(pack.search_index) || !Array.isArray(pack.search_index.shards))) {
|
|
511
|
+
invalid(code, `pack-manifest search_index 非法: ${packId}`)
|
|
512
|
+
}
|
|
513
|
+
if (pack.search_index?.shards?.length) {
|
|
514
|
+
const index = pack.search_index
|
|
515
|
+
if (!SUPPORTED_SEARCH_INDEX_ALGORITHMS[index.algorithm]) {
|
|
516
|
+
invalid('INCOMPATIBLE_RELEASE',
|
|
517
|
+
`${packId} 使用当前插件不支持的检索索引算法: ${index.algorithm || '未声明'}`)
|
|
518
|
+
}
|
|
519
|
+
const v1 = index.algorithm === 'prts-browser-trigram-postings-v1'
|
|
520
|
+
&& index.schema_version === 1 && index.normalization === 'unicode-nfc-casefold'
|
|
521
|
+
&& index.gram_size === 3 && index.format === 'varint-postings-le-v1'
|
|
522
|
+
const v2 = index.algorithm === 'prts-browser-ngram-postings-v2'
|
|
523
|
+
&& index.schema_version === 2
|
|
524
|
+
&& ['unicode-nfkc-casefold-collapse-space', 'unicode-nfkc-lower-collapse-space']
|
|
525
|
+
.includes(index.normalization)
|
|
526
|
+
&& JSON.stringify(index.gram_sizes) === '[1,2,3]'
|
|
527
|
+
&& index.format === 'varint-postings-le-v2'
|
|
528
|
+
const range = (shard) => ({ first: v2 ? shard.first_ngram : shard.first_trigram,
|
|
529
|
+
last: v2 ? shard.last_ngram : shard.last_trigram })
|
|
530
|
+
if ((!v1 && !v2) || index.shards.some((shard) => {
|
|
531
|
+
const bounds = range(shard)
|
|
532
|
+
const minimum = v2 ? 1 : 3
|
|
533
|
+
return typeof bounds.first !== 'string' || typeof bounds.last !== 'string'
|
|
534
|
+
|| [...bounds.first].length < minimum || [...bounds.first].length > 3
|
|
535
|
+
|| [...bounds.last].length < minimum || [...bounds.last].length > 3
|
|
536
|
+
|| Buffer.compare(Buffer.from(bounds.first, 'utf8'), Buffer.from(bounds.last, 'utf8')) > 0
|
|
537
|
+
})) invalid(code, `pack-manifest search_index 版本或范围非法: ${packId}`)
|
|
538
|
+
}
|
|
539
|
+
const documentCount = requireInteger(pack.document_count, `${packId}.document_count`,
|
|
540
|
+
{ minimum: 1, maximum: 10_000_000, code })
|
|
541
|
+
const lineCount = requireInteger(pack.line_count, `${packId}.line_count`,
|
|
542
|
+
{ maximum: 100_000_000, code })
|
|
543
|
+
const searchShards = pack.search_index?.shards ?? []
|
|
544
|
+
const catalog = pack.document_catalog
|
|
545
|
+
if (catalog != null && (!catalog || typeof catalog !== 'object' || Array.isArray(catalog)
|
|
546
|
+
|| catalog.algorithm !== 'prts-browser-document-catalog-v1'
|
|
547
|
+
|| catalog.schema_version !== 1 || catalog.document_count !== documentCount)) {
|
|
548
|
+
invalid(code, `pack-manifest document_catalog 非法: ${packId}`)
|
|
549
|
+
}
|
|
550
|
+
const catalogAssets = catalog ? [catalog] : []
|
|
551
|
+
if (pack.localization && packId !== 'endfield_official_game') invalid(code, '本地化附件只能属于终末地官方资料')
|
|
552
|
+
const localizedAssets = localizationAssets(pack, (message) => invalid(code, message))
|
|
553
|
+
if (pack.shards.length + searchShards.length + catalogAssets.length + localizedAssets.length
|
|
554
|
+
> CORPUS_RESOURCE_LIMITS.maxAssets
|
|
555
|
+
|| totals.assets + pack.shards.length + searchShards.length + catalogAssets.length + localizedAssets.length
|
|
556
|
+
> CORPUS_RESOURCE_LIMITS.maxAssets) {
|
|
557
|
+
invalid(code, '资料 release 的资源文件数超过上限')
|
|
558
|
+
}
|
|
559
|
+
const assets = [
|
|
560
|
+
...pack.shards.map((asset) => ({ asset, kind: 'shards/' })),
|
|
561
|
+
...searchShards.map((asset) => ({ asset, kind: 'search-index/' })),
|
|
562
|
+
...catalogAssets.map((asset) => ({ asset, kind: 'catalog/' })),
|
|
563
|
+
...localizedAssets.map((asset) => ({ asset, kind: 'localization/' })),
|
|
564
|
+
]
|
|
565
|
+
const paths = new Set()
|
|
566
|
+
let compressedSize = 0
|
|
567
|
+
let uncompressedSize = 0
|
|
568
|
+
for (const { asset, kind } of assets) {
|
|
569
|
+
const assetPath = String(asset?.path || '')
|
|
570
|
+
const compressed = asset?.compressed_size
|
|
571
|
+
const uncompressed = asset?.uncompressed_size
|
|
572
|
+
const expectedHash = String(asset?.sha256 || '')
|
|
573
|
+
if (!ASSET_PATH_PATTERN.test(assetPath) || !assetPath.startsWith(kind) || paths.has(assetPath)
|
|
574
|
+
|| !SHA256_PATTERN.test(expectedHash)) {
|
|
575
|
+
invalid(code, `分片描述非法: ${packId}/${assetPath}`)
|
|
576
|
+
}
|
|
577
|
+
requireInteger(compressed, `${packId}/${assetPath}.compressed_size`, {
|
|
578
|
+
minimum: 1, maximum: CORPUS_RESOURCE_LIMITS.maxAssetCompressedBytes, code,
|
|
579
|
+
})
|
|
580
|
+
requireInteger(uncompressed, `${packId}/${assetPath}.uncompressed_size`, {
|
|
581
|
+
minimum: 1, maximum: CORPUS_RESOURCE_LIMITS.maxAssetUncompressedBytes, code,
|
|
582
|
+
})
|
|
583
|
+
paths.add(assetPath)
|
|
584
|
+
compressedSize += compressed
|
|
585
|
+
uncompressedSize += uncompressed
|
|
586
|
+
}
|
|
587
|
+
totals.assets += assets.length
|
|
588
|
+
totals.compressed += compressedSize
|
|
589
|
+
totals.uncompressed += uncompressedSize
|
|
590
|
+
totals.documents += documentCount
|
|
591
|
+
totals.lines += lineCount
|
|
592
|
+
if (totals.assets > CORPUS_RESOURCE_LIMITS.maxAssets
|
|
593
|
+
|| totals.compressed > CORPUS_RESOURCE_LIMITS.maxReleaseCompressedBytes
|
|
594
|
+
|| totals.uncompressed > CORPUS_RESOURCE_LIMITS.maxReleaseUncompressedBytes) {
|
|
595
|
+
invalid(code, '资料 release 超过本地资源上限')
|
|
596
|
+
}
|
|
597
|
+
if (pack.compressed_size !== compressedSize || pack.uncompressed_size !== uncompressedSize) {
|
|
598
|
+
invalid(code, `pack-manifest 资源汇总不一致: ${packId}`)
|
|
599
|
+
}
|
|
600
|
+
const comparable = ['data_version', 'document_count', 'line_count', 'compressed_size',
|
|
601
|
+
'uncompressed_size']
|
|
602
|
+
for (const field of comparable) {
|
|
603
|
+
if (descriptor?.[field] != null && descriptor[field] !== pack[field]) {
|
|
604
|
+
invalid(code, `release 与 pack-manifest 的 ${field} 不一致: ${packId}`)
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
const packAuthority = String(pack.authority ?? 'official')
|
|
608
|
+
if (!packAuthority || packAuthority.length > 128 || /[\p{Cc}\p{Cf}]/u.test(packAuthority)
|
|
609
|
+
|| descriptor?.authority !== packAuthority) {
|
|
610
|
+
invalid(code, `release 与 pack-manifest 的 authority 不一致: ${packId}`)
|
|
611
|
+
}
|
|
612
|
+
if (descriptor?.shard_count != null && descriptor.shard_count !== assets.length) {
|
|
613
|
+
invalid(code, `release 与 pack-manifest 的 shard_count 不一致: ${packId}`)
|
|
614
|
+
}
|
|
615
|
+
return assets.map(({ asset }) => asset)
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
function validateReleaseTotals(manifest, totals, code = 'INVALID_MANIFEST') {
|
|
619
|
+
const fields = [
|
|
620
|
+
['document_count', 'documents', 10_000_000],
|
|
621
|
+
['line_count', 'lines', 100_000_000],
|
|
622
|
+
['compressed_size', 'compressed', CORPUS_RESOURCE_LIMITS.maxReleaseCompressedBytes],
|
|
623
|
+
['uncompressed_size', 'uncompressed', CORPUS_RESOURCE_LIMITS.maxReleaseUncompressedBytes],
|
|
624
|
+
]
|
|
625
|
+
for (const [field, total, maximum] of fields) {
|
|
626
|
+
requireInteger(manifest[field], `release.${field}`, {
|
|
627
|
+
minimum: field === 'document_count' ? 1 : 0, maximum, code,
|
|
628
|
+
})
|
|
629
|
+
if (manifest[field] !== totals[total]) invalid(code, `release.${field} 与 pack 汇总不一致`)
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
function canonicalJson(value) {
|
|
634
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`
|
|
635
|
+
if (value && typeof value === 'object') {
|
|
636
|
+
return `{${Object.keys(value).sort().map((key) =>
|
|
637
|
+
`${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(',')}}`
|
|
638
|
+
}
|
|
639
|
+
return JSON.stringify(value)
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
/**
|
|
643
|
+
* release.data_version 是构建器对 pack 逐文件哈希投影的内容根。这一步使
|
|
644
|
+
* current 返回的唯一可信根真正约束后续 versioned pack-manifest;否则
|
|
645
|
+
* 被污染的中间缓存可在保持汇总尺寸不变时替换逐文件哈希。
|
|
646
|
+
*/
|
|
647
|
+
function validateTrustedReleaseRoot(manifest, packManifests, code = 'INVALID_MANIFEST') {
|
|
648
|
+
const compilerVersion = String(manifest?.compiler_version ?? '')
|
|
649
|
+
const sourceUpdateId = String(manifest?.source_update_id ?? '')
|
|
650
|
+
const snapshotPrefix = 'local-snapshot:'
|
|
651
|
+
if (!/^[A-Za-z0-9._-]{1,128}$/u.test(compilerVersion)
|
|
652
|
+
|| !sourceUpdateId.startsWith(snapshotPrefix)
|
|
653
|
+
|| !/^[A-Za-z0-9._-]{1,160}$/u.test(sourceUpdateId.slice(snapshotPrefix.length))
|
|
654
|
+
|| manifest.corpus_version !== manifest.data_version
|
|
655
|
+
|| manifest.content_tree_sha256 !== manifest.data_version) {
|
|
656
|
+
invalid(code, 'release 缺少可验证的内容根元数据')
|
|
657
|
+
}
|
|
658
|
+
const packs = manifest.packs.map((descriptor) => {
|
|
659
|
+
const pack = packManifests.get(descriptor.pack_id)
|
|
660
|
+
if (!pack) invalid(code, `release 缺少 pack 清单: ${descriptor.pack_id}`)
|
|
661
|
+
const authority = String(pack.authority ?? 'official')
|
|
662
|
+
if (pack.localization && compareSemver(manifest.minimum_agent_version, '0.2.0') < 0) {
|
|
663
|
+
invalid(code, '本地化资料必须声明 minimum_agent_version 至少为 0.2.0')
|
|
664
|
+
}
|
|
665
|
+
if (!authority || authority.length > 128 || /[\p{Cc}\p{Cf}]/u.test(authority)) {
|
|
666
|
+
invalid(code, `pack authority 非法: ${descriptor.pack_id}`)
|
|
667
|
+
}
|
|
668
|
+
return {
|
|
669
|
+
pack_id: descriptor.pack_id,
|
|
670
|
+
data_version: pack.data_version,
|
|
671
|
+
authority,
|
|
672
|
+
shards: pack.shards.map((asset) => ({ path: asset.path, sha256: asset.sha256 })),
|
|
673
|
+
search_index_shards: (pack.search_index?.shards ?? [])
|
|
674
|
+
.map((asset) => ({ path: asset.path, sha256: asset.sha256 })),
|
|
675
|
+
...(pack.document_catalog ? { document_catalog: {
|
|
676
|
+
path: pack.document_catalog.path, sha256: pack.document_catalog.sha256,
|
|
677
|
+
} } : {}),
|
|
678
|
+
...(pack.localization ? { localization: pack.localization } : {}),
|
|
679
|
+
}
|
|
680
|
+
})
|
|
681
|
+
const calculated = createHash('sha256').update(canonicalJson({
|
|
682
|
+
compiler_version: compilerVersion,
|
|
683
|
+
source_snapshot: sourceUpdateId.slice(snapshotPrefix.length),
|
|
684
|
+
packs,
|
|
685
|
+
})).digest('hex')
|
|
686
|
+
if (calculated !== manifest.data_version) {
|
|
687
|
+
invalid(code, 'release data_version 无法约束 pack 逐文件哈希')
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* 验证一个本地 release 的全部声明 pack、资源路径、尺寸与汇总字段。调用方
|
|
693
|
+
* 可要求逐文件 SHA-256 校验;Store 和激活流程始终启用该选项。
|
|
694
|
+
*/
|
|
695
|
+
export async function validateLocalRelease(releasesDir, releaseId,
|
|
696
|
+
{ verifyHashes = false, details = false } = {}) {
|
|
697
|
+
if (!RELEASE_ID_PATTERN.test(String(releaseId || ''))) {
|
|
698
|
+
throw new InstallerFault('INVALID_RELEASE', 'releaseId 非法')
|
|
699
|
+
}
|
|
700
|
+
const releasesRoot = await realpath(resolve(releasesDir))
|
|
701
|
+
const releaseDir = await requireContainedDirectory(join(releasesRoot, releaseId), releasesRoot,
|
|
702
|
+
`release ${releaseId}`)
|
|
703
|
+
const manifest = await readContainedJson(join(releaseDir, 'release-manifest.json'), releaseDir,
|
|
704
|
+
'release-manifest', 'INVALID_RELEASE')
|
|
705
|
+
const descriptors = validateReleaseHeader(releaseId, manifest, 'INVALID_RELEASE')
|
|
706
|
+
const totals = { assets: 0, compressed: 0, uncompressed: 0, documents: 0, lines: 0 }
|
|
707
|
+
const packManifests = new Map()
|
|
708
|
+
for (const [packId, descriptor] of descriptors) {
|
|
709
|
+
const packDir = await requireContainedDirectory(join(releaseDir, packId), releaseDir,
|
|
710
|
+
`pack ${packId}`)
|
|
711
|
+
const packPath = join(packDir, 'pack-manifest.json')
|
|
712
|
+
const pack = await readContainedJson(packPath, releaseDir, `${packId}/pack-manifest.json`,
|
|
713
|
+
'INVALID_RELEASE')
|
|
714
|
+
const assets = validatePackManifest(packId, pack, descriptor, totals, 'INVALID_RELEASE')
|
|
715
|
+
packManifests.set(packId, pack)
|
|
716
|
+
for (const asset of assets) {
|
|
717
|
+
const relative = String(asset?.path || '')
|
|
718
|
+
const categoryDir = join(packDir, relative.split('/')[0])
|
|
719
|
+
const category = await lstat(categoryDir)
|
|
720
|
+
if (!category.isDirectory() || category.isSymbolicLink()) {
|
|
721
|
+
invalid('INVALID_RELEASE', `分片目录不是受管目录: ${packId}/${relative}`)
|
|
722
|
+
}
|
|
723
|
+
const assetPath = join(packDir, relative)
|
|
724
|
+
const link = await lstat(assetPath)
|
|
725
|
+
if (!link.isFile() || link.isSymbolicLink() || link.size !== asset.compressed_size) {
|
|
726
|
+
invalid('INVALID_RELEASE', `分片缺失或大小不符: ${packId}/${relative}`)
|
|
727
|
+
}
|
|
728
|
+
const actualPath = await realpath(assetPath)
|
|
729
|
+
if (!isContainedPath(releaseDir, actualPath)) {
|
|
730
|
+
invalid('INVALID_RELEASE', `分片越出 release 目录: ${packId}/${relative}`)
|
|
731
|
+
}
|
|
732
|
+
if (verifyHashes && await sha256File(assetPath) !== asset.sha256) {
|
|
733
|
+
invalid('INVALID_RELEASE', `分片 SHA-256 不符: ${packId}/${relative}`)
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
validateReleaseTotals(manifest, totals, 'INVALID_RELEASE')
|
|
738
|
+
validateTrustedReleaseRoot(manifest, packManifests, 'INVALID_RELEASE')
|
|
739
|
+
return details ? { manifest, packManifests, releaseDir } : manifest
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
async function fetchJson(url, { fetchImpl, signal }) {
|
|
743
|
+
// 显式超时 + 响应体大小上限:慢速/恶意的清单源不能无限占用下载流程,
|
|
744
|
+
// 也不能借超大清单把进程内存或磁盘当缓冲区。
|
|
745
|
+
const timeoutSignal = AbortSignal.timeout(MANIFEST_TIMEOUT_MS)
|
|
746
|
+
const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal
|
|
747
|
+
let response
|
|
748
|
+
try {
|
|
749
|
+
response = await fetchImpl(url, { redirect: 'error', signal: requestSignal })
|
|
750
|
+
} catch (error) {
|
|
751
|
+
if (signal?.aborted) throw new InstallerFault('CANCELLED', '资料下载已取消')
|
|
752
|
+
if (isTimeoutError(error)) {
|
|
753
|
+
throw new InstallerFault('DOWNLOAD_FAILED', `连接 ${new URL(url).host} 超时(${MANIFEST_TIMEOUT_MS / 1000}s)`)
|
|
754
|
+
}
|
|
755
|
+
throw new InstallerFault('DOWNLOAD_FAILED', `无法连接 ${new URL(url).host}(${error?.message ?? error})`)
|
|
756
|
+
}
|
|
757
|
+
if (!response.ok) {
|
|
758
|
+
const code = response.status === 404 ? 'RELEASE_NOT_FOUND'
|
|
759
|
+
: response.status === 403 ? 'ACCESS_DENIED' : 'DOWNLOAD_FAILED'
|
|
760
|
+
throw new InstallerFault(code, `请求失败 HTTP ${response.status}: ${url}`)
|
|
761
|
+
}
|
|
762
|
+
const declared = Number(response.headers?.get?.('content-length') ?? NaN)
|
|
763
|
+
if (Number.isFinite(declared) && declared > MAX_MANIFEST_BYTES) {
|
|
764
|
+
throw new InstallerFault('INVALID_MANIFEST', `清单超过大小上限(${MAX_MANIFEST_BYTES} 字节): ${url}`)
|
|
765
|
+
}
|
|
766
|
+
let text
|
|
767
|
+
try {
|
|
768
|
+
const bytes = await readBounded(response, MAX_MANIFEST_BYTES, url)
|
|
769
|
+
text = bytes.toString('utf8')
|
|
770
|
+
} catch (error) {
|
|
771
|
+
if (signal?.aborted) throw new InstallerFault('CANCELLED', '资料下载已取消')
|
|
772
|
+
throw error
|
|
773
|
+
}
|
|
774
|
+
try {
|
|
775
|
+
return JSON.parse(text)
|
|
776
|
+
} catch {
|
|
777
|
+
throw new InstallerFault('INVALID_MANIFEST', `返回的不是有效 JSON: ${url}`)
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/** 读取响应体到大小上限为止;超出即中止接收并报错。返回 Buffer(可直接按 utf8 解码)。 */
|
|
782
|
+
async function readBounded(response, maxBytes, url) {
|
|
783
|
+
const reader = response.body?.getReader?.()
|
|
784
|
+
if (!reader) {
|
|
785
|
+
// 无流式 body 的实现(测试桩 / 旧运行时):优先 text(),其次 arrayBuffer()。
|
|
786
|
+
const bytes = typeof response.text === 'function'
|
|
787
|
+
? Buffer.from(await response.text(), 'utf8')
|
|
788
|
+
: Buffer.from(await response.arrayBuffer())
|
|
789
|
+
if (bytes.byteLength > maxBytes) {
|
|
790
|
+
throw new InstallerFault('INVALID_MANIFEST', `响应超过大小上限(${maxBytes} 字节): ${url}`)
|
|
791
|
+
}
|
|
792
|
+
return bytes
|
|
793
|
+
}
|
|
794
|
+
const chunks = []
|
|
795
|
+
let received = 0
|
|
796
|
+
for (;;) {
|
|
797
|
+
const { done, value } = await reader.read()
|
|
798
|
+
if (done) break
|
|
799
|
+
received += value.byteLength
|
|
800
|
+
if (received > maxBytes) {
|
|
801
|
+
await reader.cancel().catch(() => {})
|
|
802
|
+
throw new InstallerFault('INVALID_MANIFEST', `响应超过大小上限(${maxBytes} 字节): ${url}`)
|
|
803
|
+
}
|
|
804
|
+
chunks.push(Buffer.from(value))
|
|
805
|
+
}
|
|
806
|
+
return Buffer.concat(chunks, received)
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
|
|
810
|
+
|
|
811
|
+
function safeDownloadRedirect(next, initial) {
|
|
812
|
+
const modelscopeOriginHost = (hostname) => /^(?:www\.)?modelscope\.cn$/u
|
|
813
|
+
.test(hostname.toLowerCase())
|
|
814
|
+
const modelscopeLfsHost = (hostname) => /^cdn-lfs-[a-z0-9-]+\.modelscope\.cn$/u
|
|
815
|
+
.test(hostname.toLowerCase())
|
|
816
|
+
if (next.username || next.password || next.hash) return false
|
|
817
|
+
if (next.protocol !== initial.protocol) return false
|
|
818
|
+
if (next.origin === initial.origin) return !next.search || modelscopeLfsHost(next.hostname)
|
|
819
|
+
// ModelScope 的 resolve 端点会把 LFS 对象跳转到官方 cdn-lfs-* 子域;
|
|
820
|
+
// LFS 的临时鉴权参数位于 query,因此仅对这个受限主机命名空间放行 query。
|
|
821
|
+
return next.protocol === 'https:'
|
|
822
|
+
&& modelscopeOriginHost(initial.hostname)
|
|
823
|
+
&& modelscopeLfsHost(next.hostname)
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
async function fetchDownload(fetchImpl, url, init) {
|
|
827
|
+
const initial = new URL(url)
|
|
828
|
+
let current = initial
|
|
829
|
+
for (let redirects = 0; redirects <= 5; redirects += 1) {
|
|
830
|
+
const response = await fetchImpl(current.toString(), { ...init, redirect: 'manual' })
|
|
831
|
+
if (!REDIRECT_STATUSES.has(response.status)) return response
|
|
832
|
+
const location = response.headers?.get?.('location')
|
|
833
|
+
if (!location || redirects === 5) {
|
|
834
|
+
await response.body?.cancel?.().catch(() => {})
|
|
835
|
+
throw new InstallerFault('DOWNLOAD_FAILED', `下载跳转链非法或过长: ${initial.host}`)
|
|
836
|
+
}
|
|
837
|
+
const next = new URL(location, current)
|
|
838
|
+
if (!safeDownloadRedirect(next, initial)) {
|
|
839
|
+
await response.body?.cancel?.().catch(() => {})
|
|
840
|
+
throw new InstallerFault('DOWNLOAD_FAILED', `下载源试图跳转到不安全地址: ${next.host}`)
|
|
841
|
+
}
|
|
842
|
+
await response.body?.cancel?.().catch(() => {})
|
|
843
|
+
current = next
|
|
844
|
+
}
|
|
845
|
+
throw new InstallerFault('DOWNLOAD_FAILED', `下载跳转链过长: ${initial.host}`)
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
/**
|
|
849
|
+
* 下载一个文件到临时路径,按清单校验(sha256 为空表示该源未提供哈希,仅落盘),
|
|
850
|
+
* 校验通过后原子改名。网络错误(含超时)重试一次。
|
|
851
|
+
* 正文流式落盘:先按 Content-Length 预检大小,超限立即中止接收;哈希在
|
|
852
|
+
* 数据流经时增量计算,任何时刻都不在内存里缓存整个文件。
|
|
853
|
+
* 清单未提供 size 的文件(站点 pack/release 清单本体)以 MAX_UNVERIFIED_BYTES
|
|
854
|
+
* 为字节上限——无哈希锚点的下载不能无限写盘。
|
|
855
|
+
*/
|
|
856
|
+
async function downloadVerified(url, targetPath, expected, env) {
|
|
857
|
+
const attempt = async (retry) => {
|
|
858
|
+
const tempPath = `${targetPath}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`
|
|
859
|
+
const timeoutMs = downloadTimeoutMs(expected.size ?? null)
|
|
860
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs)
|
|
861
|
+
const requestSignal = env.signal
|
|
862
|
+
? AbortSignal.any([env.signal, timeoutSignal]) : timeoutSignal
|
|
863
|
+
const sizeLimit = expected.size ?? MAX_UNVERIFIED_BYTES
|
|
864
|
+
let handle = null
|
|
865
|
+
try {
|
|
866
|
+
const response = await fetchDownload(env.fetchImpl, url, { signal: requestSignal })
|
|
867
|
+
if (!response.ok) {
|
|
868
|
+
throw new InstallerFault('DOWNLOAD_FAILED', `下载失败 HTTP ${response.status}: ${url}`)
|
|
869
|
+
}
|
|
870
|
+
const declared = Number(response.headers?.get?.('content-length') ?? NaN)
|
|
871
|
+
if (expected.size != null && Number.isFinite(declared) && declared !== expected.size) {
|
|
872
|
+
throw new InstallerFault('CHECKSUM_MISMATCH',
|
|
873
|
+
`${url} 大小不符(期望 ${expected.size},Content-Length ${declared})`)
|
|
874
|
+
}
|
|
875
|
+
if (expected.size == null && Number.isFinite(declared) && declared > sizeLimit) {
|
|
876
|
+
throw new InstallerFault('DOWNLOAD_FAILED',
|
|
877
|
+
`${url} 超过未校验文件大小上限(${sizeLimit} 字节)`)
|
|
878
|
+
}
|
|
879
|
+
await mkdir(dirname(targetPath), { recursive: true })
|
|
880
|
+
handle = await open(tempPath, 'wx', 0o600)
|
|
881
|
+
const hash = expected.sha256 ? createHash('sha256') : null
|
|
882
|
+
let received = 0
|
|
883
|
+
const overLimit = () => expected.size != null
|
|
884
|
+
? new InstallerFault('CHECKSUM_MISMATCH',
|
|
885
|
+
`${url} 大小不符(期望 ${expected.size},已接收 ${received})`)
|
|
886
|
+
: new InstallerFault('DOWNLOAD_FAILED',
|
|
887
|
+
`${url} 超过未校验文件大小上限(${sizeLimit} 字节,已接收 ${received})`)
|
|
888
|
+
const reader = response.body?.getReader?.()
|
|
889
|
+
if (reader) {
|
|
890
|
+
for (;;) {
|
|
891
|
+
const { done, value } = await reader.read()
|
|
892
|
+
if (done) break
|
|
893
|
+
received += value.byteLength
|
|
894
|
+
if (received > sizeLimit) {
|
|
895
|
+
await reader.cancel().catch(() => {})
|
|
896
|
+
throw overLimit()
|
|
897
|
+
}
|
|
898
|
+
await handle.write(value)
|
|
899
|
+
hash?.update(value)
|
|
900
|
+
}
|
|
901
|
+
} else {
|
|
902
|
+
// 无流式 body 的实现(测试桩 / 旧运行时):退回整段缓冲。
|
|
903
|
+
const bytes = new Uint8Array(await response.arrayBuffer())
|
|
904
|
+
received = bytes.byteLength
|
|
905
|
+
if (expected.size != null && received !== expected.size) {
|
|
906
|
+
throw new InstallerFault('CHECKSUM_MISMATCH',
|
|
907
|
+
`${url} 大小不符(期望 ${expected.size},得到 ${received})`)
|
|
908
|
+
}
|
|
909
|
+
if (expected.size == null && received > sizeLimit) throw overLimit()
|
|
910
|
+
await handle.write(bytes)
|
|
911
|
+
hash?.update(bytes)
|
|
912
|
+
}
|
|
913
|
+
if (expected.size != null && received !== expected.size) {
|
|
914
|
+
throw new InstallerFault('CHECKSUM_MISMATCH',
|
|
915
|
+
`${url} 大小不符(期望 ${expected.size},得到 ${received})`)
|
|
916
|
+
}
|
|
917
|
+
if (hash && hash.digest('hex') !== expected.sha256) {
|
|
918
|
+
throw new InstallerFault('CHECKSUM_MISMATCH', `${url} sha256 不符`)
|
|
919
|
+
}
|
|
920
|
+
await handle.close()
|
|
921
|
+
handle = null
|
|
922
|
+
await rename(tempPath, targetPath)
|
|
923
|
+
return received
|
|
924
|
+
} catch (error) {
|
|
925
|
+
if (handle) await handle.close().catch(() => {})
|
|
926
|
+
await rm(tempPath, { force: true })
|
|
927
|
+
if (env.signal?.aborted) throw error
|
|
928
|
+
if (error instanceof InstallerFault) throw error
|
|
929
|
+
// 网络错误(含显式超时)重试一次;最终失败时把超时转成可读错误。
|
|
930
|
+
if (!retry) {
|
|
931
|
+
throw isTimeoutError(error)
|
|
932
|
+
? new InstallerFault('DOWNLOAD_FAILED',
|
|
933
|
+
`下载超时(${Math.round(timeoutMs / 1000)}s): ${url}`)
|
|
934
|
+
: error
|
|
935
|
+
}
|
|
936
|
+
return attempt(retry - 1)
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
return attempt(1)
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
const requirePackId = (packId, code = 'INVALID_MANIFEST') => {
|
|
943
|
+
if (!PACK_IDS.includes(packId)) throw new InstallerFault(code, `pack_id 非法: ${packId}`)
|
|
944
|
+
return packId
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
function validateCurrentSummary(data) {
|
|
948
|
+
const releaseId = String(data?.release_id ?? '')
|
|
949
|
+
const dataVersion = String(data?.data_version ?? '')
|
|
950
|
+
if (!RELEASE_ID_PATTERN.test(releaseId) || !SHA256_PATTERN.test(dataVersion)
|
|
951
|
+
|| !Array.isArray(data?.packs) || data.packs.length === 0
|
|
952
|
+
|| data.packs.length > PACK_IDS.length) {
|
|
953
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat current 元数据不完整')
|
|
954
|
+
}
|
|
955
|
+
const minimumAgentVersion = data.minimum_agent_version
|
|
956
|
+
if (!parseSemver(minimumAgentVersion)) {
|
|
957
|
+
throw new InstallerFault('INVALID_MANIFEST',
|
|
958
|
+
'PRTS.chat current.minimum_agent_version 缺失或不是有效 SemVer')
|
|
959
|
+
}
|
|
960
|
+
if (compareSemver(AGENT_VERSION, minimumAgentVersion) < 0) {
|
|
961
|
+
throw new InstallerFault('INCOMPATIBLE_RELEASE',
|
|
962
|
+
`最新资料至少需要 prts-terrarchive ${minimumAgentVersion},当前为 ${AGENT_VERSION}`)
|
|
963
|
+
}
|
|
964
|
+
requireInteger(data.document_count, 'current.document_count', {
|
|
965
|
+
minimum: 1, maximum: 10_000_000,
|
|
966
|
+
})
|
|
967
|
+
requireInteger(data.line_count, 'current.line_count', { maximum: 100_000_000 })
|
|
968
|
+
requireInteger(data.compressed_size, 'current.compressed_size', {
|
|
969
|
+
maximum: CORPUS_RESOURCE_LIMITS.maxReleaseCompressedBytes,
|
|
970
|
+
})
|
|
971
|
+
requireInteger(data.uncompressed_size, 'current.uncompressed_size', {
|
|
972
|
+
maximum: CORPUS_RESOURCE_LIMITS.maxReleaseUncompressedBytes,
|
|
973
|
+
})
|
|
974
|
+
const packIds = new Set()
|
|
975
|
+
const packs = []
|
|
976
|
+
const totals = { document_count: 0, line_count: 0, compressed_size: 0,
|
|
977
|
+
uncompressed_size: 0 }
|
|
978
|
+
for (const rawDescriptor of data.packs) {
|
|
979
|
+
const descriptor = normalizePackDescriptor(rawDescriptor, 'current')
|
|
980
|
+
const packId = descriptor.pack_id
|
|
981
|
+
if (packIds.has(packId)) {
|
|
982
|
+
throw new InstallerFault('INVALID_MANIFEST', `PRTS.chat current pack 描述非法: ${packId}`)
|
|
983
|
+
}
|
|
984
|
+
for (const field of Object.keys(totals)) totals[field] += descriptor[field]
|
|
985
|
+
packIds.add(packId)
|
|
986
|
+
packs.push(descriptor)
|
|
987
|
+
}
|
|
988
|
+
for (const field of Object.keys(totals)) {
|
|
989
|
+
if (totals[field] !== data[field]) {
|
|
990
|
+
throw new InstallerFault('INVALID_MANIFEST', `PRTS.chat current.${field} 与 pack 汇总不一致`)
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return { releaseId, dataVersion, minimumAgentVersion, packs }
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
function validateMirrorDescriptors(mirrors, releaseId, releasePacks) {
|
|
997
|
+
if (mirrors == null) return []
|
|
998
|
+
if (!Array.isArray(mirrors) || mirrors.length > 16) {
|
|
999
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat 镜像列表非法')
|
|
1000
|
+
}
|
|
1001
|
+
const accepted = []
|
|
1002
|
+
const assignedPacks = new Set()
|
|
1003
|
+
for (const mirror of mirrors) {
|
|
1004
|
+
let base
|
|
1005
|
+
try { base = new URL(String(mirror?.base_url ?? '')) } catch {
|
|
1006
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat 镜像地址非法')
|
|
1007
|
+
}
|
|
1008
|
+
const packIds = Array.isArray(mirror?.pack_ids)
|
|
1009
|
+
? mirror.pack_ids.map((value) => requirePackId(String(value ?? ''))) : []
|
|
1010
|
+
if (mirror?.provider !== 'modelscope' || !packIds.length
|
|
1011
|
+
|| new Set(packIds).size !== packIds.length || base.protocol !== 'https:'
|
|
1012
|
+
|| !['modelscope.cn', 'www.modelscope.cn'].includes(base.hostname)
|
|
1013
|
+
|| base.username || base.password || base.search || base.hash
|
|
1014
|
+
|| !base.pathname.endsWith(`/releases/${releaseId}/`)) {
|
|
1015
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat 镜像描述非法')
|
|
1016
|
+
}
|
|
1017
|
+
if (packIds.some((packId) => !releasePacks.has(packId) || assignedPacks.has(packId))) {
|
|
1018
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat 镜像 pack 范围非法或重复')
|
|
1019
|
+
}
|
|
1020
|
+
packIds.forEach((packId) => assignedPacks.add(packId))
|
|
1021
|
+
accepted.push({ baseUrl: base.toString(), packIds })
|
|
1022
|
+
}
|
|
1023
|
+
return accepted
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
function immutableTrustedCurrent(values) {
|
|
1027
|
+
const packs = Object.freeze(values.packs.map((pack) => Object.freeze({
|
|
1028
|
+
pack_id: pack.pack_id,
|
|
1029
|
+
manifest_path: pack.manifest_path,
|
|
1030
|
+
authority: pack.authority,
|
|
1031
|
+
data_version: pack.data_version,
|
|
1032
|
+
document_count: pack.document_count,
|
|
1033
|
+
line_count: pack.line_count,
|
|
1034
|
+
compressed_size: pack.compressed_size,
|
|
1035
|
+
uncompressed_size: pack.uncompressed_size,
|
|
1036
|
+
shard_count: pack.shard_count,
|
|
1037
|
+
})))
|
|
1038
|
+
const mirrors = Object.freeze(values.mirrors.map((mirror) => Object.freeze({
|
|
1039
|
+
baseUrl: mirror.baseUrl,
|
|
1040
|
+
packIds: Object.freeze([...mirror.packIds]),
|
|
1041
|
+
})))
|
|
1042
|
+
return Object.freeze({
|
|
1043
|
+
releaseId: values.releaseId,
|
|
1044
|
+
dataVersion: values.dataVersion,
|
|
1045
|
+
minimumAgentVersion: values.minimumAgentVersion,
|
|
1046
|
+
documentCount: values.documentCount,
|
|
1047
|
+
lineCount: values.lineCount,
|
|
1048
|
+
compressedSize: values.compressedSize,
|
|
1049
|
+
uncompressedSize: values.uncompressedSize,
|
|
1050
|
+
packs,
|
|
1051
|
+
mirrors,
|
|
1052
|
+
})
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function rememberTrustedCurrent(values) {
|
|
1056
|
+
const canonical = immutableTrustedCurrent(values)
|
|
1057
|
+
const exposed = immutableTrustedCurrent(canonical)
|
|
1058
|
+
trustedCurrentSnapshots.set(exposed, canonical)
|
|
1059
|
+
return exposed
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
/**
|
|
1063
|
+
* 从 PRTS.chat 的可变 current 指针读取唯一可信的最新 release 与内容摘要。
|
|
1064
|
+
* ModelScope 的目录列表和 dataset-manifest 不参与版本选择,避免镜像仓自行
|
|
1065
|
+
* 宣称“最新”或同时替换数据与哈希。
|
|
1066
|
+
*/
|
|
1067
|
+
export async function resolveTrustedCurrentRelease(env = {}) {
|
|
1068
|
+
const fetchImpl = env.fetchImpl ?? fetch
|
|
1069
|
+
const signal = env.signal
|
|
1070
|
+
if (env.siteBaseUrl != null
|
|
1071
|
+
&& normalizeSiteBaseUrl(env.siteBaseUrl) !== DEFAULT_SITE_BASE_URL) {
|
|
1072
|
+
throw new InstallerFault('INVALID_REQUEST',
|
|
1073
|
+
`可信元数据源固定为 ${DEFAULT_SITE_BASE_URL};siteBaseUrl 仅可配置字节回退源`)
|
|
1074
|
+
}
|
|
1075
|
+
const payload = await fetchJson(
|
|
1076
|
+
`${DEFAULT_SITE_BASE_URL}/api/agent/data/releases/current`, { fetchImpl, signal })
|
|
1077
|
+
const data = payload?.data
|
|
1078
|
+
const { releaseId, dataVersion, minimumAgentVersion, packs } = validateCurrentSummary(data)
|
|
1079
|
+
return rememberTrustedCurrent({
|
|
1080
|
+
releaseId,
|
|
1081
|
+
dataVersion,
|
|
1082
|
+
minimumAgentVersion,
|
|
1083
|
+
documentCount: data.document_count,
|
|
1084
|
+
lineCount: data.line_count,
|
|
1085
|
+
compressedSize: data.compressed_size,
|
|
1086
|
+
uncompressedSize: data.uncompressed_size,
|
|
1087
|
+
packs,
|
|
1088
|
+
mirrors: validateMirrorDescriptors(data.mirrors, releaseId,
|
|
1089
|
+
new Set(packs.map((pack) => pack.pack_id))),
|
|
1090
|
+
})
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
async function loadTrustedReleaseMetadata(releaseId, env, snapshot) {
|
|
1094
|
+
const current = trustedCurrentSnapshots.get(snapshot)
|
|
1095
|
+
if (!current || current.releaseId !== releaseId) {
|
|
1096
|
+
throw new InstallerFault('INVALID_REQUEST',
|
|
1097
|
+
'远程 release 必须由同一次 PRTS.chat current 快照选定')
|
|
1098
|
+
}
|
|
1099
|
+
const url = (path) => `${DEFAULT_SITE_BASE_URL}/api/agent/data/releases/${releaseId}/${path}`
|
|
1100
|
+
const releaseManifest = await fetchJson(url('release-manifest.json'), env)
|
|
1101
|
+
const descriptors = validateReleaseHeader(releaseId, releaseManifest)
|
|
1102
|
+
{
|
|
1103
|
+
const fields = [
|
|
1104
|
+
['data_version', current.dataVersion],
|
|
1105
|
+
['minimum_agent_version', current.minimumAgentVersion],
|
|
1106
|
+
['document_count', current.documentCount],
|
|
1107
|
+
['line_count', current.lineCount],
|
|
1108
|
+
['compressed_size', current.compressedSize],
|
|
1109
|
+
['uncompressed_size', current.uncompressedSize],
|
|
1110
|
+
]
|
|
1111
|
+
if (fields.some(([field, expected]) => releaseManifest[field] !== expected)) {
|
|
1112
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat current 与 release-manifest 摘要不一致')
|
|
1113
|
+
}
|
|
1114
|
+
const currentPacks = new Map(current.packs.map((pack) => [pack.pack_id, pack]))
|
|
1115
|
+
if (currentPacks.size !== descriptors.size) {
|
|
1116
|
+
throw new InstallerFault('INVALID_MANIFEST', 'PRTS.chat current 与 release-manifest packs 不一致')
|
|
1117
|
+
}
|
|
1118
|
+
for (const [packId, descriptor] of descriptors) {
|
|
1119
|
+
const summary = currentPacks.get(packId)
|
|
1120
|
+
if (!summary || Object.entries(descriptor).some(([key, value]) => summary[key] !== value)) {
|
|
1121
|
+
throw new InstallerFault('INVALID_MANIFEST', `PRTS.chat current 与 release pack 不一致: ${packId}`)
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
const totals = { assets: 0, compressed: 0, uncompressed: 0, documents: 0, lines: 0 }
|
|
1126
|
+
const packManifests = new Map()
|
|
1127
|
+
const entries = []
|
|
1128
|
+
for (const [packId, descriptor] of descriptors) {
|
|
1129
|
+
const pack = await fetchJson(url(descriptor.manifest_path), env)
|
|
1130
|
+
const assets = validatePackManifest(packId, pack, descriptor, totals)
|
|
1131
|
+
packManifests.set(packId, pack)
|
|
1132
|
+
for (const asset of assets) {
|
|
1133
|
+
entries.push({
|
|
1134
|
+
relativePath: `${packId}/${asset.path}`,
|
|
1135
|
+
sha256: asset.sha256,
|
|
1136
|
+
size: asset.compressed_size,
|
|
1137
|
+
uncompressedSize: asset.uncompressed_size,
|
|
1138
|
+
})
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
validateReleaseTotals(releaseManifest, totals)
|
|
1142
|
+
validateTrustedReleaseRoot(releaseManifest, packManifests)
|
|
1143
|
+
if (new Set(entries.map((entry) => entry.relativePath)).size !== entries.length) {
|
|
1144
|
+
throw new InstallerFault('INVALID_MANIFEST', 'release 包含重复资源路径')
|
|
1145
|
+
}
|
|
1146
|
+
return {
|
|
1147
|
+
releaseId,
|
|
1148
|
+
dataVersion: releaseManifest.data_version,
|
|
1149
|
+
releaseManifest,
|
|
1150
|
+
packManifests,
|
|
1151
|
+
entries,
|
|
1152
|
+
mirrors: current.mirrors,
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
/** ---- 源 1:由 PRTS.chat 摘要约束的 ModelScope 固定 release ---- */
|
|
1157
|
+
|
|
1158
|
+
/**
|
|
1159
|
+
* @deprecated 使用 resolveTrustedCurrentRelease;保留导出供旧客户端模块平滑升级。
|
|
1160
|
+
*/
|
|
1161
|
+
export async function resolveModelScopeCurrentRelease(env = {}) {
|
|
1162
|
+
const trusted = await resolveTrustedCurrentRelease(env)
|
|
1163
|
+
return { releaseId: trusted.releaseId, dataVersion: trusted.dataVersion }
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
const groupForPack = (packId, composition) => composition?.layout === 'legacy-three-dataset-v1'
|
|
1167
|
+
? (packId === 'official_game' ? 'official'
|
|
1168
|
+
: packId.startsWith('endfield_') ? 'endfield' : 'community')
|
|
1169
|
+
: (packId.startsWith('endfield_') ? 'endfield' : 'arknights')
|
|
1170
|
+
|
|
1171
|
+
/** Resolve one hash-verified asset URL while honoring an immutable split-repo composition. */
|
|
1172
|
+
export function modelScopeAssetUrl(releaseId, dataVersion, relativePath, mirrorBaseUrl = null) {
|
|
1173
|
+
const packId = requirePackId(String(relativePath ?? '').split('/')[0])
|
|
1174
|
+
const composition = MODELSCOPE_RELEASE_COMPOSITIONS[releaseId]
|
|
1175
|
+
const group = groupForPack(packId, composition)
|
|
1176
|
+
if (composition && composition.dataVersion !== dataVersion) {
|
|
1177
|
+
throw new InstallerFault('INVALID_MANIFEST',
|
|
1178
|
+
`ModelScope 分仓组合与 release data_version 不匹配: ${releaseId}`)
|
|
1179
|
+
}
|
|
1180
|
+
const sourceReleaseId = composition?.releases?.[group] ?? releaseId
|
|
1181
|
+
const baseUrl = mirrorBaseUrl
|
|
1182
|
+
?? `https://modelscope.cn/datasets/${MODELSCOPE_REPOS[group]}/resolve/master/releases/${sourceReleaseId}/`
|
|
1183
|
+
return new URL(relativePath, baseUrl).toString()
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
async function listFromModelScope(trusted) {
|
|
1187
|
+
const mirrorByPack = new Map()
|
|
1188
|
+
for (const mirror of trusted.mirrors) {
|
|
1189
|
+
for (const packId of mirror.packIds) mirrorByPack.set(packId, mirror.baseUrl)
|
|
1190
|
+
}
|
|
1191
|
+
const entries = trusted.entries.map((entry) => {
|
|
1192
|
+
const packId = entry.relativePath.split('/')[0]
|
|
1193
|
+
return { ...entry, url: modelScopeAssetUrl(trusted.releaseId, trusted.dataVersion,
|
|
1194
|
+
entry.relativePath, mirrorByPack.get(packId)) }
|
|
1195
|
+
})
|
|
1196
|
+
return { dataVersion: trusted.dataVersion, entries }
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
/** ---- 源 2:PRTS.chat 站点(与浏览器前端同源) ---- */
|
|
1200
|
+
|
|
1201
|
+
async function listFromSite(trusted, siteBaseUrl) {
|
|
1202
|
+
const base = normalizeSiteBaseUrl(siteBaseUrl)
|
|
1203
|
+
const prefix = `${base}/api/agent/data/releases/${trusted.releaseId}/`
|
|
1204
|
+
return {
|
|
1205
|
+
dataVersion: trusted.dataVersion,
|
|
1206
|
+
entries: trusted.entries.map((entry) => ({ ...entry, url: `${prefix}${entry.relativePath}` })),
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
/** ---- 主流程 ---- */
|
|
1211
|
+
|
|
1212
|
+
/**
|
|
1213
|
+
* 确保本地资料就绪:指定的本地版本已完整校验则直接返回;任何需要
|
|
1214
|
+
* 联网的下载都先解析 PRTS.chat current,并且只允许该快照选定的 release。
|
|
1215
|
+
* release/pack 清单同样固定从 PRTS.chat 取得;随后按 order 从镜像或可配置
|
|
1216
|
+
* 站点取字节,所有字节均由该快照导出的内容根约束。
|
|
1217
|
+
* @param {{ releasesDir: string, releaseId?: string, order?: ('modelscope'|'site')[],
|
|
1218
|
+
* siteBaseUrl?: string, fetchImpl?: typeof fetch, signal?: AbortSignal,
|
|
1219
|
+
* logger?: { info?: Function, warn?: Function }, requireRelease?: boolean,
|
|
1220
|
+
* onProgress?: (state: { phase: 'listing'|'downloading'|'done', source: string,
|
|
1221
|
+
* releaseId: string, filesDone: number, filesTotal: number|null, bytesDone: number }) => void }} options
|
|
1222
|
+
* @returns {Promise<{ status: 'present' | 'downloaded', releaseId?: string, source?: string, files?: number, bytes?: number }>}
|
|
1223
|
+
*/
|
|
1224
|
+
export async function ensureCorpusRelease(options) {
|
|
1225
|
+
const { releasesDir, signal, logger } = options
|
|
1226
|
+
const onProgress = options.onProgress
|
|
1227
|
+
const requestedReleaseId = options.releaseId == null ? null : String(options.releaseId)
|
|
1228
|
+
if (requestedReleaseId != null && !RELEASE_ID_PATTERN.test(requestedReleaseId)) {
|
|
1229
|
+
throw new InstallerFault('INVALID_REQUEST', 'releaseId 非法')
|
|
1230
|
+
}
|
|
1231
|
+
const fetchImpl = options.fetchImpl ?? fetch
|
|
1232
|
+
const env = { fetchImpl, signal }
|
|
1233
|
+
const siteBaseUrl = normalizeSiteBaseUrl(options.siteBaseUrl ?? DEFAULT_SITE_BASE_URL)
|
|
1234
|
+
const order = options.order ?? ['modelscope', 'site']
|
|
1235
|
+
if (!Array.isArray(order) || order.length === 0 || new Set(order).size !== order.length
|
|
1236
|
+
|| order.some((source) => source !== 'modelscope' && source !== 'site')) {
|
|
1237
|
+
throw new InstallerFault('INVALID_REQUEST', 'download order 非法')
|
|
1238
|
+
}
|
|
1239
|
+
const suppliedSnapshot = options.trustedCurrent ?? null
|
|
1240
|
+
const suppliedCurrent = suppliedSnapshot == null
|
|
1241
|
+
? null : trustedCurrentSnapshots.get(suppliedSnapshot)
|
|
1242
|
+
if (suppliedSnapshot != null && (!suppliedCurrent
|
|
1243
|
+
|| requestedReleaseId == null || suppliedCurrent.releaseId !== requestedReleaseId)) {
|
|
1244
|
+
throw new InstallerFault('INVALID_REQUEST', '可信 current 快照非法或与 releaseId 不匹配')
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
const releaseLock = await acquireReleaseMutationLock(releasesDir)
|
|
1248
|
+
try {
|
|
1249
|
+
// 本地显式 pin 已就绪时仍保持零网络,但验证和返回必须在
|
|
1250
|
+
// 跨进程变更锁内,避免与另一 Host 的删除/激活操作交错后误报 present。
|
|
1251
|
+
if (requestedReleaseId && await currentReleaseReady(
|
|
1252
|
+
releasesDir, requestedReleaseId, suppliedCurrent?.dataVersion,
|
|
1253
|
+
)) {
|
|
1254
|
+
return { status: 'present' }
|
|
1255
|
+
}
|
|
1256
|
+
const trustedSnapshot = suppliedSnapshot
|
|
1257
|
+
?? await resolveTrustedCurrentRelease({ fetchImpl, signal })
|
|
1258
|
+
const trustedCurrent = trustedCurrentSnapshots.get(trustedSnapshot)
|
|
1259
|
+
if (!trustedCurrent) {
|
|
1260
|
+
throw new InstallerFault('INVALID_REQUEST', '可信 current 快照未由安装器签发')
|
|
1261
|
+
}
|
|
1262
|
+
if (requestedReleaseId && requestedReleaseId !== trustedCurrent.releaseId) {
|
|
1263
|
+
throw new InstallerFault('RELEASE_NOT_CURRENT',
|
|
1264
|
+
`PRTS.chat 最新版本为 ${trustedCurrent.releaseId},拒绝远程下载未被 current 选定的版本`)
|
|
1265
|
+
}
|
|
1266
|
+
const releaseId = trustedCurrent.releaseId
|
|
1267
|
+
|
|
1268
|
+
if (await currentReleaseReady(releasesDir, releaseId, trustedCurrent.dataVersion)) {
|
|
1269
|
+
return { status: 'present' }
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
const trusted = await loadTrustedReleaseMetadata(releaseId, env, trustedSnapshot)
|
|
1273
|
+
const { releaseDir } = await ensureManagedReleaseDirectory(releasesDir, releaseId)
|
|
1274
|
+
let lastError = null
|
|
1275
|
+
for (const source of order) {
|
|
1276
|
+
if (signal?.aborted) throw new InstallerFault('CANCELLED', '资料下载已取消')
|
|
1277
|
+
try {
|
|
1278
|
+
const listing = source === 'site'
|
|
1279
|
+
? await listFromSite(trusted, siteBaseUrl)
|
|
1280
|
+
: await listFromModelScope(trusted)
|
|
1281
|
+
|
|
1282
|
+
// 2) 跳过已校验文件,其余小并发下载
|
|
1283
|
+
const pending = []
|
|
1284
|
+
for (const entry of listing.entries) {
|
|
1285
|
+
const targetPath = await prepareAssetTarget(releaseDir, entry.relativePath)
|
|
1286
|
+
try {
|
|
1287
|
+
const existing = await lstat(targetPath)
|
|
1288
|
+
if (existing.isSymbolicLink() || !existing.isFile()) {
|
|
1289
|
+
throw new InstallerFault('INVALID_RELEASE',
|
|
1290
|
+
`现有资源不是受管普通文件: ${entry.relativePath}`)
|
|
1291
|
+
}
|
|
1292
|
+
// 哈希与大小都一致 → 已就绪跳过;否则重新下载覆盖。
|
|
1293
|
+
if (existing.size === entry.size && await sha256File(targetPath) === entry.sha256) continue
|
|
1294
|
+
} catch (error) {
|
|
1295
|
+
if (error instanceof InstallerFault) throw error
|
|
1296
|
+
// 不存在或普通 I/O 失败 → 交给原子下载覆盖。
|
|
1297
|
+
}
|
|
1298
|
+
pending.push({ ...entry, targetPath })
|
|
1299
|
+
}
|
|
1300
|
+
if (pending.length) {
|
|
1301
|
+
logger?.info?.(`prts-corpus: 从 ${source === 'site' ? siteBaseUrl : 'ModelScope'} 下载 ${releaseId}`
|
|
1302
|
+
+ `(${pending.length}/${listing.entries.length} 个文件待取,data_version=${listing.dataVersion.slice(0, 12)}…)`)
|
|
1303
|
+
}
|
|
1304
|
+
const queue = [...pending]
|
|
1305
|
+
let files = 0
|
|
1306
|
+
let bytes = 0
|
|
1307
|
+
// 任一 worker 失败后置位:其余 worker 完成手头文件即停止取新任务,
|
|
1308
|
+
// 不再产生与回退源并发写盘的孤儿下载。
|
|
1309
|
+
let failure = null
|
|
1310
|
+
onProgress?.({ phase: 'downloading', source, releaseId,
|
|
1311
|
+
filesDone: 0, filesTotal: pending.length, bytesDone: 0 })
|
|
1312
|
+
const worker = async () => {
|
|
1313
|
+
for (;;) {
|
|
1314
|
+
if (failure || signal?.aborted) return
|
|
1315
|
+
const entry = queue.shift()
|
|
1316
|
+
if (!entry) return
|
|
1317
|
+
try {
|
|
1318
|
+
const downloadedBytes = await downloadVerified(entry.url, entry.targetPath,
|
|
1319
|
+
{ sha256: entry.sha256, size: entry.size }, env)
|
|
1320
|
+
// 不要写成 `bytes += await ...`:复合赋值会在 await 前读取旧值,
|
|
1321
|
+
// 多 worker 完成顺序交错时会覆盖其他 worker 已累计的字节数。
|
|
1322
|
+
bytes += downloadedBytes
|
|
1323
|
+
} catch (error) {
|
|
1324
|
+
failure = error
|
|
1325
|
+
return
|
|
1326
|
+
}
|
|
1327
|
+
files += 1
|
|
1328
|
+
onProgress?.({ phase: 'downloading', source, releaseId,
|
|
1329
|
+
filesDone: files, filesTotal: pending.length, bytesDone: bytes })
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
await Promise.all(Array.from({ length: Math.min(6, pending.length) }, worker))
|
|
1333
|
+
if (failure) throw failure
|
|
1334
|
+
if (signal?.aborted) throw new InstallerFault('CANCELLED', '资料下载已取消')
|
|
1335
|
+
onProgress?.({ phase: 'done', source, releaseId, filesDone: files,
|
|
1336
|
+
filesTotal: pending.length, bytesDone: bytes })
|
|
1337
|
+
|
|
1338
|
+
// 3) 只落盘 PRTS.chat 预先验证过的清单;镜像返回的清单从不进入信任链。
|
|
1339
|
+
for (const [packId, manifest] of trusted.packManifests) {
|
|
1340
|
+
await writeJsonAtomically(join(releaseDir, packId, 'pack-manifest.json'), manifest)
|
|
1341
|
+
}
|
|
1342
|
+
await writeJsonAtomically(join(releaseDir, 'release-manifest.json'), trusted.releaseManifest)
|
|
1343
|
+
await validateLocalRelease(releasesDir, releaseId, { verifyHashes: true })
|
|
1344
|
+
const pointerTemp = join(releasesDir, `current.json.${randomBytes(6).toString('hex')}.tmp`)
|
|
1345
|
+
await writeFile(pointerTemp, JSON.stringify({
|
|
1346
|
+
release_id: releaseId, data_version: trusted.dataVersion,
|
|
1347
|
+
channel: source, public_download: true, schema_version: 1,
|
|
1348
|
+
downloaded_at: new Date().toISOString(),
|
|
1349
|
+
}))
|
|
1350
|
+
await rename(pointerTemp, join(releasesDir, 'current.json'))
|
|
1351
|
+
logger?.info?.(`prts-corpus: 资料包就绪 ${releaseId}(源=${source},新下载 ${files} 个文件,${Math.round(bytes / 1048576)} MiB)`)
|
|
1352
|
+
return { status: 'downloaded', releaseId, source, files, bytes }
|
|
1353
|
+
} catch (error) {
|
|
1354
|
+
lastError = error
|
|
1355
|
+
logger?.warn?.(`prts-corpus: ${source} 源失败(${error?.code ?? 'ERROR'}: ${error?.message ?? error})`)
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
throw lastError ?? new InstallerFault('DOWNLOAD_FAILED', '没有可用的下载源')
|
|
1359
|
+
} finally {
|
|
1360
|
+
await releaseLock()
|
|
1361
|
+
}
|
|
1362
|
+
}
|