prts-terrarchive 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GAME_ASSETS.md +26 -0
- package/LICENSE +21 -0
- package/README.en.md +274 -0
- package/README.md +423 -0
- package/THIRD_PARTY_NOTICES.md +61 -0
- package/bin/bake-rhine-details.mjs +34 -0
- package/bin/install.js +273 -0
- package/bin/pack-map-assets.mjs +110 -0
- package/bin/preview-rhine.mjs +71 -0
- package/bin/rhine-detail-baker.ts +84 -0
- package/contracts/cloud_tools_v1.schema.json +243 -0
- package/contracts/corpus_i18n_v1.schema.json +179 -0
- package/contracts/corpus_tools_v1.schema.json +1801 -0
- package/cordis.patch.yml +32 -0
- package/docs/endfield-official-i18n.md +103 -0
- package/lib/client.js +3912 -0
- package/lib/endfield-map/map.js.br +0 -0
- package/lib/endfield-map/map.js.gz +0 -0
- package/lib/endfield-map/resources/map-00b0d0744a1b4404.json.br +0 -0
- package/lib/endfield-map/resources/map-00b0d0744a1b4404.json.gz +0 -0
- package/lib/endfield-map/resources/map-01616f534aaaeca1.png +0 -0
- package/lib/endfield-map/resources/map-1562e0dff4be3d23.png +0 -0
- package/lib/endfield-map/resources/map-160fe040f080f6a3.png +0 -0
- package/lib/endfield-map/resources/map-17b9d8be25dc2b39.json.br +0 -0
- package/lib/endfield-map/resources/map-17b9d8be25dc2b39.json.gz +0 -0
- package/lib/endfield-map/resources/map-1c4245c4e8b80c43.json.br +0 -0
- package/lib/endfield-map/resources/map-1c4245c4e8b80c43.json.gz +0 -0
- package/lib/endfield-map/resources/map-1d30baaaef3e29cc.png +0 -0
- package/lib/endfield-map/resources/map-21a61386af79cd12.json.br +0 -0
- package/lib/endfield-map/resources/map-21a61386af79cd12.json.gz +0 -0
- package/lib/endfield-map/resources/map-24b920b8dab49b44.json.br +0 -0
- package/lib/endfield-map/resources/map-24b920b8dab49b44.json.gz +0 -0
- package/lib/endfield-map/resources/map-26caf8d71247abb7.png +0 -0
- package/lib/endfield-map/resources/map-2729942c89545e73.json.br +0 -0
- package/lib/endfield-map/resources/map-2729942c89545e73.json.gz +0 -0
- package/lib/endfield-map/resources/map-29063a9802f67014.json.br +0 -0
- package/lib/endfield-map/resources/map-29063a9802f67014.json.gz +0 -0
- package/lib/endfield-map/resources/map-2f35d09a3ab60a63.json.br +0 -0
- package/lib/endfield-map/resources/map-2f35d09a3ab60a63.json.gz +0 -0
- package/lib/endfield-map/resources/map-2f98a873da81425e.json.br +0 -0
- package/lib/endfield-map/resources/map-2f98a873da81425e.json.gz +0 -0
- package/lib/endfield-map/resources/map-3ab978ae44190f80.json.br +0 -0
- package/lib/endfield-map/resources/map-3ab978ae44190f80.json.gz +0 -0
- package/lib/endfield-map/resources/map-3b62c80d14208902.json.br +0 -0
- package/lib/endfield-map/resources/map-3b62c80d14208902.json.gz +0 -0
- package/lib/endfield-map/resources/map-3d486e887ae121ac.png +0 -0
- package/lib/endfield-map/resources/map-418dc4865608fce5.png +0 -0
- package/lib/endfield-map/resources/map-4366b84607335f76.json.br +0 -0
- package/lib/endfield-map/resources/map-4366b84607335f76.json.gz +0 -0
- package/lib/endfield-map/resources/map-4371259d3ab7aac7.png +0 -0
- package/lib/endfield-map/resources/map-48dcc0fd55e13f7f.png +0 -0
- package/lib/endfield-map/resources/map-4aabda0e41197570.json.br +0 -0
- package/lib/endfield-map/resources/map-4aabda0e41197570.json.gz +0 -0
- package/lib/endfield-map/resources/map-505f5687b287043c.json.br +0 -0
- package/lib/endfield-map/resources/map-505f5687b287043c.json.gz +0 -0
- package/lib/endfield-map/resources/map-5177e9d1d46f47e2.png +0 -0
- package/lib/endfield-map/resources/map-51baeb59b7f8c28d.png +0 -0
- package/lib/endfield-map/resources/map-549561b46e621874.png +0 -0
- package/lib/endfield-map/resources/map-56c6a0bef799c8dc.json.br +0 -0
- package/lib/endfield-map/resources/map-56c6a0bef799c8dc.json.gz +0 -0
- package/lib/endfield-map/resources/map-5a6963909423b674.png +0 -0
- package/lib/endfield-map/resources/map-5a9c533c0552d7b7.png +0 -0
- package/lib/endfield-map/resources/map-5d20ae9ffa9a02b0.png +0 -0
- package/lib/endfield-map/resources/map-626c8847393f60bf.png +0 -0
- package/lib/endfield-map/resources/map-63d43cbec3bed3b8.png +0 -0
- package/lib/endfield-map/resources/map-661b6ec56e722574.png +0 -0
- package/lib/endfield-map/resources/map-6ddd384eca190828.png +0 -0
- package/lib/endfield-map/resources/map-6fc1aa9144f8156c.json.br +0 -0
- package/lib/endfield-map/resources/map-6fc1aa9144f8156c.json.gz +0 -0
- package/lib/endfield-map/resources/map-738ebaa002c33cfc.png +0 -0
- package/lib/endfield-map/resources/map-758c13be6085ec07.json.br +0 -0
- package/lib/endfield-map/resources/map-758c13be6085ec07.json.gz +0 -0
- package/lib/endfield-map/resources/map-78929f26d6844509.png +0 -0
- package/lib/endfield-map/resources/map-7f09c142950a0f74.png +0 -0
- package/lib/endfield-map/resources/map-8385e11f39e1a51c.json.br +0 -0
- package/lib/endfield-map/resources/map-8385e11f39e1a51c.json.gz +0 -0
- package/lib/endfield-map/resources/map-881343926a568f27.json.br +0 -0
- package/lib/endfield-map/resources/map-881343926a568f27.json.gz +0 -0
- package/lib/endfield-map/resources/map-8d1ade3f01d952b0.png +0 -0
- package/lib/endfield-map/resources/map-8fef3495bb7ed60f.png +0 -0
- package/lib/endfield-map/resources/map-907a7ecf8a0ce352.png +0 -0
- package/lib/endfield-map/resources/map-9113d5951c4db903.json.br +0 -0
- package/lib/endfield-map/resources/map-9113d5951c4db903.json.gz +0 -0
- package/lib/endfield-map/resources/map-94ad6634436224e3.json.br +0 -0
- package/lib/endfield-map/resources/map-94ad6634436224e3.json.gz +0 -0
- package/lib/endfield-map/resources/map-99722e61da5f3e53.png +0 -0
- package/lib/endfield-map/resources/map-9be9596cc059abe3.png +0 -0
- package/lib/endfield-map/resources/map-a4db9a4763e47fbf.png +0 -0
- package/lib/endfield-map/resources/map-a4e8ef07159997e7.json.br +0 -0
- package/lib/endfield-map/resources/map-a4e8ef07159997e7.json.gz +0 -0
- package/lib/endfield-map/resources/map-ab35d06875ee418f.png +0 -0
- package/lib/endfield-map/resources/map-ac51bb3e7623cd8b.png +0 -0
- package/lib/endfield-map/resources/map-ac830a66296bc83a.png +0 -0
- package/lib/endfield-map/resources/map-b3fd04209718e55e.json.br +0 -0
- package/lib/endfield-map/resources/map-b3fd04209718e55e.json.gz +0 -0
- package/lib/endfield-map/resources/map-b8164f1af7c5d895.png +0 -0
- package/lib/endfield-map/resources/map-b8cde93b965384d8.json.br +0 -0
- package/lib/endfield-map/resources/map-b8cde93b965384d8.json.gz +0 -0
- package/lib/endfield-map/resources/map-b8e48511502e0e8d.json.br +0 -0
- package/lib/endfield-map/resources/map-b8e48511502e0e8d.json.gz +0 -0
- package/lib/endfield-map/resources/map-bfc58277d5ba649d.png +0 -0
- package/lib/endfield-map/resources/map-c013f82048c6891a.png +0 -0
- package/lib/endfield-map/resources/map-c49c947841ae4ab5.json.br +0 -0
- package/lib/endfield-map/resources/map-c49c947841ae4ab5.json.gz +0 -0
- package/lib/endfield-map/resources/map-c4d1b1c6d6706c81.json.br +0 -0
- package/lib/endfield-map/resources/map-c4d1b1c6d6706c81.json.gz +0 -0
- package/lib/endfield-map/resources/map-c607f8b4c8ad8967.png +0 -0
- package/lib/endfield-map/resources/map-c6f20276b8781326.json.br +0 -0
- package/lib/endfield-map/resources/map-c6f20276b8781326.json.gz +0 -0
- package/lib/endfield-map/resources/map-d51adf5952af1e8e.png +0 -0
- package/lib/endfield-map/resources/map-d695eface22f8b74.json.br +0 -0
- package/lib/endfield-map/resources/map-d695eface22f8b74.json.gz +0 -0
- package/lib/endfield-map/resources/map-db11d394456a7465.png +0 -0
- package/lib/endfield-map/resources/map-dcc4d218581e3dc6.png +0 -0
- package/lib/endfield-map/resources/map-ddd43dbb8ab8a4e3.json.br +0 -0
- package/lib/endfield-map/resources/map-ddd43dbb8ab8a4e3.json.gz +0 -0
- package/lib/endfield-map/resources/map-defe5db7dabf748a.png +0 -0
- package/lib/endfield-map/resources/map-e1aeb833e22520aa.json.br +0 -0
- package/lib/endfield-map/resources/map-e1aeb833e22520aa.json.gz +0 -0
- package/lib/endfield-map/resources/map-e25ee9af6e493500.png +0 -0
- package/lib/endfield-map/resources/map-e3824082cd6f747d.json.br +0 -0
- package/lib/endfield-map/resources/map-e3824082cd6f747d.json.gz +0 -0
- package/lib/endfield-map/resources/map-e694a91e630b1bd6.png +0 -0
- package/lib/endfield-map/resources/map-eaa93af79d12a87d.png +0 -0
- package/lib/endfield-map/resources/map-ec9575ee65d7766f.json.br +0 -0
- package/lib/endfield-map/resources/map-ec9575ee65d7766f.json.gz +0 -0
- package/lib/endfield-map/resources/map-ede421a329dc78a4.png +0 -0
- package/lib/endfield-map/resources/map-eff3bce8f6888fc5.json.br +0 -0
- package/lib/endfield-map/resources/map-eff3bce8f6888fc5.json.gz +0 -0
- package/lib/endfield-map/resources/map-fb5f4a59a181452c.png +0 -0
- package/lib/endfield-map/resources/map-fdeaa17eeba90d8e.json.br +0 -0
- package/lib/endfield-map/resources/map-fdeaa17eeba90d8e.json.gz +0 -0
- package/lib/rhine/assets/archive-assembly.glb +0 -0
- package/lib/rhine/assets/archive-cassette.glb +0 -0
- package/lib/rhine/fonts/MiSans-Bold.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-Demibold.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-Light.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-Regular.woff2 +0 -0
- package/lib/rhine/fonts/MiSans-license.pdf +0 -0
- package/lib/rhine/fonts/NOTICE.txt +6 -0
- package/lib/rhine/licenses/RhineLabUI-MIT.txt +21 -0
- package/lib/rhine/licenses/entities-BSD-2-Clause.txt +11 -0
- package/lib/rhine/licenses/linkify-it-MIT.txt +22 -0
- package/lib/rhine/licenses/markdown-it-MIT.txt +22 -0
- package/lib/rhine/licenses/mdurl-MIT.txt +45 -0
- package/lib/rhine/licenses/punycode.js-MIT.txt +20 -0
- package/lib/rhine/licenses/rolling-number-MIT.txt +21 -0
- package/lib/rhine/licenses/three-MIT.txt +21 -0
- package/lib/rhine/licenses/uc.micro-MIT.txt +20 -0
- package/lib/rhine/rhine.css +1 -0
- package/lib/rhine/rhine.js +5106 -0
- package/lib/skins/common.css +31 -0
- package/lib/skins/endfield-aic.css +141 -0
- package/lib/skins/prts-agent.css +136 -0
- package/lib/skins/rhine-lab.css +70 -0
- package/package.json +112 -0
- package/presets/definition.js +26 -0
- package/presets/prts/agent.cordis.yml +25 -0
- package/presets/prts/preset.yml +3 -0
- package/presets/register.js +86 -0
- package/resources/entity-routing.json +21 -0
- package/skills/prts-investigation/SKILL.md +121 -0
- package/skills/prts-retrieval/SKILL.md +35 -0
- package/skills/prts-retrieval/references/cloud-tools.md +13 -0
- package/skills/prts-retrieval/references/diagnostics.md +15 -0
- package/skills/prts-retrieval/references/local-tools.md +24 -0
- package/skills/prts-retrieval/references/module-arknights.md +23 -0
- package/skills/prts-retrieval/references/module-dual.md +15 -0
- package/skills/prts-retrieval/references/module-endfield.md +21 -0
- package/skills/prts-retrieval/references/retrieval-process.md +44 -0
- package/skills/prts-retrieval/references/retrieval-recipes-dual.md +34 -0
- package/skills/prts-retrieval/references/retrieval-recipes-endfield.md +37 -0
- package/skills/prts-retrieval/references/retrieval-recipes.md +39 -0
- package/skills/prts-retrieval/references/tools-runtime.md +48 -0
- package/skills/prts-retrieval/references/wiki-schema.md +38 -0
- package/src/archive-presentation.js +287 -0
- package/src/cloud-projection.js +124 -0
- package/src/cloud.js +498 -0
- package/src/entity-recognizer.js +311 -0
- package/src/entity-routing.js +82 -0
- package/src/evidence-state.js +332 -0
- package/src/i18n.js +365 -0
- package/src/index.js +1261 -0
- package/src/installer.js +1362 -0
- package/src/investigation-review.js +87 -0
- package/src/investigation-store.js +579 -0
- package/src/investigation-tools.js +113 -0
- package/src/localization-format.js +23 -0
- package/src/manual-search.js +103 -0
- package/src/read.js +1097 -0
- package/src/release-compatibility.js +73 -0
- package/src/release-watch.js +40 -0
- package/src/search-projection.js +139 -0
- package/src/search.js +1522 -0
- package/src/skill.js +98 -0
- package/src/source-map.js +348 -0
- package/src/state.js +314 -0
- package/src/store.js +1569 -0
- package/src/timeline.js +386 -0
- package/src/ui.js +1353 -0
- package/src/wiki.js +111 -0
- package/tsconfig.rhine.json +15 -0
- package/ui/rhine/archive-navigation-limit.ts +31 -0
- package/ui/rhine/archive-rack.ts +83 -0
- package/ui/rhine/assets/evidence-observatory.webp +0 -0
- package/ui/rhine/board-plane-transform.ts +24 -0
- package/ui/rhine/catalogue.ts +72 -0
- package/ui/rhine/deferred-preparation.ts +146 -0
- package/ui/rhine/evidence-board-controls.css +22 -0
- package/ui/rhine/evidence-board-history.ts +73 -0
- package/ui/rhine/evidence-board-model.ts +189 -0
- package/ui/rhine/evidence-board-panel.ts +762 -0
- package/ui/rhine/evidence-board-scene.ts +548 -0
- package/ui/rhine/evidence-board.css +191 -0
- package/ui/rhine/evidence-card-art.ts +470 -0
- package/ui/rhine/evidence-card-style.ts +17 -0
- package/ui/rhine/evidence-inbox-scene.ts +43 -0
- package/ui/rhine/evidence-inbox.css +29 -0
- package/ui/rhine/evidence-inbox.ts +104 -0
- package/ui/rhine/index.ts +12 -0
- package/ui/rhine/interaction.css +20 -0
- package/ui/rhine/investigation-board-plane.ts +5 -0
- package/ui/rhine/investigation-board.css +34 -0
- package/ui/rhine/investigation-board.ts +348 -0
- package/ui/rhine/investigation-edits.ts +46 -0
- package/ui/rhine/investigation-layout.ts +29 -0
- package/ui/rhine/label-prefetch.ts +34 -0
- package/ui/rhine/looping-title.ts +40 -0
- package/ui/rhine/manual-search.css +41 -0
- package/ui/rhine/manual-search.ts +89 -0
- package/ui/rhine/original/appearance.ts +180 -0
- package/ui/rhine/original/archive-label.ts +84 -0
- package/ui/rhine/original/archive-lighting.ts +39 -0
- package/ui/rhine/original/archive-loop.ts +82 -0
- package/ui/rhine/original/archive-refill.ts +64 -0
- package/ui/rhine/original/array-detail-data.ts +2 -0
- package/ui/rhine/original/array-detail.ts +41 -0
- package/ui/rhine/original/array-interior.ts +411 -0
- package/ui/rhine/original/array-occlusion.ts +166 -0
- package/ui/rhine/original/array-shell.ts +38 -0
- package/ui/rhine/original/array-visibility.ts +309 -0
- package/ui/rhine/original/asset-url.ts +4 -0
- package/ui/rhine/original/brand.ts +12 -0
- package/ui/rhine/original/cassette-lod.ts +81 -0
- package/ui/rhine/original/data.ts +21 -0
- package/ui/rhine/original/decryption.ts +185 -0
- package/ui/rhine/original/detail-signature.ts +18 -0
- package/ui/rhine/original/glass-reveal.ts +45 -0
- package/ui/rhine/original/internal-optics.ts +70 -0
- package/ui/rhine/original/label-renderer.ts +119 -0
- package/ui/rhine/original/label-textures.ts +117 -0
- package/ui/rhine/original/model-viewer.ts +658 -0
- package/ui/rhine/original/motion.ts +116 -0
- package/ui/rhine/original/opaque-backfaces.ts +64 -0
- package/ui/rhine/original/program-preparation.ts +111 -0
- package/ui/rhine/original/quality-renderer.ts +97 -0
- package/ui/rhine/original/quality-settings.ts +111 -0
- package/ui/rhine/original/render-quality.ts +163 -0
- package/ui/rhine/original/scene.ts +2051 -0
- package/ui/rhine/original/shelf-interior.ts +142 -0
- package/ui/rhine/original/ui-transitions.ts +111 -0
- package/ui/rhine/original/viewer-camera.ts +92 -0
- package/ui/rhine/original/viewport-layout.ts +41 -0
- package/ui/rhine/original-ui.css +2334 -0
- package/ui/rhine/readability.css +46 -0
- package/ui/rhine/reading-object.ts +96 -0
- package/ui/rhine/reading-origin.ts +28 -0
- package/ui/rhine/reading-room.css +55 -0
- package/ui/rhine/report-markdown.ts +98 -0
- package/ui/rhine/scene-edge-navigation.ts +84 -0
- package/ui/rhine/scene.ts +1726 -0
- package/ui/rhine/shelf-layout.ts +38 -0
- package/ui/rhine/snapshot-update-gate.ts +22 -0
- package/ui/rhine/source-reading.ts +15 -0
- package/ui/rhine/source-selection.css +164 -0
- package/ui/rhine/temporary-performance-observers.ts +163 -0
- package/ui/rhine/temporary-performance-panel.ts +124 -0
- package/ui/rhine/temporary-performance.css +19 -0
- package/ui/rhine/temporary-performance.ts +552 -0
- package/ui/rhine/temporary-render-diagnostics.ts +105 -0
- package/ui/rhine/tool-activity.ts +69 -0
- package/ui/rhine/types.ts +219 -0
- package/ui/rhine/visible-scene-hit.ts +13 -0
- package/ui/rhine/workbench.css +818 -0
- package/ui/rhine/workbench.ts +2245 -0
- package/ui/rhine/workspace-travel.ts +32 -0
- package/vite.rhine.config.ts +19 -0
package/src/store.js
ADDED
|
@@ -0,0 +1,1569 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CorpusStore:对 prts-browser-corpus-release-v1 资料包的只读访问层。
|
|
3
|
+
*
|
|
4
|
+
* 包结构(复用自 prts.chat/agent 的浏览器资料包,格式不变):
|
|
5
|
+
* releases/current.json → { release_id, data_version }
|
|
6
|
+
* releases/<release_id>/<pack>/pack-manifest.json → { pack_id, shards: [...] }
|
|
7
|
+
* releases/<release_id>/<pack>/shards/NNN.jsonl.gz → 每行一个文档记录
|
|
8
|
+
* { document: DocumentSummary, lines: LineRecord[], local_integrity, search_index_id }
|
|
9
|
+
*
|
|
10
|
+
* 新包初始化时读取轻量文档目录,旧包回退为扫描正文分片;两者都会建立
|
|
11
|
+
* 文档、引用、自然标题与关卡代号索引。正文分片按 LRU 缓存,避免重复解压。
|
|
12
|
+
*/
|
|
13
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
14
|
+
import { lstat, mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
15
|
+
import { gunzip } from 'node:zlib'
|
|
16
|
+
import { promisify } from 'node:util'
|
|
17
|
+
import { dirname, join } from 'node:path'
|
|
18
|
+
import { CORPUS_RESOURCE_LIMITS, readCurrentReleasePointer,
|
|
19
|
+
validateLocalRelease } from './installer.js'
|
|
20
|
+
|
|
21
|
+
const gunzipAsync = promisify(gunzip)
|
|
22
|
+
|
|
23
|
+
const TRIGRAM_MAGIC = Buffer.from([80, 82, 84, 83, 84, 71, 49, 0])
|
|
24
|
+
const NGRAM_MAGIC = Buffer.from([80, 82, 84, 83, 78, 71, 50, 0])
|
|
25
|
+
const TRIGRAM_HEADER_BYTES = TRIGRAM_MAGIC.length + 4
|
|
26
|
+
const PACK_ORDER = Object.freeze(['official_game', 'endfield_official_game', 'reviewed_wiki', 'terra_journey',
|
|
27
|
+
'endfield_reviewed_knowledge', 'entities', 'references'])
|
|
28
|
+
const MAX_SHARD_CACHE_BYTES = 96 * 1024 * 1024
|
|
29
|
+
const MAX_SEARCH_CACHE_BYTES = 64 * 1024 * 1024
|
|
30
|
+
const MAX_SHORT_LITERAL_CACHE_CANDIDATES = 65_536
|
|
31
|
+
const MAX_SHORT_LITERAL_SCAN_QUEUE = 16
|
|
32
|
+
const SHORT_LITERAL_SCAN_WORKERS = 4
|
|
33
|
+
const END_FIELD_NARRATIVE_CONTENT_TYPES = new Set([
|
|
34
|
+
'dialogue', 'cutscene', 'radio', 'remote_comm', 'black_screen',
|
|
35
|
+
'environment_talk', 'sns_topic', 'sns_chat', 'narration',
|
|
36
|
+
])
|
|
37
|
+
const END_FIELD_CONTENT_TYPE_ORDER = Object.freeze([
|
|
38
|
+
'dialogue', 'cutscene', 'black_screen', 'radio', 'remote_comm',
|
|
39
|
+
'environment_talk', 'sns_topic', 'sns_chat', 'narration',
|
|
40
|
+
])
|
|
41
|
+
export const DOCUMENT_ORDERING_VERSION = 2
|
|
42
|
+
|
|
43
|
+
/** 同时绑定 reset 代次与资料版本,避免跨 await 的读取链混用新旧记录。 */
|
|
44
|
+
export function corpusVersionSnapshot(store) {
|
|
45
|
+
return { generation: store._generation, dataVersion: store.dataVersion }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function assertCorpusVersion(store, snapshot) {
|
|
49
|
+
if (snapshot.generation !== store._generation || snapshot.dataVersion !== store.dataVersion) {
|
|
50
|
+
throw Object.assign(new Error('资料版本在读取期间发生变化,请重试'), {
|
|
51
|
+
code: 'PACKAGE_VERSION_MISMATCH', retryable: true,
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function shortLiteralScanError(code, message, retryable = false) {
|
|
57
|
+
return Object.assign(new Error(message), { code, retryable })
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function assertShortLiteralScanActive({ signal, deadline = Infinity, generation, store }) {
|
|
61
|
+
if (signal?.aborted) {
|
|
62
|
+
throw shortLiteralScanError('CANCELLED', '短字面量候选扫描已取消')
|
|
63
|
+
}
|
|
64
|
+
if (Number.isFinite(deadline) && Date.now() >= deadline) {
|
|
65
|
+
throw shortLiteralScanError('TIMEOUT', '短字面量候选扫描超时', true)
|
|
66
|
+
}
|
|
67
|
+
if (generation !== undefined && generation !== store._generation) {
|
|
68
|
+
throw shortLiteralScanError('PACKAGE_VERSION_MISMATCH',
|
|
69
|
+
'资料版本在短字面量候选扫描期间发生变化,请重试', true)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** 等待排队扫描时也及时响应调用方取消与绝对截止时间。 */
|
|
74
|
+
function waitForShortLiteralScan(operation, runtime) {
|
|
75
|
+
assertShortLiteralScanActive({ ...runtime, store: runtime.store })
|
|
76
|
+
return new Promise((resolve, reject) => {
|
|
77
|
+
let settled = false
|
|
78
|
+
let timer = null
|
|
79
|
+
const finish = (callback, value) => {
|
|
80
|
+
if (settled) return
|
|
81
|
+
settled = true
|
|
82
|
+
if (timer) clearTimeout(timer)
|
|
83
|
+
runtime.signal?.removeEventListener?.('abort', onAbort)
|
|
84
|
+
callback(value)
|
|
85
|
+
}
|
|
86
|
+
const onAbort = () => finish(reject,
|
|
87
|
+
shortLiteralScanError('CANCELLED', '短字面量候选扫描已取消'))
|
|
88
|
+
runtime.signal?.addEventListener?.('abort', onAbort, { once: true })
|
|
89
|
+
if (Number.isFinite(runtime.deadline)) {
|
|
90
|
+
timer = setTimeout(() => finish(reject,
|
|
91
|
+
shortLiteralScanError('TIMEOUT', '短字面量候选扫描排队超时', true)),
|
|
92
|
+
Math.max(1, runtime.deadline - Date.now()))
|
|
93
|
+
timer.unref?.()
|
|
94
|
+
}
|
|
95
|
+
operation.then((value) => finish(resolve, value), (error) => finish(reject, error))
|
|
96
|
+
})
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function packOrder(left, right) {
|
|
100
|
+
const leftIndex = PACK_ORDER.indexOf(left.name)
|
|
101
|
+
const rightIndex = PACK_ORDER.indexOf(right.name)
|
|
102
|
+
const leftRank = leftIndex < 0 ? PACK_ORDER.length : leftIndex
|
|
103
|
+
const rightRank = rightIndex < 0 ? PACK_ORDER.length : rightIndex
|
|
104
|
+
return leftRank - rightRank || left.name.localeCompare(right.name, 'en')
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Keep each game's internal source order, but interleave their documents in
|
|
109
|
+
* the global scan order. Literal search paginates by this ordinal; without
|
|
110
|
+
* interleaving, a frequent term can fill every scan page from the first pack
|
|
111
|
+
* before the second game's candidates are even visited.
|
|
112
|
+
*/
|
|
113
|
+
function federatedOrder(documents, sourceOrder) {
|
|
114
|
+
const byGame = new Map([['arknights', []], ['endfield', []]])
|
|
115
|
+
const other = []
|
|
116
|
+
for (const documentId of sourceOrder) {
|
|
117
|
+
const item = documents.get(documentId)
|
|
118
|
+
const game = item ? documentGame(item.document) : ''
|
|
119
|
+
const target = byGame.get(game)
|
|
120
|
+
if (target) target.push(documentId)
|
|
121
|
+
else other.push(documentId)
|
|
122
|
+
}
|
|
123
|
+
if (!byGame.get('arknights').length || !byGame.get('endfield').length) return sourceOrder
|
|
124
|
+
const ordered = []
|
|
125
|
+
const maximum = Math.max(byGame.get('arknights').length, byGame.get('endfield').length)
|
|
126
|
+
for (let index = 0; index < maximum; index += 1) {
|
|
127
|
+
for (const game of ['arknights', 'endfield']) {
|
|
128
|
+
const documentId = byGame.get(game)[index]
|
|
129
|
+
if (documentId) ordered.push(documentId)
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return ordered.concat(other)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function readVarint(bytes, state, limit) {
|
|
136
|
+
let value = 0
|
|
137
|
+
let shift = 0
|
|
138
|
+
while (state.offset < limit && shift <= 49) {
|
|
139
|
+
const byte = bytes[state.offset++]
|
|
140
|
+
value += (byte & 0x7f) * (2 ** shift)
|
|
141
|
+
if (!(byte & 0x80)) return value
|
|
142
|
+
shift += 7
|
|
143
|
+
}
|
|
144
|
+
throw new Error('CorpusStore: invalid trigram varint')
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** 与编译期 SQLite BINARY(UTF-8 字节序)保持一致。 */
|
|
148
|
+
function compareNgramKeys(left, right) {
|
|
149
|
+
return Buffer.compare(Buffer.from(String(left), 'utf8'), Buffer.from(String(right), 'utf8'))
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** 查询浏览器资料包的 PRTSTG1 / PRTSNG2 二进制倒排分片。 */
|
|
153
|
+
function lookupNgramIndex(bytes, target) {
|
|
154
|
+
if (bytes.length < TRIGRAM_HEADER_BYTES
|
|
155
|
+
|| (!bytes.subarray(0, 8).equals(TRIGRAM_MAGIC)
|
|
156
|
+
&& !bytes.subarray(0, 8).equals(NGRAM_MAGIC))) {
|
|
157
|
+
throw new Error('CorpusStore: invalid ngram index magic')
|
|
158
|
+
}
|
|
159
|
+
const count = bytes.readUInt32LE(8)
|
|
160
|
+
const payloadStart = TRIGRAM_HEADER_BYTES + (count + 1) * 4
|
|
161
|
+
if (payloadStart > bytes.length) throw new Error('CorpusStore: invalid trigram offset table')
|
|
162
|
+
const offsetAt = (index) => bytes.readUInt32LE(TRIGRAM_HEADER_BYTES + index * 4)
|
|
163
|
+
const recordAt = (index) => {
|
|
164
|
+
const start = payloadStart + offsetAt(index)
|
|
165
|
+
const end = payloadStart + offsetAt(index + 1)
|
|
166
|
+
const state = { offset: start }
|
|
167
|
+
const textLength = readVarint(bytes, state, end)
|
|
168
|
+
const keyBytes = bytes.subarray(state.offset, state.offset + textLength)
|
|
169
|
+
const trigram = keyBytes.toString('utf8')
|
|
170
|
+
state.offset += textLength
|
|
171
|
+
const postingCount = readVarint(bytes, state, end)
|
|
172
|
+
const indexes = []
|
|
173
|
+
let current = 0
|
|
174
|
+
for (let item = 0; item < postingCount; item += 1) {
|
|
175
|
+
current += readVarint(bytes, state, end)
|
|
176
|
+
indexes.push(current)
|
|
177
|
+
}
|
|
178
|
+
if (state.offset !== end) throw new Error('CorpusStore: malformed trigram record')
|
|
179
|
+
return { trigram, keyBytes, indexes }
|
|
180
|
+
}
|
|
181
|
+
const targetBytes = Buffer.from(target, 'utf8')
|
|
182
|
+
let low = 0
|
|
183
|
+
let high = count - 1
|
|
184
|
+
while (low <= high) {
|
|
185
|
+
const middle = (low + high) >> 1
|
|
186
|
+
const record = recordAt(middle)
|
|
187
|
+
if (record.trigram === target) return record.indexes
|
|
188
|
+
if (Buffer.compare(record.keyBytes, targetBytes) < 0) low = middle + 1
|
|
189
|
+
else high = middle - 1
|
|
190
|
+
}
|
|
191
|
+
return []
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function searchIndexRange(descriptor = {}) {
|
|
195
|
+
return {
|
|
196
|
+
first: String(descriptor.first_ngram ?? descriptor.first_trigram ?? ''),
|
|
197
|
+
last: String(descriptor.last_ngram ?? descriptor.last_trigram ?? ''),
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function naturalCompare(left, right) {
|
|
202
|
+
return String(left ?? '').localeCompare(String(right ?? ''), 'zh-CN', {
|
|
203
|
+
numeric: true, sensitivity: 'base',
|
|
204
|
+
})
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const sha256 = (text) => createHash('sha256').update(text, 'utf8').digest('hex')
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* 面向模型与 UI 的短篇章标识。它由 canonical document_id 确定性派生,
|
|
211
|
+
* 不携带路径信息;96 bit 摘要在装载时做碰撞检测。
|
|
212
|
+
*/
|
|
213
|
+
export function documentUid(documentId) {
|
|
214
|
+
const value = String(documentId ?? '')
|
|
215
|
+
if (!value) return ''
|
|
216
|
+
return `doc_${createHash('sha256').update(`prts-document\0${value}`, 'utf8')
|
|
217
|
+
.digest('base64url').slice(0, 16)}`
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** 模型/界面使用的自然语言资料定位;各资料类型都带上可辨认的标题与类型。 */
|
|
221
|
+
export function documentGame(document = {}) {
|
|
222
|
+
const explicit = String(document.game || '').trim().toLocaleLowerCase()
|
|
223
|
+
if (explicit === 'arknights' || explicit === 'endfield') return explicit
|
|
224
|
+
const identity = `${document.document_id || ''} ${document.source_ref_prefix || ''}`.toLocaleLowerCase()
|
|
225
|
+
return identity.includes('endfield:') ? 'endfield' : 'arknights'
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** 将玩家可见的明日方舟关卡代号归一化为索引键(如 gt-3 → GT-3)。 */
|
|
229
|
+
export function normalizeStoryStageCode(value) {
|
|
230
|
+
return String(value ?? '').normalize('NFKC')
|
|
231
|
+
.replace(/[\u2010-\u2015\u2212]/gu, '-')
|
|
232
|
+
.replace(/\s+/gu, '')
|
|
233
|
+
.toLocaleUpperCase('en-US')
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** 可安全放进模型工具参数的公开关卡代号;非法资料值不进入短定位索引。 */
|
|
237
|
+
export function publicStoryStageCode(value, { relaxedInput = false } = {}) {
|
|
238
|
+
const raw = String(value ?? '').normalize('NFKC')
|
|
239
|
+
if (!relaxedInput && /[\s\p{Cc}\p{Cf}]/u.test(raw)) return ''
|
|
240
|
+
const normalized = normalizeStoryStageCode(value)
|
|
241
|
+
return normalized.length <= 32 && /^[A-Z0-9]+(?:-[A-Z0-9]+)*$/u.test(normalized)
|
|
242
|
+
? normalized : ''
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** 清除元数据中可改写工具渲染结构的控制/换行/双向字符。 */
|
|
246
|
+
export function publicMetadataText(value, maximum = 256) {
|
|
247
|
+
return String(value ?? '').normalize('NFC')
|
|
248
|
+
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
|
|
249
|
+
.replace(/\s+/gu, ' ').trim().slice(0, maximum)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function storyStageKey(stageCode, storyPart) {
|
|
253
|
+
return `${normalizeStoryStageCode(stageCode)}\0${String(storyPart ?? '').trim().toLocaleLowerCase('en-US')}`
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** 将资料内部的 part_type 映射成 Agent 使用的玩家语义。 */
|
|
257
|
+
export function publicStoryPart(value) {
|
|
258
|
+
const part = String(value ?? '').trim().toLocaleLowerCase('en-US')
|
|
259
|
+
if (part === 'before' || part === 'after') return part
|
|
260
|
+
if (part === 'story' || part === 'interlude' || part === 'body') return 'story'
|
|
261
|
+
return ''
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** GameData 干员密录 source_story_id 末段即页面段号(目前为 1/2)。 */
|
|
265
|
+
export function operatorRecordSegment(document = {}) {
|
|
266
|
+
if (documentGame(document) !== 'arknights' || document.document_category !== 'memory') return null
|
|
267
|
+
const match = /_([1-9][0-9]*)$/u.exec(String(document.source_story_id || document.document_id || ''))
|
|
268
|
+
return match ? Number(match[1]) : null
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
const CHARACTER_MATERIALS = Object.freeze({
|
|
272
|
+
'干员档案': 'profile', '角色档案': 'profile',
|
|
273
|
+
'模组文案': 'module',
|
|
274
|
+
'干员语音': 'voice', '角色语音': 'voice',
|
|
275
|
+
'时装文案': 'skin',
|
|
276
|
+
'招聘合同': 'recruitment',
|
|
277
|
+
'潜能与信物': 'potential',
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
export function publicCharacterMaterial(document = {}) {
|
|
281
|
+
if (document.document_type !== 'character') return ''
|
|
282
|
+
return CHARACTER_MATERIALS[String(document.document_category || '')] || ''
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function normalizedLookupText(value) {
|
|
286
|
+
return String(value ?? '').normalize('NFKC').replace(/\s+/gu, ' ').trim()
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function operatorRecordKey(characterName, recordName, segment = '') {
|
|
290
|
+
return `${normalizedLookupText(characterName)}\0${normalizedLookupText(recordName)}\0${segment}`
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function characterMaterialKey(game, characterName, material) {
|
|
294
|
+
return `${String(game || '')}\0${normalizedLookupText(characterName)}\0${String(material || '')}`
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function naturalDocumentTitleBase(document = {}, { includeOperatorRecordSegment = true } = {}) {
|
|
298
|
+
const displayTitle = publicMetadataText(document.display_title)
|
|
299
|
+
if (documentGame(document) === 'endfield' && document.game) return displayTitle
|
|
300
|
+
if (document.document_type === 'entity') {
|
|
301
|
+
return displayTitle ? `${displayTitle} / 实体资料` : ''
|
|
302
|
+
}
|
|
303
|
+
if (document.document_type === 'character') return displayTitle
|
|
304
|
+
if (document.document_type === 'knowledge') {
|
|
305
|
+
if (document.document_kind === 'terra_journey') return `${displayTitle} / 大地巡旅`
|
|
306
|
+
if (document.document_kind === 'wiki') {
|
|
307
|
+
const explicitRole = String(document.wiki_role || '')
|
|
308
|
+
if (explicitRole === 'story') return `${displayTitle} / 活动 Wiki`
|
|
309
|
+
if (explicitRole === 'character_activity') {
|
|
310
|
+
const sequence = Number.isInteger(document.sequence_index) ? ` · 第 ${document.sequence_index} 篇` : ''
|
|
311
|
+
return `${displayTitle} / 角色活动 Wiki${sequence}`
|
|
312
|
+
}
|
|
313
|
+
if (explicitRole === 'other') return `${displayTitle} / 审校资料`
|
|
314
|
+
const path = String(document.path || '')
|
|
315
|
+
if (path.startsWith('stories/')) return `${displayTitle} / 活动 Wiki`
|
|
316
|
+
if (path.startsWith('char_v3/prompt_')) {
|
|
317
|
+
const sequence = Number.isInteger(document.sequence_index) ? ` · 第 ${document.sequence_index} 篇` : ''
|
|
318
|
+
return `${displayTitle} / 角色活动 Wiki${sequence}`
|
|
319
|
+
}
|
|
320
|
+
if (path.startsWith('char_v3/extended_')) return `${displayTitle} / 角色补充 Wiki`
|
|
321
|
+
return `${displayTitle} / 角色 Wiki`
|
|
322
|
+
}
|
|
323
|
+
return displayTitle
|
|
324
|
+
}
|
|
325
|
+
if (document.document_type === 'reference') {
|
|
326
|
+
const labels = { activity_timelines: '活动时间线', char_alias: '角色别名表',
|
|
327
|
+
terra_timeline: '泰拉年表' }
|
|
328
|
+
return labels[displayTitle] || displayTitle
|
|
329
|
+
}
|
|
330
|
+
if (document.document_type !== 'story') return displayTitle
|
|
331
|
+
const variation = /_variation0*(\d+)(?:\.[^./]+)?$/iu.exec(
|
|
332
|
+
String(document.source_story_id || document.document_id || ''),
|
|
333
|
+
)
|
|
334
|
+
const identity = [...new Set([document.activity_name, document.story_code, document.story_name]
|
|
335
|
+
.map((item) => publicMetadataText(item)).filter(Boolean))]
|
|
336
|
+
if (!identity.length) {
|
|
337
|
+
const labels = { rogue: '集成战略文本', system: '游戏系统文本', guide: '游戏教程文本' }
|
|
338
|
+
const label = labels[String(document.document_category || '')] || '游戏内原文'
|
|
339
|
+
const collection = publicMetadataText(
|
|
340
|
+
String(document.collection_id || '').split(':').at(-1)?.split('/').at(-1), 128)
|
|
341
|
+
const sequence = Number.isInteger(document.sequence_index) ? `第 ${document.sequence_index} 篇` : ''
|
|
342
|
+
const fallback = [label, collection && !['other', 'obt'].includes(collection) ? collection : '', sequence,
|
|
343
|
+
publicMetadataText(document.part_label, 128)].filter(Boolean).join(' · ')
|
|
344
|
+
return fallback || displayTitle
|
|
345
|
+
}
|
|
346
|
+
const memoryPrefix = document.document_category === 'memory' && document.character_name
|
|
347
|
+
? [publicMetadataText(document.character_name), '干员密录'] : []
|
|
348
|
+
const memorySegment = includeOperatorRecordSegment && memoryPrefix.length
|
|
349
|
+
? operatorRecordSegment(document) : null
|
|
350
|
+
const sequenceSuffix = document.document_category === 'activity' && !document.story_code
|
|
351
|
+
&& Number.isInteger(document.sequence_index) ? [`第 ${document.sequence_index} 篇`] : []
|
|
352
|
+
return [...memoryPrefix, ...identity, ...(memorySegment ? [`第 ${memorySegment} 段`] : []),
|
|
353
|
+
publicMetadataText(document.part_label, 128), ...sequenceSuffix,
|
|
354
|
+
...(variation ? [`分支${Number(variation[1])}`] : [])]
|
|
355
|
+
.map((item) => String(item || '').trim()).filter(Boolean).join(' · ')
|
|
356
|
+
|| displayTitle
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* v2 联合语料使用带游戏前缀的自然标题消除跨游戏同名歧义。旧 v1
|
|
361
|
+
* 文档没有 game 字段,继续保留原题面;加载器同时为显式 game 文档建立
|
|
362
|
+
* 无前缀兼容索引,因此历史数据与新联合包可以并存。
|
|
363
|
+
*/
|
|
364
|
+
export function naturalDocumentTitle(document = {}) {
|
|
365
|
+
const title = publicMetadataText(naturalDocumentTitleBase(document), 480)
|
|
366
|
+
if (!title || !document.game) return title
|
|
367
|
+
const label = documentGame(document) === 'endfield' ? '终末地' : '明日方舟'
|
|
368
|
+
return publicMetadataText(title.startsWith(`${label} · `) ? title : `${label} · ${title}`, 512)
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function legacyOperatorRecordTitle(document = {}) {
|
|
372
|
+
if (!operatorRecordSegment(document)) return ''
|
|
373
|
+
const title = publicMetadataText(naturalDocumentTitleBase(document,
|
|
374
|
+
{ includeOperatorRecordSegment: false }), 480)
|
|
375
|
+
if (!title || !document.game) return title
|
|
376
|
+
const label = documentGame(document) === 'endfield' ? '终末地' : '明日方舟'
|
|
377
|
+
return publicMetadataText(title.startsWith(`${label} · `) ? title : `${label} · ${title}`, 512)
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** 行完整性规则:sha256(全部行文本以 \n 连接) === local_integrity.sha256。 */
|
|
381
|
+
export function computeLinesIntegrity(lines) {
|
|
382
|
+
return sha256(lines.map((line) => line.text).join('\n'))
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* 收集会破坏「原始字节即匹配文本」假设的字符:单个字符经 NFKC + 小写后
|
|
387
|
+
* 发生变化(全角标点、罗马数字、上标、兼容表意文字等)。实测本语料中
|
|
388
|
+
* 此类字符只有数十个(几乎全是全角标点),但出现在约七成以上的行里;
|
|
389
|
+
* 短字面量预筛据此构造无损 needle 集合(见 findDocumentsByShortLiteral)。
|
|
390
|
+
* 展开式口径与 search.js matchesText 的匹配侧一致:NFKC 归一化后小写。
|
|
391
|
+
*/
|
|
392
|
+
function collectUnstableChars(text, stableChars, unstableChars) {
|
|
393
|
+
if (typeof text !== 'string' || !text) return
|
|
394
|
+
for (let offset = 0; offset < text.length;) {
|
|
395
|
+
const code = text.codePointAt(offset)
|
|
396
|
+
const size = code > 0xffff ? 2 : 1
|
|
397
|
+
const character = text.slice(offset, offset + size)
|
|
398
|
+
offset += size
|
|
399
|
+
if (stableChars.has(character)) continue
|
|
400
|
+
const expansion = character.normalize('NFKC').toLowerCase()
|
|
401
|
+
if (expansion === character) stableChars.add(character)
|
|
402
|
+
else unstableChars.set(character, expansion)
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* 枚举查询的规范等价形式:NFD 全分解,以及把相邻字符逐步规范复合回去的
|
|
408
|
+
* 中间形态(例如谚文音节 ↔ 字母序列)。短字面量预筛把这些形式一并作为
|
|
409
|
+
* 字节 needle,保证语料以分解形态存储时不会漏检。
|
|
410
|
+
*/
|
|
411
|
+
function canonicalVariants(query, limit = 32) {
|
|
412
|
+
const variants = new Set([query])
|
|
413
|
+
const decomposed = query.normalize('NFD')
|
|
414
|
+
if (decomposed !== query) variants.add(decomposed)
|
|
415
|
+
const agenda = [decomposed]
|
|
416
|
+
while (agenda.length && variants.size < limit) {
|
|
417
|
+
const current = agenda.pop()
|
|
418
|
+
const chars = [...current]
|
|
419
|
+
for (let index = 0; index + 1 < chars.length && variants.size < limit; index += 1) {
|
|
420
|
+
const pair = chars[index] + chars[index + 1]
|
|
421
|
+
const composed = pair.normalize('NFC')
|
|
422
|
+
if (composed.length !== 1 || composed === pair) continue
|
|
423
|
+
const next = chars.slice(0, index).concat([composed], chars.slice(index + 2)).join('')
|
|
424
|
+
if (variants.has(next)) continue
|
|
425
|
+
variants.add(next)
|
|
426
|
+
agenda.push(next)
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return variants
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
export class CorpusStore {
|
|
433
|
+
/**
|
|
434
|
+
* @param {{ releasesDir: string, cacheShards?: number, ensure?: () => Promise<void>, cursorSecretPath?: string }} options
|
|
435
|
+
* `ensure` 是调用方可选的初始化前置步骤;本插件的 Host 和工具挂载不使用它下载资料。
|
|
436
|
+
*/
|
|
437
|
+
constructor({ releasesDir, cacheShards = 24, searchCacheShards = 32, ensure, cursorSecretPath } = {}) {
|
|
438
|
+
if (!releasesDir) throw new Error('CorpusStore: releasesDir is required')
|
|
439
|
+
this.releasesDir = releasesDir
|
|
440
|
+
this.cacheShards = cacheShards
|
|
441
|
+
// 倒排分片缓存独立于正文分片缓存:trigram 查询会在一个请求内轮询
|
|
442
|
+
// 全部 search-index 分片,容量太小会导致同批分片被反复重解压。
|
|
443
|
+
this.searchCacheShards = searchCacheShards
|
|
444
|
+
this.ensure = ensure
|
|
445
|
+
this.cursorSecretPath = cursorSecretPath || join(dirname(releasesDir), 'cursor-secret.bin')
|
|
446
|
+
/** @type {Promise<void> | null} */
|
|
447
|
+
this._ready = null
|
|
448
|
+
this._generation = 0
|
|
449
|
+
this._loaded = false
|
|
450
|
+
this.releaseId = null
|
|
451
|
+
this.dataVersion = null
|
|
452
|
+
/** @type {Map<string, { packId: string, shardPath: string, index: number }>} */
|
|
453
|
+
this.documents = new Map()
|
|
454
|
+
/** @type {string[]} 全局稳定 document_ordinal → document_id。 */
|
|
455
|
+
this.documentOrder = []
|
|
456
|
+
/** @type {Map<string, string>} source_ref_prefix → document_id */
|
|
457
|
+
this.prefixIndex = new Map()
|
|
458
|
+
/** @type {Map<string, string>} 短 document_uid → canonical document_id */
|
|
459
|
+
this.uidIndex = new Map()
|
|
460
|
+
/** @type {Map<string, string[]>} display_title → document_id[](保留同名歧义) */
|
|
461
|
+
this.titleIndex = new Map()
|
|
462
|
+
/** @type {Map<string, string[]>} 自然语言完整篇章标题 → document_id[] */
|
|
463
|
+
this.naturalTitleIndex = new Map()
|
|
464
|
+
/** @type {Map<string, string[]>} 明日方舟关卡代号\0行动前后/纯剧情 → document_id[] */
|
|
465
|
+
this.storyStageIndex = new Map()
|
|
466
|
+
/** @type {Map<string, string[]>} 明日方舟关卡代号 → 全部剧情 document_id[] */
|
|
467
|
+
this.storyStageCodeIndex = new Map()
|
|
468
|
+
/** @type {Map<string, string[]>} 角色\0密录名\0段号 → 正文 document_id[] */
|
|
469
|
+
this.operatorRecordIndex = new Map()
|
|
470
|
+
/** @type {Map<string, string[]>} 游戏\0角色\0资料类型 → document_id[] */
|
|
471
|
+
this.characterMaterialIndex = new Map()
|
|
472
|
+
/** @type {Map<string, string>} path → document_id(首个命中优先) */
|
|
473
|
+
this.pathIndex = new Map()
|
|
474
|
+
/** @type {Map<string, string>} source_story_id → document_id(首个命中优先) */
|
|
475
|
+
this.sourceStoryIndex = new Map()
|
|
476
|
+
/** @type {Map<string, object>} packId → pack-manifest */
|
|
477
|
+
this.packs = new Map()
|
|
478
|
+
/** @type {Map<string, string>} packId\0search_index_id -> document_id */
|
|
479
|
+
this.searchIndexDocuments = new Map()
|
|
480
|
+
/** @type {Map<string, object[]>} shardKey → 已解析文档记录数组(LRU) */
|
|
481
|
+
this._shardCache = new Map()
|
|
482
|
+
this._shardCacheSizes = new Map()
|
|
483
|
+
this._shardCacheBytes = 0
|
|
484
|
+
/** @type {Map<string, Buffer>} search-index 分片 LRU */
|
|
485
|
+
this._searchCache = new Map()
|
|
486
|
+
this._searchCacheSizes = new Map()
|
|
487
|
+
this._searchCacheBytes = 0
|
|
488
|
+
/** @type {Map<string, string[]>} 1—2 字无损分片预筛结果(LRU) */
|
|
489
|
+
this._shortLiteralCache = new Map()
|
|
490
|
+
this._shortLiteralCacheCandidateCount = 0
|
|
491
|
+
// 不同短词都需要遍历全部正文分片。跨请求串行,单次扫描内部再使用固定
|
|
492
|
+
// worker 上限,避免并发请求把 libuv 线程池和明文 Buffer 成倍放大。
|
|
493
|
+
this._shortLiteralScanTail = Promise.resolve()
|
|
494
|
+
this._shortLiteralScanPending = 0
|
|
495
|
+
/** @type {Promise<Buffer> | null} 持久游标签名密钥。 */
|
|
496
|
+
this._cursorSecret = null
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/** 初始化(幂等):ensure → 验证 release → 读轻量目录(旧包扫描正文)建索引。 */
|
|
500
|
+
ready() {
|
|
501
|
+
if (!this._ready) {
|
|
502
|
+
const generation = this._generation
|
|
503
|
+
const operation = (this.ensure ? Promise.resolve(this.ensure()) : Promise.resolve())
|
|
504
|
+
.then(() => this._init(generation))
|
|
505
|
+
.catch((error) => {
|
|
506
|
+
if (generation !== this._generation) return false
|
|
507
|
+
throw error
|
|
508
|
+
})
|
|
509
|
+
.then(async (committed) => {
|
|
510
|
+
if (!committed) await this.ready()
|
|
511
|
+
})
|
|
512
|
+
let tracked
|
|
513
|
+
tracked = operation.catch((error) => {
|
|
514
|
+
if (this._ready === tracked) this._ready = null
|
|
515
|
+
throw error
|
|
516
|
+
})
|
|
517
|
+
this._ready = tracked
|
|
518
|
+
}
|
|
519
|
+
return this._ready
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/** 初始化是否已提交到当前 generation。 */
|
|
523
|
+
get loaded() {
|
|
524
|
+
return this._loaded
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* 丢弃全部已解析状态(版本切换/重下载后调用);下次 ready() 按新的
|
|
529
|
+
* current.json 重建索引。进行中的旧 ready() 只构建局部快照,generation
|
|
530
|
+
* 不匹配时不会提交,并会等待当前 generation 的初始化。
|
|
531
|
+
*/
|
|
532
|
+
reset() {
|
|
533
|
+
this._generation += 1
|
|
534
|
+
this._ready = null
|
|
535
|
+
this._loaded = false
|
|
536
|
+
this.releaseId = null
|
|
537
|
+
this.dataVersion = null
|
|
538
|
+
this.documents.clear()
|
|
539
|
+
this.documentOrder = []
|
|
540
|
+
this.prefixIndex.clear()
|
|
541
|
+
this.uidIndex.clear()
|
|
542
|
+
this.titleIndex.clear()
|
|
543
|
+
this.naturalTitleIndex.clear()
|
|
544
|
+
this.storyStageIndex.clear()
|
|
545
|
+
this.storyStageCodeIndex.clear()
|
|
546
|
+
this.operatorRecordIndex.clear()
|
|
547
|
+
this.characterMaterialIndex.clear()
|
|
548
|
+
this.pathIndex.clear()
|
|
549
|
+
this.sourceStoryIndex.clear()
|
|
550
|
+
this.packs.clear()
|
|
551
|
+
this.searchIndexDocuments.clear()
|
|
552
|
+
this._shardCache.clear()
|
|
553
|
+
this._shardCacheSizes.clear()
|
|
554
|
+
this._shardCacheBytes = 0
|
|
555
|
+
this._searchCache.clear()
|
|
556
|
+
this._searchCacheSizes.clear()
|
|
557
|
+
this._searchCacheBytes = 0
|
|
558
|
+
this._shortLiteralCache.clear()
|
|
559
|
+
this._shortLiteralCacheCandidateCount = 0
|
|
560
|
+
this.unstableChars = null
|
|
561
|
+
this._timelineRows = null
|
|
562
|
+
this._aliasGroups = null
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
async _init(generation) {
|
|
566
|
+
const current = await readCurrentReleasePointer(this.releasesDir)
|
|
567
|
+
const releaseId = current.release_id
|
|
568
|
+
// Store 是 Host 权限下的持久文件入口。每次装载都先验证 release id、声明的
|
|
569
|
+
// 全部 pack、资源路径/尺寸/哈希与符号链接,再读取任何压缩内容。
|
|
570
|
+
const validated = await validateLocalRelease(this.releasesDir, releaseId,
|
|
571
|
+
{ verifyHashes: true, details: true })
|
|
572
|
+
const { manifest: releaseManifest, packManifests, releaseDir } = validated
|
|
573
|
+
if (current.data_version != null && current.data_version !== releaseManifest.data_version) {
|
|
574
|
+
throw new Error('CorpusStore: current.json data_version does not match release-manifest')
|
|
575
|
+
}
|
|
576
|
+
const next = {
|
|
577
|
+
releaseId,
|
|
578
|
+
dataVersion: releaseManifest.data_version,
|
|
579
|
+
documents: new Map(),
|
|
580
|
+
documentOrder: [],
|
|
581
|
+
prefixIndex: new Map(),
|
|
582
|
+
uidIndex: new Map(),
|
|
583
|
+
titleIndex: new Map(),
|
|
584
|
+
naturalTitleIndex: new Map(),
|
|
585
|
+
storyStageIndex: new Map(),
|
|
586
|
+
storyStageCodeIndex: new Map(),
|
|
587
|
+
operatorRecordIndex: new Map(),
|
|
588
|
+
characterMaterialIndex: new Map(),
|
|
589
|
+
pathIndex: new Map(),
|
|
590
|
+
sourceStoryIndex: new Map(),
|
|
591
|
+
packs: new Map(),
|
|
592
|
+
searchIndexDocuments: new Map(),
|
|
593
|
+
unstableChars: new Map(),
|
|
594
|
+
}
|
|
595
|
+
const stableChars = new Set()
|
|
596
|
+
|
|
597
|
+
const entries = releaseManifest.packs.map((descriptor) => ({
|
|
598
|
+
name: descriptor.pack_id,
|
|
599
|
+
manifestPath: descriptor.manifest_path,
|
|
600
|
+
})).sort(packOrder)
|
|
601
|
+
for (const entry of entries) {
|
|
602
|
+
const trustedManifest = packManifests.get(entry.name)
|
|
603
|
+
if (!trustedManifest) throw new Error(`CorpusStore: missing validated pack ${entry.name}`)
|
|
604
|
+
// v3 起 pack-manifest 不再携带顶层 pack_id,以 release 清单目录名为准。
|
|
605
|
+
const packId = entry.name
|
|
606
|
+
const manifest = { ...trustedManifest, pack_id: packId }
|
|
607
|
+
next.packs.set(packId, manifest)
|
|
608
|
+
const catalog = manifest.document_catalog
|
|
609
|
+
const sources = catalog ? [catalog] : manifest.shards
|
|
610
|
+
const bodyShards = new Map(manifest.shards.map((shard) => [shard.path, shard]))
|
|
611
|
+
const seenLocations = new Set()
|
|
612
|
+
for (const shard of sources) {
|
|
613
|
+
const records = this._decodeShard(await this._readPacked(
|
|
614
|
+
packId, shard.path, releaseId, shard,
|
|
615
|
+
))
|
|
616
|
+
if (catalog && records.length !== manifest.document_count) {
|
|
617
|
+
throw new Error(`CorpusStore: document catalog count mismatch: ${packId}`)
|
|
618
|
+
}
|
|
619
|
+
// 分片间让出事件循环:初始化虽在后台,也不应长时间阻塞宿主进程。
|
|
620
|
+
await new Promise((resolve) => { setImmediate(resolve) })
|
|
621
|
+
records.forEach((record, index) => {
|
|
622
|
+
const shardPath = catalog ? String(record.shard_path || '') : shard.path
|
|
623
|
+
const recordIndex = catalog ? record.record_index : index
|
|
624
|
+
const bodyShard = bodyShards.get(shardPath)
|
|
625
|
+
if (!bodyShard || !Number.isInteger(recordIndex) || recordIndex < 0
|
|
626
|
+
|| recordIndex >= bodyShard.document_count
|
|
627
|
+
|| !record.document || typeof record.document !== 'object'
|
|
628
|
+
|| !Array.isArray(record.speakers)) {
|
|
629
|
+
throw new Error(`CorpusStore: invalid document catalog row: ${packId}#${index}`)
|
|
630
|
+
}
|
|
631
|
+
const locationKey = `${shardPath}\0${recordIndex}`
|
|
632
|
+
if (seenLocations.has(locationKey)) {
|
|
633
|
+
throw new Error(`CorpusStore: duplicate document catalog location: ${packId}#${index}`)
|
|
634
|
+
}
|
|
635
|
+
seenLocations.add(locationKey)
|
|
636
|
+
const { document_id: documentId, source_ref_prefix: prefix } = record.document
|
|
637
|
+
const ordinal = next.documentOrder.length
|
|
638
|
+
next.documentOrder.push(documentId)
|
|
639
|
+
next.documents.set(documentId, {
|
|
640
|
+
packId: manifest.pack_id, shardPath, index: recordIndex,
|
|
641
|
+
document: record.document, speakers: record.speakers ?? [],
|
|
642
|
+
searchIndexId: record.search_index_id ?? null, ordinal,
|
|
643
|
+
})
|
|
644
|
+
if (prefix) next.prefixIndex.set(prefix, documentId)
|
|
645
|
+
const uid = documentUid(documentId)
|
|
646
|
+
const existingUid = next.uidIndex.get(uid)
|
|
647
|
+
if (existingUid && existingUid !== documentId) {
|
|
648
|
+
throw new Error(`CorpusStore: document_uid collision: ${uid}`)
|
|
649
|
+
}
|
|
650
|
+
next.uidIndex.set(uid, documentId)
|
|
651
|
+
const title = record.document.display_title
|
|
652
|
+
if (title) {
|
|
653
|
+
const ids = next.titleIndex.get(title) ?? []
|
|
654
|
+
ids.push(documentId)
|
|
655
|
+
next.titleIndex.set(title, ids)
|
|
656
|
+
}
|
|
657
|
+
const naturalTitle = naturalDocumentTitle(record.document)
|
|
658
|
+
if (naturalTitle) {
|
|
659
|
+
const ids = next.naturalTitleIndex.get(naturalTitle) ?? []
|
|
660
|
+
ids.push(documentId)
|
|
661
|
+
next.naturalTitleIndex.set(naturalTitle, ids)
|
|
662
|
+
if (record.document.game) {
|
|
663
|
+
const legacyTitle = naturalDocumentTitleBase(record.document)
|
|
664
|
+
if (legacyTitle && legacyTitle !== naturalTitle) {
|
|
665
|
+
const legacyIds = next.naturalTitleIndex.get(legacyTitle) ?? []
|
|
666
|
+
legacyIds.push(documentId)
|
|
667
|
+
next.naturalTitleIndex.set(legacyTitle, legacyIds)
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
const legacyMemoryTitle = legacyOperatorRecordTitle(record.document)
|
|
672
|
+
if (legacyMemoryTitle && legacyMemoryTitle !== naturalTitle) {
|
|
673
|
+
const aliases = [legacyMemoryTitle]
|
|
674
|
+
if (record.document.game) aliases.push(naturalDocumentTitleBase(record.document,
|
|
675
|
+
{ includeOperatorRecordSegment: false }))
|
|
676
|
+
for (const alias of aliases) {
|
|
677
|
+
if (!alias) continue
|
|
678
|
+
const ids = next.naturalTitleIndex.get(alias) ?? []
|
|
679
|
+
ids.push(documentId)
|
|
680
|
+
next.naturalTitleIndex.set(alias, ids)
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
const stageCode = publicStoryStageCode(record.document.story_code)
|
|
684
|
+
const storyPart = publicStoryPart(record.document.part_type)
|
|
685
|
+
if (documentGame(record.document) === 'arknights' && record.document.document_type === 'story'
|
|
686
|
+
&& record.document.document_kind === 'story' && stageCode && storyPart) {
|
|
687
|
+
const key = storyStageKey(stageCode, storyPart)
|
|
688
|
+
const ids = next.storyStageIndex.get(key) ?? []
|
|
689
|
+
ids.push(documentId)
|
|
690
|
+
next.storyStageIndex.set(key, ids)
|
|
691
|
+
const codeIds = next.storyStageCodeIndex.get(stageCode) ?? []
|
|
692
|
+
codeIds.push(documentId)
|
|
693
|
+
next.storyStageCodeIndex.set(stageCode, codeIds)
|
|
694
|
+
}
|
|
695
|
+
const recordSegment = operatorRecordSegment(record.document)
|
|
696
|
+
if (recordSegment && record.document.document_kind === 'story'
|
|
697
|
+
&& record.document.part_type === 'body') {
|
|
698
|
+
const key = operatorRecordKey(record.document.character_name,
|
|
699
|
+
record.document.story_name, recordSegment)
|
|
700
|
+
const ids = next.operatorRecordIndex.get(key) ?? []
|
|
701
|
+
ids.push(documentId)
|
|
702
|
+
next.operatorRecordIndex.set(key, ids)
|
|
703
|
+
const allKey = operatorRecordKey(record.document.character_name,
|
|
704
|
+
record.document.story_name)
|
|
705
|
+
const allIds = next.operatorRecordIndex.get(allKey) ?? []
|
|
706
|
+
allIds.push(documentId)
|
|
707
|
+
next.operatorRecordIndex.set(allKey, allIds)
|
|
708
|
+
}
|
|
709
|
+
const material = publicCharacterMaterial(record.document)
|
|
710
|
+
if (material) {
|
|
711
|
+
const key = characterMaterialKey(documentGame(record.document),
|
|
712
|
+
record.document.character_name, material)
|
|
713
|
+
const ids = next.characterMaterialIndex.get(key) ?? []
|
|
714
|
+
ids.push(documentId)
|
|
715
|
+
next.characterMaterialIndex.set(key, ids)
|
|
716
|
+
}
|
|
717
|
+
const path = record.document.path
|
|
718
|
+
if (path && !next.pathIndex.has(path)) next.pathIndex.set(path, documentId)
|
|
719
|
+
const sourceStoryId = record.document.source_story_id
|
|
720
|
+
if (sourceStoryId && !next.sourceStoryIndex.has(sourceStoryId)) {
|
|
721
|
+
next.sourceStoryIndex.set(sourceStoryId, documentId)
|
|
722
|
+
}
|
|
723
|
+
if (record.search_index_id) {
|
|
724
|
+
next.searchIndexDocuments.set(`${manifest.pack_id}\0${record.search_index_id}`, documentId)
|
|
725
|
+
}
|
|
726
|
+
for (const line of record.lines ?? []) {
|
|
727
|
+
collectUnstableChars(line?.text, stableChars, next.unstableChars)
|
|
728
|
+
}
|
|
729
|
+
})
|
|
730
|
+
}
|
|
731
|
+
if (seenLocations.size !== manifest.document_count) {
|
|
732
|
+
throw new Error(`CorpusStore: document directory count mismatch: ${packId}`)
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if (next.documents.size === 0) {
|
|
736
|
+
throw new Error(`CorpusStore: no documents found under ${releaseDir}`)
|
|
737
|
+
}
|
|
738
|
+
next.documentOrder = federatedOrder(next.documents, next.documentOrder)
|
|
739
|
+
next.documentOrder.forEach((documentId, ordinal) => {
|
|
740
|
+
next.documents.get(documentId).ordinal = ordinal
|
|
741
|
+
})
|
|
742
|
+
if (generation !== this._generation) return false
|
|
743
|
+
this.releaseId = next.releaseId
|
|
744
|
+
this.dataVersion = next.dataVersion
|
|
745
|
+
this.documents = next.documents
|
|
746
|
+
this.documentOrder = next.documentOrder
|
|
747
|
+
this.prefixIndex = next.prefixIndex
|
|
748
|
+
this.uidIndex = next.uidIndex
|
|
749
|
+
this.titleIndex = next.titleIndex
|
|
750
|
+
this.naturalTitleIndex = next.naturalTitleIndex
|
|
751
|
+
this.storyStageIndex = next.storyStageIndex
|
|
752
|
+
this.storyStageCodeIndex = next.storyStageCodeIndex
|
|
753
|
+
this.operatorRecordIndex = next.operatorRecordIndex
|
|
754
|
+
this.characterMaterialIndex = next.characterMaterialIndex
|
|
755
|
+
this.pathIndex = next.pathIndex
|
|
756
|
+
this.sourceStoryIndex = next.sourceStoryIndex
|
|
757
|
+
this.packs = next.packs
|
|
758
|
+
this.searchIndexDocuments = next.searchIndexDocuments
|
|
759
|
+
this.unstableChars = next.unstableChars
|
|
760
|
+
this._shardCache.clear()
|
|
761
|
+
this._shardCacheSizes.clear()
|
|
762
|
+
this._shardCacheBytes = 0
|
|
763
|
+
this._searchCache.clear()
|
|
764
|
+
this._searchCacheSizes.clear()
|
|
765
|
+
this._searchCacheBytes = 0
|
|
766
|
+
this._shortLiteralCache.clear()
|
|
767
|
+
this._shortLiteralCacheCandidateCount = 0
|
|
768
|
+
this._loaded = true
|
|
769
|
+
return true
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/** 解析一个已验证解压尺寸的 JSONL 分片;昂贵扫描可传入协作式检查点。 */
|
|
773
|
+
_decodeShard(buffer, checkpoint = null) {
|
|
774
|
+
const text = buffer.toString('utf8')
|
|
775
|
+
const records = []
|
|
776
|
+
const lines = text.split('\n')
|
|
777
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
778
|
+
if ((index & 63) === 0) checkpoint?.()
|
|
779
|
+
const line = lines[index]
|
|
780
|
+
if (line) records.push(JSON.parse(line))
|
|
781
|
+
}
|
|
782
|
+
return records
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
/**
|
|
786
|
+
* 读取一个正文分片并在 zlib 分配期间强制限制输出长度。明文旁路缓存不在
|
|
787
|
+
* release 清单中,不能绕过已校验的 gzip 资源。
|
|
788
|
+
*/
|
|
789
|
+
async _readPacked(packId, shardPath, releaseId = this.releaseId, descriptor = null) {
|
|
790
|
+
const baseDir = join(this.releasesDir, releaseId, packId)
|
|
791
|
+
descriptor ??= this.packs.get(packId)?.shards?.find((item) => item.path === shardPath) ?? null
|
|
792
|
+
return this._gunzipBounded(join(baseDir, shardPath), descriptor)
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
async _gunzipBounded(path, descriptor) {
|
|
796
|
+
const expectedSize = descriptor?.uncompressed_size
|
|
797
|
+
const expectedCompressedSize = descriptor?.compressed_size
|
|
798
|
+
const expectedHash = descriptor?.sha256
|
|
799
|
+
if (!Number.isInteger(expectedSize) || expectedSize < 1
|
|
800
|
+
|| expectedSize > CORPUS_RESOURCE_LIMITS.maxAssetUncompressedBytes
|
|
801
|
+
|| !Number.isInteger(expectedCompressedSize) || expectedCompressedSize < 1
|
|
802
|
+
|| expectedCompressedSize > CORPUS_RESOURCE_LIMITS.maxAssetCompressedBytes
|
|
803
|
+
|| !/^[0-9a-f]{64}$/u.test(String(expectedHash ?? ''))) {
|
|
804
|
+
throw new Error('CorpusStore: invalid declared packed-asset metadata')
|
|
805
|
+
}
|
|
806
|
+
const info = await lstat(path)
|
|
807
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size !== expectedCompressedSize) {
|
|
808
|
+
throw new Error('CorpusStore: packed asset changed after release validation')
|
|
809
|
+
}
|
|
810
|
+
const compressed = await readFile(path)
|
|
811
|
+
if (compressed.length !== expectedCompressedSize
|
|
812
|
+
|| createHash('sha256').update(compressed).digest('hex') !== expectedHash) {
|
|
813
|
+
throw new Error('CorpusStore: packed asset checksum changed after release validation')
|
|
814
|
+
}
|
|
815
|
+
const bytes = await gunzipAsync(compressed, { maxOutputLength: expectedSize })
|
|
816
|
+
if (bytes.length !== expectedSize) {
|
|
817
|
+
throw new Error('CorpusStore: packed asset decompressed-size mismatch')
|
|
818
|
+
}
|
|
819
|
+
return bytes
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
_rememberShard(key, records, byteLength) {
|
|
823
|
+
const previous = this._shardCacheSizes.get(key) ?? 0
|
|
824
|
+
this._shardCache.delete(key)
|
|
825
|
+
this._shardCacheSizes.delete(key)
|
|
826
|
+
this._shardCacheBytes -= previous
|
|
827
|
+
this._shardCache.set(key, records)
|
|
828
|
+
this._shardCacheSizes.set(key, byteLength)
|
|
829
|
+
this._shardCacheBytes += byteLength
|
|
830
|
+
while (this._shardCache.size > this.cacheShards
|
|
831
|
+
|| this._shardCacheBytes > MAX_SHARD_CACHE_BYTES) {
|
|
832
|
+
const oldest = this._shardCache.keys().next().value
|
|
833
|
+
this._shardCache.delete(oldest)
|
|
834
|
+
this._shardCacheBytes -= this._shardCacheSizes.get(oldest) ?? 0
|
|
835
|
+
this._shardCacheSizes.delete(oldest)
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
_rememberSearchShard(key, bytes) {
|
|
840
|
+
const previous = this._searchCacheSizes.get(key) ?? 0
|
|
841
|
+
this._searchCache.delete(key)
|
|
842
|
+
this._searchCacheSizes.delete(key)
|
|
843
|
+
this._searchCacheBytes -= previous
|
|
844
|
+
this._searchCache.set(key, bytes)
|
|
845
|
+
this._searchCacheSizes.set(key, bytes.length)
|
|
846
|
+
this._searchCacheBytes += bytes.length
|
|
847
|
+
while (this._searchCache.size > this.searchCacheShards
|
|
848
|
+
|| this._searchCacheBytes > MAX_SEARCH_CACHE_BYTES) {
|
|
849
|
+
const oldest = this._searchCache.keys().next().value
|
|
850
|
+
this._searchCache.delete(oldest)
|
|
851
|
+
this._searchCacheBytes -= this._searchCacheSizes.get(oldest) ?? 0
|
|
852
|
+
this._searchCacheSizes.delete(oldest)
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/** 读取(并缓存)某包的一个分片,返回文档记录数组。 */
|
|
857
|
+
async _loadShard(packId, shardPath) {
|
|
858
|
+
const snapshot = corpusVersionSnapshot(this)
|
|
859
|
+
const key = `${packId}\0${shardPath}`
|
|
860
|
+
const cached = this._shardCache.get(key)
|
|
861
|
+
if (cached) {
|
|
862
|
+
this._shardCache.delete(key)
|
|
863
|
+
this._shardCache.set(key, cached) // 触碰 LRU
|
|
864
|
+
return cached
|
|
865
|
+
}
|
|
866
|
+
const descriptor = this.packs.get(packId)?.shards?.find((item) => item.path === shardPath)
|
|
867
|
+
let plain
|
|
868
|
+
try {
|
|
869
|
+
plain = await this._readPacked(packId, shardPath, this.releaseId, descriptor)
|
|
870
|
+
} finally {
|
|
871
|
+
// reset 后不能把旧任务的结果写入新缓存,也不能让调用方继续使用旧记录。
|
|
872
|
+
assertCorpusVersion(this, snapshot)
|
|
873
|
+
}
|
|
874
|
+
const records = this._decodeShard(plain)
|
|
875
|
+
this._rememberShard(key, records, plain.length)
|
|
876
|
+
return records
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
async _loadSearchShard(packId, shardPath) {
|
|
880
|
+
const snapshot = corpusVersionSnapshot(this)
|
|
881
|
+
const key = `${packId}\0${shardPath}`
|
|
882
|
+
const cached = this._searchCache.get(key)
|
|
883
|
+
if (cached) {
|
|
884
|
+
this._searchCache.delete(key)
|
|
885
|
+
this._searchCache.set(key, cached)
|
|
886
|
+
return cached
|
|
887
|
+
}
|
|
888
|
+
const descriptor = this.packs.get(packId)?.search_index?.shards
|
|
889
|
+
?.find((item) => item.path === shardPath)
|
|
890
|
+
let bytes
|
|
891
|
+
try {
|
|
892
|
+
bytes = await this._gunzipBounded(join(
|
|
893
|
+
this.releasesDir, this.releaseId, packId, shardPath,
|
|
894
|
+
), descriptor)
|
|
895
|
+
} finally {
|
|
896
|
+
assertCorpusVersion(this, snapshot)
|
|
897
|
+
}
|
|
898
|
+
this._rememberSearchShard(key, bytes)
|
|
899
|
+
return bytes
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
/**
|
|
903
|
+
* 按 document_id 取完整文档记录。
|
|
904
|
+
* @returns {Promise<{ record: object, packId: string } | null>}
|
|
905
|
+
*/
|
|
906
|
+
async getDocument(documentId) {
|
|
907
|
+
const snapshot = corpusVersionSnapshot(this)
|
|
908
|
+
const location = this.documents.get(documentId)
|
|
909
|
+
if (!location) return null
|
|
910
|
+
const records = await this._loadShard(location.packId, location.shardPath)
|
|
911
|
+
// 即使 _loadShard 命中缓存,await 仍会让出执行权,期间也可能切版。
|
|
912
|
+
assertCorpusVersion(this, snapshot)
|
|
913
|
+
const record = records[location.index]
|
|
914
|
+
if (!record || record.document.document_id !== documentId) return null
|
|
915
|
+
return { record, packId: location.packId }
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
/** source_ref_prefix → document_id。 */
|
|
919
|
+
getDocumentIdByPrefix(prefix) {
|
|
920
|
+
return this.prefixIndex.get(prefix) ?? null
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
/** 当前不可变资料版本中的稳定全局文档序号(0-based)。 */
|
|
924
|
+
documentOrdinal(documentId) {
|
|
925
|
+
return this.documents.get(String(documentId ?? ''))?.ordinal ?? null
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/** 将候选文档恢复成全局稳定顺序;未知 ID 被忽略。 */
|
|
929
|
+
orderedDocumentIds(documentIds = null) {
|
|
930
|
+
if (documentIds == null) return [...this.documentOrder]
|
|
931
|
+
const requested = new Set(documentIds)
|
|
932
|
+
return this.documentOrder.filter((documentId) => requested.has(documentId))
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
/** 短 document_uid → canonical document_id。 */
|
|
936
|
+
getDocumentIdByUid(uid) {
|
|
937
|
+
return this.uidIndex.get(String(uid ?? '').trim()) ?? null
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/** 按短 document_uid 取完整文档记录。 */
|
|
941
|
+
async getDocumentByUid(uid) {
|
|
942
|
+
const documentId = this.getDocumentIdByUid(uid)
|
|
943
|
+
return documentId ? this.getDocument(documentId) : null
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/**
|
|
947
|
+
* 获取跨重启持久的游标签名密钥。密钥位于 releases 的父级配置目录,
|
|
948
|
+
* 不写入不可变发布包;并发首次创建通过 wx 保证只有一个胜者。
|
|
949
|
+
*/
|
|
950
|
+
getOrCreateCursorSecret() {
|
|
951
|
+
if (this._cursorSecret) return this._cursorSecret
|
|
952
|
+
const path = this.cursorSecretPath
|
|
953
|
+
this._cursorSecret = (async () => {
|
|
954
|
+
try {
|
|
955
|
+
const existing = await readFile(path)
|
|
956
|
+
if (existing.length === 32) return existing
|
|
957
|
+
} catch (error) {
|
|
958
|
+
if (error?.code !== 'ENOENT') throw error
|
|
959
|
+
}
|
|
960
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
|
961
|
+
const created = randomBytes(32)
|
|
962
|
+
try {
|
|
963
|
+
await writeFile(path, created, { flag: 'wx', mode: 0o600 })
|
|
964
|
+
return created
|
|
965
|
+
} catch (error) {
|
|
966
|
+
if (error?.code !== 'EEXIST') throw error
|
|
967
|
+
const existing = await readFile(path)
|
|
968
|
+
if (existing.length !== 32) throw new Error('CorpusStore: invalid cursor secret')
|
|
969
|
+
return existing
|
|
970
|
+
}
|
|
971
|
+
})().catch((error) => {
|
|
972
|
+
this._cursorSecret = null
|
|
973
|
+
throw error
|
|
974
|
+
})
|
|
975
|
+
return this._cursorSecret
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
/** 按自然语言完整篇章标题或 display_title 取文档;短标题同名时要求改用完整标题。 */
|
|
979
|
+
async getDocumentByTitle(title) {
|
|
980
|
+
const normalized = String(title ?? '').trim()
|
|
981
|
+
const naturalIds = this.naturalTitleIndex.get(normalized) ?? []
|
|
982
|
+
const documentIds = naturalIds.length ? naturalIds : this.titleIndex.get(normalized) ?? []
|
|
983
|
+
if (documentIds.length > 1) {
|
|
984
|
+
// 实体来源中可能同时存在通用概念记录与后续补充的同名专页,两者的
|
|
985
|
+
// model title 也完全相同,要求模型“换完整标题”无法消歧。此时稳定选择
|
|
986
|
+
// 行数最多的完整投影;并列时按 document_id 排序保证跨进程一致。
|
|
987
|
+
const locations = documentIds.map((documentId) => this.documents.get(documentId)).filter(Boolean)
|
|
988
|
+
if (locations.length === documentIds.length
|
|
989
|
+
&& locations.every((item) => item.document.document_type === 'entity')) {
|
|
990
|
+
const preferred = [...locations].sort((left, right) =>
|
|
991
|
+
Number(right.document.line_count || 0) - Number(left.document.line_count || 0)
|
|
992
|
+
|| String(left.document.document_id).localeCompare(String(right.document.document_id)))[0]
|
|
993
|
+
return this.getDocument(preferred.document.document_id)
|
|
994
|
+
}
|
|
995
|
+
throw Object.assign(new Error(
|
|
996
|
+
`标题“${normalized}”对应 ${documentIds.length} 篇资料;请使用 corpus_search 返回的带资料类型完整标题`,
|
|
997
|
+
), { code: 'DOCUMENT_AMBIGUOUS' })
|
|
998
|
+
}
|
|
999
|
+
return documentIds.length ? this.getDocument(documentIds[0]) : null
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/** 搜索结果若无法仅靠自然标题/合集名稳定回读,就公开短 document_uid。 */
|
|
1003
|
+
requiresDocumentUid(documentId) {
|
|
1004
|
+
const location = this.documents.get(String(documentId ?? ''))
|
|
1005
|
+
const document = location?.document
|
|
1006
|
+
if (!document) return false
|
|
1007
|
+
const title = naturalDocumentTitle(document)
|
|
1008
|
+
if ((this.naturalTitleIndex.get(title) ?? []).length > 1) return true
|
|
1009
|
+
const game = documentGame(document)
|
|
1010
|
+
const streamName = normalizedLookupText(game === 'endfield'
|
|
1011
|
+
? document.collection_name : document.activity_name)
|
|
1012
|
+
if (!streamName || document.document_type !== 'story') return false
|
|
1013
|
+
const collections = new Set()
|
|
1014
|
+
for (const item of this.documents.values()) {
|
|
1015
|
+
const candidate = item.document
|
|
1016
|
+
if (documentGame(candidate) !== game || candidate.document_type !== 'story') continue
|
|
1017
|
+
const candidateName = normalizedLookupText(game === 'endfield'
|
|
1018
|
+
? candidate.collection_name : candidate.activity_name)
|
|
1019
|
+
if (candidateName !== streamName) continue
|
|
1020
|
+
collections.add(String(candidate.collection_id || candidate.activity_id || ''))
|
|
1021
|
+
if (collections.size > 1) return true
|
|
1022
|
+
}
|
|
1023
|
+
return false
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/** 按玩家可见关卡代号定位;省略部分时仅在全局唯一的情况下成功。 */
|
|
1027
|
+
async getDocumentByStoryStage(stageCode, storyPart = '') {
|
|
1028
|
+
const normalizedCode = publicStoryStageCode(stageCode)
|
|
1029
|
+
const partSupplied = storyPart !== '' && storyPart != null
|
|
1030
|
+
const normalizedPart = partSupplied ? publicStoryPart(storyPart) : ''
|
|
1031
|
+
if (!normalizedCode || (partSupplied && !normalizedPart)) return null
|
|
1032
|
+
const documentIds = normalizedPart
|
|
1033
|
+
? this.storyStageIndex.get(storyStageKey(normalizedCode, normalizedPart)) ?? []
|
|
1034
|
+
: this.storyStageCodeIndex.get(normalizedCode) ?? []
|
|
1035
|
+
if (documentIds.length > 1) {
|
|
1036
|
+
const choices = documentIds.map((documentId) => {
|
|
1037
|
+
const document = this.documents.get(documentId)?.document || {}
|
|
1038
|
+
const part = publicStoryPart(document.part_type)
|
|
1039
|
+
const label = { before: '行动前', after: '行动后', story: '剧情' }[part] || part
|
|
1040
|
+
return `${label}《${naturalDocumentTitle(document)}》`
|
|
1041
|
+
}).filter(Boolean)
|
|
1042
|
+
const requestedLabel = normalizedPart
|
|
1043
|
+
? ` 的${{ before: '行动前', after: '行动后', story: '剧情' }[normalizedPart]}` : ''
|
|
1044
|
+
throw Object.assign(new Error(
|
|
1045
|
+
`关卡 ${normalizedCode}${requestedLabel}对应 ${documentIds.length} 篇资料:${choices.join('、')};请明确 story_part`,
|
|
1046
|
+
), { code: 'DOCUMENT_AMBIGUOUS' })
|
|
1047
|
+
}
|
|
1048
|
+
return documentIds.length ? this.getDocument(documentIds[0]) : null
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/** 该文档能否用公开 stage_code + story_part 无歧义地再次定位。 */
|
|
1052
|
+
hasUniqueStoryStage(documentId) {
|
|
1053
|
+
const document = this.documents.get(String(documentId ?? ''))?.document
|
|
1054
|
+
if (!document || documentGame(document) !== 'arknights') return false
|
|
1055
|
+
const stageCode = publicStoryStageCode(document.story_code)
|
|
1056
|
+
const storyPart = publicStoryPart(document.part_type)
|
|
1057
|
+
if (!stageCode || !storyPart) return false
|
|
1058
|
+
const ids = this.storyStageIndex.get(storyStageKey(stageCode, storyPart)) ?? []
|
|
1059
|
+
return ids.length === 1 && ids[0] === document.document_id
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
/** 按角色、密录名和可选段号读取密录正文。 */
|
|
1063
|
+
async getOperatorRecord(characterName, recordName, segment = null) {
|
|
1064
|
+
const normalizedSegment = segment == null ? '' : Number(segment)
|
|
1065
|
+
if (!normalizedLookupText(characterName) || !normalizedLookupText(recordName)
|
|
1066
|
+
|| (normalizedSegment !== '' && (!Number.isInteger(normalizedSegment) || normalizedSegment < 1))) return null
|
|
1067
|
+
const ids = this.operatorRecordIndex.get(operatorRecordKey(
|
|
1068
|
+
characterName, recordName, normalizedSegment,
|
|
1069
|
+
)) ?? []
|
|
1070
|
+
if (ids.length > 1) {
|
|
1071
|
+
const segments = [...new Set(ids.map((id) => operatorRecordSegment(
|
|
1072
|
+
this.documents.get(id)?.document || {},
|
|
1073
|
+
)).filter(Boolean))].sort((a, b) => a - b)
|
|
1074
|
+
throw Object.assign(new Error(
|
|
1075
|
+
`干员“${normalizedLookupText(characterName)}”的密录“${normalizedLookupText(recordName)}”`
|
|
1076
|
+
+ `包含 ${ids.length} 段正文${segments.length ? `(${segments.join('、')})` : ''};请提供 segment`,
|
|
1077
|
+
), { code: 'DOCUMENT_AMBIGUOUS' })
|
|
1078
|
+
}
|
|
1079
|
+
return ids.length ? this.getDocument(ids[0]) : null
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
hasUniqueOperatorRecord(documentId) {
|
|
1083
|
+
const document = this.documents.get(String(documentId ?? ''))?.document
|
|
1084
|
+
const segment = operatorRecordSegment(document || {})
|
|
1085
|
+
if (!segment || document?.document_kind !== 'story' || document?.part_type !== 'body') return false
|
|
1086
|
+
const ids = this.operatorRecordIndex.get(operatorRecordKey(
|
|
1087
|
+
document.character_name, document.story_name, segment,
|
|
1088
|
+
)) ?? []
|
|
1089
|
+
return ids.length === 1 && ids[0] === document.document_id
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
/** 按玩家可见角色名与资料类别定位两款游戏的官方角色资料。 */
|
|
1093
|
+
async getCharacterMaterial(characterName, material, games = ['arknights', 'endfield']) {
|
|
1094
|
+
const ids = games.flatMap((game) => this.characterMaterialIndex.get(characterMaterialKey(
|
|
1095
|
+
game, characterName, material,
|
|
1096
|
+
)) ?? [])
|
|
1097
|
+
if (ids.length > 1) {
|
|
1098
|
+
const documents = ids.map((id) => this.documents.get(id)?.document).filter(Boolean)
|
|
1099
|
+
const hashes = new Set(documents.map((document) => document.text_sha256).filter(Boolean))
|
|
1100
|
+
if (hashes.size === 1 && documents.length === ids.length) {
|
|
1101
|
+
return this.getDocument([...ids].sort((a, b) => a.localeCompare(b, 'en'))[0])
|
|
1102
|
+
}
|
|
1103
|
+
const titles = documents.map((document) => naturalDocumentTitle(document))
|
|
1104
|
+
throw Object.assign(new Error(
|
|
1105
|
+
`角色“${normalizedLookupText(characterName)}”的 ${material} 资料对应 ${ids.length} 篇不同内容:${titles.join('、')}`,
|
|
1106
|
+
), { code: 'DOCUMENT_AMBIGUOUS' })
|
|
1107
|
+
}
|
|
1108
|
+
return ids.length ? this.getDocument(ids[0]) : null
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
hasUniqueCharacterMaterial(documentId) {
|
|
1112
|
+
const document = this.documents.get(String(documentId ?? ''))?.document
|
|
1113
|
+
const material = publicCharacterMaterial(document || {})
|
|
1114
|
+
if (!material) return false
|
|
1115
|
+
const ids = this.characterMaterialIndex.get(characterMaterialKey(
|
|
1116
|
+
documentGame(document), document.character_name, material,
|
|
1117
|
+
)) ?? []
|
|
1118
|
+
if (ids.length === 1) return ids[0] === document.document_id
|
|
1119
|
+
// 终末地的男女管理员会投影成同一个展示角色;资料正文完全相同时,
|
|
1120
|
+
// getCharacterMaterial 会稳定选 document_id 最小者,续页也应继续使用
|
|
1121
|
+
// 同一个自然定位器,而不是降级成同样会歧义的重复展示标题。
|
|
1122
|
+
const documents = ids.map((id) => this.documents.get(id)?.document).filter(Boolean)
|
|
1123
|
+
const hashes = new Set(documents.map((item) => item.text_sha256).filter(Boolean))
|
|
1124
|
+
const preferred = [...ids].sort((left, right) => left.localeCompare(right, 'en'))[0]
|
|
1125
|
+
return documents.length === ids.length && hashes.size === 1 && preferred === document.document_id
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/** 同名实体投影只让内容最完整的一份进入模型搜索结果。 */
|
|
1129
|
+
isPreferredNaturalDocument(documentId) {
|
|
1130
|
+
const location = this.documents.get(String(documentId || ''))
|
|
1131
|
+
if (!location || location.document.document_type !== 'entity') return true
|
|
1132
|
+
const title = naturalDocumentTitle(location.document)
|
|
1133
|
+
const ids = this.naturalTitleIndex.get(title) ?? []
|
|
1134
|
+
if (ids.length <= 1) return true
|
|
1135
|
+
const entityLocations = ids.map((id) => this.documents.get(id)).filter(Boolean)
|
|
1136
|
+
if (entityLocations.length !== ids.length
|
|
1137
|
+
|| entityLocations.some((item) => item.document.document_type !== 'entity')) return true
|
|
1138
|
+
entityLocations.sort((left, right) =>
|
|
1139
|
+
Number(right.document.line_count || 0) - Number(left.document.line_count || 0)
|
|
1140
|
+
|| String(left.document.document_id).localeCompare(String(right.document.document_id)))
|
|
1141
|
+
return entityLocations[0].document.document_id === location.document.document_id
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
/** 按资料内路径(如 activity_timelines.jsonl、char_alias.txt、stories/x.txt)取文档记录。 */
|
|
1145
|
+
async getDocumentByPath(path) {
|
|
1146
|
+
const documentId = this.pathIndex.get(String(path ?? ''))
|
|
1147
|
+
return documentId ? this.getDocument(documentId) : null
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
/** 按 GameData 原始 source_story_id 取文档记录(synopsis → 可读全文的桥)。 */
|
|
1151
|
+
async getDocumentBySourceStoryId(sourceStoryId) {
|
|
1152
|
+
const documentId = this.sourceStoryIndex.get(String(sourceStoryId ?? ''))
|
|
1153
|
+
return documentId ? this.getDocument(documentId) : null
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
/** 全部 pack 都声明三元倒排时,才可把其结果当作全库无损候选。 */
|
|
1157
|
+
get hasTrigramIndex() {
|
|
1158
|
+
return this.supportsNgramSize(3)
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* 指定范围内的 pack 都支持该 gram 宽度时,倒排候选才是该范围的无损超集。
|
|
1163
|
+
* packIds 省略时保持历史语义:检查全部已安装 pack。
|
|
1164
|
+
*/
|
|
1165
|
+
supportsNgramSize(size, packIds = null) {
|
|
1166
|
+
if (!Number.isInteger(size) || size < 1 || size > 3 || !this.packs.size) return false
|
|
1167
|
+
const entries = this._searchPackEntries(packIds)
|
|
1168
|
+
if (!entries?.length) return false
|
|
1169
|
+
for (const [, manifest] of entries) {
|
|
1170
|
+
const index = manifest.search_index
|
|
1171
|
+
if (!index?.shards?.length) return false
|
|
1172
|
+
const sizes = index.algorithm === 'prts-browser-ngram-postings-v2'
|
|
1173
|
+
? index.gram_sizes : [3]
|
|
1174
|
+
if (!Array.isArray(sizes) || !sizes.includes(size)) return false
|
|
1175
|
+
}
|
|
1176
|
+
return true
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
/** 把内部搜索范围解析成稳定、去重且真实存在的 pack 列表。 */
|
|
1180
|
+
_searchPackEntries(packIds = null) {
|
|
1181
|
+
if (packIds == null) return [...this.packs.entries()]
|
|
1182
|
+
if (!Array.isArray(packIds)) return null
|
|
1183
|
+
const entries = []
|
|
1184
|
+
const seen = new Set()
|
|
1185
|
+
for (const rawPackId of packIds) {
|
|
1186
|
+
const packId = String(rawPackId ?? '')
|
|
1187
|
+
if (!packId || seen.has(packId)) continue
|
|
1188
|
+
const manifest = this.packs.get(packId)
|
|
1189
|
+
if (!manifest) return null
|
|
1190
|
+
seen.add(packId)
|
|
1191
|
+
entries.push([packId, manifest])
|
|
1192
|
+
}
|
|
1193
|
+
return entries
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
/**
|
|
1197
|
+
* 使用各 pack 的 trigram 倒排分片求文档交集;查询短于 3 字符时返回 null。
|
|
1198
|
+
* 外层按分片迭代:每个分片只解压一次,再对其覆盖范围内的全部 trigram
|
|
1199
|
+
* 查询,避免多 trigram 查询反复重解压同一批分片。
|
|
1200
|
+
*/
|
|
1201
|
+
async findDocumentsByNgrams(trigrams, { signal, deadline = Infinity, packIds = null } = {}) {
|
|
1202
|
+
const generation = this._generation
|
|
1203
|
+
if (!trigrams.length) return null
|
|
1204
|
+
const entries = this._searchPackEntries(packIds)
|
|
1205
|
+
if (!entries) return null
|
|
1206
|
+
if (!entries.length) return []
|
|
1207
|
+
const gramSize = [...String(trigrams[0])].length
|
|
1208
|
+
if (!this.supportsNgramSize(gramSize, entries.map(([packId]) => packId))) return null
|
|
1209
|
+
const checkpoint = () => assertShortLiteralScanActive({ signal, deadline, generation, store: this })
|
|
1210
|
+
checkpoint()
|
|
1211
|
+
const perTrigram = new Map(trigrams.map((trigram) => [trigram, new Set()]))
|
|
1212
|
+
for (const [packId, manifest] of entries) {
|
|
1213
|
+
for (const descriptor of manifest.search_index?.shards ?? []) {
|
|
1214
|
+
checkpoint()
|
|
1215
|
+
const range = searchIndexRange(descriptor)
|
|
1216
|
+
const relevant = trigrams.filter((trigram) => compareNgramKeys(range.first, trigram) <= 0
|
|
1217
|
+
&& compareNgramKeys(trigram, range.last) <= 0)
|
|
1218
|
+
if (!relevant.length) continue
|
|
1219
|
+
const bytes = await this._loadSearchShard(packId, descriptor.path)
|
|
1220
|
+
checkpoint()
|
|
1221
|
+
for (const trigram of relevant) {
|
|
1222
|
+
const candidates = perTrigram.get(trigram)
|
|
1223
|
+
const indexes = lookupNgramIndex(bytes, trigram)
|
|
1224
|
+
for (let offset = 0; offset < indexes.length; offset += 1) {
|
|
1225
|
+
if ((offset & 1023) === 0) checkpoint()
|
|
1226
|
+
const index = indexes[offset]
|
|
1227
|
+
const documentId = this.searchIndexDocuments.get(`${packId}\0${index}`)
|
|
1228
|
+
if (documentId) candidates.add(documentId)
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
let intersection = null
|
|
1234
|
+
for (const trigram of trigrams) {
|
|
1235
|
+
checkpoint()
|
|
1236
|
+
const candidates = perTrigram.get(trigram)
|
|
1237
|
+
if (intersection === null) intersection = candidates
|
|
1238
|
+
else {
|
|
1239
|
+
const next = new Set()
|
|
1240
|
+
let checked = 0
|
|
1241
|
+
for (const documentId of intersection) {
|
|
1242
|
+
if ((checked++ & 1023) === 0) checkpoint()
|
|
1243
|
+
if (candidates.has(documentId)) next.add(documentId)
|
|
1244
|
+
}
|
|
1245
|
+
intersection = next
|
|
1246
|
+
}
|
|
1247
|
+
if (!intersection.size) break
|
|
1248
|
+
}
|
|
1249
|
+
checkpoint()
|
|
1250
|
+
return [...(intersection ?? [])]
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
/** v1 API alias;旧调用者仍可查询三元倒排。 */
|
|
1254
|
+
async findDocumentsByTrigrams(trigrams, runtime = {}) {
|
|
1255
|
+
return this.findDocumentsByNgrams(trigrams, runtime)
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
_shortLiteralCacheHit(query) {
|
|
1259
|
+
const cached = this._shortLiteralCache.get(query)
|
|
1260
|
+
if (!cached) return null
|
|
1261
|
+
this._shortLiteralCache.delete(query)
|
|
1262
|
+
this._shortLiteralCache.set(query, cached)
|
|
1263
|
+
return [...cached]
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
_rememberShortLiteralCandidates(query, candidates) {
|
|
1267
|
+
const previous = this._shortLiteralCache.get(query)
|
|
1268
|
+
if (previous) {
|
|
1269
|
+
this._shortLiteralCache.delete(query)
|
|
1270
|
+
this._shortLiteralCacheCandidateCount -= previous.length
|
|
1271
|
+
}
|
|
1272
|
+
// 单个极高频字若已超过整个缓存预算,仍可服务当前请求,但不应驱逐所有
|
|
1273
|
+
// 小结果后独占常驻内存。
|
|
1274
|
+
if (candidates.length > MAX_SHORT_LITERAL_CACHE_CANDIDATES) return
|
|
1275
|
+
this._shortLiteralCache.set(query, candidates)
|
|
1276
|
+
this._shortLiteralCacheCandidateCount += candidates.length
|
|
1277
|
+
while (this._shortLiteralCache.size > 32
|
|
1278
|
+
|| this._shortLiteralCacheCandidateCount > MAX_SHORT_LITERAL_CACHE_CANDIDATES) {
|
|
1279
|
+
const oldest = this._shortLiteralCache.keys().next().value
|
|
1280
|
+
const evicted = this._shortLiteralCache.get(oldest)
|
|
1281
|
+
this._shortLiteralCache.delete(oldest)
|
|
1282
|
+
this._shortLiteralCacheCandidateCount -= evicted?.length ?? 0
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
_enqueueShortLiteralScan(runtime, scan) {
|
|
1287
|
+
assertShortLiteralScanActive({ ...runtime, store: this })
|
|
1288
|
+
if (this._shortLiteralScanPending >= MAX_SHORT_LITERAL_SCAN_QUEUE) {
|
|
1289
|
+
throw shortLiteralScanError('BUDGET_EXCEEDED',
|
|
1290
|
+
'短字面量候选扫描队列已满,请稍后重试', true)
|
|
1291
|
+
}
|
|
1292
|
+
this._shortLiteralScanPending += 1
|
|
1293
|
+
const operation = this._shortLiteralScanTail.then(async () => {
|
|
1294
|
+
assertShortLiteralScanActive({ ...runtime, store: this })
|
|
1295
|
+
return scan()
|
|
1296
|
+
})
|
|
1297
|
+
const tracked = operation.then((value) => {
|
|
1298
|
+
this._shortLiteralScanPending -= 1
|
|
1299
|
+
return value
|
|
1300
|
+
}, (error) => {
|
|
1301
|
+
this._shortLiteralScanPending -= 1
|
|
1302
|
+
throw error
|
|
1303
|
+
})
|
|
1304
|
+
// tail 总是兑现,单个请求失败不会毒化后续队列;tracked 的拒绝同时由
|
|
1305
|
+
// 这里和调用方等待器观察,不会形成未处理拒绝。
|
|
1306
|
+
this._shortLiteralScanTail = tracked.catch(() => {})
|
|
1307
|
+
return waitForShortLiteralScan(tracked, { ...runtime, store: this })
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
/**
|
|
1311
|
+
* 为 1—2 个无大小写字符的字面量提供 grep 式候选预筛。
|
|
1312
|
+
*
|
|
1313
|
+
* 现有资料包只带 trigram 倒排,短查询不能使用它。直接逐文档扫描会再次
|
|
1314
|
+
* JSON.parse 全库并轻易越过工具时限;这里先在解压后的 JSONL Buffer 上做
|
|
1315
|
+
* 原始字节查找,只解析确实可能含命中的少量分片,再返回文档候选。
|
|
1316
|
+
*
|
|
1317
|
+
* 字节预筛必须是正式匹配(search.js 的 matchesText:NFKC + 空白折叠 +
|
|
1318
|
+
* 小写后子串匹配)的“超集”,否则会产生假阴性。本地语料并非 NFKC 纯文本
|
|
1319
|
+
* (实测七成以上行含全角标点等兼容字符),因此 needle 集合在查询原文
|
|
1320
|
+
* 之外还纳入:
|
|
1321
|
+
* 1. 查询的规范等价形式(NFD 分解及其部分重组合,如谚文音节);
|
|
1322
|
+
* 2. 初始化收集的“不稳定字符”中,展开式包含查询、或与查询首/尾字符
|
|
1323
|
+
* 相邻接者(两字查询可能横跨两个字符展开的边界)。
|
|
1324
|
+
* 不适合无损预筛的查询返回 null,由调用方走通用路径。跨字符边界的
|
|
1325
|
+
* 规范复合产物(如 "e"+U+0301 复合为 "é")属于带大小写的预组合字符,
|
|
1326
|
+
* 已被“无大小写”入参门槛排除。预筛候选仍由 search.js 逐行按正式谓词
|
|
1327
|
+
* 复核,多余候选只损失少量性能,不会污染结果。
|
|
1328
|
+
*/
|
|
1329
|
+
async findDocumentsByShortLiteral(value,
|
|
1330
|
+
{ signal, deadline = Infinity, packIds = null } = {}) {
|
|
1331
|
+
const query = String(value ?? '')
|
|
1332
|
+
const characters = [...query]
|
|
1333
|
+
if (!characters.length || characters.length > 2
|
|
1334
|
+
|| query !== query.normalize('NFC') || query !== query.normalize('NFKC')
|
|
1335
|
+
|| query.toLocaleLowerCase() !== query.toLocaleUpperCase()
|
|
1336
|
+
|| /[\s"\\\u0000-\u001f]/u.test(query)) return null
|
|
1337
|
+
if (!this.unstableChars) return null
|
|
1338
|
+
const entries = this._searchPackEntries(packIds)
|
|
1339
|
+
if (!entries) return null
|
|
1340
|
+
if (!entries.length) return []
|
|
1341
|
+
const cacheKey = `${entries.map(([packId]) => packId).sort((left, right) =>
|
|
1342
|
+
left.localeCompare(right, 'en')).join('\0')}\u0001${query}`
|
|
1343
|
+
const runtime = { signal, deadline }
|
|
1344
|
+
assertShortLiteralScanActive({ ...runtime, store: this })
|
|
1345
|
+
const cached = this._shortLiteralCacheHit(cacheKey)
|
|
1346
|
+
if (cached) return cached
|
|
1347
|
+
const generation = this._generation
|
|
1348
|
+
const releaseId = this.releaseId
|
|
1349
|
+
return this._enqueueShortLiteralScan({ ...runtime, generation }, async () => {
|
|
1350
|
+
const queuedCached = this._shortLiteralCacheHit(cacheKey)
|
|
1351
|
+
if (queuedCached) return queuedCached
|
|
1352
|
+
const checkpoint = () => assertShortLiteralScanActive({ ...runtime, generation, store: this })
|
|
1353
|
+
const needles = canonicalVariants(query)
|
|
1354
|
+
const first = characters[0]
|
|
1355
|
+
const last = characters[characters.length - 1]
|
|
1356
|
+
let unstableIndex = 0
|
|
1357
|
+
for (const [character, expansion] of this.unstableChars) {
|
|
1358
|
+
if ((unstableIndex++ & 255) === 0) checkpoint()
|
|
1359
|
+
if (expansion.includes(query) || expansion.endsWith(first) || expansion.startsWith(last)) {
|
|
1360
|
+
needles.add(character)
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
const needleStrings = [...needles]
|
|
1364
|
+
const needleBuffers = needleStrings.map((needle) => Buffer.from(needle, 'utf8'))
|
|
1365
|
+
const jobs = []
|
|
1366
|
+
for (const [packId, manifest] of entries) {
|
|
1367
|
+
checkpoint()
|
|
1368
|
+
for (const descriptor of manifest.shards ?? []) {
|
|
1369
|
+
jobs.push({ packId, descriptor, order: jobs.length })
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
// gzip 解压走 libuv 线程池。每个 worker 一次只持有一个分片的明文,
|
|
1373
|
+
// 命中分片当场解析并只保留候选文档 id 与有界的 _shardCache;跨查询
|
|
1374
|
+
// 的串行队列保证同时最多只有这一组 worker 在遍历全库。
|
|
1375
|
+
const found = []
|
|
1376
|
+
let nextJob = 0
|
|
1377
|
+
let scanFailure = null
|
|
1378
|
+
const worker = async () => {
|
|
1379
|
+
try {
|
|
1380
|
+
while (nextJob < jobs.length) {
|
|
1381
|
+
checkpoint()
|
|
1382
|
+
if (scanFailure) throw scanFailure
|
|
1383
|
+
const job = jobs[nextJob++]
|
|
1384
|
+
const bytes = await this._readPacked(job.packId, job.descriptor.path,
|
|
1385
|
+
releaseId, job.descriptor)
|
|
1386
|
+
checkpoint()
|
|
1387
|
+
let possibleMatch = false
|
|
1388
|
+
for (let index = 0; index < needleBuffers.length; index += 1) {
|
|
1389
|
+
if ((index & 7) === 0) checkpoint()
|
|
1390
|
+
if (bytes.includes(needleBuffers[index])) {
|
|
1391
|
+
possibleMatch = true
|
|
1392
|
+
break
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
if (!possibleMatch) continue
|
|
1396
|
+
// 分片间让出事件循环:JSON.parse 与逐行筛选是同步 CPU 工作。
|
|
1397
|
+
await new Promise((resolve) => { setImmediate(resolve) })
|
|
1398
|
+
checkpoint()
|
|
1399
|
+
const records = this._decodeShard(bytes, checkpoint)
|
|
1400
|
+
const key = `${job.packId}\0${job.descriptor.path}`
|
|
1401
|
+
checkpoint()
|
|
1402
|
+
this._rememberShard(key, records, bytes.length)
|
|
1403
|
+
for (let recordIndex = 0; recordIndex < records.length; recordIndex += 1) {
|
|
1404
|
+
if ((recordIndex & 31) === 0) checkpoint()
|
|
1405
|
+
const record = records[recordIndex]
|
|
1406
|
+
let title = false
|
|
1407
|
+
for (const item of [record.document?.display_title, record.document?.story_name,
|
|
1408
|
+
record.document?.activity_name, record.document?.character_name]) {
|
|
1409
|
+
if (item == null) continue
|
|
1410
|
+
const text = String(item)
|
|
1411
|
+
if (needleStrings.some((needle) => text.includes(needle))) {
|
|
1412
|
+
title = true
|
|
1413
|
+
break
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
let content = false
|
|
1417
|
+
if (!title) {
|
|
1418
|
+
const lines = record.lines ?? []
|
|
1419
|
+
for (let lineIndex = 0; lineIndex < lines.length; lineIndex += 1) {
|
|
1420
|
+
if ((lineIndex & 255) === 0) checkpoint()
|
|
1421
|
+
const text = typeof lines[lineIndex]?.text === 'string' ? lines[lineIndex].text : ''
|
|
1422
|
+
if (text !== '' && needleStrings.some((needle) => text.includes(needle))) {
|
|
1423
|
+
content = true
|
|
1424
|
+
break
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
if (title || content) found.push(record.document.document_id)
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
} catch (error) {
|
|
1432
|
+
scanFailure ||= error
|
|
1433
|
+
throw error
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
const settled = await Promise.allSettled(
|
|
1437
|
+
Array.from({ length: Math.min(SHORT_LITERAL_SCAN_WORKERS, jobs.length) }, worker),
|
|
1438
|
+
)
|
|
1439
|
+
const rejected = settled.find((item) => item.status === 'rejected')
|
|
1440
|
+
if (rejected) throw scanFailure || rejected.reason
|
|
1441
|
+
checkpoint()
|
|
1442
|
+
// worker 完成顺序不定;按全局稳定 ordinal 恢复与全量扫描一致的顺序。
|
|
1443
|
+
found.sort((left, right) =>
|
|
1444
|
+
(this.documents.get(left)?.ordinal ?? 0) - (this.documents.get(right)?.ordinal ?? 0))
|
|
1445
|
+
checkpoint()
|
|
1446
|
+
this._rememberShortLiteralCandidates(cacheKey, found)
|
|
1447
|
+
return [...found]
|
|
1448
|
+
})
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
/** 先按初始化时保留的轻量元数据过滤,再按需解压正文分片。 */
|
|
1452
|
+
async *iterateDocuments({ documentIds = null, predicate = null } = {}) {
|
|
1453
|
+
const snapshot = corpusVersionSnapshot(this)
|
|
1454
|
+
const ids = this.orderedDocumentIds(documentIds)
|
|
1455
|
+
for (const documentId of ids) {
|
|
1456
|
+
assertCorpusVersion(this, snapshot)
|
|
1457
|
+
const location = this.documents.get(documentId)
|
|
1458
|
+
if (!location || (predicate && !predicate(location.document, location.speakers))) continue
|
|
1459
|
+
const found = await this.getDocument(documentId)
|
|
1460
|
+
assertCorpusVersion(this, snapshot)
|
|
1461
|
+
if (found) yield found.record
|
|
1462
|
+
}
|
|
1463
|
+
assertCorpusVersion(this, snapshot)
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
/**
|
|
1467
|
+
* 枚举某个活动下的全部剧情 story 文档(不解压正文分片,仅用初始化保留的
|
|
1468
|
+
* 轻量元数据),按 collection_id + sequence_index 排序。
|
|
1469
|
+
* @param {{ activityId?: string, activityName?: string }} target
|
|
1470
|
+
* @returns {{ document: object, speakers: string[] }[]}
|
|
1471
|
+
*/
|
|
1472
|
+
activityStoryDocuments({ activityId = '', activityName = '', anchorDocumentId = '' }) {
|
|
1473
|
+
const anchor = anchorDocumentId ? this.documents.get(String(anchorDocumentId))?.document : null
|
|
1474
|
+
if (anchorDocumentId && (!anchor || documentGame(anchor) !== 'arknights'
|
|
1475
|
+
|| anchor.document_type !== 'story' || anchor.document_category !== 'activity')) return []
|
|
1476
|
+
const id = String(anchor?.collection_id || anchor?.activity_id || activityId || '').trim()
|
|
1477
|
+
const name = anchor ? '' : normalizedLookupText(activityName)
|
|
1478
|
+
if (!id && !name) return []
|
|
1479
|
+
const matches = []
|
|
1480
|
+
for (const location of this.documents.values()) {
|
|
1481
|
+
const document = location.document
|
|
1482
|
+
if (!document || documentGame(document) !== 'arknights'
|
|
1483
|
+
|| document.document_type !== 'story' || document.document_kind !== 'story'
|
|
1484
|
+
|| document.document_category !== 'activity') continue
|
|
1485
|
+
const source = String(document.source_story_id || document.document_id || '')
|
|
1486
|
+
// 教程与战斗教学不是活动剧情正文;它们有时沿用同一个 activity_name,
|
|
1487
|
+
// 若不剔除会把“骑兵与猎人”等活动错误合并成两个 collection。
|
|
1488
|
+
if (/(?:^|\/)(?:tutorial(?:_|\/)|training\/)/iu.test(source)) continue
|
|
1489
|
+
const byId = id && (String(document.collection_id || '') === id
|
|
1490
|
+
|| String(document.activity_id || '') === id)
|
|
1491
|
+
const byName = name && normalizedLookupText(document.activity_name) === name
|
|
1492
|
+
if (!byId && !byName) continue
|
|
1493
|
+
matches.push({ document, speakers: location.speakers })
|
|
1494
|
+
}
|
|
1495
|
+
if (!id) {
|
|
1496
|
+
const collections = new Map()
|
|
1497
|
+
for (const item of matches) {
|
|
1498
|
+
const key = String(item.document.collection_id || item.document.activity_id || '')
|
|
1499
|
+
const docs = collections.get(key) ?? []
|
|
1500
|
+
docs.push(item)
|
|
1501
|
+
collections.set(key, docs)
|
|
1502
|
+
}
|
|
1503
|
+
if (collections.size > 1) {
|
|
1504
|
+
const choices = [...collections.values()].map((docs) => {
|
|
1505
|
+
const first = [...docs].sort((left, right) =>
|
|
1506
|
+
Number(left.document.sequence_index || 0) - Number(right.document.sequence_index || 0))[0]
|
|
1507
|
+
return `document_uid=${documentUid(first.document.document_id)}(《${naturalDocumentTitle(first.document)}》等 ${docs.length} 篇)`
|
|
1508
|
+
})
|
|
1509
|
+
throw Object.assign(new Error(
|
|
1510
|
+
`活动名“${name}”对应 ${collections.size} 个不同剧情合集:${choices.join('、')};请先检索具体篇章,再用其 document_uid + mode="activity" 通读,不能自动合并`,
|
|
1511
|
+
), { code: 'DOCUMENT_AMBIGUOUS' })
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
matches.sort((left, right) =>
|
|
1515
|
+
String(left.document.collection_id || '').localeCompare(
|
|
1516
|
+
String(right.document.collection_id || ''), 'zh-CN', { numeric: true })
|
|
1517
|
+
|| (Number(left.document.sequence_index || 0) - Number(right.document.sequence_index || 0))
|
|
1518
|
+
|| String(left.document.document_id || '').localeCompare(String(right.document.document_id || '')))
|
|
1519
|
+
return matches
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
/** 按终末地任务/集合展示名枚举官方剧情碎片;同名多集合时拒绝猜测。 */
|
|
1523
|
+
endfieldCollectionDocuments({ collectionName = '', contentTypes = [], anchorDocumentId = '' }) {
|
|
1524
|
+
const anchor = anchorDocumentId ? this.documents.get(String(anchorDocumentId))?.document : null
|
|
1525
|
+
if (anchorDocumentId && (!anchor || documentGame(anchor) !== 'endfield'
|
|
1526
|
+
|| anchor.document_type !== 'story' || anchor.resource_type !== 'original_story')) return []
|
|
1527
|
+
const name = anchor ? normalizedLookupText(anchor.collection_name) : normalizedLookupText(collectionName)
|
|
1528
|
+
const collectionId = String(anchor?.collection_id || '')
|
|
1529
|
+
if (!name && !collectionId) return []
|
|
1530
|
+
const allowedTypes = contentTypes.length
|
|
1531
|
+
? new Set(contentTypes) : END_FIELD_NARRATIVE_CONTENT_TYPES
|
|
1532
|
+
const matches = []
|
|
1533
|
+
for (const location of this.documents.values()) {
|
|
1534
|
+
const document = location.document
|
|
1535
|
+
if (!document || documentGame(document) !== 'endfield'
|
|
1536
|
+
|| document.document_type !== 'story' || document.document_kind !== 'story'
|
|
1537
|
+
|| document.resource_type !== 'original_story'
|
|
1538
|
+
|| (collectionId ? String(document.collection_id || '') !== collectionId
|
|
1539
|
+
: normalizedLookupText(document.collection_name) !== name)) continue
|
|
1540
|
+
if (!allowedTypes.has(String(document.content_type || ''))) continue
|
|
1541
|
+
matches.push({ document, speakers: location.speakers })
|
|
1542
|
+
}
|
|
1543
|
+
const collections = new Map()
|
|
1544
|
+
for (const item of matches) {
|
|
1545
|
+
const key = String(item.document.collection_id || '')
|
|
1546
|
+
const docs = collections.get(key) ?? []
|
|
1547
|
+
docs.push(item)
|
|
1548
|
+
collections.set(key, docs)
|
|
1549
|
+
}
|
|
1550
|
+
if (collections.size > 1) {
|
|
1551
|
+
const choices = [...collections.values()].map((docs) => {
|
|
1552
|
+
const types = [...new Set(docs.map((item) => item.document.content_type).filter(Boolean))]
|
|
1553
|
+
const first = [...docs].sort((left, right) =>
|
|
1554
|
+
naturalCompare(left.document.source_story_id, right.document.source_story_id))[0]
|
|
1555
|
+
return `document_uid=${documentUid(first.document.document_id)}(${types.join('/')} ${docs.length} 篇)`
|
|
1556
|
+
})
|
|
1557
|
+
throw Object.assign(new Error(
|
|
1558
|
+
`终末地集合名“${name}”对应 ${collections.size} 个不同内容集合:${choices.join('、')};content_types 不能保证消歧,请先检索具体篇章,再用其 document_uid + mode="collection" 通读`,
|
|
1559
|
+
), { code: 'DOCUMENT_AMBIGUOUS' })
|
|
1560
|
+
}
|
|
1561
|
+
matches.sort((left, right) =>
|
|
1562
|
+
END_FIELD_CONTENT_TYPE_ORDER.indexOf(String(left.document.content_type || ''))
|
|
1563
|
+
- END_FIELD_CONTENT_TYPE_ORDER.indexOf(String(right.document.content_type || ''))
|
|
1564
|
+
|| naturalCompare(left.document.source_story_id || left.document.display_title,
|
|
1565
|
+
right.document.source_story_id || right.document.display_title)
|
|
1566
|
+
|| naturalCompare(left.document.document_id, right.document.document_id))
|
|
1567
|
+
return matches
|
|
1568
|
+
}
|
|
1569
|
+
}
|