recallnest 2.5.4 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +34 -9
- package/README_CN.md +37 -12
- package/package.json +4 -4
- package/src/__tests__/access-tracker-novelty.test.ts +66 -0
- package/src/__tests__/api-server-security.test.ts +102 -0
- package/src/__tests__/belief-history.test.ts +220 -0
- package/src/__tests__/capture-engine.test.ts +93 -3
- package/src/__tests__/category-min-scores.test.ts +83 -0
- package/src/__tests__/cluster-consolidation.test.ts +167 -4
- package/src/__tests__/conflict-store.test.ts +46 -2
- package/src/__tests__/consolidation-engine.test.ts +97 -0
- package/src/__tests__/contracts.test.ts +46 -0
- package/src/__tests__/decay-exemption.test.ts +45 -0
- package/src/__tests__/dream-pipeline.test.ts +464 -5
- package/src/__tests__/freshness.test.ts +186 -0
- package/src/__tests__/hotness-blend.test.ts +207 -0
- package/src/__tests__/llm-client-pivot.test.ts +628 -0
- package/src/__tests__/maintenance-locks.test.ts +15 -0
- package/src/__tests__/memory-boundaries.test.ts +32 -0
- package/src/__tests__/noise-prototype-bank.test.ts +280 -0
- package/src/__tests__/pii-redact.test.ts +19 -0
- package/src/__tests__/pivot-apply.test.ts +373 -0
- package/src/__tests__/pivot-distill-supervisor.test.ts +457 -0
- package/src/__tests__/pivot-distill.test.ts +1178 -0
- package/src/__tests__/preference-matcher.test.ts +54 -2
- package/src/__tests__/pull-from-macbook-sync-only.test.ts +679 -0
- package/src/__tests__/recall-gate.test.ts +266 -0
- package/src/__tests__/resolve-db-path.test.ts +130 -0
- package/src/__tests__/resolve-tier.test.ts +64 -0
- package/src/__tests__/retrieval-eval.test.ts +163 -0
- package/src/__tests__/retriever-lazy-init-dispatch.test.ts +123 -0
- package/src/__tests__/skill-promotion.test.ts +53 -0
- package/src/__tests__/stable-context-length-contract.test.ts +62 -0
- package/src/__tests__/store-read-consistency.test.ts +122 -0
- package/src/__tests__/store-schema-migration.test.ts +87 -0
- package/src/__tests__/store-upsert.test.ts +17 -0
- package/src/__tests__/term-registry-terminal-roster.test.ts +127 -0
- package/src/__tests__/ui-server-security.test.ts +93 -0
- package/src/access-tracker.ts +53 -3
- package/src/adaptive-retrieval.ts +11 -0
- package/src/api-server.ts +9 -2
- package/src/auto-recall.ts +61 -1
- package/src/belief-history.ts +164 -0
- package/src/capture-engine.ts +88 -11
- package/src/cli.ts +109 -14
- package/src/conflict-engine.ts +24 -3
- package/src/consolidation-engine.ts +146 -12
- package/src/context-composer-stable-selection.ts +7 -1
- package/src/data-checkup.ts +1 -1
- package/src/decay-engine.ts +59 -7
- package/src/doctor.ts +4 -2
- package/src/dream-pipeline.ts +359 -11
- package/src/env-config.ts +41 -0
- package/src/eval.ts +154 -11
- package/src/freshness.ts +192 -0
- package/src/http-request-policy.ts +148 -0
- package/src/ingest.ts +277 -2
- package/src/kg-extractor.ts +7 -1
- package/src/llm-client.ts +106 -11
- package/src/mcp-server.ts +2 -1
- package/src/mcp-tools-advanced.ts +13 -0
- package/src/mcp-tools-core.ts +5 -1
- package/src/memory-boundaries.ts +23 -2
- package/src/memory-health-rebalance.ts +4 -0
- package/src/memory-output.ts +16 -2
- package/src/memory-store-port.ts +3 -0
- package/src/noise-prototype-bank.ts +408 -0
- package/src/pii-detector.ts +52 -2
- package/src/pivot-apply.ts +399 -0
- package/src/preference-matcher.ts +37 -3
- package/src/recall-gate.ts +305 -0
- package/src/retrieval-profiles.ts +46 -4
- package/src/retriever.ts +95 -18
- package/src/retroactive-boost.ts +6 -2
- package/src/runtime-config.ts +70 -5
- package/src/skill-promotion.ts +58 -2
- package/src/store.ts +54 -19
- package/src/term-registry.ts +35 -0
- package/src/ui-server.ts +7 -0
- package/src/version.ts +19 -0
package/src/eval.ts
CHANGED
|
@@ -8,6 +8,8 @@ import { buildSessionCheckpointRecord, normalizeCheckpointScope } from "./sessio
|
|
|
8
8
|
import type { ResumeContextResponse, SessionCheckpointRecord } from "./session-schema.js";
|
|
9
9
|
import { composeResumeContext } from "./context-composer.js";
|
|
10
10
|
import { cleanText } from "./context-composer-text.js";
|
|
11
|
+
import { estimateTokens } from "./context-collapse-renderer.js";
|
|
12
|
+
import { TraceCollector } from "./retrieval-trace.js";
|
|
11
13
|
import { createComponents, loadConfig, loadDotEnv, type LocalMemoryConfig } from "./runtime-config.js";
|
|
12
14
|
import { logInfo } from "./stderr-log.js";
|
|
13
15
|
import { buildWorkflowObservationRecord } from "./workflow-observation-engine.js";
|
|
@@ -80,6 +82,30 @@ export interface CanaryEvalCase {
|
|
|
80
82
|
notes?: string;
|
|
81
83
|
}
|
|
82
84
|
|
|
85
|
+
/**
|
|
86
|
+
* Per-case cost accounting for the retrieval pipeline — the X axis of the
|
|
87
|
+
* Pareto curve (see hippo-wiki [[帕累托效率曲线]]).
|
|
88
|
+
*
|
|
89
|
+
* Scope: self-comparison across runs (did a retrieval change make it slower or
|
|
90
|
+
* fatter?). NOT for cross-system comparison — `contextTokens` comes from an
|
|
91
|
+
* estimator, not a real tokenizer, and this suite is not LoCoMo, so the numbers
|
|
92
|
+
* are not commensurable with competitors' published token counts.
|
|
93
|
+
*/
|
|
94
|
+
export interface EvalCostMetrics {
|
|
95
|
+
/** End-to-end retrieve() wall time. */
|
|
96
|
+
totalMs: number;
|
|
97
|
+
/** Number of memories returned. */
|
|
98
|
+
resultCount: number;
|
|
99
|
+
/** Estimated tokens of returned memory text — the context a caller would spend. */
|
|
100
|
+
contextTokens: number;
|
|
101
|
+
/** Duration of the rerank stage, or null when the pipeline ran no rerank.
|
|
102
|
+
* Left as a raw fact rather than an inferred "api calls" count: a local
|
|
103
|
+
* lightweight rerank and a remote cross-encoder both produce a stage here. */
|
|
104
|
+
rerankMs: number | null;
|
|
105
|
+
/** Three slowest stages, for locating the bottleneck. */
|
|
106
|
+
slowestStages: Array<{ name: string; durationMs: number }>;
|
|
107
|
+
}
|
|
108
|
+
|
|
83
109
|
export interface CanaryCaseReport {
|
|
84
110
|
mode: "canary";
|
|
85
111
|
name: string;
|
|
@@ -98,6 +124,8 @@ export interface CanaryCaseReport {
|
|
|
98
124
|
topIds: string[];
|
|
99
125
|
topSnippet: string;
|
|
100
126
|
notes?: string;
|
|
127
|
+
/** Attached by the runner after scoring; absent when scored standalone. */
|
|
128
|
+
cost?: EvalCostMetrics;
|
|
101
129
|
}
|
|
102
130
|
|
|
103
131
|
export interface RetrievalCaseReport {
|
|
@@ -115,6 +143,8 @@ export interface RetrievalCaseReport {
|
|
|
115
143
|
topScopes: string[];
|
|
116
144
|
topSnippet: string;
|
|
117
145
|
notes?: string;
|
|
146
|
+
/** Attached by the runner after scoring; absent when scored standalone. */
|
|
147
|
+
cost?: EvalCostMetrics;
|
|
118
148
|
}
|
|
119
149
|
|
|
120
150
|
export interface ContinuityCaseReport {
|
|
@@ -217,8 +247,16 @@ function matchedTerms(terms: string[] | undefined, haystack: string): string[] {
|
|
|
217
247
|
return (terms || []).filter((term) => haystack.includes(term.toLowerCase()));
|
|
218
248
|
}
|
|
219
249
|
|
|
250
|
+
/**
|
|
251
|
+
* 命中比例 × 权重。
|
|
252
|
+
*
|
|
253
|
+
* 调用方必须先滤掉 case 没声明的期望、再对剩下的权重重新归一化——下面三个
|
|
254
|
+
* 评分器都是这么做的。空期望这里返回 0 而不是 weight:万一将来有第四个调用
|
|
255
|
+
* 点忘了滤,代价是少给分而不是白送分。三个现有调用点都已滤过,所以这个分支
|
|
256
|
+
* 当前走不到,纯属兜底。
|
|
257
|
+
*/
|
|
220
258
|
function scoreExpectation(expected: string[] | undefined, matched: string[], weight: number): number {
|
|
221
|
-
if (!expected || expected.length === 0) return
|
|
259
|
+
if (!expected || expected.length === 0) return 0;
|
|
222
260
|
return (matched.length / expected.length) * weight;
|
|
223
261
|
}
|
|
224
262
|
|
|
@@ -236,10 +274,23 @@ export function scoreRetrievalCase(
|
|
|
236
274
|
const matchedScopes = (evalCase.expectScopePrefixes || []).filter((scope) => scopes.some((item) => item.startsWith(scope)));
|
|
237
275
|
const forbiddenMatches = matchedTerms(evalCase.forbid, joined);
|
|
238
276
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
277
|
+
// 动态加权:只计 case 实际声明的维度,权重在这些维度间重新归一化——
|
|
278
|
+
// 和下面 canary、continuity 两个评分器已有的写法一致。
|
|
279
|
+
// 改之前未声明的维度按满权重计入,而 eval/cases.json 里 20 个 case 一个都没写
|
|
280
|
+
// expectAll,于是每个 case 白拿 0.3:expectAny 四个词命中一个再加 scope 前缀命中,
|
|
281
|
+
// 正好落在 0.7 及格线上。
|
|
282
|
+
const expectationScores = [
|
|
283
|
+
{ expected: evalCase.expectAny, matched: matchedAny, weight: 0.4 },
|
|
284
|
+
{ expected: evalCase.expectAll, matched: matchedAll, weight: 0.3 },
|
|
285
|
+
{ expected: evalCase.expectScopePrefixes, matched: matchedScopes, weight: 0.2 },
|
|
286
|
+
].filter((item) => (item.expected || []).length > 0);
|
|
287
|
+
|
|
288
|
+
const totalExpectedWeight = expectationScores.reduce((sum, item) => sum + item.weight, 0);
|
|
289
|
+
const normalizedExpectationScore = totalExpectedWeight > 0
|
|
290
|
+
? expectationScores.reduce((sum, item) => sum + scoreExpectation(item.expected, item.matched, item.weight), 0) / totalExpectedWeight
|
|
291
|
+
: 0;
|
|
292
|
+
|
|
293
|
+
let score = normalizedExpectationScore * 0.9;
|
|
243
294
|
if (results.length > 0) score += 0.1;
|
|
244
295
|
if (forbiddenMatches.length > 0) score -= 0.3;
|
|
245
296
|
score = Math.max(0, Math.min(1, score));
|
|
@@ -552,11 +603,12 @@ function markdownRetrievalReport(reports: RetrievalCaseReport[]): string {
|
|
|
552
603
|
`- Cases: ${reports.length}`,
|
|
553
604
|
`- Passed: ${passed}/${reports.length}`,
|
|
554
605
|
`- Average score: ${formatPercent(average)}`,
|
|
606
|
+
...costSummaryLines(summarizeCost(reports), reports.length),
|
|
555
607
|
"",
|
|
556
|
-
"| Case | Profile | Score | Pass | Hits | Top scopes |",
|
|
557
|
-
"
|
|
608
|
+
"| Case | Profile | Score | Pass | Hits | ms | ~tok | Top scopes |",
|
|
609
|
+
"|------|---------|-------|------|------|----|------|------------|",
|
|
558
610
|
...reports.map((item) =>
|
|
559
|
-
`| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.hitCount} | ${item.topScopes.join(", ") || "-"} |`,
|
|
611
|
+
`| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.hitCount} | ${item.cost?.totalMs ?? "-"} | ${item.cost?.contextTokens ?? "-"} | ${item.topScopes.join(", ") || "-"} |`,
|
|
560
612
|
),
|
|
561
613
|
"",
|
|
562
614
|
"## Case Notes",
|
|
@@ -575,6 +627,7 @@ function markdownRetrievalReport(reports: RetrievalCaseReport[]): string {
|
|
|
575
627
|
lines.push(`- Matched all: ${item.matchedAll.join(", ") || "-"}`);
|
|
576
628
|
lines.push(`- Matched scopes: ${item.matchedScopes.join(", ") || "-"}`);
|
|
577
629
|
lines.push(`- Forbidden matches: ${item.forbiddenMatches.join(", ") || "-"}`);
|
|
630
|
+
if (item.cost) lines.push(`- Cost: ${formatCost(item.cost)}`);
|
|
578
631
|
if (item.notes) lines.push(`- Notes: ${item.notes}`);
|
|
579
632
|
lines.push("");
|
|
580
633
|
}
|
|
@@ -596,11 +649,12 @@ function markdownCanaryReport(reports: CanaryCaseReport[]): string {
|
|
|
596
649
|
`- Average score: ${formatPercent(average)}`,
|
|
597
650
|
`- Top1 hit: ${top1}/${reports.length}`,
|
|
598
651
|
`- Top3 hit: ${top3}/${reports.length}`,
|
|
652
|
+
...costSummaryLines(summarizeCost(reports), reports.length),
|
|
599
653
|
"",
|
|
600
|
-
"| Case | Profile | Score | Pass | Top1 | Top3 | Order | Forbid |",
|
|
601
|
-
"
|
|
654
|
+
"| Case | Profile | Score | Pass | Top1 | Top3 | Order | Forbid | ms | ~tok |",
|
|
655
|
+
"|------|---------|-------|------|------|------|-------|--------|----|------|",
|
|
602
656
|
...reports.map((item) =>
|
|
603
|
-
`| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.top1Hit ? "yes" : "-"} | ${item.top3Hit ? "yes" : "-"} | ${item.orderOk === null ? "-" : item.orderOk ? "ok" : "bad"} | ${item.forbiddenIdMatches.length + item.forbiddenMatches.length || "-"} |`,
|
|
657
|
+
`| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.top1Hit ? "yes" : "-"} | ${item.top3Hit ? "yes" : "-"} | ${item.orderOk === null ? "-" : item.orderOk ? "ok" : "bad"} | ${item.forbiddenIdMatches.length + item.forbiddenMatches.length || "-"} | ${item.cost?.totalMs ?? "-"} | ${item.cost?.contextTokens ?? "-"} |`,
|
|
604
658
|
),
|
|
605
659
|
"",
|
|
606
660
|
"## Case Notes",
|
|
@@ -619,6 +673,7 @@ function markdownCanaryReport(reports: CanaryCaseReport[]): string {
|
|
|
619
673
|
lines.push(`- Forbidden term matches: ${item.forbiddenMatches.join(", ") || "-"}`);
|
|
620
674
|
lines.push(`- Top ids: ${item.topIds.join(", ") || "-"}`);
|
|
621
675
|
lines.push(`- Top snippet: ${item.topSnippet}`);
|
|
676
|
+
if (item.cost) lines.push(`- Cost: ${formatCost(item.cost)}`);
|
|
622
677
|
if (item.notes) lines.push(`- Notes: ${item.notes}`);
|
|
623
678
|
lines.push("");
|
|
624
679
|
}
|
|
@@ -673,6 +728,88 @@ function markdownContinuityReport(reports: ContinuityCaseReport[]): string {
|
|
|
673
728
|
return lines.join("\n");
|
|
674
729
|
}
|
|
675
730
|
|
|
731
|
+
/**
|
|
732
|
+
* Turn a finished TraceCollector plus the returned memories into cost metrics.
|
|
733
|
+
*
|
|
734
|
+
* The `mode` argument to finalize() only populates a display field on the trace
|
|
735
|
+
* object; this function consumes stages / totalMs only, so it is not inferred.
|
|
736
|
+
*/
|
|
737
|
+
function collectCostMetrics(
|
|
738
|
+
trace: TraceCollector,
|
|
739
|
+
query: string,
|
|
740
|
+
results: Array<{ entry: { text: string } }>,
|
|
741
|
+
): EvalCostMetrics {
|
|
742
|
+
const finalized = trace.finalize(query, "hybrid");
|
|
743
|
+
const rerankStage = finalized.stages.find((stage) => stage.name === "rerank");
|
|
744
|
+
const slowestStages = [...finalized.stages]
|
|
745
|
+
.sort((a, b) => b.durationMs - a.durationMs)
|
|
746
|
+
.slice(0, 3)
|
|
747
|
+
.map((stage) => ({ name: stage.name, durationMs: stage.durationMs }));
|
|
748
|
+
|
|
749
|
+
return {
|
|
750
|
+
totalMs: finalized.totalMs,
|
|
751
|
+
resultCount: results.length,
|
|
752
|
+
contextTokens: results.reduce((sum, item) => sum + estimateTokens(item.entry.text), 0),
|
|
753
|
+
rerankMs: rerankStage ? rerankStage.durationMs : null,
|
|
754
|
+
slowestStages,
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
/** Aggregate cost across cases. Cases without cost data are skipped, and the
|
|
759
|
+
* count of those that did carry it is returned so the report can be honest
|
|
760
|
+
* about coverage rather than silently averaging over fewer cases. */
|
|
761
|
+
function summarizeCost(reports: Array<{ cost?: EvalCostMetrics }>): {
|
|
762
|
+
measured: number;
|
|
763
|
+
totalMs: number;
|
|
764
|
+
avgMs: number;
|
|
765
|
+
totalTokens: number;
|
|
766
|
+
avgTokens: number;
|
|
767
|
+
} | null {
|
|
768
|
+
const withCost = reports.map((item) => item.cost).filter((cost): cost is EvalCostMetrics => Boolean(cost));
|
|
769
|
+
if (withCost.length === 0) return null;
|
|
770
|
+
|
|
771
|
+
const totalMs = withCost.reduce((sum, cost) => sum + cost.totalMs, 0);
|
|
772
|
+
const totalTokens = withCost.reduce((sum, cost) => sum + cost.contextTokens, 0);
|
|
773
|
+
return {
|
|
774
|
+
measured: withCost.length,
|
|
775
|
+
totalMs,
|
|
776
|
+
avgMs: Math.round(totalMs / withCost.length),
|
|
777
|
+
totalTokens,
|
|
778
|
+
avgTokens: Math.round(totalTokens / withCost.length),
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/** Aggregate cost lines for a report header. Returns [] when no case carried
|
|
783
|
+
* cost data, so reports produced before this instrumentation still render. */
|
|
784
|
+
function costSummaryLines(
|
|
785
|
+
summary: ReturnType<typeof summarizeCost>,
|
|
786
|
+
caseCount: number,
|
|
787
|
+
): string[] {
|
|
788
|
+
if (!summary) return [];
|
|
789
|
+
const coverage = summary.measured === caseCount
|
|
790
|
+
? ""
|
|
791
|
+
: ` (measured on ${summary.measured}/${caseCount} cases)`;
|
|
792
|
+
return [
|
|
793
|
+
`- Retrieval latency: ${summary.avgMs} ms avg, ${summary.totalMs} ms total${coverage}`,
|
|
794
|
+
`- Context tokens: ~${summary.avgTokens} avg, ~${summary.totalTokens} total`
|
|
795
|
+
+ ` (estimated, not tokenizer-measured — compare across runs, not against other systems)`,
|
|
796
|
+
];
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
/** One-line per-case cost breakdown. */
|
|
800
|
+
function formatCost(cost: EvalCostMetrics): string {
|
|
801
|
+
const parts = [
|
|
802
|
+
`${cost.totalMs}ms`,
|
|
803
|
+
`~${cost.contextTokens} tok`,
|
|
804
|
+
`${cost.resultCount} results`,
|
|
805
|
+
cost.rerankMs === null ? "no rerank stage" : `rerank ${cost.rerankMs}ms`,
|
|
806
|
+
];
|
|
807
|
+
const slowest = cost.slowestStages
|
|
808
|
+
.map((stage) => `${stage.name} ${stage.durationMs}ms`)
|
|
809
|
+
.join(", ");
|
|
810
|
+
return `${parts.join(" / ")}${slowest ? ` | slowest: ${slowest}` : ""}`;
|
|
811
|
+
}
|
|
812
|
+
|
|
676
813
|
export async function runRetrievalEval(
|
|
677
814
|
cases: RetrievalEvalCase[],
|
|
678
815
|
deps: {
|
|
@@ -690,13 +827,16 @@ export async function runRetrievalEval(
|
|
|
690
827
|
const profileName = evalCase.profile || "default";
|
|
691
828
|
const { retriever, accessTracker } = createEvalComponentsForCase(profileName);
|
|
692
829
|
try {
|
|
830
|
+
const trace = new TraceCollector();
|
|
693
831
|
const results = await retriever.retrieve({
|
|
694
832
|
query: evalCase.query,
|
|
695
833
|
limit: evalCase.limit || 5,
|
|
696
834
|
scopeFilter: evalCase.scope ? [evalCase.scope] : undefined,
|
|
697
835
|
source: "auto-recall",
|
|
836
|
+
trace,
|
|
698
837
|
});
|
|
699
838
|
const report = scoreRetrievalCase(evalCase, results);
|
|
839
|
+
report.cost = collectCostMetrics(trace, evalCase.query, results);
|
|
700
840
|
reports.push(report);
|
|
701
841
|
if (cases.length > 1) {
|
|
702
842
|
logInfo(
|
|
@@ -728,13 +868,16 @@ export async function runCanaryEval(
|
|
|
728
868
|
const profileName = evalCase.profile || "default";
|
|
729
869
|
const { retriever, accessTracker } = createEvalComponentsForCase(profileName);
|
|
730
870
|
try {
|
|
871
|
+
const trace = new TraceCollector();
|
|
731
872
|
const results = await retriever.retrieve({
|
|
732
873
|
query: evalCase.query,
|
|
733
874
|
limit: evalCase.limit || 8,
|
|
734
875
|
scopeFilter: evalCase.scope ? [evalCase.scope] : undefined,
|
|
735
876
|
source: "auto-recall",
|
|
877
|
+
trace,
|
|
736
878
|
});
|
|
737
879
|
const report = scoreCanaryCase(evalCase, results);
|
|
880
|
+
report.cost = collectCostMetrics(trace, evalCase.query, results);
|
|
738
881
|
reports.push(report);
|
|
739
882
|
if (cases.length > 1) {
|
|
740
883
|
logInfo(
|
package/src/freshness.ts
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Freshness — 记忆新鲜度四态判定
|
|
3
|
+
*
|
|
4
|
+
* 出处:通用Wiki上下文记忆Agent方案借鉴审计(2026-07-21,Alice 拍板落地)。
|
|
5
|
+
*
|
|
6
|
+
* 召回的记忆按"依赖状态"判定有效性,四个离散档:
|
|
7
|
+
* - exact 依赖原封未动,可直接复用
|
|
8
|
+
* - compatible 依赖变了,但落在声明的兼容集内,不影响结论
|
|
9
|
+
* - uncertain 依赖变了且无法判定是否影响结论,复用前需验证
|
|
10
|
+
* - invalid 依赖对象已失效(文件删了 / 仓库没了)
|
|
11
|
+
*
|
|
12
|
+
* 判据是"依赖状态"不是"时间":依赖对象在不在 / revision 变没变。
|
|
13
|
+
* 失效颗粒度是结论级:每条记忆绑自己的 dependsOn,某个来源变了只降那一条。
|
|
14
|
+
*
|
|
15
|
+
* 硬边界(承重约束):
|
|
16
|
+
* 1. 判定必须廉价 —— 单条依赖 = 一次 existsSync/statSync(O(1) syscall)
|
|
17
|
+
* 或一次 `git rev-parse`(单进程、无网络、按 repo 目录 memoize)。
|
|
18
|
+
* 绝不全量扫描、绝不调模型、绝不网络请求。
|
|
19
|
+
* 2. opt-in —— 只对显式声明了 dependsOn 的记忆生效。无声明的记忆
|
|
20
|
+
* evaluateEntryFreshness 返回 null,零开销、行为与之前完全一致。
|
|
21
|
+
* 3. 向后兼容 —— dependsOn 存活在既有 metadata JSON 列里(照 emotion /
|
|
22
|
+
* privacyTier 惯例),不新增数据库字段、不改破坏性 schema。
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { z } from "zod";
|
|
26
|
+
import { execFileSync } from "node:child_process";
|
|
27
|
+
import { existsSync, statSync } from "node:fs";
|
|
28
|
+
import { dirname } from "node:path";
|
|
29
|
+
|
|
30
|
+
export const FRESHNESS_STATES = ["exact", "compatible", "uncertain", "invalid"] as const;
|
|
31
|
+
export type Freshness = (typeof FRESHNESS_STATES)[number];
|
|
32
|
+
export const FreshnessSchema = z.enum(FRESHNESS_STATES);
|
|
33
|
+
|
|
34
|
+
export const DEPENDENCY_KINDS = ["file", "git-rev"] as const;
|
|
35
|
+
export type DependencyKind = (typeof DEPENDENCY_KINDS)[number];
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* 一条依赖声明。首版只支持 file / git-rev(不支持任意 cmd 执行,安全优先)。
|
|
39
|
+
* kind='file' ref = 文件路径;expected = 文件 mtime(毫秒时间戳的整数字符串)
|
|
40
|
+
* 作为廉价变更指纹,缺省时只做存在性检查。内容级精确请用 git-rev。
|
|
41
|
+
* kind='git-rev' ref = git 仓库内的路径(文件或目录,取其所在仓库 HEAD);
|
|
42
|
+
* expected = 期望 commit hash(支持短 hash 前缀),缺省时只做仓库存在性检查。
|
|
43
|
+
* expected 单值或数组:数组首值匹配 = exact,其余值匹配 = compatible(= 声明的兼容集)。
|
|
44
|
+
*/
|
|
45
|
+
export const DependencySchema = z.object({
|
|
46
|
+
kind: z.enum(DEPENDENCY_KINDS),
|
|
47
|
+
ref: z.string().min(1).max(500),
|
|
48
|
+
expected: z
|
|
49
|
+
.union([z.string().min(1).max(200), z.array(z.string().min(1).max(200)).min(1).max(8)])
|
|
50
|
+
.optional(),
|
|
51
|
+
});
|
|
52
|
+
export type Dependency = z.infer<typeof DependencySchema>;
|
|
53
|
+
|
|
54
|
+
export const DependsOnSchema = z.array(DependencySchema).min(1).max(8);
|
|
55
|
+
|
|
56
|
+
/** 判定结果按严重度排序,聚合时取最差。 */
|
|
57
|
+
const SEVERITY: Record<Freshness, number> = { exact: 0, compatible: 1, uncertain: 2, invalid: 3 };
|
|
58
|
+
|
|
59
|
+
/** git rev-parse 结果按 repo 目录缓存:一次判定批次内同一仓库只 spawn 一次 git。 */
|
|
60
|
+
export type FreshnessCache = Map<string, string | null>;
|
|
61
|
+
|
|
62
|
+
export function createFreshnessCache(): FreshnessCache {
|
|
63
|
+
return new Map();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** 从记忆 metadata JSON 解析 dependsOn 声明;缺失或非法返回 null(照 parseEmotion 惯例)。 */
|
|
67
|
+
export function parseDependsOn(metadata?: string): Dependency[] | null {
|
|
68
|
+
if (!metadata) return null;
|
|
69
|
+
let parsed: unknown;
|
|
70
|
+
try {
|
|
71
|
+
parsed = JSON.parse(metadata);
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
if (parsed === null || typeof parsed !== "object") return null;
|
|
76
|
+
const raw = (parsed as Record<string, unknown>).dependsOn;
|
|
77
|
+
if (raw == null) return null;
|
|
78
|
+
const result = DependsOnSchema.safeParse(raw);
|
|
79
|
+
return result.success ? result.data : null;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** 校验来自 store 输入侧的 dependsOn(unknown → Dependency[] | null)。 */
|
|
83
|
+
export function parseDependsOnInput(raw: unknown): Dependency[] | null {
|
|
84
|
+
if (raw == null) return null;
|
|
85
|
+
const result = DependsOnSchema.safeParse(raw);
|
|
86
|
+
return result.success ? result.data : null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function toExpectedList(expected: Dependency["expected"]): string[] {
|
|
90
|
+
if (expected == null) return [];
|
|
91
|
+
return Array.isArray(expected) ? expected : [expected];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* 把"当前值"归类到 exact / compatible / uncertain。
|
|
96
|
+
* - 无 expected:只做存在性检查,存在即 exact。
|
|
97
|
+
* - 匹配 expected[0](首选/最新)→ exact
|
|
98
|
+
* - 匹配 expected[1..](声明的其他兼容值)→ compatible
|
|
99
|
+
* - 都不匹配 → uncertain(变了,且无法判定是否影响结论)
|
|
100
|
+
*/
|
|
101
|
+
function classifyAgainstExpected(
|
|
102
|
+
current: string,
|
|
103
|
+
expected: string[],
|
|
104
|
+
matches: (current: string, candidate: string) => boolean,
|
|
105
|
+
): Freshness {
|
|
106
|
+
if (expected.length === 0) return "exact";
|
|
107
|
+
if (matches(current, expected[0])) return "exact";
|
|
108
|
+
for (let i = 1; i < expected.length; i++) {
|
|
109
|
+
if (matches(current, expected[i])) return "compatible";
|
|
110
|
+
}
|
|
111
|
+
return "uncertain";
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function exactMatch(current: string, candidate: string): boolean {
|
|
115
|
+
return current === candidate;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** 短 hash 友好:current 是完整 HEAD(40 hex),candidate 可为完整或短前缀。 */
|
|
119
|
+
function gitRevMatch(current: string, candidate: string): boolean {
|
|
120
|
+
if (!candidate) return false;
|
|
121
|
+
const a = current.toLowerCase();
|
|
122
|
+
const b = candidate.toLowerCase();
|
|
123
|
+
return a === b || a.startsWith(b);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function resolveHeadRev(ref: string, cache: FreshnessCache): string | null {
|
|
127
|
+
let dir: string;
|
|
128
|
+
try {
|
|
129
|
+
dir = existsSync(ref) && statSync(ref).isDirectory() ? ref : dirname(ref);
|
|
130
|
+
} catch {
|
|
131
|
+
dir = dirname(ref);
|
|
132
|
+
}
|
|
133
|
+
const cached = cache.get(dir);
|
|
134
|
+
if (cached !== undefined) return cached;
|
|
135
|
+
let rev: string | null = null;
|
|
136
|
+
try {
|
|
137
|
+
const out = execFileSync("git", ["-C", dir, "rev-parse", "HEAD"], {
|
|
138
|
+
encoding: "utf8",
|
|
139
|
+
timeout: 2000,
|
|
140
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
141
|
+
}).trim();
|
|
142
|
+
rev = out.length > 0 ? out : null;
|
|
143
|
+
} catch {
|
|
144
|
+
rev = null; // 非 git 仓库 / git 不可用 / 超时 → 依赖失效
|
|
145
|
+
}
|
|
146
|
+
cache.set(dir, rev);
|
|
147
|
+
return rev;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** 单条依赖的廉价判定。 */
|
|
151
|
+
export function evaluateDependency(dep: Dependency, cache: FreshnessCache): Freshness {
|
|
152
|
+
const expected = toExpectedList(dep.expected);
|
|
153
|
+
if (dep.kind === "file") {
|
|
154
|
+
if (!existsSync(dep.ref)) return "invalid";
|
|
155
|
+
if (expected.length === 0) return "exact";
|
|
156
|
+
let mtime: string;
|
|
157
|
+
try {
|
|
158
|
+
mtime = String(Math.floor(statSync(dep.ref).mtimeMs));
|
|
159
|
+
} catch {
|
|
160
|
+
return "invalid"; // 存在性检查与 stat 之间被删
|
|
161
|
+
}
|
|
162
|
+
return classifyAgainstExpected(mtime, expected, exactMatch);
|
|
163
|
+
}
|
|
164
|
+
// git-rev
|
|
165
|
+
const head = resolveHeadRev(dep.ref, cache);
|
|
166
|
+
if (head === null) return "invalid";
|
|
167
|
+
return classifyAgainstExpected(head, expected, gitRevMatch);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** 多条依赖聚合:取最差(worst-case)——任一依赖失效,整条记忆就不可全信。 */
|
|
171
|
+
export function evaluateFreshness(deps: Dependency[], cache?: FreshnessCache): Freshness {
|
|
172
|
+
if (deps.length === 0) return "exact";
|
|
173
|
+
const revCache = cache ?? createFreshnessCache();
|
|
174
|
+
let worst: Freshness = "exact";
|
|
175
|
+
for (const dep of deps) {
|
|
176
|
+
const f = evaluateDependency(dep, revCache);
|
|
177
|
+
if (SEVERITY[f] > SEVERITY[worst]) worst = f;
|
|
178
|
+
if (worst === "invalid") break; // 已是最差档,无需继续
|
|
179
|
+
}
|
|
180
|
+
return worst;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* 便捷入口:从记忆 metadata 判定新鲜度。
|
|
185
|
+
* opt-in —— 无 dependsOn 声明返回 null(调用方据此跳过展示,零开销)。
|
|
186
|
+
* @param cache 可选的跨条目共享缓存(渲染一批结果时传入,令同仓库只 spawn 一次 git)。
|
|
187
|
+
*/
|
|
188
|
+
export function evaluateEntryFreshness(metadata?: string, cache?: FreshnessCache): Freshness | null {
|
|
189
|
+
const deps = parseDependsOn(metadata);
|
|
190
|
+
if (!deps) return null;
|
|
191
|
+
return evaluateFreshness(deps, cache);
|
|
192
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
export const LOCAL_HTTP_HOSTNAME = "127.0.0.1";
|
|
2
|
+
export const LOCAL_JSON_BODY_LIMIT_BYTES = 1024 * 1024;
|
|
3
|
+
|
|
4
|
+
const LOCAL_HOSTNAMES = new Set(["127.0.0.1", "localhost", "::1"]);
|
|
5
|
+
const DEFAULT_JSON_METHODS = ["POST"] as const;
|
|
6
|
+
|
|
7
|
+
export interface LocalHttpRequestPolicyOptions {
|
|
8
|
+
jsonMethods?: readonly string[];
|
|
9
|
+
maxJsonBodyBytes?: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export type LocalHttpRequestPolicyResult =
|
|
13
|
+
| { allowed: true }
|
|
14
|
+
| { allowed: false; status: 400 | 403 | 413 | 415; message: string };
|
|
15
|
+
|
|
16
|
+
function reject(
|
|
17
|
+
status: 400 | 403 | 413 | 415,
|
|
18
|
+
message: string,
|
|
19
|
+
): LocalHttpRequestPolicyResult {
|
|
20
|
+
return { allowed: false, status, message };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function normalizeHostname(hostname: string): string {
|
|
24
|
+
const normalized = hostname.trim().toLowerCase();
|
|
25
|
+
if (normalized.startsWith("[") && normalized.endsWith("]")) {
|
|
26
|
+
return normalized.slice(1, -1);
|
|
27
|
+
}
|
|
28
|
+
return normalized;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function hostnameFromHostHeader(hostHeader: string): string | null {
|
|
32
|
+
const authority = hostHeader.trim();
|
|
33
|
+
if (!authority || /[\s/@?#\\]/.test(authority)) return null;
|
|
34
|
+
|
|
35
|
+
try {
|
|
36
|
+
return normalizeHostname(new URL(`http://${authority}`).hostname);
|
|
37
|
+
} catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isLocalHostname(hostname: string): boolean {
|
|
43
|
+
return LOCAL_HOSTNAMES.has(normalizeHostname(hostname));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function readContentLength(request: Request): number | null {
|
|
47
|
+
const raw = request.headers.get("content-length");
|
|
48
|
+
if (raw == null) return null;
|
|
49
|
+
const trimmed = raw.trim();
|
|
50
|
+
if (!/^\d+$/.test(trimmed)) return Number.NaN;
|
|
51
|
+
return Number(trimmed);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function bodyExceedsLimit(request: Request, maxBytes: number): Promise<boolean> {
|
|
55
|
+
if (!request.body) return false;
|
|
56
|
+
|
|
57
|
+
const reader = request.clone().body?.getReader();
|
|
58
|
+
if (!reader) return false;
|
|
59
|
+
|
|
60
|
+
let totalBytes = 0;
|
|
61
|
+
try {
|
|
62
|
+
while (true) {
|
|
63
|
+
const chunk = await reader.read();
|
|
64
|
+
if (chunk.done) return false;
|
|
65
|
+
totalBytes += chunk.value.byteLength;
|
|
66
|
+
if (totalBytes > maxBytes) {
|
|
67
|
+
void reader.cancel("request body too large").catch(() => {});
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
} finally {
|
|
72
|
+
reader.releaseLock();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export async function enforceLocalHttpRequestPolicy(
|
|
77
|
+
request: Request,
|
|
78
|
+
options: LocalHttpRequestPolicyOptions = {},
|
|
79
|
+
): Promise<LocalHttpRequestPolicyResult> {
|
|
80
|
+
let requestUrl: URL;
|
|
81
|
+
try {
|
|
82
|
+
requestUrl = new URL(request.url);
|
|
83
|
+
} catch {
|
|
84
|
+
return reject(400, "Invalid request URL.");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (!isLocalHostname(requestUrl.hostname)) {
|
|
88
|
+
return reject(403, "Host is not allowed.");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const hostHeader = request.headers.get("host");
|
|
92
|
+
if (hostHeader != null) {
|
|
93
|
+
const hostname = hostnameFromHostHeader(hostHeader);
|
|
94
|
+
if (!hostname || !isLocalHostname(hostname)) {
|
|
95
|
+
return reject(403, "Host is not allowed.");
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const fetchSite = request.headers.get("sec-fetch-site")?.trim().toLowerCase();
|
|
100
|
+
if (fetchSite && fetchSite !== "same-origin" && fetchSite !== "none") {
|
|
101
|
+
return reject(403, "Cross-site requests are not allowed.");
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const origin = request.headers.get("origin");
|
|
105
|
+
if (origin != null) {
|
|
106
|
+
let originUrl: URL;
|
|
107
|
+
try {
|
|
108
|
+
originUrl = new URL(origin);
|
|
109
|
+
} catch {
|
|
110
|
+
return reject(403, "Origin is not allowed.");
|
|
111
|
+
}
|
|
112
|
+
if (originUrl.origin !== requestUrl.origin) {
|
|
113
|
+
return reject(403, "Origin is not allowed.");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const jsonMethods = options.jsonMethods ?? DEFAULT_JSON_METHODS;
|
|
118
|
+
if (!jsonMethods.some((method) => method.toUpperCase() === request.method.toUpperCase())) {
|
|
119
|
+
return { allowed: true };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const mediaType = request.headers.get("content-type")
|
|
123
|
+
?.split(";", 1)[0]
|
|
124
|
+
?.trim()
|
|
125
|
+
.toLowerCase();
|
|
126
|
+
if (mediaType !== "application/json") {
|
|
127
|
+
return reject(415, "Content-Type must be application/json.");
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const maxJsonBodyBytes = options.maxJsonBodyBytes ?? LOCAL_JSON_BODY_LIMIT_BYTES;
|
|
131
|
+
const contentLength = readContentLength(request);
|
|
132
|
+
if (contentLength != null && (!Number.isSafeInteger(contentLength) || contentLength < 0)) {
|
|
133
|
+
return reject(400, "Invalid Content-Length header.");
|
|
134
|
+
}
|
|
135
|
+
if (contentLength != null && contentLength > maxJsonBodyBytes) {
|
|
136
|
+
return reject(413, `Request body exceeds ${maxJsonBodyBytes} bytes.`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
try {
|
|
140
|
+
if (await bodyExceedsLimit(request, maxJsonBodyBytes)) {
|
|
141
|
+
return reject(413, `Request body exceeds ${maxJsonBodyBytes} bytes.`);
|
|
142
|
+
}
|
|
143
|
+
} catch {
|
|
144
|
+
return reject(400, "Unable to read request body.");
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return { allowed: true };
|
|
148
|
+
}
|