solopup-memory 0.21.0-beta.2 → 0.22.0-beta.3

package/eval/semantic/cases.json

@@ -0,0 +1,387 @@
+[
+  {
+    "id": "identity-claim",
+    "tags": [
+      "write",
+      "meaning"
+    ],
+    "title": "用户自述长期身份：存成 claim，不是已核实事实",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "我是做电视剧剧本的，长期负责剧本结构。"
+      }
+    ],
+    "gold": {
+      "type": "user",
+      "scope": "global",
+      "basis": "explicit",
+      "summary": "长期做电视剧剧本，负责剧本结构",
+      "meaning": {
+        "kind": "claim"
+      }
+    },
+    "expect": {
+      "status": "eligible",
+      "meaningMatches": true,
+      "search": [
+        {
+          "query": "电视剧剧本",
+          "hits": 1
+        }
+      ]
+    }
+  },
+  {
+    "id": "conditional-preference",
+    "tags": [
+      "write",
+      "conditions"
+    ],
+    "title": "有听众条件的偏好：条件必须落库，并在检索时过滤",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "给儿童讲解的时候先举例，其他场合不用这样。"
+      }
+    ],
+    "gold": {
+      "type": "user",
+      "scope": "global",
+      "basis": "explicit",
+      "summary": "给儿童讲解时先举例",
+      "meaning": {
+        "kind": "preference",
+        "conditions": [
+          {
+            "dimension": "audience",
+            "value": "children"
+          }
+        ]
+      }
+    },
+    "expect": {
+      "status": "eligible",
+      "meaningMatches": true,
+      "search": [
+        {
+          "query": "举例",
+          "context": {
+            "audience": "children"
+          },
+          "hits": 1
+        },
+        {
+          "query": "举例",
+          "context": {
+            "audience": "adult"
+          },
+          "hits": 0
+        }
+      ]
+    }
+  },
+  {
+    "id": "project-decision-detail",
+    "tags": [
+      "write",
+      "project"
+    ],
+    "title": "项目决定 + 正文细节：索引摘要与主题正文都要在",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "这个项目就按 26 集、每集 20 分钟来做。"
+      }
+    ],
+    "gold": {
+      "type": "project",
+      "scope": "project",
+      "basis": "explicit",
+      "summary": "项目固定 26 集、每集 20 分钟",
+      "detail": "硬格式：每集 20 分钟；不是 12 集 × 45 分钟。",
+      "meaning": {
+        "kind": "decision"
+      }
+    },
+    "expect": {
+      "status": "eligible",
+      "meaningMatches": true,
+      "detailContains": [
+        "每集 20 分钟"
+      ],
+      "search": [
+        {
+          "query": "26 集",
+          "hits": 1
+        }
+      ]
+    }
+  },
+  {
+    "id": "inferred-stays-pending",
+    "tags": [
+      "gate",
+      "attribution"
+    ],
+    "title": "推断出来的偏好只能进待审，不能直接发布",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "我最近晚上写东西比较多。"
+      }
+    ],
+    "gold": {
+      "type": "user",
+      "scope": "global",
+      "basis": "inferred",
+      "summary": "可能偏好夜间写作",
+      "meaning": {
+        "kind": "belief",
+        "alternatives": [
+          "只是最近任务较轻"
+        ]
+      }
+    },
+    "expect": {
+      "status": "candidate",
+      "reason": "inference-needs-review",
+      "published": false
+    }
+  },
+  {
+    "id": "quote-must-be-verbatim",
+    "tags": [
+      "gate",
+      "provenance"
+    ],
+    "title": "引文不是用户原话逐字子串：整条拒绝",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "我负责剪辑和声音。"
+      }
+    ],
+    "gold": {
+      "type": "user",
+      "scope": "global",
+      "basis": "explicit",
+      "summary": "负责剪辑与声音",
+      "meaning": {
+        "kind": "claim"
+      },
+      "evidence": [
+        {
+          "messageId": "m1",
+          "quote": "我负责剪辑、声音和配乐。"
+        }
+      ]
+    },
+    "expect": {
+      "status": "rejected",
+      "reason": "invalid-user-evidence",
+      "published": false
+    }
+  },
+  {
+    "id": "project-decision-cannot-be-global",
+    "tags": [
+      "gate",
+      "scope"
+    ],
+    "title": "项目约定写到 global：拒绝（scope 闸）",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "这个项目以后都用双栏格式。"
+      }
+    ],
+    "gold": {
+      "type": "project",
+      "scope": "global",
+      "basis": "explicit",
+      "summary": "项目使用双栏格式",
+      "meaning": {
+        "kind": "preference"
+      }
+    },
+    "expect": {
+      "status": "rejected",
+      "reason": "project-global",
+      "published": false
+    }
+  },
+  {
+    "id": "missing-basis",
+    "tags": [
+      "gate",
+      "attribution"
+    ],
+    "title": "没有 basis 的候选：拒绝",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "我习惯先看完所有素材再动笔。"
+      }
+    ],
+    "gold": {
+      "type": "user",
+      "scope": "global",
+      "summary": "先看完素材再动笔",
+      "meaning": {
+        "kind": "preference"
+      }
+    },
+    "expect": {
+      "status": "rejected",
+      "reason": "missing-basis",
+      "published": false
+    }
+  },
+  {
+    "id": "supersede-old-rule",
+    "tags": [
+      "write",
+      "supersede"
+    ],
+    "title": "更正旧规则：旧行消失、新行生效、历史留痕",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "改稿别直接给结论，先说理由再说结论。"
+      }
+    ],
+    "setup": [
+      {
+        "type": "feedback",
+        "line": "- [feedback] 改稿先给结论 — feedback.md"
+      }
+    ],
+    "supersedes": "- [feedback] 改稿先给结论 — feedback.md",
+    "gold": {
+      "type": "feedback",
+      "scope": "project",
+      "basis": "explicit",
+      "summary": "改稿先说理由再说结论",
+      "meaning": {
+        "kind": "preference"
+      }
+    },
+    "expect": {
+      "status": "eligible",
+      "meaningMatches": true,
+      "indexContains": [
+        "改稿先说理由再说结论"
+      ],
+      "indexExcludes": [
+        "- [feedback] 改稿先给结论 — feedback.md"
+      ],
+      "superseded": true,
+      "search": [
+        {
+          "query": "改稿理由",
+          "hits": 1
+        }
+      ]
+    }
+  },
+  {
+    "id": "condition-in-detail",
+    "tags": [
+      "write",
+      "conditions"
+    ],
+    "title": "任务条件写进摘要与正文，检索按条件过滤",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "审稿的时候只看结构，不看错别字；其他场合不看结构。"
+      }
+    ],
+    "gold": {
+      "type": "feedback",
+      "scope": "project",
+      "basis": "explicit",
+      "summary": "审稿时只看结构，不看错别字",
+      "detail": "审稿任务适用；其他任务不适用。",
+      "meaning": {
+        "kind": "preference",
+        "conditions": [
+          {
+            "dimension": "task",
+            "value": "review"
+          }
+        ]
+      }
+    },
+    "expect": {
+      "status": "eligible",
+      "meaningMatches": true,
+      "detailContains": [
+        "审稿任务适用"
+      ],
+      "search": [
+        {
+          "query": "只看结构",
+          "context": {
+            "task": "review"
+          },
+          "hits": 1
+        },
+        {
+          "query": "只看结构",
+          "context": {
+            "task": "draft"
+          },
+          "hits": 0
+        }
+      ]
+    }
+  },
+  {
+    "id": "expired-not-retrievable",
+    "tags": [
+      "lifecycle",
+      "expiry"
+    ],
+    "title": "过期条目保留在索引，但检索不返回",
+    "messages": [
+      {
+        "id": "m1",
+        "role": "user",
+        "text": "下周的内部活动只对内部开放。"
+      }
+    ],
+    "gold": {
+      "type": "reference",
+      "scope": "global",
+      "basis": "explicit",
+      "summary": "内部活动只对内部开放",
+      "expires": "2000-01-01",
+      "meaning": {
+        "kind": "reference"
+      }
+    },
+    "expect": {
+      "status": "eligible",
+      "indexContains": [
+        "(exp:2000-01-01)"
+      ],
+      "search": [
+        {
+          "query": "内部活动",
+          "hits": 0
+        }
+      ]
+    }
+  }
+]

package/eval/semantic-suite.mjs

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+/**
+ * L2 语义套件的**管道保真**层（确定性，不调用模型）。
+ *
+ * 它回答的问题是：假设抽取器给出了正确的候选（或故意错误的候选），
+ * 后续的机械闸门、写入、投影、检索会不会把它弄丢、弄错、放宽。
+ *
+ * 它**不回答**：真实模型会不会抽出正确候选。那需要 --live 主机适配器，
+ * 目前只能在真实 DSH 宿主里录一次基线再回放；不得把本套件通过写成语义验收通过。
+ *
+ *   node eval/semantic-suite.mjs            # 人类可读
+ *   node eval/semantic-suite.mjs --json     # 机器可读
+ *   node eval/semantic-suite.mjs --verbose  # 逐例输出
+ */
+
+import { createHash } from 'node:crypto'
+import { mkdtempSync, mkdirSync, readFileSync, rmSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+import { tmpdir } from 'node:os'
+import { fileURLToPath } from 'node:url'
+
+import { assessCandidate } from '../capture-evidence.mjs'
+import { normalizeMeaning } from '../meaning.mjs'
+import { normalizeText } from '../security.mjs'
+import { candidateSnapshot } from '../candidates.mjs'
+import { indexLine, searchStore, writeMemory } from '../index.mjs'
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+export const CASES = JSON.parse(readFileSync(join(HERE, 'semantic', 'cases.json'), 'utf8'))
+const hash = (text) => createHash('sha256').update(text).digest('hex')
+
+function assert(condition, message) {
+  if (!condition) throw new Error(message)
+}
+
+function runCase(testCase, room) {
+  const dir = join(room, testCase.gold.scope === 'project' ? 'project' : 'global')
+  mkdirSync(dir, { recursive: true })
+
+  for (const setup of testCase.setup ?? []) {
+    writeMemory({ type: setup.type, dir, line: setup.line })
+  }
+
+  const messages = testCase.messages.map((message) => {
+    const text = normalizeText(message.text).trim()
+    return { id: message.id, role: message.role ?? 'user', text, hash: hash(text) }
+  })
+  const evidence = testCase.gold.evidence ?? [{ messageId: messages[0].id, quote: messages[0].text.slice(0, Math.min(60, messages[0].text.length)) }]
+  const candidate = { ...testCase.gold, evidence }
+  const expect = testCase.expect ?? {}
+  const hasProject = candidate.scope === 'project'
+  const assessment = assessCandidate(candidate, messages, hasProject)
+
+  if (expect.status !== 'eligible') {
+    assert(assessment.status === expect.status, `期望 ${expect.status}，实际 ${assessment.status}`)
+    if (expect.reason !== undefined) assert(assessment.reason === expect.reason, `期望原因 ${expect.reason}，实际 ${assessment.reason}`)
+    assert(assessment.status !== 'eligible', '非 eligible 的用例不应该被写入')
+    return { assertions: 1 }
+  }
+
+  assert(assessment.status === 'eligible', `期望 eligible，实际 ${assessment.status}${assessment.reason ? `（${assessment.reason}）` : ''}`)
+  const detail = typeof candidate.detail === 'string' && candidate.detail.trim().length > 0 ? candidate.detail : undefined
+  const line = indexLine(candidate.type, candidate.summary, candidate.expires, detail !== undefined)
+  const written = writeMemory({
+    type: candidate.type,
+    dir,
+    line,
+    summary: candidate.summary,
+    scope: candidate.scope,
+    detail,
+    meaning: candidate.meaning,
+    expires: candidate.expires,
+    supersedes: testCase.supersedes,
+    provenance: { basis: 'tool-context', sessionId: `semantic-suite:${testCase.id}`, callHash: hash(testCase.id), evidence: assessment.evidence },
+  })
+  let assertions = 1
+
+  assert(written.indexLine === line, `写入行不一致：${written.indexLine}`)
+  const indexText = readFileSync(join(dir, 'MEMORY.md'), 'utf8')
+  for (const needle of expect.indexContains ?? []) assert(indexText.includes(needle), `索引缺少：${needle}`)
+  for (const needle of expect.indexExcludes ?? []) assert(!indexText.includes(needle), `索引不应包含：${needle}`)
+  if (expect.superseded === true) assert(written.superseded !== undefined, '期望返回被取代的旧行')
+  if (expect.detailContains) {
+    const topic = readFileSync(join(dir, `${candidate.type}.md`), 'utf8')
+    for (const needle of expect.detailContains) assert(topic.includes(needle), `正文缺少：${needle}`)
+  }
+  if (expect.meaningMatches === true) {
+    const snapshot = candidateSnapshot(dir, candidate.scope)
+    const record = snapshot.records.find((row) => row.indexLine === written.indexLine)
+    assert(record !== undefined, '账本里找不到刚写入的记录')
+    assert(JSON.stringify(record.meaning) === JSON.stringify(normalizeMeaning(candidate.meaning)), '认识层/条件在写入后发生变化')
+  }
+  for (const probe of expect.search ?? []) {
+    const hits = searchStore(dir, probe.query, {
+      ...(probe.context === undefined ? {} : { context: probe.context }),
+      ...(probe.task === undefined ? {} : { task: probe.task }),
+    }).hits
+    assert(hits.length === probe.hits, `检索「${probe.query}」期望 ${probe.hits} 条，实际 ${hits.length} 条`)
+    assertions += 1
+  }
+  return { assertions }
+}
+
+export function runSuite(cases = CASES) {
+  const root = mkdtempSync(join(tmpdir(), 'dsh-semantic-suite-'))
+  const previousHome = process.env.DSH_HOME
+  process.env.DSH_HOME = root
+  const results = []
+  const startedAt = Date.now()
+  try {
+    for (const testCase of cases) {
+      const room = join(root, testCase.id)
+      mkdirSync(room, { recursive: true })
+      const begin = Date.now()
+      try {
+        const outcome = runCase(testCase, room)
+        results.push({ id: testCase.id, tags: testCase.tags ?? [], ok: true, assertions: outcome.assertions, ms: Date.now() - begin })
+      } catch (error) {
+        results.push({ id: testCase.id, tags: testCase.tags ?? [], ok: false, error: error.message, ms: Date.now() - begin })
+      }
+    }
+  } finally {
+    if (previousHome === undefined) delete process.env.DSH_HOME
+    else process.env.DSH_HOME = previousHome
+    rmSync(root, { recursive: true, force: true })
+  }
+  const failed = results.filter((row) => !row.ok)
+  return {
+    kind: 'semantic-pipeline',
+    mode: 'pipeline',
+    semanticTruthEvaluated: false,
+    cases: results.length,
+    passed: results.length - failed.length,
+    failed: failed.length,
+    assertions: results.reduce((sum, row) => sum + (row.assertions ?? 0), 0),
+    ms: Date.now() - startedAt,
+    results,
+  }
+}
+
+if (process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`) {
+  const argv = process.argv.slice(2)
+  const report = runSuite()
+  if (argv.includes('--json')) process.stdout.write(JSON.stringify(report, null, 2) + '\n')
+  else {
+    const lines = [
+      `L2 语义套件（管道保真层）· ${report.cases} 例 · 通过 ${report.passed} · 失败 ${report.failed} · ${report.assertions} 项断言 · ${report.ms}ms`,
+      '说明：通过只表示“管道不会弄丢/弄错已给出的候选”，不表示真实模型的抽取质量。',
+    ]
+    for (const row of report.results) {
+      if (!row.ok || argv.includes('--verbose')) lines.push(`  ${row.ok ? '✔' : '✖'} ${row.id}${row.ok ? `（${row.assertions} 项）` : `：${row.error}`}`)
+    }
+    process.stdout.write(lines.join('\n') + '\n')
+  }
+  if (report.failed > 0) process.exit(1)
+}