ppxans-harness 2.4.0

package/src/tools/document.js

@@ -0,0 +1,253 @@
+// src/tools/document.js - 文档加载器 (对标 LangChain Document Loaders, 零依赖)
+// 支持: txt / md / json / csv / html / pdf (文字型, 扫描件需 OCR)
+// PDF 零依赖提取: 解压 FlateDecode 流 (zlib) + 提取 Tj/TJ 文本操作符
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import zlib from "node:zlib";
+import { ocrImage } from "./ocr.js";
+
+const MAX_CHARS = 20000; // 单文档返回上限
+
+// 安全路径: 阻止逃出工作目录 (防路径穿越, 与 builtin.js 同策略)
+function safePath(root, p) {
+  const resolved = path.resolve(root, p);
+  if (resolved !== root && !resolved.startsWith(root + path.sep)) {
+    throw new Error(`路径越界拒绝: ${p}`);
+  }
+  return resolved;
+}
+
+// 解码 PDF 文本字符串: 处理 UTF-16BE (FE FF BOM) 与 UTF-8 与转义字符
+function decodePdfString(latin1Str) {
+  let s = String(latin1Str || "").replace(/\\(\r?\n)/g, ""); // 续行
+  const buf = Buffer.from(s, "latin1");
+  // UTF-16BE BOM
+  if (buf.length >= 2 && buf[0] === 0xfe && buf[1] === 0xff) {
+    return buf.slice(2).toString("utf16le");
+  }
+  // 字节级反转义 \( \) \\
+  const out = [];
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] === 0x5c && i + 1 < buf.length) {
+      const n = buf[i + 1];
+      if (n === 0x28 || n === 0x29 || n === 0x5c) { out.push(n); i++; continue; }
+    }
+    out.push(buf[i]);
+  }
+  const clean = Buffer.from(out);
+  const utf8 = clean.toString("utf8");
+  return utf8.includes("\uFFFD") ? clean.toString("latin1") : utf8;
+}
+
+// 零依赖 PDF 文本提取: 遍历 stream, FlateDecode 解压, 提取 (text) Tj 与 [(a)(b)] TJ
+export function extractPdfText(buf) {
+  const raw = buf.toString("latin1");
+  const texts = [];
+  const streamRe = /stream\r?\n([\s\S]*?)endstream/g;
+  let m;
+  while ((m = streamRe.exec(raw)) !== null) {
+    let data = m[1];
+    // 尝试 FlateDecode 解压 (内容流通常是压缩的)
+    try {
+      const inf = zlib.inflateSync(Buffer.from(data, "latin1")).toString("latin1");
+      if (inf.length > 0) data = inf;
+    } catch { /* 未压缩则用原文 */ }
+    // (text) Tj
+    for (const tm of data.matchAll(/\(([^)]*)\)\s*Tj/g)) {
+      const t = decodePdfString(tm[1]).trim();
+      if (t) texts.push(t);
+    }
+    // [(a)(b)] TJ (数组形式)
+    for (const tm of data.matchAll(/\[((?:\([^)]*\)[\s<>0-9.-]*)+)\]\s*TJ/g)) {
+      for (const pm of tm[1].matchAll(/\(([^)]*)\)/g)) {
+        const t = decodePdfString(pm[1]).trim();
+        if (t) texts.push(t);
+      }
+    }
+  }
+  return texts.join(" ");
+}
+
+// 提取 PDF 内嵌 JPEG 图片 (DCTDecode 流, 扫描件 PDF 的页面图), 供 OCR
+export function extractPdfJpegs(buf) {
+  const raw = buf.toString("latin1");
+  const jpegs = [];
+  const re = /\/DCTDecode[\s\S]{0,200}?stream\r?\n([\s\S]*?)endstream/g;
+  let m;
+  while ((m = re.exec(raw)) !== null) {
+    const data = Buffer.from(m[1], "latin1");
+    // JPEG 魔数 FFD8
+    if (data.length > 2 && data[0] === 0xff && data[1] === 0xd8) jpegs.push(data);
+  }
+  return jpegs;
+}
+
+// HTML 转纯文本 (基础 strip, 与 advanced.js fetch_page 同思路)
+function htmlToText(html) {
+  return String(html || "")
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<br\s*\/?>|<\/p>|<\/div>|<\/li>|<\/h[1-6]>|<\/tr>/gi, "\n")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+// 按扩展名提取文档文本 (纯函数, 供 read_document 与 ingest_document 复用)
+export function extractDocumentText(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (!fs.existsSync(filePath)) throw new Error(`文件不存在: ${filePath}`);
+  const buf = fs.readFileSync(filePath);
+  switch (ext) {
+    case ".txt": case ".md": case ".csv": case ".json": case ".log":
+      return buf.toString("utf8");
+    case ".html": case ".htm":
+      return htmlToText(buf.toString("utf8"));
+    case ".pdf":
+      return extractPdfText(buf);
+    default:
+      throw new Error(`不支持的文档类型: ${ext} (支持 txt/md/csv/json/html/pdf)`);
+  }
+}
+
+// 分块: 按段落切, 每块约 chunkSize 字 (供 ingest 向量化)
+export function splitChunks(text, chunkSize = 500) {
+  const clean = String(text || "").replace(/\r/g, "");
+  const paras = clean.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
+  const chunks = [];
+  let cur = "";
+  for (const p of paras) {
+    if ((cur + p).length > chunkSize && cur) {
+      chunks.push(cur.trim());
+      cur = p;
+    } else {
+      cur = cur ? cur + "\n" + p : p;
+    }
+  }
+  if (cur.trim()) chunks.push(cur.trim());
+  return chunks.filter((c) => c.length >= 10);
+}
+
+// 从 config 构造 OCR 选项 (未显式关闭时默认启用本地 tesseract 自动 OCR)
+function ocrOptsFromConfig(cfg) {
+  const c = (cfg && cfg.ocr) || {};
+  if (c.auto === false) return null; // 显式关闭自动 OCR
+  return { tesseract: c.tesseract || "tesseract", lang: c.lang || "chi_sim", cloud: c.cloud || null };
+}
+
+// 读文档文本, PDF 无文本层(扫描件)时自动提取内嵌图片 OCR (_ocrFn 供测试注入)
+export async function readDocumentText(filePath, ocrOpts, _ocrFn = ocrImage) {
+  let text = extractDocumentText(filePath);
+  if (path.extname(filePath).toLowerCase() === ".pdf" && !text.trim() && ocrOpts) {
+    const jpegs = extractPdfJpegs(fs.readFileSync(filePath));
+    if (jpegs.length) {
+      const parts = [];
+      const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "ppx-ocr-pdf-"));
+      try {
+        for (let i = 0; i < jpegs.length; i++) {
+          const tmp = path.join(tmpDir, `page-${i + 1}.jpg`);
+          fs.writeFileSync(tmp, jpegs[i]);
+          try { parts.push(await _ocrFn(tmp, ocrOpts)); } catch {}
+        }
+      } finally {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+      }
+      text = parts.filter(Boolean).join("\n").trim();
+    }
+  }
+  return text;
+}
+
+// 注册文档工具
+export function registerDocumentTools(catalog, { rootDir }) {
+  // 1. 读文档 (加载器, PDF 扫描件自动 OCR)
+  catalog.register({
+    name: "read_document",
+    description: "读取本地文档并转纯文本。支持 .txt/.md/.json/.csv/.html/.pdf (文字型 PDF 直接提取, 扫描件 PDF 自动 OCR)。用于读文档/报告/数据文件后回答问题。",
+    parameters: {
+      type: "object",
+      properties: {
+        path: { type: "string", description: "文档路径 (相对工作目录)" },
+        maxChars: { type: "number", description: "返回最大字符数, 默认 20000" },
+      },
+      required: ["path"],
+    },
+    execute: async (args, ctx) => {
+      try {
+        const p = safePath(rootDir, args.path);
+        const cfg = (ctx && ctx.agent && ctx.agent.config) || {};
+        const text = await readDocumentText(p, ocrOptsFromConfig(cfg));
+        const max = Math.min(args.maxChars || MAX_CHARS, 40000);
+        const truncated = text.length > max ? text.slice(0, max) + `\n...[已截断, 共 ${text.length} 字符]` : text;
+        return truncated || "(文档无文本内容, 且未识别出扫描件文字)";
+      } catch (e) {
+        return JSON.stringify({ error: "read_document 失败: " + e.message });
+      }
+    },
+  });
+
+  // 2. OCR 识别图片/扫描件文字
+  catalog.register({
+    name: "ocr_image",
+    description: "识别图片或扫描件里的文字 (OCR)。需系统安装 tesseract (含中文语言包) 或配置 config.ocr 云 key。用于 read_image/read_document 读到图片却无法理解文字时。",
+    parameters: {
+      type: "object",
+      properties: {
+        path: { type: "string", description: "图片或扫描件路径 (相对工作目录)" },
+        lang: { type: "string", description: "识别语言, 默认 chi_sim (中文)" },
+      },
+      required: ["path"],
+    },
+    execute: async (args, ctx) => {
+      const agent = ctx && ctx.agent;
+      const cfg = (agent && agent.config && agent.config.ocr) || {};
+      try {
+        const p = safePath(rootDir, args.path);
+        const text = await ocrImage(p, {
+          tesseract: cfg.tesseract || "tesseract",
+          lang: args.lang || cfg.lang || "chi_sim",
+          cloud: cfg.cloud || null,
+        });
+        return text || "(未识别出文字)";
+      } catch (e) {
+        return JSON.stringify({ error: "ocr_image 失败: " + e.message });
+      }
+    },
+  });
+
+  // 3. 文档入库 (RAG): 读文档 → 分块 → 存记忆 (带 scope 隔离)
+  catalog.register({
+    name: "ingest_document",
+    description: "读取文档, 分块后写入长期记忆 (RAG 入库), 之后可被语义检索命中。scope 用于隔离文档来源 (如 '公司制度'/'项目文档'), 避免与其他记忆混淆。",
+    parameters: {
+      type: "object",
+      properties: {
+        path: { type: "string", description: "文档路径 (相对工作目录)" },
+        scope: { type: "string", description: "文档来源标签 (可选, 便于按来源检索)" },
+      },
+      required: ["path"],
+    },
+    execute: async (args, ctx) => {
+      const agent = ctx && ctx.agent;
+      if (!agent || !agent.facts) return JSON.stringify({ error: "ingest_document: 缺少 agent 上下文" });
+      try {
+        const p = safePath(rootDir, args.path);
+        const text = await readDocumentText(p, ocrOptsFromConfig(agent.config));
+        const chunks = splitChunks(text, 500);
+        if (!chunks.length) return JSON.stringify({ error: "文档无可入库的文本 (可能是扫描件 PDF, 且 OCR 不可用)" });
+        let added = 0;
+        for (const c of chunks) {
+          const f = agent.facts.add(c, { source: "document", scope: args.scope || null, dedupe: false });
+          if (f) added++;
+        }
+        return JSON.stringify({ ok: true, chunks: chunks.length, added, scope: args.scope || null });
+      } catch (e) {
+        return JSON.stringify({ error: "ingest_document 失败: " + e.message });
+      }
+    },
+  });
+
+  return catalog;
+}

package/src/tools/governance.js

@@ -0,0 +1,260 @@
+// src/tools/governance.js - 记忆治理 + 审计 + 运维工具集 (吸收自 ppx-v2)
+// 来源: ppx-v2 v0.4.0 bundles/ppx-tools/index.js 的 5 个治理工具 (memory_forget/restore/export/import/clear_layer)
+//        + audit_verify, 以及 ppx-memory/ppx-selfheal 暴露的 persona_build/persona_read/selfheal_run。
+// 改写: ppx-v2 原版用 cordis 的 defineTool + @deepseek-ai/schemastery (z.object) 声明,
+//       这里改写为 ppx-agent 原生的 ToolCatalog.register 风格, 剥离全部外部依赖。
+import fs from "node:fs";
+import path from "node:path";
+import { safePath } from "./builtin.js";
+import { ensureDir, nowISO } from "../utils/store.js";
+
+// 记忆治理 + 审计 + 运维工具注册
+// deps: { rootDir, facts, audit, personaStore, healer, experience, dataDir }
+export function registerGovernanceTools(catalog, { rootDir, facts, audit, personaStore, healer, experience, dataDir } = {}) {
+  const exportsDir = path.join(rootDir || dataDir || ".", "exports");
+
+  // 1. 遗忘 (软删, 可回滚) —— ppx-agent 原版只有不可逆硬删, 这是最大的治理缺口
+  catalog.register({
+    name: "memory_forget",
+    description: "遗忘一条记忆 (软删, 数据保留可用 memory_restore 回滚)。传 id 或内容关键字定位。",
+    parameters: {
+      type: "object",
+      properties: {
+        id_or_content: { type: "string", description: "记忆 id 或内容 (按内容精确匹配, 去记忆动词前缀后比对)" },
+        reason: { type: "string", description: "遗忘原因 (记入审计, 便于后续复核)" },
+      },
+      required: ["id_or_content"],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: true,
+    execute: async (args) => {
+      if (!facts) return JSON.stringify({ error: "记忆未初始化" });
+      const f = facts.forget(args.id_or_content, { reason: args.reason || null });
+      if (!f) return JSON.stringify({ error: "未找到匹配记忆", target: args.id_or_content });
+      return JSON.stringify({ ok: true, id: f.id, content: f.content, status: f.status, note: "已软删, 可用 memory_restore 回滚" });
+    },
+  });
+
+  // 2. 回滚遗忘
+  catalog.register({
+    name: "memory_restore",
+    description: "恢复一条被遗忘 (软删) 的记忆。",
+    parameters: {
+      type: "object",
+      properties: { id: { type: "string", description: "记忆 id" } },
+      required: ["id"],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: true,
+    execute: async (args) => {
+      if (!facts) return JSON.stringify({ error: "记忆未初始化" });
+      const f = facts.restore(args.id);
+      if (!f) return JSON.stringify({ error: "未找到该记忆", id: args.id });
+      return JSON.stringify({ ok: true, id: f.id, content: f.content, status: f.status });
+    },
+  });
+
+  // 3. 查看已遗忘的记忆 (复核入口)
+  catalog.register({
+    name: "memory_list_deleted",
+    description: "列出已遗忘 (软删) 的记忆, 供人工/审计复核, 避免误删无法发现。",
+    parameters: {
+      type: "object",
+      properties: { limit: { type: "number", description: "最多返回条数, 默认 20" } },
+      required: [],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: true,
+    execute: async (args) => {
+      if (!facts) return JSON.stringify({ error: "记忆未初始化" });
+      const list = facts.deletedList().slice(0, args.limit || 20);
+      if (!list.length) return "(无已遗忘的记忆)";
+      return list.map((f) => `- ${f.id} | ${f.deletedAt || "?"} | ${f.deleteReason || "无原因"} | ${f.content}`).join("\n");
+    },
+  });
+
+  // 4. 导出记忆 (备份/迁移)
+  catalog.register({
+    name: "memory_export",
+    description: "导出全量记忆到 JSON 文件 (含软删/归档条目), 用于备份与跨机迁移。",
+    parameters: {
+      type: "object",
+      properties: {
+        file: { type: "string", description: "相对工作区的输出路径, 省略则写到 exports/memory-<时间戳>.json" },
+        include_deleted: { type: "boolean", description: "是否包含已软删/归档条目, 默认 true" },
+      },
+      required: [],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: true,
+    execute: async (args) => {
+      if (!facts) return JSON.stringify({ error: "记忆未初始化" });
+      const dump = facts.exportAll({ includeDeleted: args.include_deleted !== false });
+      let out;
+      if (args.file) {
+        out = safePath(rootDir || ".", args.file);
+      } else {
+        ensureDir(exportsDir);
+        out = path.join(exportsDir, `memory-${Date.now()}.json`);
+      }
+      ensureDir(path.dirname(out));
+      fs.writeFileSync(out, JSON.stringify(dump, null, 2), "utf8");
+      return JSON.stringify({ ok: true, file: out, count: dump.count, exportedAt: dump.exportedAt });
+    },
+  });
+
+  // 5. 导入记忆 (merge 去重 / replace 整体替换)
+  catalog.register({
+    name: "memory_import",
+    description: "从 JSON 文件导入记忆。mode=merge 按内容去重跳过重复 (默认), mode=replace 整体替换。",
+    parameters: {
+      type: "object",
+      properties: {
+        file: { type: "string", description: "相对工作区的 JSON 文件路径 (memory_export 的产物)" },
+        mode: { type: "string", description: "merge (默认) 或 replace" },
+      },
+      required: ["file"],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: false,
+    execute: async (args) => {
+      if (!facts) return JSON.stringify({ error: "记忆未初始化" });
+      const fp = safePath(rootDir || ".", args.file);
+      if (!fs.existsSync(fp)) return JSON.stringify({ error: "文件不存在", file: args.file });
+      let payload;
+      try { payload = JSON.parse(fs.readFileSync(fp, "utf8")); } catch (e) { return JSON.stringify({ error: "JSON 解析失败: " + e.message }); }
+      const r = facts.importAll(payload, { mode: args.mode === "replace" ? "replace" : "merge" });
+      return JSON.stringify(r);
+    },
+  });
+
+  // 6. 按层清空 (L1 事实 / L4 程序性记忆)
+  catalog.register({
+    name: "memory_clear_layer",
+    description: "按记忆层级批量清空。layer=1 是事实/用户记忆, layer=4 是程序性记忆(技能/流程)。默认软删可回滚, hard=true 才物理删除。",
+    parameters: {
+      type: "object",
+      properties: {
+        layer: { type: "number", description: "记忆层级: 1 (事实) 或 4 (程序性)" },
+        hard: { type: "boolean", description: "true 则物理删除不可回滚, 默认 false (软删)" },
+      },
+      required: ["layer"],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: false,
+    execute: async (args) => {
+      if (!facts) return JSON.stringify({ error: "记忆未初始化" });
+      const r = facts.clearLayer(Number(args.layer), { hard: args.hard === true });
+      return JSON.stringify({ ok: true, ...r, note: r.hard ? "已物理删除" : "已软删, 可用 memory_restore 逐条回滚" });
+    },
+  });
+
+  // 7. 审计链校验 (吸收自 ppx-v2 audit.js: SHA-256 哈希链防篡改验证)
+  catalog.register({
+    name: "audit_verify",
+    description: "校验工具调用审计日志的 SHA-256 哈希链完整性, 定位首个被篡改/截断的位置。quarantine=true 时隔离损坏段并重建空链。",
+    parameters: {
+      type: "object",
+      properties: {
+        quarantine: { type: "boolean", description: "校验失败时是否自动隔离损坏日志并重建, 默认 false" },
+        tail: { type: "number", description: "附带返回最近 n 条审计记录, 默认 0 (不返回)" },
+      },
+      required: [],
+    },
+    category: "system",
+    power: "user",
+    idempotent: true,
+    execute: async (args) => {
+      if (!audit) return JSON.stringify({ error: "审计未启用 (config.audit.enabled = false)" });
+      let v = audit.verify();
+      let quarantined = false;
+      let backup = null;
+      if (!v.ok && args.quarantine) {
+        const { quarantineBroken } = await import("../audit/audit-chain.js");
+        const q = quarantineBroken(audit.dataDir);
+        quarantined = !!q.quarantined;
+        backup = q.backup || null;
+        v = audit.verify();
+      }
+      const out = { ok: v.ok, total: v.total, brokenAt: v.brokenAt, detail: v.detail, quarantined, backup };
+      if (args.tail) out.recent = audit.tail(Number(args.tail) || 10);
+      return JSON.stringify(out);
+    },
+  });
+
+  // 8. 画像重建 (ppx-v2 ppx-memory 的 persona_build, 这里薄包装 l3 PersonaStore)
+  catalog.register({
+    name: "persona_build",
+    description: "从当前记忆与经验重新提炼用户画像 / agent 人格, 写入 L3 层。",
+    parameters: {
+      type: "object",
+      properties: {
+        target: { type: "string", description: "user (默认) 或 agent" },
+        force: { type: "boolean", description: "true 则忽略频率限制强制重建" },
+      },
+      required: [],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: false,
+    execute: async (args) => {
+      if (!personaStore) return JSON.stringify({ error: "画像存储未初始化" });
+      const target = args.target === "agent" ? "agent" : "user";
+      if (target === "agent") {
+        const lessons = experience ? (experience.list?.() || []) : [];
+        const r = personaStore.buildAgentPersona(lessons, { force: args.force === true });
+        return JSON.stringify({ ok: true, target, ...(typeof r === "object" ? r : { built: true }) });
+      }
+      const factsList = facts ? facts.list() : [];
+      const r = personaStore.buildUserPersona(factsList, { force: args.force === true });
+      return JSON.stringify({ ok: true, target, ...(typeof r === "object" ? r : { built: true }) });
+    },
+  });
+
+  // 9. 画像读取
+  catalog.register({
+    name: "persona_read",
+    description: "读取 L3 层沉淀的用户画像 / agent 人格全文。",
+    parameters: {
+      type: "object",
+      properties: { target: { type: "string", description: "user (默认) 或 agent" } },
+      required: [],
+    },
+    category: "memory",
+    power: "user",
+    idempotent: true,
+    execute: async (args) => {
+      if (!personaStore) return JSON.stringify({ error: "画像存储未初始化" });
+      const target = args.target === "agent" ? "agent" : "user";
+      const text = target === "agent" ? personaStore.agentPersona() : personaStore.userPersona();
+      return text || "(画像尚未生成, 可用 persona_build 提炼)";
+    },
+  });
+
+  // 10. 自愈体检 (ppx-v2 selfheal_run, 薄包装 Healer)
+  catalog.register({
+    name: "selfheal_run",
+    description: "手动跑一次自愈体检: 补建缺失目录、修复损坏 JSON、清理崩溃残留与过期备份。",
+    parameters: {
+      type: "object",
+      properties: {},
+      required: [],
+    },
+    category: "system",
+    power: "user",
+    idempotent: true,
+    execute: async () => {
+      if (!healer) return JSON.stringify({ error: "自愈引擎未初始化" });
+      const health = healer.heal();
+      return JSON.stringify({ ok: true, healedAt: nowISO(), health });
+    },
+  });
+
+  return catalog;
+}

package/src/tools/index.js

@@ -0,0 +1,11 @@
+// src/tools/index.js - 工具系统统一出口
+export { ToolCatalog, TOOL_ERROR_PREFIX } from "./catalog.js";
+export { normalizeMeta, runWithPolicy, toDescriptor } from "./seam.js";
+export { registerBuiltinTools } from "./builtin.js";
+export { registerAdvancedTools, Scheduler } from "./advanced.js";
+export { registerMethodTools } from "./methods.js";
+export { registerSelfmodTools } from "./selfmod.js";
+export { registerCustomTools } from "./custom.js";
+export { registerDocumentTools } from "./document.js";
+// 记忆治理 + 审计 + 运维 (吸收自 ppx-v2)
+export { registerGovernanceTools } from "./governance.js";