pi-tool-repair 0.1.3 → 0.1.4

package/README.md

@@ -168,7 +168,7 @@ Safety gates:
 
 - `requireKnownTool: true` only recovers calls whose name is in pi's active tool registry.
 - Markup inside fenced code blocks is ignored so syntax discussions and examples are preserved.
-- Incomplete or unparseable blocks are left alone.
+- Incomplete or unparseable blocks are not recovered as tool calls. For DSML, dangling or truncated marker tokens (e.g. a stream that died at `<｜DSML｜tool_calls` with no closing `>`) are still stripped from visible text so the raw marker doesn't persist in the transcript.
 - If the provider already emitted native `toolCall` blocks, leaked shadow text is stripped but duplicate calls are not added.
 
 Covered grammar families: DeepSeek DSML, MiniMax/Anthropic `<invoke>`, Qwen/Hermes `<tool_call>`, Kimi sentinels, Mistral `[TOOL_CALLS]`, Llama `<|python_tag|>`, GLM `arg_key`/`arg_value`, Granite JSON `<tool_call>`, MiniMax-Text-01 TypeScript calls, and OLMo3 `<function_calls>` pythonic calls. See [`docs/tool-call-grammar-leakage-survey.md`](./docs/tool-call-grammar-leakage-survey.md) for the survey.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pi-tool-repair",
-  "version": "0.1.3",
-  "description": "Validate-then-repair extension for pi \u2014 fixes common LLM tool-call mistakes (null fields, stringified arrays, wrong field names, anchor bleed) before tools execute",
+  "version": "0.1.4",
+  "description": "Validate-then-repair extension for pi — fixes common LLM tool-call mistakes (null fields, stringified arrays, wrong field names, anchor bleed) before tools execute",
   "type": "module",
   "author": "Tom X Nguyen",
   "license": "MIT",

package/src/grammar-repair.ts

@@ -38,6 +38,7 @@ export interface RecoveredToolCall {
 
 interface Candidate extends RecoveredToolCall {
   range: Range;
+  stripOnly?: boolean;
 }
 
 interface Range {
@@ -147,13 +148,14 @@ export function repairAssistantMessageGrammarLeaks(
     if (text === undefined) return part;
 
     const candidates = selectCandidates(parseToolGrammarCandidates(text, enabled))
-      .filter((candidate) => isAllowedTool(candidate.name, config, knownTools));
+      .filter((candidate) => candidate.stripOnly || isAllowedTool(candidate.name, config, knownTools));
 
     if (candidates.length === 0) return part;
 
     strippedRanges += candidates.length;
     changed = true;
     for (const candidate of candidates) {
+      if (candidate.stripOnly) continue;
       recoveredCalls.push({
         name: candidate.name,
         arguments: candidate.arguments,
@@ -201,16 +203,21 @@ export function repairAssistantMessageGrammarLeaks(
 
 export function parseToolGrammarLeaks(text: string, grammars: Iterable<GrammarName> = ALL_GRAMMARS): RecoveredToolCall[] {
   const enabled = new Set(grammars);
-  return selectCandidates(parseToolGrammarCandidates(text, enabled)).map((candidate) => ({
-    name: candidate.name,
-    arguments: candidate.arguments,
-    grammar: candidate.grammar,
-  }));
+  return selectCandidates(parseToolGrammarCandidates(text, enabled))
+    .filter((candidate) => !candidate.stripOnly)
+    .map((candidate) => ({
+      name: candidate.name,
+      arguments: candidate.arguments,
+      grammar: candidate.grammar,
+    }));
 }
 
 function parseToolGrammarCandidates(text: string, enabled: Set<GrammarName>): Candidate[] {
   const candidates: Candidate[] = [];
-  if (enabled.has("dsml")) candidates.push(...parseDsml(text));
+  if (enabled.has("dsml")) {
+    candidates.push(...parseDsml(text));
+    candidates.push(...parseDsmlDanglingMarkers(text));
+  }
   if (enabled.has("kimi")) candidates.push(...parseKimi(text));
   if (enabled.has("mistral")) {
     candidates.push(...parseMistral(text));
@@ -252,6 +259,28 @@ function parseDsml(text: string): Candidate[] {
   return candidates;
 }
 
+function parseDsmlDanglingMarkers(text: string): Candidate[] {
+  if (!text.includes("DSML")) return [];
+  const prefix = "(?:｜{1,2}DSML｜{1,2}|DSML｜|\\s*\\|\\s*DSML\\s*\\|\\s*)";
+  const markerRe = new RegExp(
+    `</?${prefix}(?:tool_calls|function_calls|invoke|parameter)(?:\\s+[^>\\n]*)?>?`,
+    "giu",
+  );
+  const candidates: Candidate[] = [];
+  for (const match of text.matchAll(markerRe)) {
+    if (match.index === undefined) continue;
+    if (isInsideCodeFence(text, match.index)) continue;
+    candidates.push({
+      name: "",
+      arguments: {},
+      grammar: "dsml",
+      range: { start: match.index, end: match.index + match[0].length },
+      stripOnly: true,
+    });
+  }
+  return candidates;
+}
+
 function findDsmlClose(text: string, from: number, outerName: string): Range | undefined {
   const prefix = "(?:｜{1,2}DSML｜{1,2}|DSML｜|\\s*\\|\\s*DSML\\s*\\|\\s*)";
   const closeRe = new RegExp(`</${prefix}${outerName}>`, "giu");