prefix-safe-json 0.0.1-alpha.0

package/LICENSE-APACHE

@@ -0,0 +1,114 @@
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity authorized by
+   the copyright owner that is granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form.
+
+   "Work" shall mean the work of authorship made available under the License.
+
+   "Contribution" shall mean any work of authorship submitted to the
+   Licensor for inclusion in the Work.
+
+   "Contributor" shall mean Licensor and any Legal Entity on behalf of whom
+   a Contribution has been received by the Licensor.
+
+2. Grant of Copyright License.
+
+   Subject to the terms and conditions of this License, each Contributor
+   hereby grants to You a perpetual, worldwide, non-exclusive, no-charge,
+   royalty-free, irrevocable copyright license to reproduce, prepare
+   Derivative Works of, publicly display, publicly perform, sublicense,
+   and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+   Subject to the terms and conditions of this License, each Contributor
+   hereby grants to You a perpetual, worldwide, non-exclusive, no-charge,
+   royalty-free, irrevocable patent license to make, have made, use, offer
+   to sell, sell, import, and otherwise transfer the Work.
+
+4. Redistribution.
+
+   You may reproduce and distribute copies of the Work or Derivative Works
+   thereof in any medium, with or without modifications, and in Source or
+   Object form, provided that You meet the following conditions:
+
+   (a) You must give any other recipients of the Work or Derivative Works
+       a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices stating
+       that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works that You
+       distribute, all copyright, patent, trademark, and attribution notices
+       from the Source form of the Work; and
+
+   (d) If the Work includes a "NOTICE" text file, You must include a readable
+       copy of the attribution notices contained within such NOTICE file.
+
+5. Submission of Contributions.
+
+   Unless You explicitly state otherwise, any Contribution intentionally
+   submitted for inclusion in the Work by You to the Licensor shall be
+   under the terms and conditions of this License, without any additional
+   terms or conditions.
+
+6. Trademarks.
+
+   This License does not grant permission to use the trade names, trademarks,
+   service marks, or product names of the Licensor.
+
+7. Disclaimer of Warranty.
+
+   Unless required by applicable law or agreed to in writing, Licensor
+   provides the Work on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+   OF ANY KIND.
+
+8. Limitation of Liability.
+
+   In no event and under no legal theory shall any Contributor be liable to
+   You for damages, including any direct, indirect, special, incidental, or
+   consequential damages of any character arising as a result of this License
+   or out of the use or inability to use the Work.
+
+9. Accepting Warranty or Additional Liability.
+
+   While redistributing the Work or Derivative Works thereof, You may choose
+   to offer, and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations consistent with this License.
+
+END OF TERMS AND CONDITIONS
+
+Copyright (c) 2026 Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

package/LICENSE-MIT

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,169 @@
+# prefix-safe-json
+
+> ⚠️ **Alpha** — This project is under active development. APIs may change without notice. Do not use in production.
+
+Incremental LLM tool-call JSON parser and deterministic repair engine.
+
+## Problem
+
+LLM providers (OpenAI, Anthropic, Gemini, OpenRouter) stream tool-call arguments as small JSON chunks. During streaming, the JSON is frequently incomplete: strings split mid-word, UTF-8 characters split mid-byte, containers left unclosed, numbers terminated at chunk boundaries.
+
+This library parses each chunk incrementally and provides:
+
+- **Committed values**: Fields whose values are definitively complete
+- **Pending tokens**: Partially received values awaiting more data
+- **Monotonic semantic events**: JSON Pointer-based events that are never retracted
+- **Parse diagnostics**: Structured error reporting with severity and recoverability
+- **Deterministic repair**: When the stream ends, a clear accounting of what was repaired and why
+- **Execution safety**: Whether the parsed result is safe to use for tool invocation
+
+## Core Principle
+
+> The parser never fabricates missing data. It clearly distinguishes what is definite, what is pending, what was deterministically repaired, and what is lost.
+
+## Why not a general "partial JSON" parser?
+
+Most streaming JSON helpers (including ones already used in the wild for
+LLM tool-call arguments) are designed to show *something* as early as
+possible, even if that means guessing. Concretely, given a stream that has
+so far delivered `{"city":"Tok` (the model is still typing "Tokyo"):
+
+| | `city` field after this chunk |
+|---|---|
+| [vercel/ai's `fixJson`](https://github.com/vercel/ai/blob/main/packages/ai/src/util/fix-json.ts) | `"Tok"` — closes the open string as-is and reports a successful parse |
+| [langchain's `parsePartialJson`](https://github.com/langchain-ai/langchainjs/blob/main/libs/langchain-core/src/utils/json.ts) | `"Tok"` — its recursive-descent parser returns whatever string content it collected before running out of input |
+| **this library** (`snapshot().stableValue`) | *(absent — `city` is not committed yet)* |
+
+Both of those are real, current, well-engineered implementations — verified
+by cloning `vercel/ai` (commit `fbb154a0`) and `langchain-ai/langchainjs`
+(commit `555d6f14`) and running their actual code side-by-side with this
+library. Neither is a bug: they're built for progressively rendering a
+value in a UI, where showing `"Tok"` and then `"Tokyo"` a moment later is
+good UX. This library is built for the different question — *is it safe to
+act on this value yet* — where treating `"Tok"` as the city would be
+wrong. It's why `city` only appears in `stableValue` once its closing
+quote genuinely arrives, and why `executable` only becomes `true` once the
+whole document is unambiguously, definitively complete.
+
+If you need live "filling in..." UI text, a partial-JSON renderer is the
+right tool. If you need to know precisely when it's safe to execute a tool
+call with the parsed arguments, that's what this library is for.
+
+### A concrete case where this matters: silent execution on truncated data
+
+The risk above isn't hypothetical. [Cline](https://github.com/cline/cline)
+(a coding agent that writes files and runs terminal commands from LLM tool
+calls — verified by cloning it, commit `81cce3d70e1`) wires a JSON-repair
+step directly into tool execution:
+[`sdk/packages/llms/src/providers/ai-sdk.ts:1332`](https://github.com/cline/cline/blob/81cce3d70e10244cdde40dbd0eb0bb711c93006d/sdk/packages/llms/src/providers/ai-sdk.ts#L1332)
+registers `repairMalformedToolCall` as the Vercel AI SDK's
+`experimental_repairToolCall` hook — whatever it returns *is* the tool call
+that runs. That function's own docstring names its target: "truncated
+payloads ... common with weaker models." Its repair path
+([`sdk/packages/shared/src/parse/json.ts:42`](https://github.com/cline/cline/blob/81cce3d70e10244cdde40dbd0eb0bb711c93006d/sdk/packages/shared/src/parse/json.ts#L42))
+runs the raw argument text through
+[`jsonrepair`](https://github.com/josdejong/jsonrepair), which — like the
+two implementations above — closes an unterminated string as-is rather
+than rejecting it.
+
+Feeding `parseJsonStream` (copied verbatim from that file) and this
+library the same truncated `write_file` tool call — a `content` argument
+cut off mid-value, the exact scenario the docstring describes:
+
+```
+input: {"path":"config/database.yml","content":"production:\n  host: db.prod.internal\n  password: correct-horse-battery-sta
+
+Cline's parseJsonStream() -> jsonrepair:
+  { "path": "config/database.yml",
+    "content": "production:\n  host: db.prod.internal\n  password: correct-horse-battery-sta" }
+  No error, no warning. This is what gets written to disk.
+
+This library:
+  outcome: "truncated", executable: false
+  stableValue: { "path": "config/database.yml" }   // content is absent, not guessed
+```
+
+To be clear about scope: `jsonrepair` also fixes things this library
+intentionally doesn't (single quotes, trailing commas, unquoted keys) — it
+isn't a strict RFC 8259 parser and isn't trying to be, so this library
+can't simply replace it there. The gap is narrower and specific: nothing
+in that repair path currently distinguishes *"the model's JSON syntax is
+wrong"* from *"the model's JSON was cut off mid-value and the rest is
+unknown"* — `jsonrepair` handles both the same way, silently. This library
+draws exactly that distinction (`outcome: "truncated"`) and could gate
+execution on it without touching the syntax-repair behavior for the other
+case.
+
+## Current Status (Alpha)
+
+### Implemented
+
+- Incremental UTF-8 decoder with split-byte handling
+- Lexical JSON state machine (17 states)
+- Grammar stack with object/array tracking
+- Semantic event emission for committed scalar values
+- Duplicate key detection and rejection
+- `push()` / `snapshot()` / `drainEvents()` / `finish()` API
+- Configurable resource limits (depth, input size, string length, queued events)
+- Machine-readable test corpus with 25+ canonical fixtures
+- Chunk invariance verification
+- Provider stream adapters for OpenAI (legacy `function_call` and Responses API), Anthropic, Gemini, OpenRouter, and generic OpenAI-compatible endpoints, plus a coordinator for tracking multiple concurrent tool-call streams
+
+### Not Yet Implemented
+
+- Advanced repair engine (structural/lossy repairs)
+- JSON Schema validation
+- CLI tool
+- Network/SSE client integration
+
+## Installation
+
+```bash
+npm install prefix-safe-json
+```
+
+## Quick Start
+
+```typescript
+import { createParser } from "prefix-safe-json";
+
+const parser = createParser();
+
+// Feed chunks as they arrive from the LLM stream
+parser.push('{"tool":"calc",');
+parser.push('"args":{"x":42}}');
+
+// Get committed values at any point
+const snap = parser.snapshot();
+console.log(snap.stableValue);
+
+// Drain semantic events
+const events = parser.drainEvents();
+
+// Signal stream completion
+const result = parser.finish({ reason: "complete" });
+console.log(result.executable); // true if safe to execute
+```
+
+### End-to-end example (real API, no mocked internals)
+
+The snippet above is illustrative; for a runnable, verified demonstration
+against the actual provider-adapter + coordinator public API — including
+the exact truncation scenario described above — see
+[`examples/anthropic-truncation-safety.mjs`](examples/anthropic-truncation-safety.mjs)
+(`node examples/anthropic-truncation-safety.mjs` after `pnpm run build`).
+It feeds `AnthropicStreamAdapter` a realistic Anthropic Messages API SSE
+sequence — including one cut off mid-argument by `max_tokens`, matching
+Anthropic's real streaming shape — through `createToolCallStreamCoordinator()`,
+and asserts the truncated call is never reported executable while a
+genuinely complete one is. This example runs in CI on every push, so it
+can't silently rot into a stale claim.
+
+## License
+
+Licensed under either of:
+
+- MIT License ([LICENSE-MIT](LICENSE-MIT))
+- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE))
+
+at your option.

package/dist/coordinator/coordinator.d.ts

@@ -0,0 +1,43 @@
+import { ParserOptions } from "../types.js";
+import { NormalizedToolStreamEvent, StreamEndReason } from "./protocol.js";
+import { ToolCallStreamCoordinator, ToolCallCoordinatorSnapshot, ToolCallCoordinatorEvent, ToolCallCoordinatorFinalResult, CoordinatorPushResult, CoordinatorLimits } from "./types.js";
+export declare class DefaultToolCallStreamCoordinator implements ToolCallStreamCoordinator {
+    private readonly limits;
+    private readonly parserOptions;
+    private calls;
+    private sourceKeyToInternalId;
+    private callCounter;
+    private eventQueue;
+    private diagnostics;
+    private eventsProcessed;
+    private globalSequence;
+    private isFinished;
+    /**
+     * @param limits Coordinator-level limits (concurrent tool calls, tool name
+     *   size, etc.) — distinct from `parserOptions`, which configures each
+     *   underlying per-call IncrementalJsonParser (limits, repairs). Previously
+     *   there was no way to pass the latter through at all; every parser the
+     *   coordinator created was hardcoded to library defaults.
+     */
+    constructor(limits?: Partial<CoordinatorLimits>, parserOptions?: ParserOptions);
+    push(event: NormalizedToolStreamEvent): CoordinatorPushResult;
+    private handleStart;
+    private handleIdentity;
+    private handleNameDelta;
+    private handleArgumentsDelta;
+    private handleCallEnd;
+    private finishCall;
+    private hasCallConflict;
+    private handleProviderDiagnostic;
+    private handleStreamEnd;
+    private getCall;
+    private addDiagnostic;
+    snapshot(): ToolCallCoordinatorSnapshot;
+    drainEvents(): readonly ToolCallCoordinatorEvent[];
+    finish(meta?: {
+        reason?: StreamEndReason;
+        providerReason?: string;
+    }): ToolCallCoordinatorFinalResult;
+}
+export declare function createToolCallStreamCoordinator(limits?: Partial<CoordinatorLimits>, parserOptions?: ParserOptions): ToolCallStreamCoordinator;
+//# sourceMappingURL=coordinator.d.ts.map

package/dist/coordinator/coordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/coordinator/coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,aAAa,EAAE,MAAM,aAAa,CAAC;AAEnE,OAAO,EAAE,yBAAyB,EAAgB,eAAe,EAAE,MAAM,eAAe,CAAC;AACzF,OAAO,EACL,yBAAyB,EACzB,2BAA2B,EAC3B,wBAAwB,EACxB,8BAA8B,EAC9B,qBAAqB,EACrB,iBAAiB,EAIlB,MAAM,YAAY,CAAC;AAwBpB,qBAAa,gCAAiC,YAAW,yBAAyB;IAChF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA4B;IAC1D,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,qBAAqB,CAAkC;IAC/D,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,WAAW,CAA+B;IAClD,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,UAAU,CAAS;IAE3B;;;;;;OAMG;gBACS,MAAM,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,EAAE,aAAa,CAAC,EAAE,aAAa;IAK9E,IAAI,CAAC,KAAK,EAAE,yBAAyB,GAAG,qBAAqB;IAgD7D,OAAO,CAAC,WAAW;IAqDnB,OAAO,CAAC,cAAc;IAgDtB,OAAO,CAAC,eAAe;IAqCvB,OAAO,CAAC,oBAAoB;IA8B5B,OAAO,CAAC,aAAa;IA6BrB,OAAO,CAAC,UAAU;IA8ClB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,wBAAwB;IAUhC,OAAO,CAAC,eAAe;IAyBvB,OAAO,CAAC,OAAO;IAcf,OAAO,CAAC,aAAa;IASrB,QAAQ,IAAI,2BAA2B;IAkBvC,WAAW,IAAI,SAAS,wBAAwB,EAAE;IAMlD,MAAM,CAAC,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,eAAe,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,8BAA8B;CAcrG;AAED,wBAAgB,+BAA+B,CAC7C,MAAM,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,EACnC,aAAa,CAAC,EAAE,aAAa,GAC5B,yBAAyB,CAE3B"}