llmparse-py 0.1.1__py3-none-any.whl

llmparse/__init__.py

@@ -0,0 +1,12 @@
+"""llmparse — parse the JSON your LLM meant to send.
+
+Zero-dependency extraction and repair for JSON embedded in raw LLM
+output: markdown fences, prose, single quotes, trailing commas,
+Python literals, comments and truncated streams.
+"""
+
+from .core import LLMParseError, extract, loads, parse
+from .repair import repair
+
+__version__ = "0.1.1"
+__all__ = ["LLMParseError", "extract", "loads", "parse", "repair", "__version__"]

llmparse/core.py

@@ -0,0 +1,93 @@
+"""Extraction and parsing: find the JSON inside LLM output and load it."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .repair import RULES, repair
+from .scanner import find_balanced_end
+
+__all__ = ["LLMParseError", "extract", "parse", "loads", "repair"]
+
+
+class LLMParseError(ValueError):
+    """Raised when no JSON value can be recovered from the text."""
+
+
+def _try_loads(text: str) -> tuple[bool, Any]:
+    try:
+        return True, json.loads(text)
+    except (ValueError, TypeError):
+        return False, None
+
+
+def _load_with_repairs(text: str) -> tuple[bool, Any]:
+    """Try plain parsing, then re-try after each repair stage."""
+    ok, value = _try_loads(text.strip())
+    if ok:
+        return True, value
+    current = text
+    for rule in RULES:
+        current = rule(current)
+        ok, value = _try_loads(current.strip())
+        if ok:
+            return True, value
+    return False, None
+
+
+def extract(text: str) -> str | None:
+    """Return the first recoverable JSON substring of ``text``, or None.
+
+    Candidates start at every ``{`` or ``[`` outside the previous
+    candidate; each candidate is cut at its balanced end (or at end of
+    text when truncated) and tested with the full repair pipeline.
+    """
+    if not text:
+        return None
+    ok, _ = _load_with_repairs(text)
+    if ok:
+        return repair(text).strip() if "```" in text else text.strip()
+    for start, ch in enumerate(text):
+        if ch not in "{[":
+            continue
+        end = find_balanced_end(text, start)
+        candidate = text[start:end] if end is not None else text[start:]
+        ok, _ = _load_with_repairs(candidate)
+        if ok:
+            return candidate
+    return None
+
+
+_SENTINEL = object()
+
+
+def parse(text: str, *, default: Any = _SENTINEL) -> Any:
+    """Parse JSON out of raw LLM output.
+
+    Handles markdown fences, surrounding prose, single quotes, bare
+    keys, Python literals, comments, trailing commas and truncation.
+    Returns ``default`` when given and nothing can be recovered;
+    otherwise raises :class:`LLMParseError`.
+    """
+    if isinstance(text, (dict, list)):
+        return text
+    if not isinstance(text, str) or not text.strip():
+        if default is not _SENTINEL:
+            return default
+        raise LLMParseError("no JSON found: empty input")
+    ok, value = _load_with_repairs(text)
+    if ok:
+        return value
+    candidate = extract(text)
+    if candidate is not None:
+        ok, value = _load_with_repairs(candidate)
+        if ok:
+            return value
+    if default is not _SENTINEL:
+        return default
+    raise LLMParseError(f"could not recover JSON from: {text[:120]!r}")
+
+
+# Familiar alias for users coming from the json module.
+loads = parse

llmparse/repair.py

@@ -0,0 +1,192 @@
+"""Repair rules for malformed JSON commonly produced by LLMs.
+
+Each rule is a small ``str -> str`` function that fixes one class of
+damage and is safe to apply to already-valid JSON (it must not change
+valid input). Rules run in the order listed in ``RULES``; the parser
+tries ``json.loads`` after every stage and stops as soon as it works.
+
+Adding a rule is the easiest meaningful contribution to llmparse:
+write the function, register it in ``RULES``, and add a test case with
+a real-world-style broken input. See CONTRIBUTING.md.
+"""
+
+from __future__ import annotations
+
+import re
+
+from .scanner import in_string_at_end, iter_chars, unclosed_brackets
+
+
+def strip_code_fences(text: str) -> str:
+    """Remove surrounding markdown ``` fences, if present."""
+    stripped = text.strip()
+    if "```" not in stripped:
+        return text
+    match = re.search(r"```(?:json|JSON)?\s*\n?(.*?)(?:```|$)", stripped, re.DOTALL)
+    return match.group(1).strip() if match else text
+
+
+def remove_comments(text: str) -> str:
+    """Remove ``//`` line and ``/* */`` block comments outside strings."""
+    out: list[str] = []
+    i = 0
+    in_string = False
+    quote = ""
+    escaped = False
+    while i < len(text):
+        ch = text[i]
+        if in_string:
+            out.append(ch)
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == quote:
+                in_string = False
+            i += 1
+            continue
+        if ch in ("'", '"'):
+            in_string, quote = True, ch
+            out.append(ch)
+            i += 1
+            continue
+        if ch == "/" and i + 1 < len(text) and text[i + 1] == "/":
+            while i < len(text) and text[i] != "\n":
+                i += 1
+            continue
+        if ch == "/" and i + 1 < len(text) and text[i + 1] == "*":
+            i += 2
+            while i + 1 < len(text) and not (text[i] == "*" and text[i + 1] == "/"):
+                i += 1
+            i += 2
+            continue
+        out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+def single_to_double_quotes(text: str) -> str:
+    """Convert single-quoted strings to double-quoted strings."""
+    out: list[str] = []
+    in_string = False
+    quote = ""
+    escaped = False
+    for ch in text:
+        if in_string:
+            if escaped:
+                out.append(ch)
+                escaped = False
+            elif ch == "\\":
+                out.append(ch)
+                escaped = True
+            elif ch == quote:
+                out.append('"')
+                in_string = False
+            elif ch == '"' and quote == "'":
+                out.append('\\"')
+            else:
+                out.append(ch)
+        else:
+            if ch in ("'", '"'):
+                in_string, quote = True, ch
+                out.append('"')
+            else:
+                out.append(ch)
+    return "".join(out)
+
+
+_LITERAL_MAP = {
+    "True": "true",
+    "False": "false",
+    "None": "null",
+    "undefined": "null",
+    "NaN": "null",
+    "Infinity": "null",
+    "-Infinity": "null",
+}
+_LITERAL_RE = re.compile(r"\b(True|False|None|undefined|NaN|Infinity|-Infinity)\b")
+
+
+def fix_literals(text: str) -> str:
+    """Replace Python/JavaScript literals with JSON ones, outside strings."""
+    out: list[str] = []
+    buffer: list[str] = []
+    for _, ch, in_string, _ in iter_chars(text):
+        if in_string:
+            if buffer:
+                out.append(_LITERAL_RE.sub(lambda m: _LITERAL_MAP[m.group(1)], "".join(buffer)))
+                buffer = []
+            out.append(ch)
+        else:
+            buffer.append(ch)
+    if buffer:
+        out.append(_LITERAL_RE.sub(lambda m: _LITERAL_MAP[m.group(1)], "".join(buffer)))
+    return "".join(out)
+
+
+_UNQUOTED_KEY_RE = re.compile(r"([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)(\s*:)")
+
+
+def quote_unquoted_keys(text: str) -> str:
+    """Quote bare identifier keys: ``{name: 1}`` -> ``{"name": 1}``."""
+    out: list[str] = []
+    buffer: list[str] = []
+    for _, ch, in_string, _ in iter_chars(text):
+        if in_string:
+            if buffer:
+                out.append(_UNQUOTED_KEY_RE.sub(r'\1"\2"\3', "".join(buffer)))
+                buffer = []
+            out.append(ch)
+        else:
+            buffer.append(ch)
+    if buffer:
+        out.append(_UNQUOTED_KEY_RE.sub(r'\1"\2"\3', "".join(buffer)))
+    return "".join(out)
+
+
+def remove_trailing_commas(text: str) -> str:
+    """Drop commas that immediately precede ``}`` or ``]`` outside strings."""
+    chars = [(ch, in_string) for _, ch, in_string, _ in iter_chars(text)]
+    out: list[str] = []
+    for idx, (ch, in_string) in enumerate(chars):
+        if ch == "," and not in_string:
+            lookahead = idx + 1
+            while lookahead < len(chars) and chars[lookahead][0] in " \t\r\n":
+                lookahead += 1
+            if lookahead < len(chars) and chars[lookahead][0] in "}]":
+                continue
+        out.append(ch)
+    return "".join(out)
+
+
+def close_truncated(text: str) -> str:
+    """Close an unterminated string and any unclosed brackets, in order."""
+    result = text.rstrip()
+    if not result:
+        return result
+    # A dangling escape at the very end would swallow the closing quote.
+    if result.endswith("\\") and not result.endswith("\\\\"):
+        result = result[:-1]
+    closers = ""
+    if in_string_at_end(result):
+        closers += '"'
+    closers += "".join(unclosed_brackets(result))
+    return result + closers
+
+
+RULES = [
+    strip_code_fences,
+    remove_comments,
+    single_to_double_quotes,
+    fix_literals,
+    quote_unquoted_keys,
+    remove_trailing_commas,
+    close_truncated,
+]
+
+
+def repair(text: str) -> str:
+    """Apply every repair rule in order and return the repaired text."""
+    for rule in RULES:
+        text = rule(text)
+    return text

llmparse/scanner.py

@@ -0,0 +1,92 @@
+"""Character scanners shared by extraction and repair.
+
+Everything here walks the text once while tracking whether we are
+inside a string, so brackets or comment markers inside string values
+are never mistaken for structure. Strings may be double-quoted,
+single-quoted, or (in truncated output) unterminated.
+"""
+
+from __future__ import annotations
+
+
+def iter_chars(text: str):
+    """Yield ``(index, char, in_string, quote_char)`` for each position.
+
+    ``in_string`` describes the state *at* the character: opening and
+    closing quote characters themselves are yielded with
+    ``in_string == False`` for the opener and ``True`` for the closer.
+    Backslash escapes inside strings are honoured.
+    """
+    in_string = False
+    quote = ""
+    escaped = False
+    for i, ch in enumerate(text):
+        if in_string:
+            yield i, ch, True, quote
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == quote:
+                in_string = False
+                quote = ""
+        else:
+            yield i, ch, False, ""
+            if ch in ("'", '"'):
+                in_string = True
+                quote = ch
+
+
+def find_balanced_end(text: str, start: int) -> int | None:
+    """Return the index just past the bracket matching ``text[start]``.
+
+    ``text[start]`` must be ``{`` or ``[``. Returns ``None`` when the
+    structure is truncated (no matching close before end of text).
+    """
+    pairs = {"{": "}", "[": "]"}
+    stack: list[str] = []
+    for i, ch, in_string, _ in iter_chars(text):
+        if i < start or in_string:
+            continue
+        if ch in pairs:
+            stack.append(pairs[ch])
+        elif ch in ("}", "]"):
+            if not stack or stack[-1] != ch:
+                return None
+            stack.pop()
+            if not stack:
+                return i + 1
+    return None
+
+
+def unclosed_brackets(text: str) -> list[str]:
+    """Return the closing brackets needed to balance ``text``, in order."""
+    pairs = {"{": "}", "[": "]"}
+    stack: list[str] = []
+    for _, ch, in_string, _ in iter_chars(text):
+        if in_string:
+            continue
+        if ch in pairs:
+            stack.append(pairs[ch])
+        elif ch in ("}", "]") and stack and stack[-1] == ch:
+            stack.pop()
+    return list(reversed(stack))
+
+
+def in_string_at_end(text: str) -> bool:
+    """True when ``text`` ends inside an unterminated string."""
+    in_string = False
+    quote = ""
+    escaped = False
+    for ch in text:
+        if in_string:
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == quote:
+                in_string = False
+        elif ch in ("'", '"'):
+            in_string = True
+            quote = ch
+    return in_string

llmparse_py-0.1.1.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: llmparse-py
+Version: 0.1.1
+Summary: Parse the JSON your LLM meant to send — zero-dependency extraction and repair for raw LLM output.
+Author: gptfei
+License: MIT
+Project-URL: Homepage, https://github.com/gptfei/llmparse
+Project-URL: Repository, https://github.com/gptfei/llmparse
+Project-URL: Issues, https://github.com/gptfei/llmparse/issues
+Keywords: llm,json,parser,repair,openai,chatgpt,claude,structured-output
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# llmparse
+
+**Parse the JSON your LLM meant to send.**
+
+LLMs promise JSON and deliver markdown fences, apologies, single
+quotes, trailing commas, Python literals — and, when the token budget
+runs out, JSON that stops mid-string. `llmparse` is a zero-dependency
+Python library that extracts the JSON from raw model output and repairs
+the damage, so `json.loads` stops being the most fragile line in your
+pipeline.
+
+```python
+from llmparse import parse
+
+parse('Sure! Here you go:\n```json\n{name: "ada", tags: ["x", "y",], ok: True}\n```')
+# {'name': 'ada', 'tags': ['x', 'y'], 'ok': True}
+
+parse('{"name": "ada", "tags": ["x", "y"')   # truncated stream
+# {'name': 'ada', 'tags': ['x', 'y']}
+```
+
+## Install
+
+```bash
+pip install llmparse-py     # once published to PyPI
+pip install git+https://github.com/gptfei/llmparse
+```
+
+Python 3.9+. No third-party dependencies, ever.
+
+## What it fixes
+
+| Damage in LLM output | Example |
+| --- | --- |
+| Markdown fences | ` ```json ... ``` ` |
+| Prose around the JSON | `Here is the result: {...} hope it helps` |
+| Single-quoted strings | `{'a': 'b'}` |
+| Unquoted keys | `{name: "ada"}` |
+| Python / JS literals | `True`, `None`, `undefined`, `NaN` |
+| Comments | `{"a": 1} // note` |
+| Trailing commas | `{"a": [1, 2,]}` |
+| Truncation | `{"msg": "hello wor` → closed and parsed |
+
+Brackets and comment markers *inside* string values are never touched.
+
+## API
+
+- `parse(text, default=...)` — return the recovered Python object.
+  Raises `LLMParseError` when nothing is recoverable, unless `default`
+  is given. `loads` is an alias.
+- `extract(text)` — return the first recoverable JSON substring, or `None`.
+- `repair(text)` — return repaired JSON *text* (for when you need the
+  string, not the object).
+
+Valid JSON is returned untouched: every repair rule is a no-op on
+input that already parses.
+
+## Contributing
+
+The repair pipeline is a list of small, independent rules — adding one
+is the easiest meaningful contribution in open source right now:
+
+1. Find a way an LLM broke JSON that llmparse does not fix yet.
+2. Write the rule in `src/llmparse/repair.py`, register it in `RULES`.
+3. Add a test with the broken input.
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) and issues labelled
+`good first issue`.
+
+## 中文简介
+
+大模型说好返回 JSON，实际经常带 markdown 围栏、解释文字、单引号、
+尾随逗号、Python 的 `True/None`，甚至输出到一半被截断。`llmparse`
+是一个零依赖的 Python 库：从原始输出里提取 JSON、逐项修复这些损坏、
+再解析成 Python 对象。合法 JSON 不会被改动。
+
+## License
+
+MIT — see [LICENSE](LICENSE).

llmparse_py-0.1.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+llmparse/__init__.py,sha256=YbcAqlgvNMQTRghyvOQP410hx3Vr_OtiIXzi1nq2exI,428
+llmparse/core.py,sha256=UUp4659XQwAumRHSwKS8DZXYcFp1EhhMTUigHzs9tHM,2774
+llmparse/repair.py,sha256=h10sSYRKoVMBXnGNlYFllEcBHOADQxxRWOhn4QV9c-8,5851
+llmparse/scanner.py,sha256=ocIMyCSeiq-Cc_fno-2Cy1gflmXf5B0TLWg8H4IfMd8,2856
+llmparse_py-0.1.1.dist-info/licenses/LICENSE,sha256=Cj0XVFIO6j89kpPNcfpffKbP5HMCbWcewAPMdY8yNVA,1063
+llmparse_py-0.1.1.dist-info/METADATA,sha256=1ooBStwl7afNJpkY23tLomNioNw6r-LoySjMESaj11k,3505
+llmparse_py-0.1.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+llmparse_py-0.1.1.dist-info/top_level.txt,sha256=DpTx5hVsGMrhzNpXRrrWbQcYUtEihokf2IyX2P-MGeM,9
+llmparse_py-0.1.1.dist-info/RECORD,,

llmparse_py-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

llmparse_py-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 gptfei
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llmparse_py-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+llmparse