llmclean 0.1.0__py3-none-any.whl

llmclean/__init__.py

@@ -0,0 +1,18 @@
+"""
+llmclean — utilities for cleaning and normalizing raw LLM output.
+
+Quick start::
+
+    from llmclean import strip_fences, enforce_json, trim_repetition
+
+    clean = strip_fences(raw_output)
+    data  = enforce_json(raw_output)
+    text  = trim_repetition(raw_output)
+"""
+
+from .fences import strip_fences
+from .json_utils import enforce_json
+from .repetition import trim_repetition
+
+__all__ = ["strip_fences", "enforce_json", "trim_repetition"]
+__version__ = "0.1.0"

llmclean/fences.py

@@ -0,0 +1,155 @@
+"""
+fences.py — strip markdown code fences from LLM output.
+
+LLMs frequently wrap their output in fences like:
+    ```json
+    { "key": "value" }
+    ```
+
+This module handles:
+  - Named fences:      ```json ... ```
+  - Anonymous fences:  ``` ... ```
+  - Tilde fences:      ~~~ ... ~~~
+  - Indented fences:   leading whitespace before the backticks
+  - Multiple fences in one string (strips all of them)
+  - Nested / back-to-back fences
+  - Fences with no closing marker (strips opening line, returns rest)
+  - Stray language tags left on their own line after stripping
+"""
+
+import re
+
+# ---------------------------------------------------------------------------
+# Patterns
+# ---------------------------------------------------------------------------
+
+# Matches an opening fence line, e.g.  ```json  or  ```  or  ~~~python
+# Group 1: the fence characters (``` or ~~~)
+# Group 2: optional language identifier (may be empty)
+_OPEN_FENCE_RE = re.compile(
+    r"^[ \t]*(?P<fence>`{3,}|~{3,})[ \t]*(?P<lang>[a-zA-Z0-9_+\-.]*)[ \t]*$",
+    re.MULTILINE,
+)
+
+# Matches a closing fence: same or more fence characters on its own line
+_CLOSE_FENCE_RE = re.compile(
+    r"^[ \t]*(?P<fence>`{3,}|~{3,})[ \t]*$",
+    re.MULTILINE,
+)
+
+# After all fences are stripped, clean up lone language-tag lines that were
+# sitting right above the content, e.g. a line that is just "json" or "python"
+# This is intentionally narrow to avoid removing real content.
+_LONE_LANG_TAG_RE = re.compile(
+    r"^[ \t]*(?:json|python|javascript|js|typescript|ts|bash|sh|shell|"
+    r"html|xml|css|yaml|yml|toml|ini|markdown|md|text|txt|plaintext|"
+    r"sql|graphql|rust|go|java|c|cpp|c\+\+|csharp|cs|ruby|rb|php|"
+    r"swift|kotlin|scala|r|lua|perl|haskell|elixir|erlang|clojure|"
+    r"dart|objc|output|console|log)[ \t]*$",
+    re.MULTILINE | re.IGNORECASE,
+)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def strip_fences(text: str) -> str:
+    """Remove all markdown/tilde code fences from *text*.
+
+    Strips every fenced block found, replacing each fence with the raw
+    content that was inside it.  If no fences are present the original
+    text is returned unchanged.  The function never raises.
+
+    Parameters
+    ----------
+    text:
+        Raw LLM output that may contain one or more fenced code blocks.
+
+    Returns
+    -------
+    str
+        The cleaned text with fence markers removed and inner content
+        preserved.  Leading/trailing whitespace that was *outside* all
+        fences is also stripped.
+    """
+    if not isinstance(text, str):
+        return text  # defensive: wrong type → return as-is
+
+    original = text
+
+    try:
+        result = _strip_all_fences(text)
+        # Collapse any excessive blank lines introduced by fence removal
+        result = _normalize_blank_lines(result)
+        return result
+    except Exception:
+        # Safety net: never crash the caller
+        return original
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+def _strip_all_fences(text: str) -> str:
+    """Iteratively strip fences until no more remain."""
+    # We loop because fences can be back-to-back; each pass removes one layer.
+    for _ in range(50):  # guard against pathological inputs
+        new_text = _strip_one_pass(text)
+        if new_text == text:
+            break
+        text = new_text
+
+    # Clean up orphaned language-tag lines (e.g. a bare "json" line)
+    text = _LONE_LANG_TAG_RE.sub("", text)
+    return text.strip()
+
+
+def _strip_one_pass(text: str) -> str:
+    """Remove every complete (or unclosed) fence block found in *text*."""
+    result_parts: list[str] = []
+    cursor = 0
+
+    for open_match in _OPEN_FENCE_RE.finditer(text):
+        open_start = open_match.start()
+        open_end = open_match.end()
+        fence_chars = open_match.group("fence")
+        fence_type = fence_chars[0]  # ` or ~
+
+        # Append everything before this opening fence
+        result_parts.append(text[cursor:open_start])
+
+        # Look for a matching closing fence (same fence character, >= same length)
+        close_match = _find_closing_fence(text, open_end, fence_type, len(fence_chars))
+
+        if close_match:
+            # Content between open and close fences
+            inner = text[open_end:close_match.start()]
+            result_parts.append(inner)
+            cursor = close_match.end()
+        else:
+            # Unclosed fence: drop the opening line, keep the rest of the text
+            inner = text[open_end:]
+            result_parts.append(inner)
+            cursor = len(text)
+            break  # nothing left to process
+
+    # Append any trailing text after the last fence
+    result_parts.append(text[cursor:])
+
+    return "".join(result_parts)
+
+
+def _find_closing_fence(text: str, search_from: int, fence_type: str, min_len: int):
+    """Return the first closing fence match at or after *search_from*."""
+    for m in _CLOSE_FENCE_RE.finditer(text, search_from):
+        fc = m.group("fence")
+        if fc[0] == fence_type and len(fc) >= min_len:
+            return m
+    return None
+
+
+def _normalize_blank_lines(text: str) -> str:
+    """Collapse 3+ consecutive blank lines down to 2."""
+    return re.sub(r"\n{3,}", "\n\n", text)

llmclean/json_utils.py

@@ -0,0 +1,354 @@
+"""
+json_utils.py — extract and repair valid JSON from messy LLM output.
+
+LLMs routinely return JSON wrapped in prose, fences, or with small syntax
+errors.  This module tries a pipeline of increasingly aggressive strategies
+to get back a valid, parse-able JSON string.
+
+Strategy pipeline (stops at first success):
+  1. Parse as-is (already valid JSON)
+  2. Strip fences then parse
+  3. Strip leading/trailing prose, leaving only the JSON substring
+     (handles "Sure! Here is your JSON: {...} Hope that helps!")
+  4. Remove trailing commas before ] or }
+  5. Attempt to fix unquoted keys  (moderate repair)
+  6. Attempt to close unclosed brackets/braces
+  7. Combination of fixes 4+5+6
+
+If every strategy fails the original text is returned unchanged so the
+caller can decide what to do.
+"""
+
+import json
+import re
+from .fences import strip_fences
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def enforce_json(text: str) -> str:
+    """Attempt to extract and return valid JSON from *text*.
+
+    Applies a pipeline of cleaning strategies.  The first strategy that
+    produces parse-able JSON wins; its output (the cleaned JSON string) is
+    returned.  If nothing works the original text is returned unchanged.
+
+    The returned string, when a strategy succeeds, is re-serialized with
+    ``json.dumps`` so it is always consistently formatted.
+
+    Parameters
+    ----------
+    text:
+        Raw LLM output that should contain JSON somewhere.
+
+    Returns
+    -------
+    str
+        A valid JSON string, or the original *text* if extraction failed.
+    """
+    if not isinstance(text, str):
+        return text
+
+    original = text
+
+    try:
+        return _run_pipeline(text.strip())
+    except Exception:
+        return original
+
+
+# ---------------------------------------------------------------------------
+# Pipeline
+# ---------------------------------------------------------------------------
+
+def _run_pipeline(text: str) -> str:
+    strategies = [
+        _try_parse_direct,
+        _try_strip_fences,
+        _try_extract_json_substring,
+        _try_fix_trailing_commas,
+        _try_fix_python_literals,
+        _try_fix_unquoted_keys,
+        _try_close_open_brackets,
+        _try_combined_fixes,
+    ]
+    for strategy in strategies:
+        result = strategy(text)
+        if result is not None:
+            return result
+    # Nothing worked — return original
+    return text
+
+
+# ---------------------------------------------------------------------------
+# Strategies (each returns a clean JSON *string* or None)
+# ---------------------------------------------------------------------------
+
+def _try_parse_direct(text: str):
+    """Strategy 1: already valid JSON."""
+    return _parse_and_serialize(text)
+
+
+def _try_strip_fences(text: str):
+    """Strategy 2: strip code fences then parse."""
+    stripped = strip_fences(text)
+    if stripped == text:
+        return None  # no change, skip
+    return _parse_and_serialize(stripped)
+
+
+def _try_extract_json_substring(text: str):
+    """Strategy 3: find the first JSON object or array in the text.
+
+    Scans for the first '{' or '[' and tries progressively larger substrings
+    until one parses.  Also tries from the *last* '}' or ']' backwards.
+    This handles patterns like:
+        'Sure, here is the data: {"key": "value"} Let me know if...'
+    """
+    # Try object extraction
+    result = _extract_by_brackets(text, "{", "}")
+    if result is not None:
+        return result
+    # Try array extraction
+    return _extract_by_brackets(text, "[", "]")
+
+
+def _extract_by_brackets(text: str, open_char: str, close_char: str):
+    """Find the outermost balanced bracket pair and try to parse it."""
+    start = text.find(open_char)
+    if start == -1:
+        return None
+
+    end = text.rfind(close_char)
+    if end == -1 or end <= start:
+        return None
+
+    # Try from outermost to innermost close bracket
+    candidate = text[start:end + 1]
+    result = _parse_and_serialize(candidate)
+    if result is not None:
+        return result
+
+    # Walk inward if the outer attempt fails (handles trailing junk)
+    for i in range(end - 1, start, -1):
+        if text[i] == close_char:
+            result = _parse_and_serialize(text[start:i + 1])
+            if result is not None:
+                return result
+    return None
+
+
+def _try_fix_trailing_commas(text: str):
+    """Strategy 4: remove trailing commas before closing brackets."""
+    cleaned = _remove_trailing_commas(text)
+    if cleaned == text:
+        return None
+    return _parse_and_serialize(cleaned)
+
+
+def _try_fix_python_literals(text: str):
+    """Strategy 5: replace Python literals that LLMs emit instead of JSON.
+
+    LLMs frequently output Python-style values inside otherwise valid JSON:
+        True / False / None  ->  true / false / null
+        Single-quoted strings: {'key': 'val'}  ->  {"key": "val"}
+
+    Applied word-boundary-aware so legitimate content is not corrupted.
+    """
+    cleaned = _replace_python_literals(text)
+    if cleaned == text:
+        return None
+    return _parse_and_serialize(cleaned)
+
+
+def _try_fix_unquoted_keys(text: str):
+    """Strategy 6: quote bare word keys like {key: value} -> {"key": value}."""
+    cleaned = _quote_unquoted_keys(text)
+    if cleaned == text:
+        return None
+    return _parse_and_serialize(cleaned)
+
+
+def _try_close_open_brackets(text: str):
+    """Strategy 7: append missing closing brackets/braces."""
+    cleaned = _close_open_structures(text)
+    if cleaned == text:
+        return None
+    return _parse_and_serialize(cleaned)
+
+
+def _try_combined_fixes(text: str):
+    """Strategy 8: apply all fixers in sequence."""
+    cleaned = _replace_python_literals(text)
+    cleaned = _remove_trailing_commas(cleaned)
+    cleaned = _quote_unquoted_keys(cleaned)
+    cleaned = _close_open_structures(cleaned)
+    if cleaned == text:
+        return None
+    # Also try substring extraction on the combined fix
+    result = _parse_and_serialize(cleaned)
+    if result is not None:
+        return result
+    return _try_extract_json_substring(cleaned)
+
+
+# ---------------------------------------------------------------------------
+# Fixers
+# ---------------------------------------------------------------------------
+
+# Trailing comma before } or ]  (also handles whitespace/newlines between)
+_TRAILING_COMMA_RE = re.compile(r",\s*([}\]])")
+
+def _remove_trailing_commas(text: str) -> str:
+    return _TRAILING_COMMA_RE.sub(r"\1", text)
+
+
+# Bare (unquoted) object keys:  { key: ... }  →  { "key": ... }
+# Only matches word-characters; won't disturb already-quoted keys.
+_UNQUOTED_KEY_RE = re.compile(r'(?<!["\w])(\b[a-zA-Z_][a-zA-Z0-9_]*\b)\s*(?=:)')
+
+def _quote_unquoted_keys(text: str) -> str:
+    # Only operate inside what looks like a JSON object
+    start = text.find("{")
+    if start == -1:
+        return text
+    prefix = text[:start]
+    body = text[start:]
+    fixed = _UNQUOTED_KEY_RE.sub(r'"\1"', body)
+    return prefix + fixed
+
+
+def _close_open_structures(text: str) -> str:
+    """Append any missing closing } or ] characters."""
+    stack = []
+    in_string = False
+    escape_next = False
+    pairs = {"{": "}", "[": "]"}
+    closers = set(pairs.values())
+
+    for ch in text:
+        if escape_next:
+            escape_next = False
+            continue
+        if ch == "\\" and in_string:
+            escape_next = True
+            continue
+        if ch == '"':
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if ch in pairs:
+            stack.append(pairs[ch])
+        elif ch in closers:
+            if stack and stack[-1] == ch:
+                stack.pop()
+
+    # Append missing closers in reverse order
+    return text + "".join(reversed(stack))
+
+
+# ---------------------------------------------------------------------------
+# Utility
+# ---------------------------------------------------------------------------
+
+def _parse_and_serialize(text: str):
+    """Try to parse *text* as JSON; return re-serialized string or None."""
+    try:
+        parsed = json.loads(text.strip())
+        return json.dumps(parsed, ensure_ascii=False, indent=2)
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Python-literal fixer (added separately for clarity)
+# ---------------------------------------------------------------------------
+
+# Word-boundary replacements for Python boolean/None literals
+_PYTHON_LITERAL_RE = re.compile(r'\bTrue\b|\bFalse\b|\bNone\b')
+_PYTHON_LITERAL_MAP = {"True": "true", "False": "false", "None": "null"}
+
+def _replace_python_literals(text: str) -> str:
+    """Replace Python True/False/None with JSON true/false/null.
+
+    Also converts single-quoted strings to double-quoted where safe.
+    Only operates outside of already-valid JSON strings to avoid
+    corrupting content that legitimately contains these words.
+    """
+    # Step 1: True / False / None  (simple word-boundary swap)
+    result = _PYTHON_LITERAL_RE.sub(lambda m: _PYTHON_LITERAL_MAP[m.group()], text)
+
+    # Step 2: single-quoted strings -> double-quoted
+    # Strategy: only replace 'value' patterns that look like JSON string values
+    # or keys (preceded by { , or : and optional whitespace).
+    # This is intentionally conservative to avoid mangling prose.
+    result = _single_to_double_quotes(result)
+    return result
+
+
+def _single_to_double_quotes(text: str) -> str:
+    """Convert single-quoted JSON-ish strings to double-quoted JSON."""
+    result = []
+    i = 0
+    in_single = False
+    in_double = False
+    sq = chr(39)   # '
+    dq = chr(34)   # "
+    bs = chr(92)   # \
+    while i < len(text):
+        ch = text[i]
+        if ch == bs and (in_single or in_double):
+            result.append(ch)
+            i += 1
+            if i < len(text):
+                result.append(text[i])
+                i += 1
+            continue
+        if ch == sq and not in_double:
+            in_single = not in_single
+            result.append(dq)
+        elif ch == dq and not in_single:
+            in_double = not in_double
+            result.append(ch)
+        elif ch == dq and in_single:
+            result.append(bs + dq)
+        else:
+            result.append(ch)
+        i += 1
+    return "".join(result)
+
+def _parse_and_serialize(text: str):
+    """Try to parse *text* as JSON; return re-serialized string or None."""
+    try:
+        parsed = json.loads(text.strip())
+        return json.dumps(parsed, ensure_ascii=False, indent=2)
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Python-literal fixer (added separately for clarity)
+# ---------------------------------------------------------------------------
+
+# Word-boundary replacements for Python boolean/None literals
+_PYTHON_LITERAL_RE = re.compile(r'\bTrue\b|\bFalse\b|\bNone\b')
+_PYTHON_LITERAL_MAP = {"True": "true", "False": "false", "None": "null"}
+
+def _replace_python_literals(text: str) -> str:
+    """Replace Python True/False/None with JSON true/false/null.
+
+    Also converts single-quoted strings to double-quoted where safe.
+    Only operates outside of already-valid JSON strings to avoid
+    corrupting content that legitimately contains these words.
+    """
+    # Step 1: True / False / None  (simple word-boundary swap)
+    result = _PYTHON_LITERAL_RE.sub(lambda m: _PYTHON_LITERAL_MAP[m.group()], text)
+
+    # Step 2: single-quoted strings -> double-quoted
+    # Strategy: only replace 'value' patterns that look like JSON string values
+    # or keys (preceded by { , or : and optional whitespace).
+    # This is intentionally conservative to avoid mangling prose.
+    result = _single_to_double_quotes(result)
+    return result

llmclean/repetition.py

@@ -0,0 +1,257 @@
+"""
+repetition.py — detect and trim repetitive content at the end of LLM output.
+
+LLMs sometimes loop: they repeat the same sentence, phrase, or paragraph
+multiple times — especially near the end of a generation.  This module
+trims that tail while preserving the legitimate content before it.
+
+Detection strategies (applied in order, first match wins):
+  1. Exact sentence repetition  — the same sentence appears 2+ times in a row
+     at the tail of the text.
+  2. Near-duplicate sentences   — sentences that are ≥ N% similar (Jaccard on
+     word tokens) appear 2+ times in a row.
+  3. N-gram phrase repetition   — a multi-word phrase of 4+ tokens repeats
+     3+ times anywhere in the last quarter of the text.
+  4. Paragraph-level repetition — same paragraph (≥ 20 chars) repeated 2+
+     times at the end.
+
+The function is conservative by design: it only trims from the *end* of the
+text and requires repetition to be clearly back-to-back or densely clustered.
+"""
+
+import re
+from typing import Optional
+
+# ---------------------------------------------------------------------------
+# Tunables (all have sensible defaults)
+# ---------------------------------------------------------------------------
+
+# Minimum sentence length (chars) before we consider it for dedup
+_MIN_SENTENCE_LEN = 12
+
+# Jaccard similarity threshold to call two sentences "near-duplicates"
+_JACCARD_THRESHOLD = 0.82
+
+# Minimum n-gram size for phrase repetition detection
+_NGRAM_SIZE = 5
+
+# How many times an n-gram must repeat before we act
+_NGRAM_MIN_REPEAT = 3
+
+# Minimum paragraph length (chars) for paragraph-level dedup
+_MIN_PARA_LEN = 20
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def trim_repetition(text: str, *, similarity_threshold: float = _JACCARD_THRESHOLD) -> str:
+    """Detect and remove repetitive content from the tail of *text*.
+
+    Only removes content from the *end* of the text.  If no repetition is
+    detected the original text is returned unchanged.  Never raises.
+
+    Parameters
+    ----------
+    text:
+        Raw LLM output to clean.
+    similarity_threshold:
+        Jaccard similarity (0–1) above which two sentences are considered
+        duplicates.  Default 0.82.  Lower = more aggressive trimming.
+
+    Returns
+    -------
+    str
+        Cleaned text with repetitive tail removed, or original if no
+        repetition is found.
+    """
+    if not isinstance(text, str) or not text.strip():
+        return text
+
+    original = text
+    try:
+        result = _trim(text, similarity_threshold)
+        return result if result.strip() else original
+    except Exception:
+        return original
+
+
+# ---------------------------------------------------------------------------
+# Core trimmer
+# ---------------------------------------------------------------------------
+
+def _trim(text: str, sim_threshold: float) -> str:
+    strategies = [
+        lambda t: _trim_exact_sentence_repeat(t),
+        lambda t: _trim_near_duplicate_sentences(t, sim_threshold),
+        lambda t: _trim_ngram_repeat(t),
+        lambda t: _trim_paragraph_repeat(t),
+    ]
+    for strategy in strategies:
+        cleaned = strategy(text)
+        if cleaned != text:
+            # Apply recursively — sometimes repetition is layered
+            return _trim(cleaned, sim_threshold)
+    return text
+
+
+# ---------------------------------------------------------------------------
+# Strategy 1: exact sentence repetition
+# ---------------------------------------------------------------------------
+
+def _trim_exact_sentence_repeat(text: str) -> str:
+    """Remove consecutive duplicate sentences from the tail."""
+    sentences = _split_sentences(text)
+    if len(sentences) < 2:
+        return text
+
+    # Find the last run of 2+ identical sentences
+    cut_idx = _find_repeat_run(sentences, key=lambda s: s.strip().lower())
+    if cut_idx is None:
+        return text
+
+    return _rejoin_sentences(sentences[:cut_idx]).rstrip()
+
+
+# ---------------------------------------------------------------------------
+# Strategy 2: near-duplicate sentence detection
+# ---------------------------------------------------------------------------
+
+def _trim_near_duplicate_sentences(text: str, threshold: float) -> str:
+    """Remove near-duplicate consecutive sentences from the tail."""
+    sentences = _split_sentences(text)
+    if len(sentences) < 2:
+        return text
+
+    cut_idx = _find_repeat_run(
+        sentences,
+        key=lambda s: frozenset(_tokenize(s)),
+        similarity=lambda a, b: _jaccard(a, b) >= threshold,
+    )
+    if cut_idx is None:
+        return text
+
+    return _rejoin_sentences(sentences[:cut_idx]).rstrip()
+
+
+# ---------------------------------------------------------------------------
+# Strategy 3: n-gram phrase repetition
+# ---------------------------------------------------------------------------
+
+def _trim_ngram_repeat(text: str) -> str:
+    """Detect a repeated multi-word phrase and trim from its second occurrence."""
+    words = _tokenize(text)
+    if len(words) < _NGRAM_SIZE * _NGRAM_MIN_REPEAT:
+        return text
+
+    # Only look in the last 75% of the word list to stay conservative
+    search_start = len(words) // 4
+
+    ngram_positions: dict[tuple, list[int]] = {}
+    for i in range(search_start, len(words) - _NGRAM_SIZE + 1):
+        ngram = tuple(words[i:i + _NGRAM_SIZE])
+        ngram_positions.setdefault(ngram, []).append(i)
+
+    # Find the n-gram with most repeats; trim before its second occurrence
+    best: Optional[tuple] = None
+    best_positions: list[int] = []
+    for ngram, positions in ngram_positions.items():
+        if len(positions) >= _NGRAM_MIN_REPEAT:
+            if not best or positions[0] < best_positions[0]:
+                best = ngram
+                best_positions = positions
+
+    if best is None:
+        return text
+
+    # Reconstruct text up to (but not including) the second occurrence
+    cut_word_idx = best_positions[1]
+    return _words_to_text_approx(text, words, cut_word_idx)
+
+
+# ---------------------------------------------------------------------------
+# Strategy 4: paragraph-level repetition
+# ---------------------------------------------------------------------------
+
+def _trim_paragraph_repeat(text: str) -> str:
+    """Remove repeated paragraphs at the tail."""
+    paras = [p.strip() for p in re.split(r"\n{2,}", text)]
+    paras = [p for p in paras if len(p) >= _MIN_PARA_LEN]
+    if len(paras) < 2:
+        return text
+
+    cut_idx = _find_repeat_run(paras, key=lambda p: p.lower())
+    if cut_idx is None:
+        return text
+
+    # Rejoin paragraphs up to the cut point
+    rejoined = "\n\n".join(paras[:cut_idx])
+    return rejoined.rstrip()
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _split_sentences(text: str) -> list[str]:
+    """Split text into sentences, keeping the delimiter."""
+    parts = re.split(r"(?<=[.!?])\s+", text)
+    return [p for p in parts if len(p.strip()) >= _MIN_SENTENCE_LEN]
+
+
+def _rejoin_sentences(sentences: list[str]) -> str:
+    return " ".join(sentences)
+
+
+def _tokenize(text: str) -> list[str]:
+    """Lowercase word tokens, no punctuation."""
+    return re.findall(r"\b[a-z0-9]+\b", text.lower())
+
+
+def _jaccard(a: frozenset, b: frozenset) -> float:
+    if not a and not b:
+        return 1.0
+    intersection = len(a & b)
+    union = len(a | b)
+    return intersection / union if union else 0.0
+
+
+def _find_repeat_run(items: list, *, key=None, similarity=None) -> Optional[int]:
+    """Return the cut index: keep items[:cut], which preserves ONE copy of the
+    repeated item and drops all subsequent duplicates.
+
+    Scans from the end.  Returns None if no such run exists.
+    ``key`` maps an item to a comparable value.
+    ``similarity`` is an optional callable(key(a), key(b)) -> bool; if
+    omitted, equality is used.
+    """
+    if not items:
+        return None
+
+    keys = [key(item) if key else item for item in items]
+    eq = similarity if similarity else (lambda a, b: a == b)
+
+    n = len(keys)
+    # Walk backwards looking for a run of duplicates
+    i = n - 1
+    while i > 0:
+        if eq(keys[i], keys[i - 1]):
+            # Found a duplicate pair — find the start of this run
+            run_start = i - 1
+            while run_start > 0 and eq(keys[run_start], keys[run_start - 1]):
+                run_start -= 1
+            # Keep one copy: cut point is run_start + 1
+            return run_start + 1
+        i -= 1
+    return None
+
+
+def _words_to_text_approx(original: str, words: list[str], cut_word_idx: int) -> str:
+    """Return original text truncated approximately at the *cut_word_idx*-th word."""
+    # Find the character offset of the cut_word_idx-th word in the original
+    pattern = re.compile(r"\b[a-z0-9]+\b", re.IGNORECASE)
+    for i, m in enumerate(pattern.finditer(original)):
+        if i == cut_word_idx:
+            return original[:m.start()].rstrip()
+    return original

llmclean-0.1.0.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: llmclean
+Version: 0.1.0
+Summary: Utilities for cleaning and normalizing raw LLM output
+License: MIT
+Project-URL: Homepage, https://github.com/Tushar-9802/llmclean
+Keywords: llm,cleaning,json,text,ai,output,normalization
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+
+# llmclean
+
+**A zero-dependency Python library for cleaning and normalizing raw LLM output.**
+
+LLMs are inconsistent: they wrap JSON in markdown fences, add prose around code, repeat themselves, and produce subtly broken JSON. `llmclean` handles all of that with three focused utilities.
+
+---
+
+## Install
+
+```bash
+pip install llmclean
+```
+
+---
+
+## Quick start
+
+```python
+from llmclean import strip_fences, enforce_json, trim_repetition
+
+# Remove ```json ... ``` wrappers
+strip_fences('```json\n{"name": "Alice"}\n```')
+# → '{"name": "Alice"}'
+
+# Extract valid JSON from messy output
+enforce_json('Here you go: {"ok": True, "items": [1,2,3,]}')
+# → '{\n  "ok": true,\n  "items": [1, 2, 3]\n}'
+
+# Remove repeated sentences/paragraphs at the end
+trim_repetition("The answer is 42. This is final. This is final.")
+# → 'The answer is 42. This is final.'
+```
+
+For full examples and edge cases see **[USAGE.md](USAGE.md)**.
+
+---
+
+## Functions
+
+| Function | What it fixes |
+|---|---|
+| `strip_fences(text)` | Removes ` ```lang ` / ` ``` ` / `~~~` code fences |
+| `enforce_json(text)` | Extracts valid JSON from fences, prose, trailing commas, Python literals, unquoted keys, unclosed brackets |
+| `trim_repetition(text)` | Removes repeated sentences, near-duplicates, and repeated paragraphs from the tail |
+
+---
+
+## Design principles
+
+- **Zero dependencies** — pure Python standard library
+- **Never throws** — every function returns the original input if cleaning fails
+- **Non-destructive** — unchanged input when nothing needs cleaning
+- **Composable** — chain freely
+
+```python
+# Full pipeline
+data = enforce_json(trim_repetition(strip_fences(raw_output)))
+```
+
+---
+
+## Running tests
+
+```bash
+# With pytest
+pip install "llmclean[dev]"
+pytest -v
+
+# Without pytest
+python run_tests.py
+```
+
+---
+
+## License
+
+MIT

llmclean-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+llmclean/__init__.py,sha256=roZ27a2NbVhf8ZkocUvsarpO1zFH1Lq3Z5jsbT-ADAs,488
+llmclean/fences.py,sha256=af619jjPrKC_zeM898TlXJxfVSBB0caRECWCYVUmT5s,5585
+llmclean/json_utils.py,sha256=5Vs1M14hC9SGtU3lNWChc8eVuCo_LGNkKQbqfMUHG8Q,12293
+llmclean/repetition.py,sha256=A8hUe9gmGS6WZYvP51uFTlISu-1li9tyxqdlLdV-Glo,9385
+llmclean-0.1.0.dist-info/METADATA,sha256=CDFa9SO5uEUuECO_DP3iIgMEANwxad_cytPDh4U8wks,2795
+llmclean-0.1.0.dist-info/WHEEL,sha256=YCfwYGOYMi5Jhw2fU4yNgwErybb2IX5PEwBKV4ZbdBo,91
+llmclean-0.1.0.dist-info/top_level.txt,sha256=JCCk4TFXSl0znMtj0lvPjfnRj04Qd7OAYJg681Svl40,9
+llmclean-0.1.0.dist-info/RECORD,,

llmclean-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

llmclean-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+llmclean