stream-replace 0.1.0__py3-none-any.whl

stream_replace/__init__.py

@@ -0,0 +1,4 @@
+from stream_replace._replacer import Replacer
+from stream_replace._functional import stream_replace, astream_replace
+
+__all__ = ["Replacer", "stream_replace", "astream_replace"]

stream_replace/_functional.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import AsyncIterable, Iterable
+
+from stream_replace._replacer import Replacer
+
+
+def stream_replace(
+    iterable: Iterable[str],
+    rules: list[tuple],
+) -> Iterable[str]:
+    """Convenience wrapper — apply replacement *rules* to a sync chunk stream."""
+    return Replacer(rules).wrap(iterable)
+
+
+async def astream_replace(
+    iterable: AsyncIterable[str],
+    rules: list[tuple],
+) -> AsyncIterable[str]:
+    """Convenience wrapper — apply replacement *rules* to an async chunk stream."""
+    async for chunk in Replacer(rules).wrap_async(iterable):
+        yield chunk

stream_replace/_replacer.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from typing import AsyncIterable, Iterable
+
+from stream_replace._rule import Rule, parse_rule
+
+
+class Replacer:
+    """Streaming text replacer that correctly handles partial matches across
+    chunk boundaries.
+
+    Usage::
+
+        r = Replacer([("hello", "world")])
+        for chunk in token_stream:
+            yield r.feed(chunk)
+        yield r.flush()
+    """
+
+    __slots__ = ("_rules", "_buffer")
+
+    def __init__(self, rules: list[tuple]) -> None:
+        self._rules: list[Rule] = [parse_rule(r) for r in rules]
+        self._buffer: str = ""
+
+    # ------------------------------------------------------------------
+    # Core API
+    # ------------------------------------------------------------------
+
+    def feed(self, chunk: str) -> str:
+        """Process an incoming chunk.  Returns text that is safe to emit."""
+        if not chunk:
+            return ""
+        self._buffer += chunk
+        return self._process(flushing=False)
+
+    def flush(self) -> str:
+        """Flush any remaining buffered text.  Call once after the stream ends."""
+        return self._process(flushing=True)
+
+    def reset(self) -> None:
+        """Clear internal state so the instance can be reused."""
+        self._buffer = ""
+
+    # ------------------------------------------------------------------
+    # Convenience wrappers
+    # ------------------------------------------------------------------
+
+    def wrap(self, iterable: Iterable[str]) -> Iterable[str]:
+        """Wrap a sync iterable of chunks, yielding replaced text."""
+        self.reset()
+        for chunk in iterable:
+            out = self.feed(chunk)
+            if out:
+                yield out
+        out = self.flush()
+        if out:
+            yield out
+
+    async def wrap_async(self, iterable: AsyncIterable[str]) -> AsyncIterable[str]:
+        """Wrap an async iterable of chunks, yielding replaced text."""
+        self.reset()
+        async for chunk in iterable:
+            out = self.feed(chunk)
+            if out:
+                yield out
+        out = self.flush()
+        if out:
+            yield out
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _process(self, *, flushing: bool) -> str:
+        result: list[str] = []
+        buf = self._buffer
+
+        # Phase 1: find and apply all complete matches iteratively
+        pos = 0
+        while pos < len(buf):
+            earliest = self._find_earliest_match(buf, pos)
+            if earliest is None:
+                break
+            result.append(buf[pos:earliest.start])
+            result.append(earliest.replacement_text)
+            pos = earliest.end
+
+        remaining = buf[pos:]
+
+        if flushing:
+            # No more data coming — do one final replacement pass on the
+            # remaining tail, then emit everything.
+            final = self._replace_all(remaining)
+            result.append(final)
+            self._buffer = ""
+        else:
+            # Determine safe-to-emit boundary in *remaining*.
+            safe = self._safe_boundary(remaining)
+            result.append(remaining[:safe])
+            self._buffer = remaining[safe:]
+
+        return "".join(result)
+
+    def _find_earliest_match(self, text: str, pos: int):
+        """Return the earliest MatchResult among all rules, or None."""
+        from stream_replace._rule import MatchResult
+
+        best: MatchResult | None = None
+        for rule in self._rules:
+            m = rule.search(text, pos)
+            if m is not None and (best is None or m.start < best.start):
+                best = m
+        return best
+
+    def _safe_boundary(self, remaining: str) -> int:
+        """How many characters from *remaining* can be safely emitted."""
+        if not remaining:
+            return 0
+
+        safe = len(remaining)
+        for rule in self._rules:
+            p = rule.find_partial_start(remaining)
+            if p is not None and p < safe:
+                safe = p
+        return safe
+
+    def _replace_all(self, text: str) -> str:
+        """Run all rules on *text* until no more matches (for flush)."""
+        buf = text
+        changed = True
+        while changed:
+            changed = False
+            pos = 0
+            parts: list[str] = []
+            while pos < len(buf):
+                m = self._find_earliest_match(buf, pos)
+                if m is None:
+                    break
+                parts.append(buf[pos:m.start])
+                parts.append(m.replacement_text)
+                pos = m.end
+                changed = True
+            parts.append(buf[pos:])
+            buf = "".join(parts)
+        return buf

stream_replace/_rule.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Callable, Union
+
+from stream_replace._utils import extract_literal_prefix, find_suffix_prefix_overlap
+
+Replacement = Union[str, Callable]
+
+
+@dataclass(slots=True)
+class MatchResult:
+    start: int
+    end: int
+    replacement_text: str
+
+
+@dataclass(slots=True)
+class StringRule:
+    pattern: str
+    replacement: Replacement
+
+    def search(self, text: str, pos: int = 0) -> MatchResult | None:
+        idx = text.find(self.pattern, pos)
+        if idx == -1:
+            return None
+        matched = self.pattern
+        rep = self.replacement(matched) if callable(self.replacement) else self.replacement
+        return MatchResult(start=idx, end=idx + len(self.pattern), replacement_text=rep)
+
+    def find_partial_start(self, text: str) -> int | None:
+        return find_suffix_prefix_overlap(text, self.pattern)
+
+
+@dataclass(slots=True)
+class RegexRule:
+    regex: re.Pattern[str]
+    replacement: Replacement
+    literal_prefix: str = field(init=False)
+    _fallback_lookback: int = 0
+
+    def __post_init__(self) -> None:
+        self.literal_prefix = extract_literal_prefix(self.regex)
+        if not self.literal_prefix and self._fallback_lookback == 0:
+            self._fallback_lookback = 32
+
+    def search(self, text: str, pos: int = 0) -> MatchResult | None:
+        m = self.regex.search(text, pos)
+        if m is None:
+            return None
+        if callable(self.replacement):
+            rep = self.replacement(m)
+        else:
+            rep = m.expand(self.replacement)
+        return MatchResult(start=m.start(), end=m.end(), replacement_text=rep)
+
+    def find_partial_start(self, text: str) -> int | None:
+        prefix = self.literal_prefix
+        if prefix:
+            # Check whether a complete literal prefix exists without a full
+            # regex match — that means an open (unfinished) match is in
+            # progress and we must hold back from that position.
+            search_start = 0
+            while True:
+                idx = text.find(prefix, search_start)
+                if idx == -1:
+                    break
+                if self.regex.search(text, idx) is None:
+                    return idx
+                search_start = idx + 1
+
+            # Also check if the very tail is a *partial* literal prefix
+            # (token split mid-prefix).
+            return find_suffix_prefix_overlap(text, prefix)
+
+        if self._fallback_lookback > 0 and len(text) > 0:
+            return max(0, len(text) - self._fallback_lookback)
+        return None
+
+
+Rule = StringRule | RegexRule
+
+
+def parse_rule(spec: tuple) -> Rule:
+    """Convert a user-supplied tuple into a Rule object.
+
+    Accepted forms:
+        (str,   str | callable)
+        (re.Pattern, str | callable)
+        (str,   str | callable, dict)      -- extra options forwarded
+        (re.Pattern, str | callable, dict)
+    """
+    if len(spec) == 2:
+        pattern, replacement = spec
+        opts: dict = {}
+    elif len(spec) == 3:
+        pattern, replacement, opts = spec
+    else:
+        raise ValueError(f"Rule tuple must have 2 or 3 elements, got {len(spec)}")
+
+    if isinstance(pattern, str):
+        return StringRule(pattern=pattern, replacement=replacement)
+    if isinstance(pattern, re.Pattern):
+        lookback = opts.get("lookback", 0)
+        rule = RegexRule(regex=pattern, replacement=replacement, _fallback_lookback=lookback)
+        return rule
+
+    raise TypeError(f"Pattern must be str or re.Pattern, got {type(pattern).__name__}")

stream_replace/_utils.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import re
+
+try:
+    import re._parser as _sre_parse  # Python 3.13+
+except ImportError:
+    import sre_parse as _sre_parse  # type: ignore[no-redef]
+
+
+def extract_literal_prefix(pattern: re.Pattern[str]) -> str:
+    """Extract the leading literal character sequence from a compiled regex.
+
+    Examples:
+        r"<think>[\\s\\S]*?</think>" -> "<think>"
+        r"hello\\s+world"            -> "hello"
+        r"\\d{11}"                   -> ""
+    """
+    try:
+        parsed = _sre_parse.parse(pattern.pattern)
+    except Exception:
+        return ""
+
+    chars: list[str] = []
+    for op, av in parsed:
+        if op == _sre_parse.LITERAL:
+            chars.append(chr(av))
+        else:
+            break
+    return "".join(chars)
+
+
+def find_suffix_prefix_overlap(text: str, pattern: str) -> int | None:
+    """Find the earliest position in *text* where a suffix of *text* equals
+    a proper prefix of *pattern*.
+
+    Returns the start index in *text*, or None if no overlap exists.
+
+    Example:
+        text="abc hel", pattern="hello" -> 4  (suffix "hel" is prefix of "hello")
+    """
+    if not pattern:
+        return None
+
+    max_check = min(len(text), len(pattern) - 1)
+    for length in range(max_check, 0, -1):
+        suffix = text[-length:]
+        if pattern[:length] == suffix:
+            return len(text) - length
+    return None

stream_replace-0.1.0.dist-info/METADATA

@@ -0,0 +1,126 @@
+Metadata-Version: 2.4
+Name: stream-replace
+Version: 0.1.0
+Summary: Streaming text replacement for AI token streams — handles partial matches across chunk boundaries
+License-Expression: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Typing :: Typed
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# stream-replace
+
+Streaming text replacement for AI token streams — correctly handles partial matches across chunk boundaries.
+
+## Install
+
+```bash
+pip install stream-replace
+```
+
+## Quick Start
+
+```python
+import re
+from stream_replace import Replacer
+
+r = Replacer([
+    ("敏感词", "***"),                                        # string → string
+    ("secret", lambda s: s[0] + "***"),                       # string → callable
+    (re.compile(r"1[3-9]\d{9}"), "[PHONE]"),                  # regex  → string
+    (re.compile(r"<think>[\s\S]*?</think>"), ""),              # regex  → remove
+    (re.compile(r"(\d+)"), lambda m: str(int(m.group()) * 2)), # regex  → callable
+])
+
+for chunk in ai_stream:
+    safe_text = r.feed(chunk)
+    print(safe_text, end="")
+
+print(r.flush(), end="")
+```
+
+## Why?
+
+AI models stream tokens incrementally. A word you want to replace may be split across chunks:
+
+```
+chunk 1: "hel"
+chunk 2: "lo world"
+```
+
+Naive per-chunk replacement would miss `"hello"`. **stream-replace** buffers just enough text at chunk boundaries to detect partial matches, while emitting safe text as early as possible.
+
+## API
+
+### `Replacer(rules)`
+
+Create a replacer with a list of `(pattern, replacement)` tuples.
+
+| Pattern | Replacement | Description |
+|---|---|---|
+| `str` | `str` | Exact string replacement |
+| `str` | `callable(matched_str) → str` | Dynamic string replacement |
+| `re.Pattern` | `str` | Regex replacement (supports `\1` backrefs) |
+| `re.Pattern` | `callable(re.Match) → str` | Dynamic regex replacement |
+
+#### `r.feed(chunk: str) → str`
+
+Process one incoming chunk. Returns text that is safe to emit (fully resolved, no pending partial matches).
+
+#### `r.flush() → str`
+
+Flush the internal buffer after the stream ends. Must be called once to get any remaining text.
+
+#### `r.reset()`
+
+Clear internal state so the replacer can be reused for another stream.
+
+#### `r.wrap(iterable) → Iterable[str]`
+
+Convenience wrapper for a sync chunk stream. Handles `feed` + `flush` automatically.
+
+```python
+for text in r.wrap(chunks):
+    print(text, end="")
+```
+
+#### `r.wrap_async(async_iterable) → AsyncIterable[str]`
+
+Same as `wrap`, but for async iterables.
+
+```python
+async for text in r.wrap_async(async_chunks):
+    print(text, end="")
+```
+
+### Functional API
+
+For one-off use without creating a `Replacer` instance:
+
+```python
+from stream_replace import stream_replace, astream_replace
+
+# sync
+for text in stream_replace(chunks, [("hello", "world")]):
+    print(text, end="")
+
+# async
+async for text in astream_replace(async_chunks, [("hello", "world")]):
+    print(text, end="")
+```
+
+## How It Works
+
+1. **Buffer**: Incoming chunks accumulate in an internal buffer.
+2. **Match**: On each `feed()`, the buffer is scanned for complete matches across all rules. The earliest match wins.
+3. **Replace**: Matched text is replaced; scanning continues from after the replacement.
+4. **Hold back**: After all matches, the buffer tail is checked for *potential* partial matches (a suffix that could be the start of a pattern). This tail is held back for the next `feed()`.
+5. **Flush**: On `flush()`, the remaining buffer is processed without holding anything back.
+
+For regex rules, the library automatically extracts literal prefixes from the pattern (e.g., `"<think>"` from `r"<think>[\s\S]*?</think>"`) to detect both partial prefix matches and open-but-unclosed matches spanning multiple chunks.
+
+## License
+
+MIT

stream_replace-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+stream_replace/__init__.py,sha256=9kR47PP6CpFrBOe53TxdjwfE3jELI_YnmfuAGsis8Vo,178
+stream_replace/_functional.py,sha256=lpE5qWoGqPos3tiCB7i46rpFYFbSiYbkarc2EZUnAvw,626
+stream_replace/_replacer.py,sha256=79wyOiywBWnm6dPOrOi4srD4IjDgisw4eQHENNTe3No,4753
+stream_replace/_rule.py,sha256=USrAayLXt7bkatontMwmzFFHDAVcqfAml0UYZg2-tCA,3589
+stream_replace/_utils.py,sha256=qRvEpeHuezRhyJZ-uHJWH1uw0Ylcx9Kg8Mxh2rkjF6g,1386
+stream_replace-0.1.0.dist-info/METADATA,sha256=SfvlmHwZ4m3XBaDKzCkwhu7eA2sVrIsisAFAuECKifk,3902
+stream_replace-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+stream_replace-0.1.0.dist-info/RECORD,,

stream_replace-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any