code-muse 0.0.1__py3-none-any.whl

code_muse/stream_parser/inline_hidden_tag_parser.py

@@ -0,0 +1,236 @@
+"""Streaming parser for inline hidden tags.
+
+Scans text for user-defined open/close delimiter pairs, extracts the content
+between them as hidden payloads, and strips the delimiters from visible text.
+Tags are matched literally and are not nested.
+
+Partial delimiters that span chunk boundaries are buffered correctly using
+suffix/prefix overlap checks so that a split ``<oai-mem-citation>`` is still
+recognised when the chunks are ``'<oai-'`` and ``'mem-citation>'``.
+"""
+
+from dataclasses import dataclass
+from typing import TypeVar
+
+from code_muse.stream_parser.stream_text_chunk import StreamTextChunk
+from code_muse.stream_parser.stream_text_parser import StreamTextParser
+
+T = TypeVar("T")
+
+
+@dataclass
+class InlineTagSpec[T]:
+    """Specification for a single hidden inline tag type.
+
+    Attributes:
+        tag: Payload type / identifier emitted in extracted results.
+        open: Literal opening delimiter (e.g. ``"<oai-mem-citation>"``).
+        close: Literal closing delimiter (e.g. ``"</oai-mem-citation>"``).
+    """
+
+    tag: T
+    open: str
+    close: str
+
+
+@dataclass
+class ExtractedInlineTag[T]:
+    """Payload produced when an inline tag is fully closed.
+
+    Attributes:
+        tag: The tag identifier from the matching :class:`InlineTagSpec`.
+        content: Text that appeared between the open and close delimiters.
+    """
+
+    tag: T
+    content: str
+
+
+def _longest_suffix_prefix_len(s: str, needle: str) -> int:
+    """Return the largest ``k > 0`` such that ``s`` ends with ``needle[:k]``.
+
+    This tells us how many characters at the end of ``s`` might be a partial
+    occurrence of ``needle`` when more input arrives in the next chunk.
+
+    Args:
+        s: The string to inspect (usually the pending buffer).
+        needle: The delimiter we are searching for.
+
+    Returns:
+        Length of the longest overlap, or ``0`` when there is none.
+    """
+    if not needle:
+        return 0
+    max_k = min(len(s), len(needle))
+    for k in range(max_k, 0, -1):
+        if s.endswith(needle[:k]):
+            return k
+    return 0
+
+
+class InlineHiddenTagParser(StreamTextParser[T]):
+    """Streaming parser that extracts hidden inline tags from text.
+
+    * Searches for the earliest open delimiter; when two open delimiters start
+      at the same position, the longer one wins (longest-match tiebreaker).
+    * Once inside a tag, everything up to the matching close delimiter is
+      treated as literal content—**nested tags are not parsed**.
+    * On chunk boundaries, characters that could be a partial delimiter are
+      kept in the pending buffer rather than being emitted as visible text.
+    * Unterminated tags are auto-closed at :meth:`finish`; their buffered
+      content becomes the extracted payload.
+    """
+
+    def __init__(self, specs: list[InlineTagSpec[T]]) -> None:
+        if not specs:
+            raise ValueError("InlineHiddenTagParser requires at least one tag spec")
+        for spec in specs:
+            if not spec.open:
+                raise ValueError(
+                    "InlineHiddenTagParser requires non-empty open delimiters"
+                )
+            if not spec.close:
+                raise ValueError(
+                    "InlineHiddenTagParser requires non-empty close delimiters"
+                )
+        self.specs = specs
+        self._pending: str = ""
+        self._active_spec: InlineTagSpec[T] | None = None
+        self._active_content: str = ""
+
+    def push_str(self, chunk: str) -> StreamTextChunk[T]:
+        """Feed a new text chunk and scan for tag delimiters.
+
+        The chunk is appended to any buffered pending text, then the parser
+        loops until it can no longer make forward progress:
+
+        1. If a tag is open, search for its close delimiter.
+           * Found → emit the extracted tag and drain through the close.
+           * Not found → keep a suffix that might be a partial close,
+             drain the rest into the tag's content buffer.
+        2. Otherwise, search for the next open delimiter.
+           * Found → emit visible text before it, drain through the open,
+             start tracking tag content.
+           * Not found → keep a suffix that might be a partial open,
+             drain the rest as visible text.
+
+        Args:
+            chunk: Incoming text delta.
+
+        Returns:
+            Visible text and any fully-closed tags found in this delta.
+        """
+        self._pending += chunk
+        visible_parts: list[str] = []
+        extracted_parts: list[ExtractedInlineTag[T]] = []
+
+        while True:
+            if self._active_spec is not None:
+                close_pos = self._pending.find(self._active_spec.close)
+                if close_pos != -1:
+                    # Close delimiter found.
+                    content = self._active_content + self._pending[:close_pos]
+                    extracted_parts.append(
+                        ExtractedInlineTag(self._active_spec.tag, content)
+                    )
+                    # Drain through the close delimiter.
+                    close_len = len(self._active_spec.close)
+                    self._pending = self._pending[close_pos + close_len :]
+                    self._active_spec = None
+                    self._active_content = ""
+                    continue
+
+                # No close yet — keep characters that could be a partial
+                # close delimiter at the end of the buffer.
+                keep = _longest_suffix_prefix_len(
+                    self._pending, self._active_spec.close
+                )
+                drain_len = len(self._pending) - keep
+                self._active_content += self._pending[:drain_len]
+                self._pending = self._pending[drain_len:]
+                break
+
+            # No active tag — look for the next open delimiter.
+            next_open = self._find_next_open()
+            if next_open is not None:
+                pos, spec_idx = next_open
+                spec = self.specs[spec_idx]
+                # Emit visible text before the open delimiter.
+                visible_parts.append(self._pending[:pos])
+                # Drain through the open delimiter.
+                open_len = len(spec.open)
+                self._pending = self._pending[pos + open_len :]
+                self._active_spec = spec
+                self._active_content = ""
+                continue
+
+            # No open delimiter found — keep characters that could be a
+            # partial open delimiter at the end of the buffer.
+            keep = self._max_open_prefix_suffix_len()
+            drain_len = len(self._pending) - keep
+            visible_parts.append(self._pending[:drain_len])
+            self._pending = self._pending[drain_len:]
+            break
+
+        return StreamTextChunk(
+            visible_text="".join(visible_parts),
+            extracted=extracted_parts,
+        )
+
+    def finish(self) -> StreamTextChunk[T]:
+        """Flush any remaining state.
+
+        If a tag is still open, its accumulated content is emitted as an
+        :class:`ExtractedInlineTag` with the tag's identifier.  Any leftover
+        pending text (without an active tag) is emitted as visible text.
+
+        Returns:
+            Final visible text and any auto-closed extracted tags.
+        """
+        visible = ""
+        extracted: list[ExtractedInlineTag[T]] = []
+
+        if self._active_spec is not None:
+            content = self._active_content + self._pending
+            extracted.append(ExtractedInlineTag(self._active_spec.tag, content))
+            self._active_spec = None
+            self._active_content = ""
+            self._pending = ""
+        else:
+            visible = self._pending
+            self._pending = ""
+
+        return StreamTextChunk(visible_text=visible, extracted=extracted)
+
+    def _find_next_open(self) -> tuple[int, int] | None:
+        """Find the earliest open delimiter in the pending buffer.
+
+        Returns:
+            ``(position, spec_index)`` of the earliest open delimiter, or
+            ``None`` when no open delimiter is present.  Tie-breaking rules:
+
+            1. Smallest position (earliest in the buffer).
+            2. Longest open delimiter at that position.
+            3. Lowest spec index (stable ordering).
+        """
+        candidates: list[tuple[int, int, int]] = []
+        for i, spec in enumerate(self.specs):
+            pos = self._pending.find(spec.open)
+            if pos != -1:
+                candidates.append((pos, len(spec.open), i))
+        if not candidates:
+            return None
+        best = min(candidates, key=lambda c: (c[0], -c[1], c[2]))
+        return best[0], best[2]
+
+    def _max_open_prefix_suffix_len(self) -> int:
+        """Maximum overlap between the end of ``pending`` and any open delimiter.
+
+        Returns:
+            Largest ``k > 0`` such that ``pending`` ends with the first ``k``
+            characters of at least one open delimiter.
+        """
+        max_len = 0
+        for spec in self.specs:
+            max_len = max(max_len, _longest_suffix_prefix_len(self._pending, spec.open))
+        return max_len

code_muse/stream_parser/proposed_plan_parser.py

@@ -0,0 +1,158 @@
+"""Parser for ``<proposed_plan>…</proposed_plan>`` line-based blocks.
+
+Wraps :class:`TaggedLineParser` with a single tag spec and exposes a
+:class:`StreamTextParser` interface that emits
+:class:`ProposedPlanSegment` values.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import TypeVar
+
+from code_muse.stream_parser.stream_text_chunk import StreamTextChunk
+from code_muse.stream_parser.stream_text_parser import StreamTextParser
+from code_muse.stream_parser.tagged_line_parser import (
+    TaggedLineParser,
+    TaggedLineSegment,
+    TaggedLineSegmentNormal,
+    TaggedLineSegmentTagDelta,
+    TaggedLineSegmentTagEnd,
+    TaggedLineSegmentTagStart,
+    TagSpec,
+)
+
+T = TypeVar("T")
+
+
+class ProposedPlanSegmentType(Enum):
+    """Kind of segment produced by :class:`ProposedPlanParser`."""
+
+    NORMAL = "normal"
+    PLAN_START = "plan_start"
+    PLAN_DELTA = "plan_delta"
+    PLAN_END = "plan_end"
+
+
+@dataclass
+class ProposedPlanSegment:
+    """Single semantic piece of a parsed assistant response.
+
+    Attributes:
+        type: Which kind of segment this is.
+        text: Literal text content.  Only populated for ``NORMAL`` and
+            ``PLAN_DELTA`` segments.
+    """
+
+    type: ProposedPlanSegmentType
+    text: str = ""
+
+
+class ProposedPlanParser(StreamTextParser[ProposedPlanSegment]):
+    """Streaming parser that identifies ``<proposed_plan>`` blocks.
+
+    Lines that exactly equal ``"<proposed_plan>"`` or ``"</proposed_plan>"``
+    (after trimming) are removed from visible text and replaced by
+    :class:`ProposedPlanSegment` boundary markers.  Content lines between
+    those boundaries become ``PLAN_DELTA`` segments.  Everything else is
+    ``NORMAL``.
+    """
+
+    def __init__(self) -> None:
+        self._parser = TaggedLineParser(
+            [TagSpec(open="<proposed_plan>", close="</proposed_plan>", tag="plan")]
+        )
+
+    def push_str(self, chunk: str) -> StreamTextChunk[ProposedPlanSegment]:
+        """Feed a new text chunk.
+
+        Args:
+            chunk: Incoming text delta (may contain partial lines).
+
+        Returns:
+            Visible text outside plan blocks, plus any plan segments
+            (``PLAN_START``, ``PLAN_DELTA``, ``PLAN_END``) extracted from
+            the chunk.
+        """
+        segments = self._parser.parse(chunk)
+        return self._build_chunk(segments)
+
+    def finish(self) -> StreamTextChunk[ProposedPlanSegment]:
+        """Flush any remaining buffered state.
+
+        Unterminated plan blocks are auto-closed with a ``PLAN_END`` segment.
+
+        Returns:
+            Final visible text and any trailing plan segments.
+        """
+        segments = self._parser.finish()
+        return self._build_chunk(segments)
+
+    @staticmethod
+    def _map_segment(seg: TaggedLineSegment) -> ProposedPlanSegment:
+        """Convert a raw :class:`TaggedLineSegment` to a plan segment."""
+        if isinstance(seg, TaggedLineSegmentNormal):
+            return ProposedPlanSegment(ProposedPlanSegmentType.NORMAL, seg.text)
+        if isinstance(seg, TaggedLineSegmentTagStart):
+            return ProposedPlanSegment(ProposedPlanSegmentType.PLAN_START)
+        if isinstance(seg, TaggedLineSegmentTagDelta):
+            return ProposedPlanSegment(ProposedPlanSegmentType.PLAN_DELTA, seg.text)
+        if isinstance(seg, TaggedLineSegmentTagEnd):
+            return ProposedPlanSegment(ProposedPlanSegmentType.PLAN_END)
+        # Exhaustive because TaggedLineSegment is a union of the four classes.
+        raise TypeError(f"unexpected segment type: {type(seg)}")
+
+    def _build_chunk(
+        self, segments: list[TaggedLineSegment]
+    ) -> StreamTextChunk[ProposedPlanSegment]:
+        """Turn raw line segments into a :class:`StreamTextChunk`."""
+        mapped = [self._map_segment(s) for s in segments]
+        visible = "".join(
+            s.text for s in mapped if s.type == ProposedPlanSegmentType.NORMAL
+        )
+        return StreamTextChunk(visible_text=visible, extracted=mapped)
+
+
+def extract_proposed_plan_text(text: str) -> str | None:
+    """Extract the raw plan text from a complete string.
+
+    Runs the parser over the full text and concatenates all ``PLAN_DELTA``
+    segments that appear inside ``PLAN_START`` / ``PLAN_END`` pairs.
+
+    Args:
+        text: Full assistant response text.
+
+    Returns:
+        The concatenated plan text, or ``None`` when no plan block is
+        present.
+    """
+    parser = ProposedPlanParser()
+    out = parser.push_str(text)
+    tail = parser.finish()
+    all_segments = out.extracted + tail.extracted
+
+    parts: list[str] = []
+    in_plan = False
+    for seg in all_segments:
+        if seg.type == ProposedPlanSegmentType.PLAN_START:
+            in_plan = True
+        elif seg.type == ProposedPlanSegmentType.PLAN_END:
+            in_plan = False
+        elif seg.type == ProposedPlanSegmentType.PLAN_DELTA and in_plan:
+            parts.append(seg.text)
+
+    return "".join(parts) if parts else None
+
+
+def strip_proposed_plan_blocks(text: str) -> str:
+    """Remove all ``<proposed_plan>…</proposed_plan>`` blocks from text.
+
+    Args:
+        text: Full assistant response text.
+
+    Returns:
+        Visible text with plan blocks and their content stripped.
+    """
+    parser = ProposedPlanParser()
+    out = parser.push_str(text)
+    tail = parser.finish()
+    return out.visible_text + tail.visible_text

code_muse/stream_parser/stream_text_chunk.py

@@ -0,0 +1,23 @@
+"""Incremental parser result for one pushed chunk (or final flush)."""
+
+from dataclasses import dataclass, field
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass
+class StreamTextChunk[T]:
+    """Result from feeding a text chunk to a StreamTextParser.
+
+    Attributes:
+        visible_text: Text safe to render immediately.
+        extracted: Hidden payloads extracted from the chunk.
+    """
+
+    visible_text: str = ""
+    extracted: list[T] = field(default_factory=list)
+
+    def is_empty(self) -> bool:
+        """Return True when no visible text or extracted payloads were produced."""
+        return not self.visible_text and not self.extracted

code_muse/stream_parser/stream_text_parser.py

@@ -0,0 +1,27 @@
+"""Abstract base class for composable streaming text parsers."""
+
+from abc import ABC, abstractmethod
+from typing import TypeVar
+
+from code_muse.stream_parser.stream_text_chunk import StreamTextChunk
+
+T = TypeVar("T")
+
+
+class StreamTextParser[T](ABC):
+    """Base class for parsers that consume streamed text and emit visible text
+    plus extracted payloads.
+
+    Parsers are composable: one parser can wrap another, delegating and
+    merging output.
+    """
+
+    @abstractmethod
+    def push_str(self, chunk: str) -> StreamTextChunk[T]:
+        """Feed a new text chunk. Returns visible text + extracted payloads."""
+        ...
+
+    @abstractmethod
+    def finish(self) -> StreamTextChunk[T]:
+        """Flush any buffered state at end-of-stream (or end-of-item)."""
+        ...

code_muse/stream_parser/tagged_line_parser.pyx

@@ -0,0 +1,251 @@
+# cython: language_level=3
+"""Line-based tag-block parser for streamed text.
+
+A tag must appear alone on a line after trimming (e.g.
+``"<proposed_plan>"`` or ``"</proposed_plan>"``).  Lines inside a tag block
+are emitted as :class:`TaggedLineSegmentTagDelta`; lines outside are emitted as
+:class:`TaggedLineSegmentNormal`.
+
+The parser buffers text until it can disprove that the current partial line is
+a tag line—once the trimmed prefix is no longer a prefix of any known open or
+close delimiter, the line is emitted immediately so that visible text is not
+held back unnecessarily.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class TagSpec:
+    """Specification for a line-based tag block.
+
+    Attributes:
+        open: Exact opening line text (e.g. ``"<proposed_plan>"``).
+        close: Exact closing line text (e.g. ``"</proposed_plan>"``).
+        tag: Tag identifier emitted in segment results.
+    """
+
+    open: str
+    close: str
+    tag: Any
+
+
+@dataclass
+class TaggedLineSegmentNormal:
+    """Plain text that lives outside any tag block."""
+
+    text: str
+
+
+@dataclass
+class TaggedLineSegmentTagStart:
+    """Emitted when a line exactly matches a tag's ``open`` delimiter."""
+
+    tag: Any
+
+
+@dataclass
+class TaggedLineSegmentTagDelta:
+    """Text that belongs inside an open tag block.
+
+    Consecutive deltas with the same tag are coalesced by the parser.
+    """
+
+    tag: Any
+    text: str
+
+
+@dataclass
+class TaggedLineSegmentTagEnd:
+    """Emitted when a line exactly matches a tag's ``close`` delimiter."""
+
+    tag: Any
+
+
+TaggedLineSegment = (
+    TaggedLineSegmentNormal
+    | TaggedLineSegmentTagStart
+    | TaggedLineSegmentTagDelta
+    | TaggedLineSegmentTagEnd
+)
+
+
+class TaggedLineParser:
+    """Streaming line-based tag-block parser.
+
+    * Buffers partial lines and emits them only once they can no longer be a
+      tag line (the trimmed prefix is not a prefix of any open or close).
+    * Complete lines (ending in ``\\n``) are classified immediately.
+    * Tag lines are stripped from visible output and replaced by
+      :class:`TaggedLineSegmentTagStart` / :class:`TaggedLineSegmentTagEnd`.
+    * Unterminated tag blocks are auto-closed at :meth:`finish`.
+    """
+
+    def __init__(self, specs: list[TagSpec]) -> None:
+        self.specs = specs
+        self._line_buffer: str = ""
+        self._active_tag: Any = None
+
+    def parse(self, delta: str) -> list[TaggedLineSegment]:
+        """Process a text delta and emit any newly-resolved segments.
+
+        The delta is appended to the internal line buffer, then every complete
+        line (up to and including ``\\n``) is classified.  If the remaining
+        partial line can no longer become a tag line, it is emitted as well.
+
+        Args:
+            delta: Incoming text chunk (may contain zero or more newlines).
+
+        Returns:
+            List of segments produced by this delta.
+        """
+        cdef int newline_idx
+        cdef str line
+        cdef str buf
+        cdef list segments
+
+        self._line_buffer += delta
+        segments = []
+        buf = self._line_buffer
+
+        # Drain complete lines.
+        while True:
+            newline_idx = buf.find("\n")
+            if newline_idx == -1:
+                break
+            line = buf[: newline_idx + 1]
+            buf = buf[newline_idx + 1 :]
+            self._line_buffer = buf
+            self._finish_line(line, segments)
+
+        self._line_buffer = buf
+
+        # If the remaining partial line can never become a tag line,
+        # flush it immediately so it does not stall visible output.
+        if buf and not self._is_tag_prefix(buf.strip()):
+            self._push_text(buf, segments)
+            self._line_buffer = ""
+
+        return segments
+
+    def finish(self) -> list[TaggedLineSegment]:
+        """Flush any remaining buffered state.
+
+        If the pending partial line exactly matches an ``open`` or ``close``
+        delimiter, the appropriate boundary segment is emitted.  Otherwise the
+        text is emitted as Normal or TagDelta (depending on whether a tag is
+        currently open).  Finally, any still-open tag block is auto-closed with
+        a :class:`TaggedLineSegmentTagEnd`.
+
+        Returns:
+            List of final segments.
+        """
+        cdef list segments
+        cdef str slug
+        cdef object open_spec
+        cdef object close_spec
+
+        segments = []
+
+        if self._line_buffer:
+            slug = self._line_buffer.strip()
+            open_spec = self._match_open(slug)
+            if open_spec is not None:
+                segments.append(TaggedLineSegmentTagStart(open_spec.tag))
+                self._active_tag = open_spec.tag
+            else:
+                close_spec = self._match_close(slug)
+                if close_spec is not None:
+                    segments.append(TaggedLineSegmentTagEnd(close_spec.tag))
+                    self._active_tag = None
+                else:
+                    self._push_text(self._line_buffer, segments)
+            self._line_buffer = ""
+
+        if self._active_tag is not None:
+            segments.append(TaggedLineSegmentTagEnd(self._active_tag))
+            self._active_tag = None
+
+        return segments
+
+    def _push_text(self, text: str, segments: list[TaggedLineSegment]) -> None:
+        """Emit ``text`` as the appropriate segment type, coalescing when possible.
+
+        If a tag is currently active, ``text`` becomes a
+        :class:`TaggedLineSegmentTagDelta`; otherwise it becomes a
+        :class:`TaggedLineSegmentNormal`.  Consecutive segments of the same
+        type (and same tag, for deltas) are merged into one segment so the
+        output stays compact.
+        """
+        cdef object last
+        cdef object active = self._active_tag
+
+        if active is not None:
+            if segments:
+                last = segments[-1]
+                if isinstance(last, TaggedLineSegmentTagDelta) and last.tag == active:
+                    last.text += text
+                    return
+            segments.append(TaggedLineSegmentTagDelta(active, text))
+        else:
+            if segments:
+                last = segments[-1]
+                if isinstance(last, TaggedLineSegmentNormal):
+                    last.text += text
+                    return
+            segments.append(TaggedLineSegmentNormal(text))
+
+    def _finish_line(self, line: str, segments: list[TaggedLineSegment]) -> None:
+        """Classify a complete line (including its trailing ``\\n``).
+
+        The line is trimmed and checked against ``open`` / ``close``
+        delimiters in order.  Tag lines are consumed entirely; non-tag lines
+        are forwarded to :meth:`_push_text`.
+        """
+        cdef str slug
+        cdef object open_spec
+        cdef object close_spec
+
+        slug = line.strip()
+        open_spec = self._match_open(slug)
+        if open_spec is not None:
+            segments.append(TaggedLineSegmentTagStart(open_spec.tag))
+            self._active_tag = open_spec.tag
+            return
+
+        close_spec = self._match_close(slug)
+        if close_spec is not None:
+            segments.append(TaggedLineSegmentTagEnd(close_spec.tag))
+            self._active_tag = None
+            return
+
+        self._push_text(line, segments)
+
+    def _is_tag_prefix(self, slug: str) -> bool:
+        """Return ``True`` if ``slug`` is a prefix of any ``open`` or ``close``."""
+        cdef object spec
+        cdef str open_str
+        cdef str close_str
+        for spec in self.specs:
+            open_str = spec.open
+            close_str = spec.close
+            if open_str.startswith(slug) or close_str.startswith(slug):
+                return True
+        return False
+
+    def _match_open(self, slug: str) -> TagSpec | None:
+        """Return the first spec whose ``open`` exactly equals ``slug``."""
+        cdef object spec
+        for spec in self.specs:
+            if spec.open == slug:
+                return spec
+        return None
+
+    def _match_close(self, slug: str) -> TagSpec | None:
+        """Return the first spec whose ``close`` exactly equals ``slug``."""
+        cdef object spec
+        for spec in self.specs:
+            if spec.close == slug:
+                return spec
+        return None