partial-python 0.1.0a1__py3-none-any.whl

partial_python/__init__.py

@@ -0,0 +1,7 @@
+"""Safe syntactic boundaries for streaming Python."""
+
+from .executor import Execution, Executor, ExecutorState, ParserState, UnsupportedSyntaxError, incremental_exec
+from .parser import CodeUnit, Parser
+from .preview import Allow, Completion, complete
+
+__all__ = ["Allow", "CodeUnit", "Completion", "Execution", "Executor", "ExecutorState", "Parser", "ParserState", "UnsupportedSyntaxError", "complete", "incremental_exec"]

partial_python/executor.py

@@ -0,0 +1,358 @@
+"""Sequential execution of original source under explicit interactive semantics."""
+
+import __future__
+
+import ast
+import asyncio
+import inspect
+import sys
+from collections import deque
+from collections.abc import AsyncGenerator, AsyncIterable
+from contextlib import aclosing
+from dataclasses import dataclass
+from time import monotonic
+from types import CodeType
+from typing import Any, Literal, cast
+
+from .parser import _FUTURES, CodeUnit, Parser, _source_line, _tree
+
+
+def _character_column(text: str, byte_column: int | None) -> int | None:
+    if byte_column is None or byte_column <= 0:
+        return byte_column
+    return len(text.encode()[: byte_column - 1].decode(errors="replace")) + 1
+
+
+ExecutorState = Literal["open", "finished", "failed", "aborted"]
+
+
+class UnsupportedSyntaxError(SyntaxError):
+    """Syntax whose cross-unit semantics cannot be preserved by this executor."""
+
+
+@dataclass(frozen=True, slots=True)
+class ParserState:
+    """Immutable parser snapshot; read ``executor.parser_state`` again for current values.
+
+    ``committed`` counts source characters sealed by parsing, not executed code.
+    ``closed`` records parser EOF, not the executor's terminal state.
+    """
+
+    pending: str
+    committed: int
+    closed: bool
+
+
+@dataclass(frozen=True, slots=True)
+class Execution:
+    """One successfully completed unit, with zero-based index and monotonic-second timestamps.
+
+    Timing includes compilation and scheduling; an absent event does not prove
+    that a failed or cancelled unit had no effects.
+    """
+
+    index: int
+    unit: CodeUnit
+    started_at: float
+    completed_at: float
+
+
+class _ModuleScope:
+    def __init__(self) -> None:
+        self.annotations: list[ast.AnnAssign] = []
+        self.globals: list[ast.Global] = []
+
+    def visit(self, tree: ast.AST) -> None:
+        pending = [tree]
+        while pending:
+            node = pending.pop()
+            # Declarations inside another scope cannot affect earlier units.
+            # Expressions cannot contain either declaration; skipping them also
+            # avoids Python recursion limits on otherwise valid deep expressions.
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.expr)):
+                continue
+            if isinstance(node, ast.AnnAssign):
+                # Even non-simple targets can reset deferred module metadata on
+                # 3.14+, despite not creating a new __annotate__ function themselves.
+                self.annotations.append(node)
+            elif isinstance(node, ast.Global):
+                self.globals.append(node)
+            else:
+                pending.extend(reversed(list(ast.iter_child_nodes(node))))
+
+
+class Executor:
+    """One append-only execution stream sharing a caller-owned global namespace.
+
+    Failure and cancellation are terminal: earlier effects remain, and accepted
+    statements are never retried. This is an in-process executor, not a sandbox.
+    Concurrent or recursive writes are rejected rather than silently reordered.
+    """
+
+    def __init__(self, namespace: dict[str, Any] | None = None, *, filename: str = "<partial-python>"):
+        self._namespace = namespace if namespace is not None else {"__name__": "__main__"}
+        self._parser = Parser(filename=filename)
+        self._events: list[Execution] = []
+        self._flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
+        self._prologue = True
+        self._first = True
+        self._busy = False
+        self._state: ExecutorState = "open"
+        self._failed_unit: CodeUnit | None = None
+        self._error: BaseException | None = None
+        self._queued: deque[CodeUnit] = deque()
+
+    @property
+    def namespace(self) -> dict[str, Any]:
+        return self._namespace
+
+    @property
+    def error(self) -> BaseException | None:
+        return self._error
+
+    @property
+    def failed_unit(self) -> CodeUnit | None:
+        return self._failed_unit
+
+    @property
+    def parser_state(self) -> ParserState:
+        """Snapshot of parsing progress, separate from completed execution events."""
+        return ParserState(self._parser.pending, self._parser.committed, self._parser.closed)
+
+    @property
+    def queued(self) -> tuple[CodeUnit, ...]:
+        """Sealed units not yet started; excludes the unsealed ``parser_state.pending`` tail."""
+        return tuple(self._queued)
+
+    @property
+    def events(self) -> tuple[Execution, ...]:
+        return tuple(self._events)
+
+    @property
+    def state(self) -> ExecutorState:
+        return self._state
+
+    def _enter(self) -> None:
+        if self._busy:
+            raise RuntimeError("executor already has an active operation")
+        if self._state != "open":
+            raise RuntimeError(f"executor is {self._state}")
+        self._busy = True
+
+    def _syntax_error(self, error_type: type[SyntaxError], message: str, statement: CodeUnit, node: ast.stmt, *, shifted: bool = False) -> SyntaxError:
+        line = node.lineno - (statement.start_line - 1 if shifted else 0)
+        text = _source_line(statement.source, line)
+        column = _character_column(text, node.col_offset + 1)
+        return error_type(message, (self._parser.filename, statement.start_line + line - 1, column, text))
+
+    def _compile(self, statement: CodeUnit) -> CodeType:
+        tree = _tree(statement.source, self._parser.filename, self._flags, statement.start_line)
+        scope = _ModuleScope()
+        scope.visit(tree)
+        if scope.globals:
+            raise self._syntax_error(UnsupportedSyntaxError, "module-scope global declarations are unsupported across incremental compilation units", statement, scope.globals[0])
+        future = [node for node in tree.body if isinstance(node, ast.ImportFrom) and node.module == "__future__"]
+        if future and not self._prologue:
+            raise self._syntax_error(SyntaxError, "from __future__ imports must occur at the beginning of the stream", statement, future[0])
+        docstring = bool(tree.body and isinstance(tree.body[0], ast.Expr) and isinstance(tree.body[0].value, ast.Constant) and isinstance(tree.body[0].value.value, str))
+        if docstring and not self._first:
+            # Each code object otherwise treats its first string as a new module docstring.
+            tree.body.insert(0, ast.copy_location(ast.Pass(), tree.body[0]))
+        ast.increment_lineno(tree, statement.start_line - 1)
+        try:
+            code = compile(tree, self._parser.filename, "exec", flags=self._flags, dont_inherit=True)
+        except SyntaxError as error:
+            if error.lineno is not None:
+                error.text = _source_line(statement.source, error.lineno - statement.start_line + 1)
+                # AST compiler ranges are UTF-8 byte columns. Once source text
+                # is attached, convert both endpoints for diagnostic renderers.
+                error.offset = _character_column(error.text, error.offset)
+                if error.end_lineno is not None:
+                    end_text = _source_line(statement.source, error.end_lineno - statement.start_line + 1)
+                    error.end_offset = _character_column(end_text, error.end_offset)
+                error.args = (error.msg, (self._parser.filename, error.lineno, error.offset, error.text, error.end_lineno, error.end_offset))
+            raise
+        if sys.version_info >= (3, 14) and scope.annotations and not code.co_flags & __future__.annotations.compiler_flag:
+            raise self._syntax_error(
+                UnsupportedSyntaxError, "incremental module annotations require `from __future__ import annotations` on Python 3.14+", statement, scope.annotations[0], shifted=True
+            )
+        # Read feature requests from this unit, not code-object bits such as
+        # CO_NESTED that overlap legacy future flags. Never borrow future flags
+        # from later queued units accepted by the parser in the same feed.
+        for future_node in future:
+            for name in future_node.names:
+                self._flags |= _FUTURES.get(name.name, 0)
+        for index, node in enumerate(tree.body):
+            if isinstance(node, ast.ImportFrom) and node.module == "__future__":
+                continue
+            if self._first and index == 0 and docstring:
+                continue
+            self._prologue = False
+            break
+        if tree.body:
+            self._first = False
+        return cast(CodeType, code)
+
+    async def _run(self, statement: CodeUnit) -> Execution:
+        started = monotonic()
+        try:
+            task = asyncio.current_task()
+            cancellations = task.cancelling() if task is not None else 0
+            # Deliver an already-pending cancellation before running user code.
+            # A historical caught cancellation count alone does not poison a new operation.
+            await asyncio.sleep(0)
+            code = self._compile(statement)
+            # eval returns the top-level-await coroutine; exec would discard it.
+            result = eval(code, self.namespace, self.namespace)
+            if inspect.isawaitable(result):
+                await result
+            # User code may catch cancellation. Do not start another unit while
+            # the owning task still carries an outstanding cancellation request.
+            if task is not None and task.cancelling() > cancellations:
+                raise asyncio.CancelledError
+        except BaseException:
+            # The owning operation records terminal state; this layer identifies the attempted unit.
+            self._failed_unit = statement
+            raise
+        event = Execution(len(self._events), statement, started, monotonic())
+        self._events.append(event)
+        return event
+
+    async def feed(self, delta: str) -> tuple[Execution, ...]:
+        """Append new source and execute its sealed units in order.
+
+        Return only this call's completed events. On failure, earlier effects
+        remain; inspect ``events``, ``failed_unit``, ``queued`` and ``parser_state``.
+        """
+        self._enter()
+        try:
+            self._queued.extend(self._parser.feed(delta))
+            events = []
+            while self._queued:
+                events.append(await self._run(self._queued.popleft()))
+            return tuple(events)
+        except BaseException as error:
+            self._error = error
+            self._state = "failed"
+            raise
+        finally:
+            self._busy = False
+
+    async def finish(self) -> tuple[Execution, ...]:
+        """Declare successful EOF and execute the remaining source, then close.
+
+        Never use as unconditional cleanup: this can execute a pending loop.
+        Invalid EOF or execution failure is terminal and preserves earlier effects.
+        """
+        self._enter()
+        try:
+            self._queued.extend(self._parser.finish())
+            events = []
+            while self._queued:
+                events.append(await self._run(self._queued.popleft()))
+            self._state = "finished"
+            return tuple(events)
+        except BaseException as error:
+            self._error = error
+            self._state = "failed"
+            raise
+        finally:
+            self._busy = False
+
+    def abort(self) -> None:
+        """Abandon pending source after a transport failure, retaining it for diagnostics."""
+        if self._busy:
+            raise RuntimeError("cancel the active operation before aborting")
+        if self._state == "open":
+            self._state = "aborted"
+
+    async def stream(self, chunks: AsyncIterable[str]) -> AsyncGenerator[Execution, None]:
+        """Execute a source iterator, retaining diagnostics on this executor.
+
+        Normal exhaustion declares EOF. Adapters must raise for truncation or
+        failed transport. Every exit attempts to await the acquired iterator’s
+        ``aclose()`` when provided.
+        This operation owns the executor until exhausted or explicitly closed.
+        """
+        self._enter()
+        iterator = None
+        yielding = False
+        try:
+            iterator = aiter(chunks)
+            while True:
+                # Each anext may be resumed by a different task (e.g. wait_for).
+                task = asyncio.current_task()
+                cancellations = task.cancelling() if task is not None else 0
+                try:
+                    chunk = await anext(iterator)
+                except StopAsyncIteration:
+                    if task is not None and task.cancelling() > cancellations:
+                        raise asyncio.CancelledError
+                    break
+                if task is not None and task.cancelling() > cancellations:
+                    raise asyncio.CancelledError
+                self._queued.extend(self._parser.feed(chunk))
+                if not self._queued:
+                    # Buffered input can return immediately without suspending.
+                    # Keep empty/unfinished streams cancellable between chunks.
+                    await asyncio.sleep(0)
+                while self._queued:
+                    event = await self._run(self._queued.popleft())
+                    yielding = True
+                    yield event
+                    yielding = False
+            self._queued.extend(self._parser.finish())
+            while self._queued:
+                event = await self._run(self._queued.popleft())
+                yielding = True
+                yield event
+                yielding = False
+            self._state = "finished"
+        except BaseException as error:
+            # Only GeneratorExit injected at our event yield is consumer closure.
+            # The same exception from a source or executed unit is a real failure.
+            if not (yielding and isinstance(error, GeneratorExit)):
+                self._error = error
+                self._state = "failed"
+            raise
+        finally:
+            if self._state == "open":
+                self._state = "aborted"
+            try:
+                cleanup_task = asyncio.current_task()
+                cancellations = cleanup_task.cancelling() if cleanup_task is not None else 0
+                close = getattr(iterator, "aclose", None)
+                if close is not None:
+                    await close()
+                if cleanup_task is not None and cleanup_task.cancelling() > cancellations:
+                    raise asyncio.CancelledError
+            except BaseException as cleanup_error:
+                # The entry guard excludes stale errors from earlier terminal operations.
+                if self._error is None or not isinstance(cleanup_error, Exception):
+                    self._error = cleanup_error
+                    self._state = "failed"
+                    raise
+                try:
+                    BaseException.add_note(self._error, f"Source iterator cleanup also failed: {cleanup_error!r}")
+                except Exception:  # noqa: BLE001, S110 - secondary diagnostics must not replace the primary error.
+                    # A broken exception repr or notes attribute must not replace
+                    # the execution failure merely while attaching diagnostics.
+                    pass
+                except BaseException as note_error:
+                    self._error = note_error
+                    raise
+            finally:
+                self._busy = False
+
+
+async def incremental_exec(chunks: AsyncIterable[str], namespace: dict[str, Any] | None = None, *, filename: str = "<partial-python>") -> dict[str, Any]:
+    """Execute validated source as it arrives and return the namespace used.
+
+    Normal iterator exhaustion authorizes EOF. The source adapter must raise on
+    truncation or failure. Use ``Executor.stream`` for events and diagnostics.
+    """
+    executor = Executor(namespace, filename=filename)
+    async with aclosing(executor.stream(chunks)) as events:
+        async for _ in events:
+            pass
+    return executor.namespace

partial_python/parser.py

@@ -0,0 +1,405 @@
+"""Conservative boundaries for append-only Python source."""
+
+import __future__
+
+import ast
+import io
+import re
+import tokenize
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from typing import Literal, cast
+
+
+@dataclass(frozen=True, slots=True)
+class CodeUnit:
+    """An exact source slice; offsets count Unicode characters, lines are one-based."""
+
+    source: str
+    start: int
+    end: int
+    start_line: int
+    end_line: int
+
+
+_COMPOUND = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.With, ast.AsyncWith, ast.Try, ast.TryStar, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Match)
+_CLAUSES = frozenset({"elif", "else", "except", "finally"})
+_OPENERS = {")": "(", "]": "[", "}": "{"}
+_LINE_END = re.compile(r"[\r\n]")
+_FUTURES = {name: getattr(__future__, name).compiler_flag for name in __future__.all_feature_names}
+_TRIVIA = frozenset({tokenize.COMMENT, tokenize.NL, tokenize.ENCODING})
+
+
+def _tree(source: str, filename: str, flags: int = 0, start_line: int = 1) -> ast.Module:
+    try:
+        return cast(ast.Module, compile(source, filename, "exec", flags=flags | ast.PyCF_ONLY_AST | ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, dont_inherit=True))
+    except SyntaxError:
+        if start_line == 1:
+            raise
+        # Only the error path pays for leading lines. Let CPython construct both
+        # serialized locations and messages that themselves mention a line number.
+        return cast(ast.Module, compile("\n" * (start_line - 1) + source, filename, "exec", flags=flags | ast.PyCF_ONLY_AST | ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, dont_inherit=True))
+
+
+def _source_line(source: str, line: int) -> str:
+    # str.splitlines also splits formfeeds and Unicode separators, unlike Python.
+    start = 0
+    for _ in range(max(0, line - 1)):
+        match = _LINE_END.search(source, start)
+        if match is None:
+            return ""
+        start = match.end() + (match[0] == "\r" and source[match.end() : match.end() + 1] == "\n")
+    match = _LINE_END.search(source, start)
+    if match is None:
+        return source[start:]
+    end = match.end() + (match[0] == "\r" and source[match.end() : match.end() + 1] == "\n")
+    return source[start:end]
+
+
+def _line_end(source: str, start: int) -> int:
+    match = _LINE_END.search(source, start)
+    if match is None:
+        return len(source)
+    # Include one lookahead character even after a bare CR: _normalize holds a
+    # trailing CR until it can distinguish CRLF. This is a probe window, not a unit.
+    return min(match.end() + (match[0] == "\r"), len(source))
+
+
+def _normalize(source: str) -> tuple[str, Sequence[int]]:
+    if "\r" not in source:
+        return source, range(len(source) + 1)
+    normalized = []
+    offsets = [0]
+    index = 0
+    while index < len(source):
+        character = source[index]
+        if character == "\r":
+            if index + 1 == len(source):
+                break  # CR may be half of CRLF; do not commit it before the next chunk.
+            index += 2 if source[index + 1] == "\n" else 1
+            normalized.append("\n")
+        else:
+            index += 1
+            normalized.append(character)
+        offsets.append(index)
+    return "".join(normalized), offsets
+
+
+_STRING_STARTS = frozenset(getattr(tokenize, name) for name in ("FSTRING_START", "TSTRING_START") if hasattr(tokenize, name))
+_STRING_ENDS = frozenset(getattr(tokenize, name) for name in ("FSTRING_END", "TSTRING_END") if hasattr(tokenize, name))
+
+
+@dataclass(frozen=True, slots=True)
+class _Boundary:
+    offset: int
+    kind: Literal["successor", "newline", "resume", "continuation", "started", "string"]
+    delimiters: tuple[str, ...] = ()
+    logical_root: bool = True
+    quote: str = ""
+
+
+def _quote_delimiter(token_text: str) -> str:
+    marker = token_text.lstrip("rRuUbBfFtT")
+    return marker[:3] if marker[:3] in ("'''", '"""') else marker[:1]
+
+
+def _escaped_before(text: str, index: int, before_window: bool) -> bool:
+    start = index
+    while start and text[start - 1] == "\\":
+        start -= 1
+    return bool((index - start) % 2) ^ (before_window if start == 0 else False)
+
+
+def _quote_window(text: str, before_window: bool) -> tuple[str, bool]:
+    # Two quote characters suffice for a split triple delimiter, but a slash
+    # run can span any number of chunks. Carry its parity before this window.
+    start = max(len(text) - 2, 0)
+    return text[start:], _escaped_before(text, start, before_window)
+
+
+def _has_unescaped_closer(text: str, quote: str, before_window: bool) -> bool:
+    start = 0
+    while (candidate := text.find(quote, start)) >= 0:
+        if not _escaped_before(text, candidate, before_window):
+            return True
+        # An escaped quote followed by three more quotes closes at the second
+        # quote. Preserve overlapping candidates instead of skipping a delimiter.
+        start = candidate + 1
+    return False
+
+
+def _unfinished_quote(source: str, last_end: tuple[int, int]) -> tuple[str, tuple[int, int]] | None:
+    eof = (source.count("\n") + 1, len(source.rsplit("\n", 1)[-1]))
+    # Ask the host tokenizer to identify an unfinished STRING crossing actual EOF.
+    # Four quotes also close a trailing backslash. Never reinterpret an already
+    # emitted token (e.g. r'' becoming r''' merely because of the probe suffix).
+    for quote in ("'", '"'):
+        try:
+            for token in tokenize.generate_tokens(io.StringIO(source + quote * 4).readline):
+                if token.type == tokenize.STRING and last_end <= token.start < eof < token.end:
+                    return _quote_delimiter(token.string), token.start
+        except (tokenize.TokenError, IndentationError, SyntaxError, UnicodeError):
+            pass
+    return None
+
+
+def _candidates(source: str, delimiters: tuple[str, ...], logical_root: bool) -> Iterator[_Boundary]:
+    source, original_offsets = _normalize(source)
+    starts = [0]
+    starts.extend(index + 1 for index, character in enumerate(source) if character == "\n")
+    prefix = "".join(delimiters) + "\n" if delimiters else ""
+    row_shift = bool(prefix)
+
+    def offset(position: tuple[int, int]) -> int:
+        row, column = position
+        row -= row_shift
+        return starts[row - 1] + column if row <= len(starts) else len(source)
+
+    def root_prefix(position: tuple[int, int]) -> bool:
+        if position[0] - row_shift == 1:
+            return True
+        # Python 3.11's pure-Python tokenizer can miss indentation after an
+        # explicit join. Ask the host grammar whether this prefix is truly root.
+        # The sentinel is only a lexical-context probe, never executable source.
+        try:
+            tree = _tree(source[: offset(position)] + "pass\n", "<partial-python-boundary>")
+        except (SyntaxError, ValueError, UnicodeError):
+            return False
+        return len(tree.body) == 1 and isinstance(tree.body[0], ast.Pass)
+
+    # This synthetic prefix restores lexical bracket context only. It never
+    # enters a CodeUnit or the compiler. Checkpoints carry exact opener kinds
+    # and the ORIGINAL logical line's indentation, not the prefix's column zero.
+    stack = list(delimiters)
+    string_quotes: list[str] = []
+    last_end = (1, 0)
+    lexical_error = False
+    logical_start = not delimiters
+    try:
+        for token in tokenize.generate_tokens(io.StringIO(prefix + source).readline):
+            if token.start[0] <= row_shift:
+                continue
+            kind = token.type
+            if token.string and kind not in {tokenize.DEDENT, tokenize.ENDMARKER}:
+                last_end = token.end
+            if kind in _STRING_STARTS:
+                string_quotes.append(_quote_delimiter(token.string))
+            elif kind in _STRING_ENDS:
+                string_quotes.pop()
+            elif kind == tokenize.ERRORTOKEN:
+                lexical_error = True
+                # An incomplete short quote can expose leading whitespace as
+                # ERRORTOKEN on 3.11. It cannot prove a successor's indentation.
+                if token.string in {" ", "\t", "\f"}:
+                    continue
+            elif kind == tokenize.OP:
+                if token.string in "([{":
+                    stack.append(token.string)
+                elif token.string in ")]}" and (not stack or stack.pop() != _OPENERS[token.string]):
+                    return
+            if kind == tokenize.INDENT:
+                logical_root = False
+            elif kind == tokenize.DEDENT:
+                continue
+            elif kind == tokenize.NEWLINE:
+                # tokenize invents NEWLINE and DEDENT at EOF. Neither closes a live stream.
+                if token.string and logical_root:
+                    yield _Boundary(original_offsets[offset(token.end)], "newline")
+                if token.string:
+                    yield _Boundary(original_offsets[offset(token.end)], "resume")
+                    return
+                logical_start = True
+            elif kind == tokenize.NL and token.string:
+                if logical_start and not stack:
+                    yield _Boundary(original_offsets[offset(token.end)], "resume")
+                    return
+                if stack and not string_quotes and not lexical_error:
+                    # Never checkpoint f/t-string lexical state or a partial
+                    # token. Persist only this completed NL snapshot, not any
+                    # transient delimiter changes observed at temporary EOF.
+                    yield _Boundary(original_offsets[offset(token.end)], "continuation", tuple(stack), logical_root)
+                    return
+            elif kind not in _TRIVIA and kind != tokenize.ENDMARKER:
+                if logical_start and logical_root:
+                    # A literal may grow, but cannot become a continuation clause.
+                    # An ERRORTOKEN at temporary EOF is not enough: a trailing
+                    # backslash can disappear into a join before a real else.
+                    stable = (
+                        offset(token.end) < len(source)
+                        or kind in {tokenize.OP, tokenize.NUMBER, tokenize.STRING}
+                        or kind in _STRING_STARTS
+                        or (kind == tokenize.NAME and not any(clause.startswith(token.string) for clause in _CLAUSES))
+                    )
+                    if stable and token.string not in _CLAUSES and root_prefix(token.start):
+                        # This window starts at the logical line's beginning.
+                        # Keep formfeeds and explicit joins with the successor
+                        # so its source columns and physical lines stay intact.
+                        yield _Boundary(0, "successor")
+                    if stable:
+                        yield _Boundary(0, "started")
+                elif logical_start:
+                    yield _Boundary(0, "started")
+                logical_start = False
+    except (tokenize.TokenError, IndentationError, SyntaxError, UnicodeError):
+        if string_quotes:
+            yield _Boundary(0, "string", quote=string_quotes[0])
+        elif opening := _unfinished_quote(prefix + source, last_end):
+            quote, position = opening
+            if logical_start:
+                # A host-proven string opener already excludes continuation
+                # clauses; its unfinished contents need not delay a predecessor.
+                if logical_root and root_prefix(position):
+                    yield _Boundary(0, "successor")
+                yield _Boundary(0, "started")
+            yield _Boundary(0, "string", quote=quote)
+        # Valid earlier tokens remain useful even when the unfinished suffix is not tokenizable.
+        return
+
+
+class Parser:
+    """Emit only syntactically sealed source, never guessed closures.
+
+    ``finish`` declares actual EOF. A failed finish closes the parser, because a
+    transport EOF cannot subsequently be reinterpreted as another text chunk.
+    """
+
+    def __init__(self, *, filename: str = "<partial-python>"):
+        self.filename = filename
+        self._pending = ""
+        self._chunks: list[str] = []
+        self._await_quote = ""
+        self._quote_tail = ""
+        self._quote_tail_escaped = False
+        self._offset = 0
+        self._line = 1
+        self._closed = False
+        self._flags = 0
+        self._scan = 0
+        self._retry_full_window = False
+        self._line_started = False
+        self._delimiters: tuple[str, ...] = ()
+        self._logical_root = True
+        self._checked = 0
+        self._checked_successor = False
+
+    @property
+    def pending(self) -> str:
+        if self._chunks:
+            self._pending += "".join(self._chunks)
+            self._chunks.clear()
+        return self._pending
+
+    @property
+    def closed(self) -> bool:
+        return self._closed
+
+    @property
+    def committed(self) -> int:
+        return self._offset
+
+    def _ensure_open(self) -> None:
+        if self._closed:
+            raise RuntimeError("parser is closed")
+
+    def _statement(self, source: str) -> CodeUnit:
+        statement = CodeUnit(source, self._offset, self._offset + len(source), self._line, self._line + source.count("\n") + source.count("\r") - source.count("\r\n"))
+        self._offset = statement.end
+        self._line = statement.end_line
+        return statement
+
+    def feed(self, delta: str) -> tuple[CodeUnit, ...]:
+        """Append a delta and emit original-source units stable under further appends."""
+        self._ensure_open()
+        if not isinstance(delta, str):
+            raise TypeError("delta must be str")
+        if self._await_quote:
+            window = self._quote_tail + delta
+            # Even f/t strings need an unescaped outer closer. Nested fields
+            # can cause extra retries; only the host tokenizer decides closure.
+            if not _has_unescaped_closer(window, self._await_quote, self._quote_tail_escaped):
+                if delta:
+                    self._chunks.append(delta)
+                    self._quote_tail, self._quote_tail_escaped = _quote_window(window, self._quote_tail_escaped)
+                return ()
+        self._await_quote = ""
+        resolves_cr = (self._chunks[-1] if self._chunks else self._pending).endswith("\r")
+        # Once the first token is unambiguous, only a physical line ending can
+        # produce NEWLINE. Skip growing operands/literals until that can happen.
+        if self._line_started and not resolves_cr and "\n" not in delta and "\r" not in delta:
+            if delta:
+                # Join only when scanning or reading diagnostics; copying the
+                # growing operand on every small delta is otherwise quadratic.
+                self._chunks.append(delta)
+            return ()
+        self._pending = self.pending + delta
+        result = []
+        consumed = 0
+        scan = self._scan
+        window_end = len(self._pending) if self._retry_full_window else _line_end(self._pending, scan)
+        self._retry_full_window = False
+        while scan < len(self._pending):
+            checkpoint = None
+            for boundary in _candidates(self._pending[scan:window_end], self._delimiters, self._logical_root):
+                end = scan + boundary.offset
+                kind = boundary.kind
+                if kind == "string":
+                    if window_end == len(self._pending):
+                        self._await_quote = boundary.quote
+                        self._quote_tail, self._quote_tail_escaped = _quote_window(self._pending, False)
+                    continue
+                if kind == "started":
+                    self._line_started = True
+                    continue
+                if kind in {"resume", "continuation"}:
+                    self._line_started = kind == "continuation"
+                    self._delimiters = boundary.delimiters
+                    self._logical_root = boundary.logical_root
+                    checkpoint = end
+                    break
+                successor = kind == "successor"
+                if end <= consumed or end < self._checked or (end == self._checked and successor == self._checked_successor):
+                    continue
+                self._checked = end
+                self._checked_successor = successor
+                source = self._pending[consumed:end]
+                try:
+                    tree = _tree(source, self.filename, self._flags)
+                except (SyntaxError, ValueError, UnicodeError):
+                    continue
+                if not tree.body or (not successor and isinstance(tree.body[-1], _COMPOUND)):
+                    continue
+                for node in tree.body:
+                    if isinstance(node, ast.ImportFrom) and node.module == "__future__":
+                        for name in node.names:
+                            self._flags |= _FUTURES.get(name.name, 0)
+                result.append(self._statement(source))
+                consumed = end
+            if checkpoint is None or checkpoint <= scan:
+                if window_end == len(self._pending):
+                    self._retry_full_window = True
+                    break
+                # Exponential windows avoid revisiting every prior physical line
+                # of an unfinished list/string in one large transport chunk.
+                window_end = min(len(self._pending), scan + max(1, window_end - scan) * 2)
+            else:
+                scan = checkpoint
+                window_end = _line_end(self._pending, scan)
+        # Lexer checkpoints retain only reconstructible state. Original-source
+        # compilation still validates the complete suite and every source unit.
+        self._scan = max(scan - consumed, 0)
+        self._checked = max(self._checked - consumed, 0)
+        self._pending = self._pending[consumed:]
+        return tuple(result)
+
+    def finish(self) -> tuple[CodeUnit, ...]:
+        """Declare successful EOF and emit the final unit; even invalid EOF closes the parser."""
+        self._ensure_open()
+        self._closed = True
+        # Parse even a trivia-only tail: an unterminated comment string is harmless,
+        # but a stray backslash or bracket must never disappear silently.
+        _tree(self.pending, self.filename, self._flags, self._line)
+        if not self._pending:
+            return ()
+        statement = self._statement(self._pending)
+        self._pending = ""
+        return (statement,)

partial_python/preview.py

@@ -0,0 +1,176 @@
+"""Inspectable display completions. No generated text is an execution boundary."""
+
+import ast
+from dataclasses import dataclass
+from enum import IntFlag, auto
+from typing import cast
+
+from .parser import Parser
+
+
+class Allow(IntFlag):
+    """Opt into provisional display; these options never affect Parser or Executor."""
+
+    NONE = 0
+    TAIL = auto()
+    STRING = auto()
+    BRACKET = auto()
+    SUITE = auto()
+    ALL = TAIL | STRING | BRACKET | SUITE
+
+
+@dataclass(frozen=True, slots=True)
+class Completion:
+    """``prefix + withheld`` is the input; ``prefix + suffix`` is display source.
+
+    Only ``committed`` characters have passed Parser's streaming boundary rules.
+    Even a preview with no synthetic suffix may contain an uncommitted statement.
+    ``diagnostic`` describes the original input, and can simply mean incomplete.
+    """
+
+    prefix: str
+    suffix: str
+    withheld: str
+    committed: int
+    diagnostic: SyntaxError | UnicodeError | None = None
+
+    @property
+    def source(self) -> str:
+        return self.prefix + self.suffix
+
+    @property
+    def provisional(self) -> bool:
+        return bool(self.suffix) or len(self.prefix) > self.committed
+
+
+def _parse(source: str, flags: int, start_line: int = 1) -> ast.Module:
+    # Snapshot validation can afford padding: warnings, errors and returned AST
+    # nodes then all use the same original line coordinates.
+    return cast(ast.Module, compile("\n" * (start_line - 1) + source, "<partial-python-preview>", "exec", flags=flags | ast.PyCF_ONLY_AST | ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, dont_inherit=True))
+
+
+def _closures(source: str, allow: Allow) -> str | None:
+    # This scanner only proposes suffixes. CPython validates every proposal below;
+    # unsupported syntax stays pending instead of being rewritten into other code.
+    source = source.replace("\r\n", "\n").replace("\r", "\n")
+    stack: list[str] = []
+    index = 0
+    last = -1
+    suffix = ""
+    while index < len(source):
+        char = source[index]
+        if char == "#":
+            end = source.find("\n", index)
+            if end < 0:
+                if stack:
+                    suffix = "\n"
+                break
+            index = end + 1
+            continue
+        if char in "\"'":
+            prefix_start = index
+            while prefix_start and (source[prefix_start - 1].isalnum() or source[prefix_start - 1] == "_"):
+                prefix_start -= 1
+            prefix = source[prefix_start:index].lower()
+            if prefix in {"f", "fr", "rf", "t", "tr", "rt"}:
+                return None  # f/t strings have nested grammar; do not guess their braces or format specs.
+            quote = char * (3 if source.startswith(char * 3, index) else 1)
+            cursor = index + len(quote)
+            closed = False
+            partial_closer = 0
+            while cursor < len(source):
+                if source[cursor] == "\\":
+                    cursor += 2
+                elif source.startswith(quote, cursor):
+                    cursor += len(quote)
+                    closed = True
+                    break
+                elif len(quote) == 3 and len(source) - cursor < 3 and source[cursor:] in {char, char * 2}:
+                    partial_closer = len(source) - cursor
+                    cursor = len(source)
+                elif len(quote) == 1 and source[cursor] in "\r\n":
+                    return None
+                else:
+                    cursor += 1
+            if not closed:
+                if not allow & Allow.STRING or cursor > len(source):
+                    return None
+                # One or two quote characters at the end of a triple string are
+                # the start of its closer; append only the missing characters.
+                suffix = quote[partial_closer:]
+                break
+            index = cursor
+            last = index - 1
+            continue
+        if char in "([{":
+            stack.append(char)
+        elif char in ")]}":
+            if not stack or stack.pop() != {")": "(", "]": "[", "}": "{"}[char]:
+                return None
+        elif char == "\\" and index + 1 >= len(source):
+            # Explicit line continuations need real source; do not invent an operand.
+            return None
+        if not char.isspace():
+            last = index
+        index += 1
+    if stack:
+        if not allow & Allow.BRACKET:
+            return None
+        suffix += "".join({"(": ")", "[": "]", "{": "}"}[char] for char in reversed(stack))
+    if not suffix and last >= 0 and source[last] == ":" and allow & Allow.SUITE:
+        header = source[source.rfind("\n", 0, last) + 1 : last]
+        indent = header[: len(header) - len(header.lstrip(" \t"))]
+        suffix = "\n" + indent + "    pass\n"
+    return suffix or None
+
+
+def complete(source: str, allow: Allow = Allow.NONE) -> Completion:
+    """Complete a preview without altering or deleting interior source.
+
+    Defaults to the exact committed prefix. Opt-in completion is display-only:
+    STRING closes plain literals, BRACKET closes data/grouping delimiters, SUITE
+    adds ``pass`` to an empty final suite, and TAIL includes already valid but
+    unsealed source. Calls needing synthetic closers/arguments stay withheld.
+    """
+    if not isinstance(source, str):
+        raise TypeError("source must be str")
+    allow = Allow(allow)
+    if int(allow) & ~int(Allow.ALL):
+        raise ValueError("unknown preview flags")
+    parser = Parser()
+    parser.feed(source)
+    committed = parser.committed
+    pending = parser.pending
+    diagnostic = None
+    try:
+        _parse(pending, parser._flags, parser._line)
+    except (SyntaxError, UnicodeError) as error:
+        diagnostic = error
+        if isinstance(error, UnicodeEncodeError):
+            # Padding is only for line coordinates. Encoding diagnostics must
+            # retain the actual full input and its original character offsets.
+            # Encoding fails before grammar, so current flags cannot reinterpret history.
+            try:
+                _parse(source, parser._flags)
+            except (SyntaxError, UnicodeError) as original_error:
+                diagnostic = original_error
+    else:
+        if allow:
+            return Completion(source, "", "", committed)
+    suffix = _closures(pending, allow) if allow else None
+    if suffix:
+        try:
+            tree = _parse(pending + suffix, parser._flags, parser._line)
+        except (SyntaxError, ValueError, UnicodeError):
+            pass
+        else:
+            # Closing a call would guess its final argument list, even when the
+            # existing arguments happen to parse. Keep the entire pending unit.
+            normalized = pending.replace("\r\n", "\n").replace("\r", "\n")
+            boundary = (parser._line + normalized.count("\n"), len(normalized.rsplit("\n", 1)[-1].encode()))
+            closes_call = any(isinstance(node, ast.Call) and (node.end_lineno, node.end_col_offset) > boundary for node in ast.walk(tree))
+            if not closes_call:
+                return Completion(source, suffix, "", committed, diagnostic)
+    # Parser already checked each committed unit under its historical flags;
+    # a later future import must not reinterpret that accepted prefix.
+    return Completion(source[:committed], "", pending, committed, diagnostic)

partial_python-0.1.0a1.dist-info/METADATA

@@ -0,0 +1,96 @@
+Metadata-Version: 2.5
+Name: partial-python
+Version: 0.1.0a1
+Summary: Append-only Python parsing and incremental execution
+Project-URL: Repository, https://github.com/promplate/partial-python
+Project-URL: Documentation, https://github.com/promplate/partial-python/blob/main/docs/api.md
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# partial-python
+
+Execute append-only Python while source is still arriving. Python 3.11+, no runtime dependencies. Experimental API.
+
+```sh
+uv add 'partial-python==0.1.0a1'
+```
+
+## Input and completion
+
+Use an `Executor` for each logical response. Feed source as it arrives, then explicitly finish or abort according to the provider's completion status.
+
+```python
+from partial_python import Executor
+
+executor = Executor({"foo": foo, "bar": bar, "baz": baz})
+await executor.feed("await foo(1)\n")  # Executes now.
+await executor.feed("await bar(2)")  # Waits: an operator or attribute access may follow.
+await executor.feed("\nawait baz(3)")  # Executes bar, buffers baz.
+await executor.finish()  # Successful EOF permits baz.
+```
+
+**Units execute in source order, at most once each.** New input never reruns earlier units. A loop still performs its ordinary iterations, but its whole body must be sealed before execution starts.
+
+`feed()` accepts new text deltas, not accumulated snapshots. `feed()` and `finish()` return completed `Execution` events. The executor retains `events`, `failed_unit`, sealed but unstarted `queued` units, and an immutable `parser_state` snapshot containing the unsealed `pending` tail. Read results from `executor.namespace`.
+
+`finish()` declares successful source completion. On disconnects, token limits, cancellation, or failed responses, use `abort()` for an idle executor; cancel and await an active operation first. Failure is terminal: earlier effects remain, later units do not start, and nothing is retried. [Lifecycle and integration](docs/api.md).
+
+## Async text iterators
+
+When your adapter already supplies a validated async text iterator, `incremental_exec()` is a convenience wrapper. `validated_deltas` must yield raw source and end normally **only after successful generation**. Its adapter must raise on truncation, failure, or a missing completion signal.
+
+```python
+from partial_python import incremental_exec
+
+namespace = await incremental_exec(validated_deltas, {"foo": foo, "bar": bar})
+```
+
+Execution starts while source is arriving; the await completes after execution and returns the namespace used. To consume events as each unit completes, use `Executor.stream()` and retain the executor for diagnostics. Close a partially consumed event iterator with `aclosing` in its owning task/context. [Streaming adapter](docs/api.md#stateful-streaming).
+
+## Stable boundaries
+
+| Input | Execution boundary |
+| --- | --- |
+| Top-level simple statement | A real logical newline or successful EOF. |
+| `foo()` at temporary EOF | Wait; it can still become `foo() + bar()`. |
+| `for`, `if`, `try`, `def`, and other compounds | Hold the entire statement until an unambiguous top-level successor or successful EOF. |
+| `else`, `elif`, `except`, `finally`, or an ambiguous prefix such as `e` | Keep clauses attached; wait for ambiguous prefixes to resolve. |
+| Blank lines and comments | Preserve them; they do not close a suite. |
+| Decorators, brackets, line joins, and strings | Keep their complete logical statement together. |
+
+`Parser.feed()` / `finish()` expose the same boundaries as immutable `CodeUnit` source slices without executing them. `pending` retains the uncommitted suffix. Every unit preserves original characters and source positions; no repaired text is executed. [Precise semantic contract](docs/semantics.md).
+
+## Display completion
+
+```python
+from partial_python import Allow, complete
+
+assert complete("ready = 1\nvalue = ['hel").source == "ready = 1\n"
+preview = complete("value = ['hel", Allow.STRING | Allow.BRACKET)
+assert preview.source == "value = ['hel']"
+assert preview.provisional
+```
+
+The default returns only the stable prefix. `Allow` flags opt into provisional display; `prefix`, `suffix`, and `withheld` distinguish original text from synthetic text. A preview never authorizes execution, even when its suffix is empty. [Preview options and limits](docs/preview.md).
+
+## Execution scope
+
+Units share one namespace and support native top-level `await`. Execution requires an asyncio event loop; use the asyncio backend with AnyIO. `Parser` and `complete` are synchronous.
+
+This uses sequential compilation units, with [explicit differences from whole-module execution](docs/semantics.md#execution-is-a-sequence-of-compilation-units). Future imports remain in the prologue, only the initial string sets the module docstring, and module-scope `global` declarations are rejected. On Python 3.14+, module annotations require an explicit `from __future__ import annotations`.
+
+In-process execution provides no sandbox or rollback. Hosts own permissions, background tasks, source limits, and hard deadlines. The [integration examples](docs/api.md#host-examples) demonstrate transport completion, concurrent reception, and traceback source lifetime.
+
+## Development
+
+```sh
+uv sync
+uv run pytest
+uv run basedpyright
+uv run ruff check .
+uv run ruff format --check .
+uv run examples/streaming_tools.py
+uv build
+```
+
+[API and transport](docs/api.md) · [Design](docs/design.md) · [Validation](docs/validation.md) · [Related implementations](docs/prior-art.md)

partial_python-0.1.0a1.dist-info/RECORD

@@ -0,0 +1,8 @@
+partial_python/__init__.py,sha256=Xhc7Yuhm7ZLdLBe56Jcm6-84QN3Htwa5IskrJf640M8,425
+partial_python/executor.py,sha256=157LxUn095Ge97LYaRq8dFtnWRbiZifWup60LIW8duk,15962
+partial_python/parser.py,sha256=AX-dvWhmtwai8YcNz2DvgZpma--ELSatMIuctzJIJf8,18437
+partial_python/preview.py,sha256=1NszfwFuGYGCHLMdEEaL5Mm-Tk8W39NsykWSI9CJzeQ,7324
+partial_python/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+partial_python-0.1.0a1.dist-info/METADATA,sha256=NID8lC7Yw3k_PYUxsdaBGFBzh0OVZFqMw_M-Z1nqybg,5483
+partial_python-0.1.0a1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+partial_python-0.1.0a1.dist-info/RECORD,,

partial_python-0.1.0a1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any