partial-python 0.1.0a1__tar.gz

partial_python-0.1.0a1/.github/workflows/ci.yml

@@ -0,0 +1,66 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.15"
+      - name: Check release version
+        if: startsWith(github.ref, 'refs/tags/')
+        run: test "${GITHUB_REF_NAME#v}" = "$(uv version --short)"
+      - run: uv run --locked ruff check .
+      - run: uv run --locked ruff format --check .
+      - run: uv run --locked basedpyright
+      - run: uv run --locked basedpyright --verifytypes partial_python --ignoreexternal
+      - run: uv build --no-sources
+      - name: Test installed distributions
+        run: |
+          for version in 3.11 3.15; do
+            for package in dist/*; do
+              uv run --isolated --no-project --python "$version" --with "$package" python -I -c 'from importlib.metadata import requires; from importlib.resources import files; assert not requires("partial-python"); assert files("partial_python").joinpath("py.typed").is_file()'
+              uv run --isolated --no-project --python "$version" --with "$package" python -I examples/streaming_tools.py
+            done
+          done
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  test:
+    name: Python ${{ matrix.python-version }} / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"]
+        include:
+          - os: macos-latest
+            python-version: "3.15"
+          - os: windows-latest
+            python-version: "3.15"
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv run --locked pytest

partial_python-0.1.0a1/.github/workflows/publish.yml

@@ -0,0 +1,26 @@
+name: Publish
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    uses: ./.github/workflows/ci.yml
+
+  publish:
+    needs: checks
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+      - uses: astral-sh/setup-uv@v7
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - run: uv publish --trusted-publishing always

partial_python-0.1.0a1/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+.pytest_cache/
+.hypothesis/
+.ruff_cache/
+dist/
+*.egg-info/

partial_python-0.1.0a1/AGENTS.md

@@ -0,0 +1,3 @@
+# Development
+
+Use IPython MCP for every Python operation, including tests and environment management. Use uv; no pip or npx. Prefer gh MCP for GitHub. Keep commits focused and small. Review 80 prior relevant commit/PR names before naming commits. Default to 200 character lines; comment on surprising invariants. Never execute repaired source: preview and execution have different contracts. Do not publish packages or push without a concrete authorized destination.

partial_python-0.1.0a1/PKG-INFO

@@ -0,0 +1,96 @@
+Metadata-Version: 2.5
+Name: partial-python
+Version: 0.1.0a1
+Summary: Append-only Python parsing and incremental execution
+Project-URL: Repository, https://github.com/promplate/partial-python
+Project-URL: Documentation, https://github.com/promplate/partial-python/blob/main/docs/api.md
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# partial-python
+
+Execute append-only Python while source is still arriving. Python 3.11+, no runtime dependencies. Experimental API.
+
+```sh
+uv add 'partial-python==0.1.0a1'
+```
+
+## Input and completion
+
+Use an `Executor` for each logical response. Feed source as it arrives, then explicitly finish or abort according to the provider's completion status.
+
+```python
+from partial_python import Executor
+
+executor = Executor({"foo": foo, "bar": bar, "baz": baz})
+await executor.feed("await foo(1)\n")  # Executes now.
+await executor.feed("await bar(2)")  # Waits: an operator or attribute access may follow.
+await executor.feed("\nawait baz(3)")  # Executes bar, buffers baz.
+await executor.finish()  # Successful EOF permits baz.
+```
+
+**Units execute in source order, at most once each.** New input never reruns earlier units. A loop still performs its ordinary iterations, but its whole body must be sealed before execution starts.
+
+`feed()` accepts new text deltas, not accumulated snapshots. `feed()` and `finish()` return completed `Execution` events. The executor retains `events`, `failed_unit`, sealed but unstarted `queued` units, and an immutable `parser_state` snapshot containing the unsealed `pending` tail. Read results from `executor.namespace`.
+
+`finish()` declares successful source completion. On disconnects, token limits, cancellation, or failed responses, use `abort()` for an idle executor; cancel and await an active operation first. Failure is terminal: earlier effects remain, later units do not start, and nothing is retried. [Lifecycle and integration](docs/api.md).
+
+## Async text iterators
+
+When your adapter already supplies a validated async text iterator, `incremental_exec()` is a convenience wrapper. `validated_deltas` must yield raw source and end normally **only after successful generation**. Its adapter must raise on truncation, failure, or a missing completion signal.
+
+```python
+from partial_python import incremental_exec
+
+namespace = await incremental_exec(validated_deltas, {"foo": foo, "bar": bar})
+```
+
+Execution starts while source is arriving; the await completes after execution and returns the namespace used. To consume events as each unit completes, use `Executor.stream()` and retain the executor for diagnostics. Close a partially consumed event iterator with `aclosing` in its owning task/context. [Streaming adapter](docs/api.md#stateful-streaming).
+
+## Stable boundaries
+
+| Input | Execution boundary |
+| --- | --- |
+| Top-level simple statement | A real logical newline or successful EOF. |
+| `foo()` at temporary EOF | Wait; it can still become `foo() + bar()`. |
+| `for`, `if`, `try`, `def`, and other compounds | Hold the entire statement until an unambiguous top-level successor or successful EOF. |
+| `else`, `elif`, `except`, `finally`, or an ambiguous prefix such as `e` | Keep clauses attached; wait for ambiguous prefixes to resolve. |
+| Blank lines and comments | Preserve them; they do not close a suite. |
+| Decorators, brackets, line joins, and strings | Keep their complete logical statement together. |
+
+`Parser.feed()` / `finish()` expose the same boundaries as immutable `CodeUnit` source slices without executing them. `pending` retains the uncommitted suffix. Every unit preserves original characters and source positions; no repaired text is executed. [Precise semantic contract](docs/semantics.md).
+
+## Display completion
+
+```python
+from partial_python import Allow, complete
+
+assert complete("ready = 1\nvalue = ['hel").source == "ready = 1\n"
+preview = complete("value = ['hel", Allow.STRING | Allow.BRACKET)
+assert preview.source == "value = ['hel']"
+assert preview.provisional
+```
+
+The default returns only the stable prefix. `Allow` flags opt into provisional display; `prefix`, `suffix`, and `withheld` distinguish original text from synthetic text. A preview never authorizes execution, even when its suffix is empty. [Preview options and limits](docs/preview.md).
+
+## Execution scope
+
+Units share one namespace and support native top-level `await`. Execution requires an asyncio event loop; use the asyncio backend with AnyIO. `Parser` and `complete` are synchronous.
+
+This uses sequential compilation units, with [explicit differences from whole-module execution](docs/semantics.md#execution-is-a-sequence-of-compilation-units). Future imports remain in the prologue, only the initial string sets the module docstring, and module-scope `global` declarations are rejected. On Python 3.14+, module annotations require an explicit `from __future__ import annotations`.
+
+In-process execution provides no sandbox or rollback. Hosts own permissions, background tasks, source limits, and hard deadlines. The [integration examples](docs/api.md#host-examples) demonstrate transport completion, concurrent reception, and traceback source lifetime.
+
+## Development
+
+```sh
+uv sync
+uv run pytest
+uv run basedpyright
+uv run ruff check .
+uv run ruff format --check .
+uv run examples/streaming_tools.py
+uv build
+```
+
+[API and transport](docs/api.md) · [Design](docs/design.md) · [Validation](docs/validation.md) · [Related implementations](docs/prior-art.md)

partial_python-0.1.0a1/README.md

@@ -0,0 +1,87 @@
+# partial-python
+
+Execute append-only Python while source is still arriving. Python 3.11+, no runtime dependencies. Experimental API.
+
+```sh
+uv add 'partial-python==0.1.0a1'
+```
+
+## Input and completion
+
+Use an `Executor` for each logical response. Feed source as it arrives, then explicitly finish or abort according to the provider's completion status.
+
+```python
+from partial_python import Executor
+
+executor = Executor({"foo": foo, "bar": bar, "baz": baz})
+await executor.feed("await foo(1)\n")  # Executes now.
+await executor.feed("await bar(2)")  # Waits: an operator or attribute access may follow.
+await executor.feed("\nawait baz(3)")  # Executes bar, buffers baz.
+await executor.finish()  # Successful EOF permits baz.
+```
+
+**Units execute in source order, at most once each.** New input never reruns earlier units. A loop still performs its ordinary iterations, but its whole body must be sealed before execution starts.
+
+`feed()` accepts new text deltas, not accumulated snapshots. `feed()` and `finish()` return completed `Execution` events. The executor retains `events`, `failed_unit`, sealed but unstarted `queued` units, and an immutable `parser_state` snapshot containing the unsealed `pending` tail. Read results from `executor.namespace`.
+
+`finish()` declares successful source completion. On disconnects, token limits, cancellation, or failed responses, use `abort()` for an idle executor; cancel and await an active operation first. Failure is terminal: earlier effects remain, later units do not start, and nothing is retried. [Lifecycle and integration](docs/api.md).
+
+## Async text iterators
+
+When your adapter already supplies a validated async text iterator, `incremental_exec()` is a convenience wrapper. `validated_deltas` must yield raw source and end normally **only after successful generation**. Its adapter must raise on truncation, failure, or a missing completion signal.
+
+```python
+from partial_python import incremental_exec
+
+namespace = await incremental_exec(validated_deltas, {"foo": foo, "bar": bar})
+```
+
+Execution starts while source is arriving; the await completes after execution and returns the namespace used. To consume events as each unit completes, use `Executor.stream()` and retain the executor for diagnostics. Close a partially consumed event iterator with `aclosing` in its owning task/context. [Streaming adapter](docs/api.md#stateful-streaming).
+
+## Stable boundaries
+
+| Input | Execution boundary |
+| --- | --- |
+| Top-level simple statement | A real logical newline or successful EOF. |
+| `foo()` at temporary EOF | Wait; it can still become `foo() + bar()`. |
+| `for`, `if`, `try`, `def`, and other compounds | Hold the entire statement until an unambiguous top-level successor or successful EOF. |
+| `else`, `elif`, `except`, `finally`, or an ambiguous prefix such as `e` | Keep clauses attached; wait for ambiguous prefixes to resolve. |
+| Blank lines and comments | Preserve them; they do not close a suite. |
+| Decorators, brackets, line joins, and strings | Keep their complete logical statement together. |
+
+`Parser.feed()` / `finish()` expose the same boundaries as immutable `CodeUnit` source slices without executing them. `pending` retains the uncommitted suffix. Every unit preserves original characters and source positions; no repaired text is executed. [Precise semantic contract](docs/semantics.md).
+
+## Display completion
+
+```python
+from partial_python import Allow, complete
+
+assert complete("ready = 1\nvalue = ['hel").source == "ready = 1\n"
+preview = complete("value = ['hel", Allow.STRING | Allow.BRACKET)
+assert preview.source == "value = ['hel']"
+assert preview.provisional
+```
+
+The default returns only the stable prefix. `Allow` flags opt into provisional display; `prefix`, `suffix`, and `withheld` distinguish original text from synthetic text. A preview never authorizes execution, even when its suffix is empty. [Preview options and limits](docs/preview.md).
+
+## Execution scope
+
+Units share one namespace and support native top-level `await`. Execution requires an asyncio event loop; use the asyncio backend with AnyIO. `Parser` and `complete` are synchronous.
+
+This uses sequential compilation units, with [explicit differences from whole-module execution](docs/semantics.md#execution-is-a-sequence-of-compilation-units). Future imports remain in the prologue, only the initial string sets the module docstring, and module-scope `global` declarations are rejected. On Python 3.14+, module annotations require an explicit `from __future__ import annotations`.
+
+In-process execution provides no sandbox or rollback. Hosts own permissions, background tasks, source limits, and hard deadlines. The [integration examples](docs/api.md#host-examples) demonstrate transport completion, concurrent reception, and traceback source lifetime.
+
+## Development
+
+```sh
+uv sync
+uv run pytest
+uv run basedpyright
+uv run ruff check .
+uv run ruff format --check .
+uv run examples/streaming_tools.py
+uv build
+```
+
+[API and transport](docs/api.md) · [Design](docs/design.md) · [Validation](docs/validation.md) · [Related implementations](docs/prior-art.md)

partial_python-0.1.0a1/docs/api.md

@@ -0,0 +1,131 @@
+# API and transport integration
+
+## Public entry points
+
+Start with `Executor.feed()` / `finish()` / `abort()` when integrating a model response. They keep source reception and the provider's completion status explicit. The async iterator APIs adapt sources that already validate successful completion.
+
+| Entry point | Input | Output | Purpose |
+| --- | --- | --- | --- |
+| `Executor.feed(delta)` / `finish()` | Append-only raw Python deltas / explicit EOF | Tuple of completed `Execution` events | Own provider status handling explicitly. |
+| `Executor.abort()` | No arguments; executor must be idle | `None`; retains diagnostic state | Abandon pending source without executing it. |
+| `await incremental_exec(deltas)` | A validated async text iterator | The namespace used, after successful execution | Convenience wrapper when per-unit events are unnecessary. |
+| `Executor.stream(deltas)` | The same validated iterator | Completed events with retained executor diagnostics | Consume events as each unit completes. |
+| `Parser.feed(delta)` / `finish()` | Append-only raw Python deltas / explicit EOF | Immutable original-source `CodeUnit` ranges | Integrate with another execution runtime. |
+| `complete(source, allow=...)` | Current full source snapshot | `Completion` with original prefix, synthetic suffix, withheld source and committed offset | Display an explicitly provisional preview. Never feed its generated suffix to execution. |
+
+A validated iterator ends normally only after authoritative successful generation. Truncation, failure, or a missing completion signal must raise; normal exhaustion authorizes `finish()`.
+
+`feed` always means a new delta. It must not accept whole accumulated snapshots implicitly: guessing whether repeated text is a retransmission would make legitimate repeated Python indistinguishable from duplication. A snapshot adapter should independently enforce `new.startswith(previous)` and emit only the new suffix; a changed prefix is terminal. A reconnect adapter needs its own durable replay/offset policy and must not feed an already executed prefix again.
+
+The execution rule should not gain an eager mode that closes an unfinished loop or call. Configuration belongs to preview detail, caller namespace, diagnostics, host resource limits, and transport ownership. Waiting longer is safe; inventing an execution boundary is not a tuning option.
+
+## Namespace and results
+
+`Executor(namespace=None, *, filename="<partial-python>")` uses a supplied dictionary as-is. Without one, it creates a namespace with `__name__ = "__main__"`. Include that binding explicitly when a supplied namespace should behave as a script. No module is registered in `sys.modules`.
+
+`await incremental_exec(validated_deltas, namespace=None, *, filename="<partial-python>")` follows the same namespace rules and returns that exact dictionary on success. It executes while source arrives; the await finishes only after execution and source cleanup complete. Keep an `Executor` when you need diagnostics after failure; use its `stream()` when you also need events as each unit completes.
+
+Each `Execution` records its index, original `unit`, and monotonic `started_at` / `completed_at` times. It does not capture expression results or stdout. Assign results in source (`result = await foo()`) and read them from `executor.namespace`.
+
+Reuse the namespace with a new executor for another logical model response. Bindings persist; compiler flags do not. Each new stream must repeat its required future imports. See [compilation-unit semantics](semantics.md#execution-is-a-sequence-of-compilation-units).
+
+## Lifecycle and diagnostics
+
+`finish()` declares authoritative successful source completion. It can release a final call without a newline or an entire compound suite. It is not a generic cleanup method. `abort()` abandons the stream while preserving evidence; `finally: await finish()` is therefore the wrong integration pattern.
+
+The current diagnostic model is useful when its distinctions are explicit:
+
+| Field | Meaning |
+| --- | --- |
+| `events` | Units that completed successfully; immutable snapshots. |
+| `failed_unit` | The attempted unit whose completion was not acknowledged; partial or complete effects may exist. A parser EOF error has no fabricated attempted unit. |
+| `queued` | Accepted, sealed units that have not started. Excludes the unsealed tail; retained for diagnostics, not automatic retry. |
+| `parser_state` | Immutable `ParserState` snapshot of `pending`, `committed`, and `closed`; read the property again for current values. |
+| `parser_state.pending` | Source still awaiting a grammatical boundary. |
+| `parser_state.committed` | Number of source characters sealed by the parser; this is not an execution watermark. |
+| `parser_state.closed` | Whether the parser received EOF, including invalid EOF; use executor `state` for execution lifecycle. |
+| `error` | The recorded terminal failure; rejected later calls must not replace it. Native generator control exceptions may be wrapped when propagated; see [exception semantics](semantics.md#failures-cancellation-and-effects). |
+| `state` | `ExecutorState`: `"open"`, `"finished"`, `"failed"`, or `"aborted"`. |
+
+Successful events plus the failed range, queued ranges and pending tail preserve the accepted source after a mid-chunk failure. Syntax errors retain the original filename and line numbers. Cooperative cancellation is terminal, including when a producer or statement catches `CancelledError` without clearing the new cancellation request. This is not a hard deadline for code that loops forever or deliberately removes cancellation state.
+
+Catching `CancelledError` can let a unit run its remaining body, assign its result, and return normally. Without `uncancel()`, the outstanding new cancellation still causes `failed_unit` and no success event. The event journal is an acknowledgement record, not a rollback log or proof that an unacknowledged unit had no effects.
+
+Source ranges use half-open character offsets and one-based cursor lines. A final newline advances `end_line`; without it, `end_line` stays on the final physical line. EOF may emit a trivia-only `CodeUnit`, so event counts are neither AST statement counts nor tool-call counts.
+
+### Stateful streaming
+
+`Executor.stream(chunks) -> AsyncGenerator[Execution, None]` gives streaming callers per-unit events and access to diagnostics after failure. `incremental_exec()` consumes the same stream in the calling task, handling closure internally without a second execution state machine.
+
+The stream reserves the executor from its first consumption through exhaustion or explicit close, including source waits and suspended event yields. Interleaved `feed`, `finish`, `abort`, or another stream are rejected. Merely constructing an unstarted stream does not acquire the source or reserve the executor. It can continue a prefix already accepted through `feed`.
+
+The method maintains the same queue/error journal as direct calls and attempts to close its acquired source iterator on exit. Normal validated source exhaustion permits `finish`; early consumer close abandons pending source and retains the unexecuted queue. Runtime failure, invalid EOF, or source failure retain their original exception and source diagnostics. Ordinary cleanup errors must not replace an existing execution failure. New cancellation or process-level interrupts during cleanup propagate with the original failure in Python’s exception chain and leave a failed terminal state.
+
+An async generator that yields while processing `aclose()` violates Python's close protocol and raises `RuntimeError`. With no earlier failure, the executor records that error and enters `failed`; after an execution failure, an ordinary cleanup exception is attached to the original error as a best-effort note. Broken exception formatting cannot replace that primary failure; a new cancellation or process-level interrupt still propagates. The library does not suppress the violation or promise that a broken iterator has finished releasing its resources.
+
+```python
+from contextlib import aclosing
+from partial_python import Executor
+
+executor = Executor(namespace)
+try:
+    async with aclosing(executor.stream(validated_deltas)) as events:
+        async for event in events:
+            display_completed(event)
+except Exception:
+    inspect_failure(executor.error, executor.failed_unit, executor.queued, executor.parser_state.pending)
+    raise
+```
+
+Exceptions from `display_completed()` belong to the caller; `executor.error` records source, execution and cleanup failures inside the executor. `aclosing` also closes the stream when the consumer raises or stops early.
+
+## Minimal transport adapter
+
+`examples/transport.py` demonstrates two NDJSON records over a single loopback TCP connection:
+
+```json
+{"type": "delta", "text": "await tool('early')\n"}
+{"type": "done", "status": "completed", "source": "await tool('early')\n"}
+```
+
+The adapter yields only `text` from delta records. A completed done record must contain the exact concatenation of those deltas. It then ends the text iterator normally, permitting the executor to finalize Python. The terminal record ends this logical stream; the example does not multiplex further streams on the connection.
+
+`truncated`, `cancelled`, and `failed` terminal statuses raise `SourceAborted`. A closed socket without a completed done record, a partial wire record, invalid JSON/UTF-8, wrong field types, unknown status, or different final source raises `SourceProtocolError`. None of these paths turns an unfinished suite into a completed program. An explicit completed record is required even for an empty source stream.
+
+Exact final source equality is a consistency check, not authorization or rollback. A mismatch discovered at the end cannot undo a previously executed statement. Hosts must select the intended tool call, validate its identity and permissions before feeding source, and keep nested tool authorization intact. The small example intentionally has no start-record handshake, sequence numbers, reconnects, hashes or provider dependencies because one ordered connection already owns one known source stream.
+
+The source reader bounds each wire record, and the example bounds total source characters because it retains text for final equality. These are example host limits, not universal library limits. They fail rather than forcing a flush. The socket and server are closed in the same run; no background daemon remains.
+
+### Raw source versus streamed JSON tool arguments
+
+The parser accepts Python characters, not a JSON envelope or markdown. A raw custom-tool input delta is the smallest integration. With a JSON tool schema such as `{"code": "..."}`, a separate decoder must emit only irrevocably decoded characters from the `code` string. It must retain incomplete escape sequences and surrogate pairs and reject changes to an already emitted prefix. Do not execute a repaired partial-JSON value, strip code fences speculatively, or feed a provider's final full source again after its deltas.
+
+The example's JSON is record framing: each record is decoded completely before its Python string is emitted. It is not an implementation of partial JSON argument decoding.
+
+## Task ownership
+
+The standalone executor does not own tasks created by user code. `asyncio.create_task()` must survive parser flushes so a later statement can await the resulting task. Closing or failing a stream does not imply canceling every task in the process: that would also kill unrelated application work. Applications needing scoped cleanup should supply their own task registry or task group around a logical model call. Hard timeouts for CPU-bound code require a process boundary; canceling an asyncio task only works when execution cooperates.
+
+Units execute in the calling task's context, with no hidden task per unit. A single consuming task therefore preserves `ContextVar` changes and tokens across units, which matters for output routing. A shared globals dictionary does not transfer task-local context between separate calls to `asyncio.create_task(anext(events))`. Consume the logical stream in one task and place `asyncio.timeout(...)` around that scope when its context must persist; a source iterator's own context-manager tokens may also require that lifetime.
+
+For notebook integration, retain a single logical cell's output/history lifecycle while executing its sealed statements. Calling `run_cell_async()` separately for each statement changes `_`, `Out[n]`, final-expression display, transform semantics, and task cleanup. A raw-Python execution backend is the initial default; IPython `%`, `!`, and `%%` forms need their own cell-level contract.
+
+## Receive/execution concurrency
+
+The default is pull-based: obtaining the next completed event permits execution to continue and eventually requests more source. It naturally applies backpressure and has no hidden producer task. The TCP reader and operating system may buffer incoming bytes; that does not mean Python execution is concurrent or reordered.
+
+For a host that needs reception while an awaitable tool is running, use an explicit bounded producer queue with separate successful-completion, producer-failure and cancellation signals. Queue exhaustion alone is not EOF. If failure is observed while a tool is running, the host must decide whether to let that already-started unit finish or cancel it; either choice must prevent later queued units from starting. Preserve FIFO order, do not automatically retry side effects, and close both source and owning task lifetimes. A bounded queue is an adapter concern until a real integration demonstrates the right public abstraction.
+
+## Host examples
+
+Run an example from a checkout with `uv run examples/<name>.py`. They are not installed as public package modules; a checkout can import them as `examples.transport`, `examples.overlap`, and `examples.tracebacks`.
+
+| Example | Host responsibility |
+| --- | --- |
+| [streaming_tools](../examples/streaming_tools.py) | Show early execution, whole-compound holdback, ordered awaits, and explicit EOF. |
+| [compare](../examples/compare.py) | Compare live incremental and whole-module execution on the same simulated source stream: `calls`, `loop`, `syntax`, or `truncated`. |
+| [transport](../examples/transport.py) | Validate completion over a real loopback TCP connection with fragmented records. |
+| [overlap](../examples/overlap.py) | Receive through a bounded queue while a tool awaits; propagate source failure and cancellation. |
+| [tracebacks](../examples/tracebacks.py) | Write original source before execution and retain the file while generated functions or tracebacks remain in use. |
+
+The traceback recipe writes UTF-8 without newline translation; it requires an absent or UTF-8 source encoding cookie, because traceback readers honor cookies even though `compile(str)` does not.

partial_python-0.1.0a1/docs/design.md

@@ -0,0 +1,47 @@
+# Design decisions
+
+## Contract before implementation
+
+The product has three layers: a syntactic commit frontier, an effectful executor, and a provisional display completer. Keeping them separate is essential: a parser accepting a prefix at EOF does not prove future append-only text cannot extend it, and a renderable repaired string must never authorize side effects.
+
+An emitted `CodeUnit` is an exact original-source slice whose AST cannot be extended by a valid continuation. This is syntactic stability, not equivalence to compiling an entire future module or a promise of runtime isolation. See [the semantic contract](semantics.md) for counterexamples and explicit restrictions.
+
+## Why Python and the standard library
+
+The execution host's `tokenize` supplies lexical candidate boundaries; its compiler supplies the authoritative grammar and scope checks. An AST is sufficient because original source slices retain formatting and comments. A full CST is useful for editing, but it does not prove an append-only commit boundary. A Rust/TypeScript backend would add grammar/version synchronization and packaging costs before a measured need exists.
+
+`tokenize` is not trusted as a recovery oracle: it is specified for valid Python and synthesizes EOF tokens. We ignore synthetic line endings, retain invalid/incomplete tails, and validate candidate source with CPython. Adversarial source prefixes and a Python version matrix are part of the implementation contract.
+
+## Boundary rules
+
+The [semantic contract](semantics.md#boundaries) defines when an original-source slice is stable. No execution option may relax that boundary; configurable completion belongs to [display preview](preview.md).
+
+A partial name such as `e` or exact `else` at temporary EOF is still ambiguous. A name that cannot grow into a continuation clause, such as `next`, already proves a successor. Known operator, number and string tokens establish the same boundary, even when the literal itself may still grow. Actual f/t-string start tokens and host-proven opaque string openers also establish successors before their contents finish. Unresolved lexical errors remain conservative. A successor keeps its entire logical-line prefix, including formfeeds and explicit joins, so original columns and physical lines remain intact. For a prefix spanning explicit joins, a native AST probe with a placeholder `pass` verifies that it is truly at module indentation; Python 3.11’s pure-Python tokenizer can disagree with the compiler here. The placeholder never enters a `CodeUnit` or execution.
+
+## Incremental scanning
+
+The scanner retains an uncommitted source window, lexical checkpoint, and the last candidate/reason checked. It resumes after actual logical newlines. Inside containers, a completed physical newline can also provide a checkpoint: exact opener kinds and the original logical line's indentation restore lexical context for the next probe. A synthetic opener prefix is used only by `tokenize`; it never enters a `CodeUnit` or compilation. Strings, mismatched delimiters, lexical errors and backslash joins cannot create those checkpoints. Compilation of the entire original unit validates the complete suite.
+
+Physical-line windows avoid scanning the entire remaining buffer for every line in one batch. When a logical line spans physical lines, probe windows grow geometrically; the previous unfinished probe position persists across feed calls. A candidate's reason is part of its cache key: a compound held at newline must be reconsidered when a successor later proves that same offset stable.
+
+Once a line's first token resolves the preceding boundary, a chunk without CR/LF cannot close that logical line. Such chunks accumulate without copying the growing source; scanning or reading `pending` joins them on demand. This removes repeated parsing of large one-line values/calls without maintaining a second Python lexer. A trailing CR always gets lookahead because it may be half of CRLF. Normalized scanning offsets map back to exact original characters.
+
+For a proven unfinished string, a retry gate skips fragments containing no possible closing delimiter. The host tokenizer supplies f/t start tokens; a tokenizer-only quote suffix probe identifies opaque STRING tokens crossing actual EOF without reinterpreting any emitted token. Probe suffixes never enter compilation or execution. A two-character overlap and the preceding backslash-run parity reject escaped candidates while retaining overlapping closers. This necessary condition applies to ordinary/raw/bytes and f/t strings: a real outer closer cannot begin after an odd backslash run, and leaving a replacement field introduces a non-backslash `}`. It does not parse nested expressions or format specifications; a possible closer always triggers authoritative original-source tokenization. Frequent unescaped candidates in nested grammars can still cause repeated rescans. Growing multiline suites and repeatedly reading the full growing `pending` value can also require repeated copying, so this is not a blanket linear-time claim. Structural work-count tests prevent the avoidable cubic/batch-quadratic regressions; future changes should follow representative measurements, not a custom lexer by default.
+
+## Execution lifecycle
+
+A logical response owns one executor and namespace. Units run sequentially; source reception uses pull backpressure. The core creates no reception task. An optional host adapter can overlap reception and execution with explicit queue and task ownership; `examples/overlap.py` verifies that arrangement. No detached tasks are created or cancelled at unit boundaries. A host may own/cancel tasks at the end of the whole logical response.
+
+Failure and cancellation are terminal. The journal separates completed units, the failed attempted unit, sealed units never started, and the unsealed suffix. Parser diagnostics are immutable snapshots so a caller cannot accidentally consume source around the execution journal. Accepted code is never automatically replayed.
+
+`stream` owns its acquired iterator and attempts to await its `aclose()` on every exit when provided. Its write guard stays active across yielded events. Each accepted chunk reaches a cooperative checkpoint even if it is empty or has no sealed unit; an immediately returning buffered source cannot starve cancellation by endlessly supplying unfinished code. Cancellation checks surround each current task's receive/execution/cleanup boundary; historical cancellation counts alone do not poison a fresh operation. Ordinary cleanup errors do not mask an existing execution error. New cancellation or process-level exceptions during cleanup remain observable failures. CPU-bound or cancellation-defeating code requires a process boundary.
+
+## Compiler context
+
+Future flags persist across units with stream-prologue checks. Only the initial string can become a module docstring. Module `global` declarations are rejected. On Python 3.14+, module annotated assignments require source-explicit future annotations so separate lazy annotation functions cannot overwrite one another. Function/class scopes remain intact. Policy traversal skips expressions and uses an explicit stack, avoiding artificial recursion limits on native-compilable expressions.
+
+## Validation and remaining scope
+
+The [validation guide](validation.md) covers interpreter versions, generated grammar, exact AST-position conformance on real source, package installs, and host examples. Shared mutable AST caches would add ownership and mutation rules; optimize this path only when measurements justify that cost.
+
+The core deliberately leaves source-size limits, hard execution deadlines and process isolation to its host. Large multiline inputs and frequent unescaped quote candidates remain performance limits, not reasons to weaken the commit boundary.

partial_python-0.1.0a1/docs/preview.md

@@ -0,0 +1,67 @@
+# Display completion
+
+`partial_python.complete()` is a separate, display-only API. It never executes source and never configures `Parser` or `Executor`. The runtime executes original `CodeUnit` slices sealed by actual streaming boundaries.
+
+```python
+from partial_python import Allow, complete
+
+preview = complete('await foo(1)\nvalues = ["hel', Allow.STRING | Allow.BRACKET)
+assert preview.prefix == 'await foo(1)\nvalues = ["hel'
+assert preview.suffix == '"]'
+assert preview.withheld == ""
+assert preview.committed == len("await foo(1)\n")
+assert preview.provisional
+```
+
+The immutable `Completion` preserves provenance:
+
+- `prefix + withheld` is exactly the original input.
+- `source` is `prefix + suffix`.
+- `committed` is the number of original characters that passed the streaming parser's boundary rules.
+- `provisional` says whether display includes characters beyond that boundary or synthetic text.
+- `diagnostic` is the original `SyntaxError` or `UnicodeError` raised while parsing the input, or `None`. During a stream a syntax diagnostic can mean incomplete, not necessarily irreparable. A lone surrogate cannot be encoded as Python source; its original Unicode error is retained while the malformed tail stays in `withheld`.
+
+A preview without a synthetic suffix can still be provisional. For example, `await foo(1)` is valid source but can still grow into `await foo(1) + other()`. It remains uncommitted until its real logical newline or declared EOF.
+
+## Options
+
+The default `Allow.NONE` returns only the committed prefix and retains the rest as `withheld`. Any nonzero policy enables provisional display and retains a valid pending tail under the parser's current grammar. Additional flags allow specific repairs:
+
+| Flag | Display behavior |
+| --- | --- |
+| `TAIL` | Enable provisional display of already valid source, with no synthetic repairs. |
+| `STRING` | Close ordinary, raw, bytes and triple-quoted string literals. |
+| `BRACKET` | Close data-container and grouping delimiters. |
+| `SUITE` | Append an indented `pass` for an empty final suite. |
+| `ALL` | Enable all four display permissions. |
+
+These flags authorize different repairs. An open string inside a list requires both `STRING` and `BRACKET`. Complete source remains visible under either flag: a preview must not disappear when its real closing quote or bracket arrives. `TAIL` is useful when no repairs are wanted; combining it with another flag adds no behavior.
+
+```python
+assert complete("value = 'hello", Allow.STRING).source == "value = 'hello'"
+assert complete("value = 'hello'", Allow.STRING).source == "value = 'hello'"
+```
+
+Both previews remain provisional until a real streaming boundary arrives. A synthetic `pass` is likewise replaced by the author's real suite without treating that display update as an execution commitment.
+
+Preview follows `Parser`'s ordered AST grammar, with top-level await enabled. Each committed unit was checked under the future flags active at that point; only the pending tail and proposed suffix are checked under the flags established by those units. A later future import never reinterprets earlier accepted source. The combined display is therefore not promised to parse as one module in a single compiler call; Python 3.11's `barry_as_FLUFL` handling is one such difference. Preview adds no executor prologue or scope checks.
+
+Syntax diagnostics retain their original snapshot line numbers. Encoding diagnostics retain the full original input and character offsets. These checks do not imply successful runtime execution, a stable final value, sandboxing or reversible effects.
+
+## Conservative boundaries
+
+The completer only appends suffixes to the entire input, or returns the parser-committed exact prefix. It does not rewrite interior tokens or search for a shorter slice that happens to parse. An invalid interior stays in `withheld` with a diagnostic, including everything after it; later valid-looking code is never spliced around it.
+
+No operands, dictionary values, identifiers, call arguments or exception handlers are invented. An incomplete call stays withheld even when closing its parenthesis would parse: `send(1` might still acquire more arguments. Fully written calls may be displayed under any nonzero policy, but are still not execution commitments.
+
+F-strings and template strings have nested grammars and are never repaired. A complete source containing them can still be displayed under any nonzero policy. A previously committed f-string or template string does not block an independent pending tail. If an f/t string remains inside the uncommitted unit, even when that literal itself is complete, the suffix scanner still conservatively declines repairs. Incomplete escape sequences and odd trailing backslashes remain withheld. A complete compound body may be displayed under a nonzero policy; its execution boundary is unchanged.
+
+`SUITE` is deliberately a preview permission. `for item in items:` can display with a synthetic `pass`, but the real loop remains entirely pending in the execution parser. No API automatically converts a `Completion` object into an execution unit.
+
+`complete()` is a stateless snapshot operation. For streaming execution use `Executor.feed()` or `Executor.stream()`, and inspect completed events through `Execution.unit`. Calling the snapshot completer on every growing prefix necessarily rereads prior source; benchmark that separately from the incremental runtime. Display completions do not belong in `executor.queued` or its completed-event journal.
+
+## References and validation
+
+See [the partial family](prior-art.md#the-partial-family) for pinned references and the distinction between provisional display and committed effects.
+
+Tests cover every character prefix of representative Unicode, CRLF, nested-suite, decorator, async and f-string sources under every flag combination, plus explicit repair/hold cases.

partial_python-0.1.0a1/docs/prior-art.md

@@ -0,0 +1,55 @@
+# Related implementations
+
+Streaming execution already has direct precedents. This prototype focuses on an append-only Python boundary contract, original-source provenance, top-level await, and explicit terminal state. The comparisons below describe inspected versions; they are not claims about every version or benchmark results for this project. External implementations were inspected without installing or executing them, and no implementation code was copied.
+
+## The partial family
+
+The family suggests keeping provisional display separate from committed effects:
+
+| Reference | Useful idea and boundary |
+| --- | --- |
+| [`partial-json-parser` API](https://github.com/promplate/partial-json-parser/blob/e0aa5e2ec52ed81c2c5d7692d5fe81d01c4b0bc6/src/partial_json_parser/core/api.py#L11-L26) | Separates value parsing from string completion. [`Allow`](https://github.com/promplate/partial-json-parser/blob/e0aa5e2ec52ed81c2c5d7692d5fe81d01c4b0bc6/src/partial_json_parser/core/options.py#L4-L21) controls incomplete values; [`fix`](https://github.com/promplate/partial-json-parser/blob/e0aa5e2ec52ed81c2c5d7692d5fe81d01c4b0bc6/src/partial_json_parser/core/complete.py#L12-L28) separates retained text and synthetic suffix. This motivates `complete()` and its display-only permissions. |
+| [`partial-tsx`](https://github.com/MindLab-Research/macaron-genui-demo/blob/e960b17a460e7ddf7f03045c2a67aabbf5f79f67/lib/partial-tsx/src/partial.ts#L1454-L1480) | Tracks original-source provenance; [continuation handling](https://github.com/MindLab-Research/macaron-genui-demo/blob/e960b17a460e7ddf7f03045c2a67aabbf5f79f67/lib/partial-tsx/src/partial.ts#L1900-L1938) retains pending control structures. Display exceptions for JSX or function shells do not authorize Python effects. |
+| [`partial-react`](https://github.com/MindLab-Research/macaron-genui-demo/blob/e960b17a460e7ddf7f03045c2a67aabbf5f79f67/lib/partial-react/src/runtime.ts#L309-L413) | Separates scheduling, compilation and committed rendering. A renderer can coalesce updates; an executor must preserve accepted source order and must not replay effects. |
+| [`partial-vue`](https://github.com/MindLab-Research/macaron-genui-demo/blob/e960b17a460e7ddf7f03045c2a67aabbf5f79f67/lib/partial-vue/src/partial.ts#L137-L172) | Structural directives wait for a real closing quote: a temporarily valid expression can still change. Python statements likewise need actual boundary evidence. |
+| [`partial-svelte`](https://github.com/MindLab-Research/macaron-genui-demo/blob/e960b17a460e7ddf7f03045c2a67aabbf5f79f67/lib/partial-svelte/README.md#L34-L39) | Documents limitations of state preservation through internal compiler-generated APIs. The Python parser delegates final syntax acceptance to the host compiler. |
+| [`partial-solidjs`](https://github.com/MindLab-Research/macaron-genui-demo/blob/e960b17a460e7ddf7f03045c2a67aabbf5f79f67/lib/partial-solidjs/README.md#L26-L44) | Preserves visible output and reactive state across updates. Such restoration cannot undo a Python operation's external effects. |
+
+The inspected JSON revision is `e0aa5e2ec52ed81c2c5d7692d5fe81d01c4b0bc6`, with an [MIT license](https://github.com/promplate/partial-json-parser/blob/e0aa5e2ec52ed81c2c5d7692d5fe81d01c4b0bc6/LICENSE). Renderer references are pinned to GenUI revision `e960b17a460e7ddf7f03045c2a67aabbf5f79f67`; this audit does not establish a standalone license for those in-repository packages. [Preview design](preview.md) describes the resulting API in more detail.
+
+## EAGER / StreamExec
+
+[Executing as You Generate](https://arxiv.org/html/2604.00491v2), arXiv `2604.00491v2` (22 June 2026), describes AST boundary detection with lookahead, persistent execution, dynamic batching, gating and optional early error interruption. This is directly related prior art. Batching is useful when interpreter invocation has meaningful overhead; its benefits need measurement for a particular host.
+
+The inspected [public artifact](https://doi.org/10.6084/m9.figshare.31869469.v1) is an earlier version, published 27 March 2026: [`EAGER-main.zip`](https://ndownloader.figshare.com/files/63180436), MD5 `f441473c12c18fd106eb80926059ec75`. It has no Git commit metadata. The archive's `LICENSE:1–21` is MIT, copyright 2026 EAGER Authors; the [deposit metadata](https://api.figshare.com/v2/articles/31869469) separately labels the deposit CC BY 4.0. Paper and artifact dates differ, so implementation differences below should not be read as an audit of the later paper's implementation.
+
+Useful source locations within that archive:
+
+| Location | Inspected behavior |
+| --- | --- |
+| `eager/parser.py:117–172,289–344,397–433` | Uses `code.compile_command` and `ast.parse` on pending physical lines. Compounds wait for a later parsed statement or finalization. It reparses the growing pending source, and receives lookahead through complete LF-terminated lines. |
+| `eager/parser.py:446–473`; `eager/consumer.py:303–324` | Invalid/incomplete EOF becomes a chunk carrying `parse_error`; the consumer records and excludes such chunks from executable batches. |
+| `eager/consumer.py:62–133,479–516` | Coalesces available chunks, with optional size/delay limits and halt/continue error policies. |
+| `eager/executors.py:17–46,50–85,152–207` | Shared namespaces in local threads or a persistent subprocess. The subprocess has its own timeout/kill mechanism. Plain compilation does not enable top-level await or retain future flags across subprocess requests. |
+| `eager/pipeline.py:41,94–125` | Disables rewriting by default, gathers producer/consumer, then runs a separate full-program baseline: this entry point is an experiment harness. |
+| `eager/parser.py:26,48–50`; `eager/rewriters.py:101–117,139–173` | Direct parser construction instead enables an if-chain rewriter by default. It injects bookkeeping globals and fixed-indentation source. |
+
+Our contract deliberately avoids that rewriting, retains native exceptions, and makes EOF, abort and failure explicit. The examined artifact's runtime-error halt marks later chunks skipped while generation continues; its timeout path cancels both tasks (`clean_benchmark/runtime.py:310–351`). The later paper describes additional gating and interruption behavior that should not be inferred from these older paths.
+
+Static inspection also highlights why source reconstruction alone cannot prove general module equivalence: compiler flags, module docstrings and compilation context survive differently across separate `exec` calls. See [our semantic limits](semantics.md). The archive contains benchmark runners and an overhead experiment (`scripts/test_exec_overhead.py`); no dedicated parser unit/property suite was found. None of its reported benchmark results are results for partial-python.
+
+## Playbooks
+
+[Playbooks](https://github.com/playbooks-ai/playbooks/tree/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8) contains a direct streaming Python executor integrated with agent call frames. Inspected revision: `1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8`; [MIT license](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/LICENSE), copyright 2024 Playbooks AI.
+
+- [`CodeBuffer`](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/src/playbooks/execution/incremental_code_buffer.py#L38-L115) finds a parsable prefix using physical newlines and indentation. Its markdown stripping and `$variable` preprocessing serve that host's language, rather than exact Python-source preservation.
+- [`add_chunk`](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/src/playbooks/execution/streaming_python_executor.py#L106-L192) executes parsed statements and records errors. [`Await handling`](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/src/playbooks/execution/streaming_python_executor.py#L321-L383) wraps a statement in an async function, then copies selected locals into the agent frame. Partial-python instead uses native top-level-await compilation and one namespace.
+- [`Tests`](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/tests/unit/execution/test_streaming_variable_capture.py#L110-L177) exercise variable persistence and mixed sync/async work. Host-specific [safety-filter tests](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/tests/unit/execution/test_streaming_executor_safety_filters.py#L68-L115) intentionally permit skipping some source statements, unlike this project's execution contract.
+
+Static counterexamples help define our regression requirements: `else:` must not close and emit the preceding `if`; a final suite needs explicit EOF handling; `x = x + await foo()` must keep module scope; and successful statements before a later failure need individual provenance. At the inspected revision, `CodeBuffer:95–102` can select the prefix before `else`, [`finalize`](https://github.com/playbooks-ai/playbooks/blob/1f6ee0cc760f245aa3af262b0fd0e6d0d70060f8/src/playbooks/execution/streaming_python_executor.py#L388-L419) only retries the normal buffer path, and the prefix journal updates after the entire parsed statement loop succeeds. These are source-derived observations, not execution results from that project.
+
+## Partial expression parsing
+
+vLLM's [`make_valid_python`](https://github.com/vllm-project/vllm/blob/b8c04dd3b455affd8d91f0ab436b815491c9af38/vllm/tool_parsers/utils.py#L982-L1077) completes streaming tool-call expressions and returns completed text plus an added suffix, or `None`. The inspected revision is `b8c04dd3b455affd8d91f0ab436b815491c9af38`; the file [declares Apache-2.0](https://github.com/vllm-project/vllm/blob/b8c04dd3b455affd8d91f0ab436b815491c9af38/vllm/tool_parsers/utils.py#L1-L2). It can strip trailing whitespace/comma and synthesize delimiters; it also rejects synthesized sets that could really be incomplete dictionaries. This is useful preview prior art, with a different contract from executing committed original Python.
+
+Python's [`codeop`](https://docs.python.org/3/library/codeop.html) supplies complete/incomplete/error classification and a stateful compiler that remembers future directives. That is useful machinery, but temporary syntactic completeness does not establish append stability. [`Parso`](https://parso.readthedocs.io/en/latest/docs/usage.html) supplies error recovery and optional incremental reparsing; recovering a syntax tree likewise does not authorize irreversible execution.