repl-session 0.2.1__tar.gz → 0.3.1__tar.gz

{repl_session-0.2.1 → repl_session-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: repl-session
-Version: 0.2.1
+Version: 0.3.1
 Summary: Runs a session with a Read-Eval-Print-Loop (REPL). Takes YAML as input gives JSON as output.
 Classifier: Development Status :: 3 - Alpha
 Classifier: Environment :: Console
@@ -14,6 +14,9 @@ License-File: LICENSE
 Requires-Dist: click>=8.1
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: pexpect>=4.9.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: jupyter
+Requires-Dist: jupyter-client>=8.6; extra == "jupyter"
 Dynamic: license-file
 
 # REPL Session
@@ -33,7 +36,7 @@ This is only really useful if you're hacking together a literate programming env
 
 The REPL of your choice is started and interacted with through the `pexpect` library. All I/O is dealt with through `msgspec`.
 
-The preferred way to solve this is by using `jupyter-client` in combination with an existing Jupyter kernel. However, not all languages have a Jupyter kernel available, and developing one takes a bit more than configuring `pexpect` for an existing REPL. I may still include Jupyter support in this application at a later stage.
+The preferred way to solve this is by using `jupyter-client` in combination with an existing Jupyter kernel. However, not all languages have a Jupyter kernel available, and developing one takes a bit more than configuring `pexpect` for an existing REPL. If a kernel is available for your language, you can use it through the optional Jupyter backend: `pip install repl-session[jupyter]`, then see [the Jupyter backend docs](https://entangled.github.io/repl-session/jupyter).
 
 ## Install
 
@@ -169,12 +172,13 @@ null
 
 ## Input/Output structure
 
-The user can configure how the REPL is called and interpreted.
+The user can configure how the REPL is called and interpreted. The `backend` field picks between this `pexpect` based REPL backend (`backend: repl`, the default) and the Jupyter backend (`backend: jupyter`).
 
 ```python
 #| id: input-data
-class ReplConfig(msgspec.Struct):
-    """Configuration
+class ReplConfig(msgspec.Struct, tag_field="backend", tag="repl"):
+    """Configuration for the `pexpect` backend; selected with `backend: repl`,
+    which is also the default when `backend` is absent.
 
     Attributes:
         command (str): Command to start the REPL
@@ -201,33 +205,61 @@ class ReplConfig(msgspec.Struct):
 
 Then, a session is a list of commands. Each command should be a UTF-8 string, and we allow to attach some meta-data like expected MIME type for the output. We can also pass an expected output in the case of a documentation test. If `output` was already given on the input, it is  moved to `expected`. This way it becomes really easy to setup regression tests on your documentation. Just rerun on the generated output file.
 
+The `output` field only holds the representation matching `output_type`. The `outputs` field records everything a command produced, in order. Each item has a `kind` (`result`, `display`, `stdout`, `stderr` or `error`) and a MIME bundle `data` with one or more representations of the same output. A Jupyter kernel may return `text/plain`, `text/html` and `image/png` together, for example. The REPL backend only sees terminal text, so it always gives a single `result`.
+
 ```python
 #| id: input-data
+OutputKind = Literal["result", "display", "stdout", "stderr", "error"]
+
+
+class ReplOutput(msgspec.Struct, omit_defaults=True):
+    """A single piece of output produced by a command.
+
+    Attributes:
+        kind (str): One of `result` (the value of the command), `display`
+            (rich output produced along the way, e.g. a plot), `stdout`,
+            `stderr` or `error` (a traceback). The `repl` backend cannot tell
+            these apart and reports everything as a single `result`.
+        data (dict[str, str]): MIME bundle: the same output in one or more
+            representations, keyed by MIME type. Binary data is base64 encoded;
+            JSON data is serialized to a string.
+        metadata (dict[str, Any]): Per MIME type metadata, e.g. image sizes.
+    """
+
+    kind: OutputKind
+    data: dict[str, str]
+    metadata: dict[str, Any] = msgspec.field(default_factory=dict)
+
+
 class ReplCommand(msgspec.Struct):
     """A command to be sent to the REPL.
 
     Attributes:
         command (str): the command.
         output_type (str): MIME type of expected output.
-        output (str | None): evaluated output.
+        output (str | None): evaluated output for `output_type`.
         expected (str | None): expected output.
+        outputs (list[ReplOutput] | None): everything the command produced, in
+            order, with all available MIME types.
     """
 
     command: str
     output_type: str = "text/plain"
     output: str | None = None
     expected: str | None = None
+    outputs: list[ReplOutput] | None = None
 
 
 class ReplSession(msgspec.Struct):
     """A REPL session.
 
     Attributes:
-        config (ReplConfig): Config for setting up a REPL session.
+        config (ReplConfig | JupyterConfig): Config for setting up a REPL
+            session, the `backend` field selects which.
         commands (list[ReplCommand]): List of commands in the session.
     """
 
-    config: ReplConfig
+    config: ReplConfig | JupyterConfig
     commands: list[ReplCommand]
 ```
 

{repl_session-0.2.1 → repl_session-0.3.1}/README.md

@@ -15,7 +15,7 @@ This is only really useful if you're hacking together a literate programming env
 
 The REPL of your choice is started and interacted with through the `pexpect` library. All I/O is dealt with through `msgspec`.
 
-The preferred way to solve this is by using `jupyter-client` in combination with an existing Jupyter kernel. However, not all languages have a Jupyter kernel available, and developing one takes a bit more than configuring `pexpect` for an existing REPL. I may still include Jupyter support in this application at a later stage.
+The preferred way to solve this is by using `jupyter-client` in combination with an existing Jupyter kernel. However, not all languages have a Jupyter kernel available, and developing one takes a bit more than configuring `pexpect` for an existing REPL. If a kernel is available for your language, you can use it through the optional Jupyter backend: `pip install repl-session[jupyter]`, then see [the Jupyter backend docs](https://entangled.github.io/repl-session/jupyter).
 
 ## Install
 
@@ -151,12 +151,13 @@ null
 
 ## Input/Output structure
 
-The user can configure how the REPL is called and interpreted.
+The user can configure how the REPL is called and interpreted. The `backend` field picks between this `pexpect` based REPL backend (`backend: repl`, the default) and the Jupyter backend (`backend: jupyter`).
 
 ```python
 #| id: input-data
-class ReplConfig(msgspec.Struct):
-    """Configuration
+class ReplConfig(msgspec.Struct, tag_field="backend", tag="repl"):
+    """Configuration for the `pexpect` backend; selected with `backend: repl`,
+    which is also the default when `backend` is absent.
 
     Attributes:
         command (str): Command to start the REPL
@@ -183,33 +184,61 @@ class ReplConfig(msgspec.Struct):
 
 Then, a session is a list of commands. Each command should be a UTF-8 string, and we allow to attach some meta-data like expected MIME type for the output. We can also pass an expected output in the case of a documentation test. If `output` was already given on the input, it is  moved to `expected`. This way it becomes really easy to setup regression tests on your documentation. Just rerun on the generated output file.
 
+The `output` field only holds the representation matching `output_type`. The `outputs` field records everything a command produced, in order. Each item has a `kind` (`result`, `display`, `stdout`, `stderr` or `error`) and a MIME bundle `data` with one or more representations of the same output. A Jupyter kernel may return `text/plain`, `text/html` and `image/png` together, for example. The REPL backend only sees terminal text, so it always gives a single `result`.
+
 ```python
 #| id: input-data
+OutputKind = Literal["result", "display", "stdout", "stderr", "error"]
+
+
+class ReplOutput(msgspec.Struct, omit_defaults=True):
+    """A single piece of output produced by a command.
+
+    Attributes:
+        kind (str): One of `result` (the value of the command), `display`
+            (rich output produced along the way, e.g. a plot), `stdout`,
+            `stderr` or `error` (a traceback). The `repl` backend cannot tell
+            these apart and reports everything as a single `result`.
+        data (dict[str, str]): MIME bundle: the same output in one or more
+            representations, keyed by MIME type. Binary data is base64 encoded;
+            JSON data is serialized to a string.
+        metadata (dict[str, Any]): Per MIME type metadata, e.g. image sizes.
+    """
+
+    kind: OutputKind
+    data: dict[str, str]
+    metadata: dict[str, Any] = msgspec.field(default_factory=dict)
+
+
 class ReplCommand(msgspec.Struct):
     """A command to be sent to the REPL.
 
     Attributes:
         command (str): the command.
         output_type (str): MIME type of expected output.
-        output (str | None): evaluated output.
+        output (str | None): evaluated output for `output_type`.
         expected (str | None): expected output.
+        outputs (list[ReplOutput] | None): everything the command produced, in
+            order, with all available MIME types.
     """
 
     command: str
     output_type: str = "text/plain"
     output: str | None = None
     expected: str | None = None
+    outputs: list[ReplOutput] | None = None
 
 
 class ReplSession(msgspec.Struct):
     """A REPL session.
 
     Attributes:
-        config (ReplConfig): Config for setting up a REPL session.
+        config (ReplConfig | JupyterConfig): Config for setting up a REPL
+            session, the `backend` field selects which.
         commands (list[ReplCommand]): List of commands in the session.
     """
 
-    config: ReplConfig
+    config: ReplConfig | JupyterConfig
     commands: list[ReplCommand]
 ```
 

{repl_session-0.2.1 → repl_session-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "repl-session"
-version = "0.2.1"
+version = "0.3.1"
 description = "Runs a session with a Read-Eval-Print-Loop (REPL). Takes YAML as input gives JSON as output."
 readme = "README.md"
 classifiers = [
@@ -16,12 +16,14 @@ dependencies = [
     "click>=8.1",
     "msgspec>=0.19.0",
     "pexpect>=4.9.0",
+    "pyyaml>=6.0",
 ]
 
 [dependency-groups]
 dev = [
     "basedpyright>=1.31.4",
     "entangled-cli>=2.1.13",
+    "ipykernel>=7.4.0",
     "mkdocs-macros-plugin>=1.3.9",
     "mkdocs-material>=9.6.20",
     "pytest>=9.2.0",
@@ -34,5 +36,14 @@ dev = [
 [project.scripts]
 repl-session = "repl_session:main"
 
+[project.optional-dependencies]
+jupyter = [
+    "jupyter-client>=8.6",
+]
+
 [tool.uv]
 package = true
+
+[tool.ruff.lint]
+# Imports are grouped by theme (stdlib, dependencies, internal), not sorted.
+ignore = ["I001"]

repl_session-0.3.1/src/repl_session/__init__.py

@@ -0,0 +1,115 @@
+# ~/~ begin <<docs/index.md#src/repl_session/__init__.py>>[init]
+"""
+`repl-session` is a command-line tool to evaluate a given session
+in any REPL, and store the results.
+"""
+
+# ~/~ begin <<docs/index.md#imports>>[init]
+# from datetime import datetime, tzinfo
+from typing import IO, Any
+
+# import re
+from contextlib import AbstractContextManager
+import sys
+import logging
+
+import msgspec
+import click
+import importlib.metadata
+
+from .schema import (
+    Evaluator,
+    JupyterConfig,
+    OutputKind,
+    ReplCommand,
+    ReplConfig,
+    ReplOutput,
+    ReplSession,
+)
+from .repl import repl
+
+__all__ = [
+    "Evaluator",
+    "JupyterConfig",
+    "OutputKind",
+    "ReplCommand",
+    "ReplConfig",
+    "ReplOutput",
+    "ReplSession",
+    "evaluator",
+    "read_session",
+    "repl",
+    "run_session",
+    "select_output",
+    "write_session",
+]
+
+__version__ = importlib.metadata.version("repl-session")
+# ~/~ end
+# ~/~ begin <<docs/index.md#run-session>>[init]
+def evaluator(config: ReplConfig | JupyterConfig) -> AbstractContextManager[Evaluator]:
+    """Start the backend selected by `config`. The Jupyter backend is only
+    imported when needed, so its dependencies remain optional."""
+    if isinstance(config, JupyterConfig):
+        from .jupyter import kernel
+
+        return kernel(config)
+    return repl(config)
+
+
+def select_output(outputs: list[ReplOutput], output_type: str) -> str | None:
+    """Pick the representation of type `output_type` from the outputs. Text
+    is concatenated in order; for other types the last one wins."""
+    values = [out.data[output_type] for out in outputs if output_type in out.data]
+    if not values:
+        return None
+    if output_type.startswith("text/"):
+        return "\n".join(v.strip("\n") for v in values).strip() or None
+    return values[-1]
+
+
+def run_session(session: ReplSession):
+    with evaluator(session.config) as run:
+        for cmd in session.commands:
+            expected = cmd.expected or cmd.output
+            cmd.outputs = run(cmd)
+            cmd.output = select_output(cmd.outputs, cmd.output_type)
+            cmd.expected = expected
+
+    return session
+# ~/~ end
+# ~/~ begin <<docs/index.md#io>>[init]
+def read_session(port: IO[str] = sys.stdin) -> ReplSession:
+    data: Any = msgspec.yaml.decode(port.read())
+    # Sessions written before there were multiple backends have no `backend` tag.
+    if isinstance(data, dict) and isinstance(config := data.get("config"), dict):
+        _ = config.setdefault("backend", "repl")
+    return msgspec.convert(data, type=ReplSession)
+
+
+def write_session(session: ReplSession, port: IO[str] = sys.stdout):
+    data = msgspec.json.encode(session)
+    _ = port.write(data.decode())
+# ~/~ end
+
+@click.command()
+@click.version_option(
+    __version__, "-v", "--version",
+    prog_name="repl-session", message="%(prog)s %(version)s",
+    help="show version and exit")
+@click.option("-l", "--log-enable", is_flag=True, help="show debugging output")
+def repl_session(log_enable: bool = False):
+    """
+    repl-session runs a REPL session, reading JSON from standard input and
+    writing to standard output. Both the input and output follow the same
+    schema.
+    """
+    if log_enable:
+        logging.basicConfig(level=logging.DEBUG)
+
+    write_session(run_session(read_session()))
+
+
+def main():
+    repl_session()
+# ~/~ end

repl_session-0.3.1/src/repl_session/jupyter.py

@@ -0,0 +1,105 @@
+# ~/~ begin <<docs/jupyter.md#src/repl_session/jupyter.py>>[init]
+"""
+Jupyter backend for `repl-session`. Requires the `jupyter` extra.
+"""
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import Any
+import logging
+import os
+
+import msgspec
+
+try:
+    from jupyter_client.manager import start_new_kernel
+except ImportError as e:
+    raise ImportError(
+        "The Jupyter backend needs the `jupyter` extra: "
+        "`pip install repl-session[jupyter]`"
+    ) from e
+
+from .schema import ANSI_ESCAPE, Evaluator, JupyterConfig, ReplCommand, ReplOutput
+
+logger = logging.getLogger(__name__)
+
+# ~/~ begin <<docs/jupyter.md#jupyter-collect>>[init]
+def mime_bundle(data: dict[str, Any]) -> dict[str, str]:
+    return {
+        mime: value if isinstance(value, str) else msgspec.json.encode(value).decode()
+        for mime, value in data.items()
+    }
+
+
+class OutputCollector:
+    """Collects IOPub messages into a list of `ReplOutput`."""
+
+    def __init__(self):
+        self.outputs: list[ReplOutput] = []
+        self.clear_pending: bool = False
+
+    def append(self, output: ReplOutput):
+        if self.clear_pending:
+            self.outputs.clear()
+            self.clear_pending = False
+
+        last = self.outputs[-1] if self.outputs else None
+        if (
+            last is not None
+            and output.kind in ("stdout", "stderr")
+            and last.kind == output.kind
+        ):
+            last.data["text/plain"] += output.data["text/plain"]
+        else:
+            self.outputs.append(output)
+
+    def __call__(self, msg: dict[str, Any]):
+        msg_type = msg["header"]["msg_type"]
+        content = msg["content"]
+        match msg_type:
+            case "stream":
+                self.append(ReplOutput(content["name"], {"text/plain": content["text"]}))
+            case "execute_result" | "display_data":
+                kind = "result" if msg_type == "execute_result" else "display"
+                self.append(
+                    ReplOutput(kind, mime_bundle(content["data"]), content.get("metadata", {}))
+                )
+            case "error":
+                traceback = ANSI_ESCAPE.sub("", "\n".join(content["traceback"]))
+                self.append(ReplOutput("error", {"text/plain": traceback}))
+            case "clear_output":
+                if content.get("wait"):
+                    self.clear_pending = True
+                else:
+                    self.outputs.clear()
+            case _:
+                logger.debug("ignoring %s message", msg_type)
+# ~/~ end
+# ~/~ begin <<docs/jupyter.md#jupyter-kernel>>[init]
+@contextmanager
+def kernel(config: JupyterConfig) -> Generator[Evaluator]:
+    km, kc = start_new_kernel(
+        startup_timeout=config.startup_timeout,
+        kernel_name=config.kernel,
+        cwd=config.cwd,
+        env=dict(os.environ) | config.environment,
+    )
+
+    def send(cmd: ReplCommand) -> list[ReplOutput]:
+        logger.debug("executing: %s", cmd.command)
+        collect = OutputCollector()
+        _ = kc.execute_interactive(
+            cmd.command,
+            timeout=config.timeout,
+            output_hook=collect,
+            allow_stdin=False,
+        )
+        return collect.outputs
+
+    try:
+        yield send
+    finally:
+        kc.stop_channels()
+        km.shutdown_kernel(now=True)
+# ~/~ end
+# ~/~ end

repl_session-0.3.1/src/repl_session/repl.py

@@ -0,0 +1,121 @@
+# ~/~ begin <<docs/index.md#src/repl_session/repl.py>>[init]
+"""
+REPL backend for `repl-session`: drives any interactive REPL through `pexpect`.
+"""
+
+from collections.abc import Generator
+from contextlib import contextmanager
+import uuid
+import re
+import logging
+import os
+
+import pexpect
+
+from .schema import ANSI_ESCAPE, Evaluator, ReplCommand, ReplConfig, ReplOutput
+
+logger = logging.getLogger(__name__)
+
+# ~/~ begin <<docs/index.md#repl-contextmanager>>[init]
+def spawn(config: ReplConfig):
+    env = dict(os.environ) | config.environment
+    child: pexpect.spawn[str] = pexpect.spawn(
+        config.command,
+        timeout=config.timeout,
+        echo=False,
+        encoding="utf-8",
+        env=env,
+    )
+    return child
+
+
+def clean_output(config: ReplConfig, output: str) -> str | None:
+    """Remove ANSI escapes (if requested), normalize the CRLF line endings
+    that the pseudo-terminal produces, and trim surrounding whitespace.
+    Returns `None` if nothing is left."""
+    if config.strip_ansi:
+        output = ANSI_ESCAPE.sub("", output)
+    output = output.replace("\r\n", "\n").replace("\r", "").strip()
+    return output or None
+
+
+@contextmanager
+def repl(config: ReplConfig) -> Generator[Evaluator]:
+    key = str(uuid.uuid4())
+    change_prompt_cmd = config.change_prompt.format(key=key)
+    prompt = config.prompt.format(key=key)
+    continuation_prompt = (
+        config.continuation_prompt.format(key=key)
+        if config.continuation_prompt is not None
+        else None
+    )
+
+    child: pexpect.spawn[str]
+    with spawn(config) as child:
+        _ = child.expect(config.first_prompt)
+        _ = child.sendline(change_prompt_cmd)
+        # if config.strip_command:
+        #    child.expect(key)
+        _ = child.expect(prompt)
+
+        if continuation_prompt is not None:
+            def send_text(msg: str) -> str | None:
+                lines = msg.splitlines()
+                answer: list[str] = []
+
+                if not lines:
+                    return None
+
+                still_waiting: bool = True
+                for line in lines:
+                    logger.debug("sending: '%s'", line)
+                    _ = child.sendline(line)
+                    logger.debug("waiting for prompt or continuation")
+                    _ = child.expect(
+                        f"(?P<norm>{prompt})|(?P<cont>{continuation_prompt})"
+                    )
+                    if not isinstance(child.match, re.Match):
+                        continue
+                    if child.match.group("cont") is not None:
+                        logger.debug("continuation: %s -- %s", child.before, child.after)
+                        still_waiting = True
+                    else:
+                        logger.debug("done: %s", child.before)
+                        if child.before is not None:
+                            answer.append(child.before)
+                        still_waiting = False
+
+                if still_waiting:
+                    logger.debug("waiting for last prompt")
+                    _ = child.sendline("")
+                    _ = child.expect(prompt)
+                    logger.debug("got: %s", child.before)
+                    if child.before:
+                        answer.append(child.before)
+
+                if not answer:
+                    return None
+
+                return clean_output(config, answer[-1])
+
+        else:
+            def send_text(msg: str) -> str | None:
+                logger.debug("sending: %s", msg)
+
+                _ = child.sendline(msg)
+                _ = child.expect(prompt)
+
+                if not child.before:
+                    return None
+
+                return clean_output(config, child.before)
+
+        def send(cmd: ReplCommand) -> list[ReplOutput]:
+            text = send_text(cmd.command)
+            if text is None:
+                return []
+            return [ReplOutput("result", {cmd.output_type: text})]
+
+        yield send
+# ~/~ end
+# ~/~ end

repl_session-0.3.1/src/repl_session/schema.py

@@ -0,0 +1,117 @@
+# ~/~ begin <<docs/index.md#src/repl_session/schema.py>>[init]
+"""
+Data types for `repl-session`: the session format and the interface shared
+by the backends.
+"""
+
+from typing import Any, Literal
+from collections.abc import Callable
+import re
+
+import msgspec
+
+# ~/~ begin <<docs/jupyter.md#jupyter-config>>[init]
+class JupyterConfig(msgspec.Struct, tag_field="backend", tag="jupyter"):
+    """Configuration for the Jupyter backend; selected with `backend: jupyter`.
+
+    Attributes:
+        kernel (str): Name of the Jupyter kernel, see `jupyter kernelspec list`.
+        timeout (float): Command timeout in seconds.
+        startup_timeout (float): Timeout for starting the kernel in seconds.
+        cwd (str | None): Working directory of the kernel.
+        environment (dict[str, str]): Extra environment variables for the kernel.
+    """
+
+    kernel: str
+    timeout: float = 30.0
+    startup_timeout: float = 60.0
+    cwd: str | None = None
+    environment: dict[str, str] = msgspec.field(default_factory=dict)
+# ~/~ end
+# ~/~ begin <<README.md#input-data>>[init]
+class ReplConfig(msgspec.Struct, tag_field="backend", tag="repl"):
+    """Configuration for the `pexpect` backend; selected with `backend: repl`,
+    which is also the default when `backend` is absent.
+
+    Attributes:
+        command (str): Command to start the REPL
+        first_prompt (str): Regex to match the first prompt
+        change_prompt (str): Command to change prompt; should contain '{key}' as an
+            argument.
+        next_prompt (str): Regex to match the changed prompts; should contain '{key}'
+            as an argument.
+        append_newline (bool): Whether to append a newline to given commands.
+        strip_command (bool): Whether to strip the original command from the gotten
+            output; useful if the REPL echoes your input before answering.
+        timeout (float): Command timeout for this session in seconds.
+    """
+
+    command: str
+    first_prompt: str
+    change_prompt: str
+    prompt: str
+    continuation_prompt: str | None = None
+    strip_ansi: bool = False
+    environment: dict[str, str] = msgspec.field(default_factory=dict)
+    timeout: float = 5.0
+# ~/~ end
+# ~/~ begin <<README.md#input-data>>[1]
+OutputKind = Literal["result", "display", "stdout", "stderr", "error"]
+
+
+class ReplOutput(msgspec.Struct, omit_defaults=True):
+    """A single piece of output produced by a command.
+
+    Attributes:
+        kind (str): One of `result` (the value of the command), `display`
+            (rich output produced along the way, e.g. a plot), `stdout`,
+            `stderr` or `error` (a traceback). The `repl` backend cannot tell
+            these apart and reports everything as a single `result`.
+        data (dict[str, str]): MIME bundle: the same output in one or more
+            representations, keyed by MIME type. Binary data is base64 encoded;
+            JSON data is serialized to a string.
+        metadata (dict[str, Any]): Per MIME type metadata, e.g. image sizes.
+    """
+
+    kind: OutputKind
+    data: dict[str, str]
+    metadata: dict[str, Any] = msgspec.field(default_factory=dict)
+
+
+class ReplCommand(msgspec.Struct):
+    """A command to be sent to the REPL.
+
+    Attributes:
+        command (str): the command.
+        output_type (str): MIME type of expected output.
+        output (str | None): evaluated output for `output_type`.
+        expected (str | None): expected output.
+        outputs (list[ReplOutput] | None): everything the command produced, in
+            order, with all available MIME types.
+    """
+
+    command: str
+    output_type: str = "text/plain"
+    output: str | None = None
+    expected: str | None = None
+    outputs: list[ReplOutput] | None = None
+
+
+class ReplSession(msgspec.Struct):
+    """A REPL session.
+
+    Attributes:
+        config (ReplConfig | JupyterConfig): Config for setting up a REPL
+            session, the `backend` field selects which.
+        commands (list[ReplCommand]): List of commands in the session.
+    """
+
+    config: ReplConfig | JupyterConfig
+    commands: list[ReplCommand]
+# ~/~ end
+# ~/~ begin <<docs/index.md#schema-shared>>[init]
+Evaluator = Callable[[ReplCommand], list[ReplOutput]]
+
+ANSI_ESCAPE = re.compile(r"(\u001b|\x1B)(\[[0-?]*[ -\/]*[@-~]|[\>\=])")
+# ~/~ end
+# ~/~ end

{repl_session-0.2.1 → repl_session-0.3.1}/src/repl_session.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: repl-session
-Version: 0.2.1
+Version: 0.3.1
 Summary: Runs a session with a Read-Eval-Print-Loop (REPL). Takes YAML as input gives JSON as output.
 Classifier: Development Status :: 3 - Alpha
 Classifier: Environment :: Console
@@ -14,6 +14,9 @@ License-File: LICENSE
 Requires-Dist: click>=8.1
 Requires-Dist: msgspec>=0.19.0
 Requires-Dist: pexpect>=4.9.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: jupyter
+Requires-Dist: jupyter-client>=8.6; extra == "jupyter"
 Dynamic: license-file
 
 # REPL Session
@@ -33,7 +36,7 @@ This is only really useful if you're hacking together a literate programming env
 
 The REPL of your choice is started and interacted with through the `pexpect` library. All I/O is dealt with through `msgspec`.
 
-The preferred way to solve this is by using `jupyter-client` in combination with an existing Jupyter kernel. However, not all languages have a Jupyter kernel available, and developing one takes a bit more than configuring `pexpect` for an existing REPL. I may still include Jupyter support in this application at a later stage.
+The preferred way to solve this is by using `jupyter-client` in combination with an existing Jupyter kernel. However, not all languages have a Jupyter kernel available, and developing one takes a bit more than configuring `pexpect` for an existing REPL. If a kernel is available for your language, you can use it through the optional Jupyter backend: `pip install repl-session[jupyter]`, then see [the Jupyter backend docs](https://entangled.github.io/repl-session/jupyter).
 
 ## Install
 
@@ -169,12 +172,13 @@ null
 
 ## Input/Output structure
 
-The user can configure how the REPL is called and interpreted.
+The user can configure how the REPL is called and interpreted. The `backend` field picks between this `pexpect` based REPL backend (`backend: repl`, the default) and the Jupyter backend (`backend: jupyter`).
 
 ```python
 #| id: input-data
-class ReplConfig(msgspec.Struct):
-    """Configuration
+class ReplConfig(msgspec.Struct, tag_field="backend", tag="repl"):
+    """Configuration for the `pexpect` backend; selected with `backend: repl`,
+    which is also the default when `backend` is absent.
 
     Attributes:
         command (str): Command to start the REPL
@@ -201,33 +205,61 @@ class ReplConfig(msgspec.Struct):
 
 Then, a session is a list of commands. Each command should be a UTF-8 string, and we allow to attach some meta-data like expected MIME type for the output. We can also pass an expected output in the case of a documentation test. If `output` was already given on the input, it is  moved to `expected`. This way it becomes really easy to setup regression tests on your documentation. Just rerun on the generated output file.
 
+The `output` field only holds the representation matching `output_type`. The `outputs` field records everything a command produced, in order. Each item has a `kind` (`result`, `display`, `stdout`, `stderr` or `error`) and a MIME bundle `data` with one or more representations of the same output. A Jupyter kernel may return `text/plain`, `text/html` and `image/png` together, for example. The REPL backend only sees terminal text, so it always gives a single `result`.
+
 ```python
 #| id: input-data
+OutputKind = Literal["result", "display", "stdout", "stderr", "error"]
+
+
+class ReplOutput(msgspec.Struct, omit_defaults=True):
+    """A single piece of output produced by a command.
+
+    Attributes:
+        kind (str): One of `result` (the value of the command), `display`
+            (rich output produced along the way, e.g. a plot), `stdout`,
+            `stderr` or `error` (a traceback). The `repl` backend cannot tell
+            these apart and reports everything as a single `result`.
+        data (dict[str, str]): MIME bundle: the same output in one or more
+            representations, keyed by MIME type. Binary data is base64 encoded;
+            JSON data is serialized to a string.
+        metadata (dict[str, Any]): Per MIME type metadata, e.g. image sizes.
+    """
+
+    kind: OutputKind
+    data: dict[str, str]
+    metadata: dict[str, Any] = msgspec.field(default_factory=dict)
+
+
 class ReplCommand(msgspec.Struct):
     """A command to be sent to the REPL.
 
     Attributes:
         command (str): the command.
         output_type (str): MIME type of expected output.
-        output (str | None): evaluated output.
+        output (str | None): evaluated output for `output_type`.
         expected (str | None): expected output.
+        outputs (list[ReplOutput] | None): everything the command produced, in
+            order, with all available MIME types.
     """
 
     command: str
     output_type: str = "text/plain"
     output: str | None = None
     expected: str | None = None
+    outputs: list[ReplOutput] | None = None
 
 
 class ReplSession(msgspec.Struct):
     """A REPL session.
 
     Attributes:
-        config (ReplConfig): Config for setting up a REPL session.
+        config (ReplConfig | JupyterConfig): Config for setting up a REPL
+            session, the `backend` field selects which.
         commands (list[ReplCommand]): List of commands in the session.
     """
 
-    config: ReplConfig
+    config: ReplConfig | JupyterConfig
     commands: list[ReplCommand]
 ```
 

{repl_session-0.2.1 → repl_session-0.3.1}/src/repl_session.egg-info/SOURCES.txt

@@ -2,7 +2,10 @@ LICENSE
 README.md
 pyproject.toml
 src/repl_session/__init__.py
+src/repl_session/jupyter.py
 src/repl_session/py.typed
+src/repl_session/repl.py
+src/repl_session/schema.py
 src/repl_session.egg-info/PKG-INFO
 src/repl_session.egg-info/SOURCES.txt
 src/repl_session.egg-info/dependency_links.txt

repl_session-0.3.1/src/repl_session.egg-info/requires.txt

@@ -0,0 +1,7 @@
+click>=8.1
+msgspec>=0.19.0
+pexpect>=4.9.0
+pyyaml>=6.0
+
+[jupyter]
+jupyter-client>=8.6

repl_session-0.3.1/test/test_examples.py

@@ -0,0 +1,65 @@
+"""Run the example sessions from the README and check their outputs."""
+
+import importlib.util
+import os
+import shlex
+import shutil
+from pathlib import Path
+
+import pytest
+
+from repl_session import ReplConfig, ReplSession, read_session, run_session
+
+
+TEST_DIR = Path(__file__).parent
+
+
+EXPECTED = {
+    "scheme": ["42", None, "3628800"],
+    "lua": ["42", "Hello, World!", None, "3628800"],
+    "python": ["Hello, World!", "42", None, "3628800"],
+}
+
+
+def load_session(name: str) -> ReplSession:
+    with open(TEST_DIR / f"{name}.yml") as f:
+        return read_session(f)
+
+
+def require(available: bool, what: str):
+    if available:
+        return
+    # In CI everything must be available; locally we allow skipping.
+    if os.environ.get("CI"):
+        pytest.fail(f"{what} not found")
+    pytest.skip(f"{what} not found")
+
+
+@pytest.mark.parametrize("name", EXPECTED.keys())
+def test_example(name: str):
+    session = load_session(name)
+    assert isinstance(session.config, ReplConfig)
+    executable = shlex.split(session.config.command)[0]
+    require(shutil.which(executable) is not None, f"`{executable}`")
+
+    result = run_session(session)
+    assert [cmd.output for cmd in result.commands] == EXPECTED[name]
+
+
+def test_jupyter():
+    require(
+        importlib.util.find_spec("jupyter_client") is not None
+        and importlib.util.find_spec("ipykernel") is not None,
+        "`jupyter_client` and `ipykernel`",
+    )
+    result = run_session(load_session("jupyter"))
+    hello, answer, define, fac, html, error = result.commands
+
+    assert [cmd.output for cmd in (hello, answer, define, fac, html)] == [
+        "Hello, World!", "42", None, "3628800", "<b>Hello</b>, World!"
+    ]
+    assert [[out.kind for out in cmd.outputs or []] for cmd in result.commands] == [
+        ["stdout"], ["result"], [], ["result"], ["result"], ["error"]
+    ]
+    assert html.outputs and set(html.outputs[0].data) == {"text/plain", "text/html"}
+    assert error.output and error.output.endswith("ZeroDivisionError: division by zero")

repl_session-0.2.1/src/repl_session/__init__.py

@@ -1,223 +0,0 @@
-# ~/~ begin <<docs/index.md#src/repl_session/__init__.py>>[init]
-"""
-`repl-session` is a command-line tool to evaluate a given session
-in any REPL, and store the results.
-"""
-
-# ~/~ begin <<docs/index.md#imports>>[init]
-# from datetime import datetime, tzinfo
-from typing import IO, cast
-from collections.abc import Generator, Callable
-
-# import re
-from contextlib import contextmanager
-import uuid
-import sys
-import re
-import logging
-import os
-
-import pexpect
-import msgspec
-import click
-import importlib.metadata
-
-
-__version__ = importlib.metadata.version("repl-session")
-# ~/~ end
-# ~/~ begin <<README.md#input-data>>[init]
-class ReplConfig(msgspec.Struct):
-    """Configuration
-
-    Attributes:
-        command (str): Command to start the REPL
-        first_prompt (str): Regex to match the first prompt
-        change_prompt (str): Command to change prompt; should contain '{key}' as an
-            argument.
-        next_prompt (str): Regex to match the changed prompts; should contain '{key}'
-            as an argument.
-        append_newline (bool): Whether to append a newline to given commands.
-        strip_command (bool): Whether to strip the original command from the gotten
-            output; useful if the REPL echoes your input before answering.
-        timeout (float): Command timeout for this session in seconds.
-    """
-
-    command: str
-    first_prompt: str
-    change_prompt: str
-    prompt: str
-    continuation_prompt: str | None = None
-    strip_ansi: bool = False
-    environment: dict[str, str] = msgspec.field(default_factory=dict)
-    timeout: float = 5.0
-# ~/~ end
-# ~/~ begin <<README.md#input-data>>[1]
-class ReplCommand(msgspec.Struct):
-    """A command to be sent to the REPL.
-
-    Attributes:
-        command (str): the command.
-        output_type (str): MIME type of expected output.
-        output (str | None): evaluated output.
-        expected (str | None): expected output.
-    """
-
-    command: str
-    output_type: str = "text/plain"
-    output: str | None = None
-    expected: str | None = None
-
-
-class ReplSession(msgspec.Struct):
-    """A REPL session.
-
-    Attributes:
-        config (ReplConfig): Config for setting up a REPL session.
-        commands (list[ReplCommand]): List of commands in the session.
-    """
-
-    config: ReplConfig
-    commands: list[ReplCommand]
-# ~/~ end
-# ~/~ begin <<docs/index.md#repl-contextmanager>>[init]
-def spawn(config: ReplConfig):
-    env = dict(os.environ) | config.environment
-    child: pexpect.spawn[str] = pexpect.spawn(
-        config.command,
-        timeout=config.timeout,
-        echo=False,
-        encoding="utf-8",
-        env=dict(os.environ) | config.environment,
-    )
-    return child
-
-
-ANSI_ESCAPE = re.compile(r"(\u001b|\x1B)(\[[0-?]*[ -\/]*[@-~]|[\>\=])")
-
-
-def clean_output(config: ReplConfig, output: str) -> str | None:
-    """Remove ANSI escapes (if requested), normalize the CRLF line endings
-    that the pseudo-terminal produces, and trim surrounding whitespace.
-    Returns `None` if nothing is left."""
-    if config.strip_ansi:
-        output = ANSI_ESCAPE.sub("", output)
-    output = output.replace("\r\n", "\n").replace("\r", "").strip()
-    return output or None
-
-
-@contextmanager
-def repl(config: ReplConfig) -> Generator[Callable[[str], str | None]]:
-    key = str(uuid.uuid4())
-    change_prompt_cmd = config.change_prompt.format(key=key)
-    prompt = config.prompt.format(key=key)
-    continuation_prompt = (
-        config.continuation_prompt.format(key=key)
-        if config.continuation_prompt is not None
-        else None
-    )
-
-    child: pexpect.spawn[str]
-    with spawn(config) as child:
-        _ = child.expect(config.first_prompt)
-        _ = child.sendline(change_prompt_cmd)
-        # if config.strip_command:
-        #    child.expect(key)
-        _ = child.expect(prompt)
-
-        if continuation_prompt is not None:
-            def send(msg: str) -> str | None:
-                lines = msg.splitlines()
-                answer: list[str] = []
-
-                if not lines:
-                    return None
-
-                still_waiting: bool = True
-                for line in lines:
-                    logging.debug("sending: '%s'", line)
-                    _ = child.sendline(line)
-                    logging.debug("waiting for prompt or continuation")
-                    _ = child.expect(
-                        f"(?P<norm>{prompt})|(?P<cont>{continuation_prompt})"
-                    )
-                    if not isinstance(child.match, re.Match):
-                        continue
-                    if child.match.group("cont") is not None:
-                        logging.debug("continuation: %s -- %s", child.before, child.after)
-                        still_waiting = True
-                    else:
-                        logging.debug("done: %s", child.before)
-                        if child.before is not None:
-                            answer.append(child.before)
-                        still_waiting = False
-
-                if still_waiting:
-                    logging.debug(f"waiting for last prompt")
-                    _ = child.sendline("")
-                    _ = child.expect(prompt)
-                    logging.debug(f"got: %s", child.before)
-                    if child.before:
-                        answer.append(child.before)
-
-                if not answer:
-                    return None
-
-                return clean_output(config, answer[-1])
-
-        else:
-            def send(msg: str) -> str | None:
-                logging.debug("sending: %s", msg)
-
-                _ = child.sendline(msg)
-                _ = child.expect(prompt)
-
-                if not child.before:
-                    return None
-
-                return clean_output(config, child.before)
-
-        yield send
-# ~/~ end
-# ~/~ begin <<docs/index.md#run-session>>[init]
-def run_session(session: ReplSession):
-    with repl(session.config) as run:
-        for cmd in session.commands:
-            expected = cmd.expected or cmd.output
-            output = run(cmd.command)
-            cmd.output = output
-            cmd.expected = expected
-
-    return session
-# ~/~ end
-# ~/~ begin <<docs/index.md#io>>[init]
-def read_session(port: IO[str] = sys.stdin) -> ReplSession:
-    data: str = port.read()
-    return msgspec.yaml.decode(data, type=ReplSession)
-
-
-def write_session(session: ReplSession, port: IO[str] = sys.stdout):
-    data = msgspec.json.encode(session)
-    _ = port.write(data.decode())
-# ~/~ end
-
-@click.command()
-@click.version_option(
-    __version__, "-v", "--version",
-    prog_name="repl-session", message="%(prog)s %(version)s",
-    help="show version and exit")
-@click.option("-l", "--log-enable", is_flag=True, help="show debugging output")
-def repl_session(log_enable: bool = False):
-    """
-    repl-session runs a REPL session, reading JSON from standard input and
-    writing to standard output. Both the input and output follow the same
-    schema.
-    """
-    if log_enable:
-        logging.basicConfig(level=logging.DEBUG)
-
-    write_session(run_session(read_session()))
-
-
-def main():
-    repl_session()
-# ~/~ end

repl_session-0.2.1/src/repl_session.egg-info/requires.txt

@@ -1,3 +0,0 @@
-click>=8.1
-msgspec>=0.19.0
-pexpect>=4.9.0

repl_session-0.2.1/test/test_examples.py

@@ -1,40 +0,0 @@
-"""Run the example sessions from the README and check their outputs."""
-
-import os
-import shlex
-import shutil
-from pathlib import Path
-
-import msgspec
-import pytest
-
-from repl_session import ReplSession, run_session
-
-
-TEST_DIR = Path(__file__).parent
-
-
-EXPECTED = {
-    "scheme": ["42", None, "3628800"],
-    "lua": ["42", "Hello, World!", None, "3628800"],
-    "python": ["Hello, World!", "42", None, "3628800"],
-}
-
-
-def load_session(name: str) -> ReplSession:
-    data = (TEST_DIR / f"{name}.yml").read_bytes()
-    return msgspec.yaml.decode(data, type=ReplSession)
-
-
-@pytest.mark.parametrize("name", EXPECTED.keys())
-def test_example(name: str):
-    session = load_session(name)
-    executable = shlex.split(session.config.command)[0]
-    if shutil.which(executable) is None:
-        # In CI every interpreter must be available; locally we allow skipping.
-        if os.environ.get("CI"):
-            pytest.fail(f"`{executable}` not found")
-        pytest.skip(f"`{executable}` not found")
-
-    result = run_session(session)
-    assert [cmd.output for cmd in result.commands] == EXPECTED[name]