zettcode-tools 0.4.1__tar.gz

zettcode_tools-0.4.1/.gitignore

@@ -0,0 +1,42 @@
+**/.venv/
+**/__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.smoke/
+.coverage
+**/.pytest_cache/
+**/.ruff_cache/
+.zett-tool-output/
+.env
+.DS_Store
+
+# viztracer writes its trace here by default.
+/result.json
+
+# The docs site: dependencies, build output, and VitePress' cache.
+node_modules/
+docs/.vitepress/dist/
+docs/.vitepress/cache/
+docs/.vitepress/.temp/
+
+# The TypeScript client is built from its sources; only dist/ is published.
+clients/typescript/dist/
+
+# The gateway's page is built from clients/gateway into the package when a distribution is made (`make gateway-ui`);
+# the build output is not kept in the repository. pyproject's `artifacts` puts it in the wheel all the same.
+packages/zettcode-gateway/src/zettcode/gateway/static/
+
+# The palette gallery is generated by `make palettes`: every widget painted in
+# every palette, written by docs/scripts/palette_gallery.py. Regenerate it
+# rather than committing a 2 MB copy that goes stale the moment a colour moves.
+docs/internal/palettes.html
+
+# The release order is generated from the pyproject.toml files (`make order`); it is not kept in git.
+packages/order.txt
+
+# The desktop and phone apps build the gateway's page into their own folders, and electron-builder writes its
+# installers under release/.
+clients/desktop/renderer/
+clients/desktop/release/

zettcode_tools-0.4.1/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.5
+Name: zettcode-tools
+Version: 0.4.1
+Summary: The coding tools of ZettCode: read, search, write, edit and the shell, as a part of the agent that lives apart from it
+Project-URL: Repository, https://github.com/Chang-LeHung/zettcode
+Author: Chang-LeHung
+License-Expression: MIT
+Keywords: agent,tools,zettcode
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: zett-agent<0.3,>=0.2.10

zettcode_tools-0.4.1/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "zettcode-tools"
+description = "The coding tools of ZettCode: read, search, write, edit and the shell, as a part of the agent that lives apart from it"
+requires-python = ">=3.10"
+dynamic = ["version"]
+license = "MIT"
+authors = [{name = "Chang-LeHung"}]
+keywords = ["zettcode", "tools", "agent"]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Programming Language :: Python :: 3.15",
+  "Typing :: Typed",
+]
+dependencies = [
+  # The tool contract, the message types and the @tool builder.
+  "zett-agent>=0.2.10,<0.3",
+  # A tool's parameters are described with the schema library the framework uses.
+  "pydantic>=2,<3",
+]
+
+[project.urls]
+Repository = "https://github.com/Chang-LeHung/zettcode"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+# One version for the whole repository: the engine ships the file.
+path = "../zettcode-engine/src/zettcode/version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/zettcode"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/src", "/pyproject.toml"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"

zettcode_tools-0.4.1/src/zettcode/tools/__init__.py

@@ -0,0 +1,7 @@
+"""The coding tools the agent offers, written here: read, search, write, edit and the shell.
+
+``files`` holds the six file tools and ``shell`` the seventh; ``workspace``, ``output`` and ``images`` are what they are
+built from; ``boundary`` is the contract the shell tool has with the permission level. Nothing here is re-exported,
+so importing the package costs nothing: ``boundary`` is imported by the permission package at start-up and has to stay
+free of the schema library the tools themselves need.
+"""

zettcode_tools-0.4.1/src/zettcode/tools/boundary.py

@@ -0,0 +1,56 @@
+"""What the shell tool asks of the permission level, and what it asks of a sandbox.
+
+Both are abstract base classes, and the permission package subclasses them: the tools sit below it in the import
+order, so the contract has to live down here, and a subclass that forgets a method fails when it is built instead of
+when a command first runs. Nothing in this module imports the framework or a model library, so the permission
+package can import it at start-up for free.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class Sandboxer(ABC):
+    """A sandbox that rewrites a shell command so it runs inside it."""
+
+    @property
+    @abstractmethod
+    def available(self) -> bool:
+        """Return whether this machine can sandbox a command at all."""
+
+    @abstractmethod
+    def wrap(self, command: str, cwd: Path | None = None) -> str:
+        """Return ``command`` as a shell line that runs sandboxed, in ``cwd`` or in the working directory.
+
+        The directory is not a detail: the policy a sandbox writes allows one directory, and a Linux sandbox
+        moves the process there itself, so a wrapper that is given the working directory runs the command where
+        it is not meant to run.
+
+        Raises:
+            RuntimeError: The machine has no sandbox; check :attr:`available` first.
+        """
+
+    @abstractmethod
+    def hint(self) -> str:
+        """Return the note that goes after a failed sandboxed command, naming the boundary it hit."""
+
+
+class Boundary(ABC):
+    """The decision the shell tool defers to at every command: run it sandboxed, or as it was given.
+
+    ``ShellPolicy`` is the implementation. The decision is read at each call and not when the tool is built, so
+    ``/permission`` takes effect on the next command of a request that is already running.
+    """
+
+    @abstractmethod
+    def sandbox_now(self) -> Sandboxer | None:
+        """Return the sandbox the next command has to run in, or ``None`` to run it as given.
+
+        One answer, not a flag and a sandbox read separately: a level changed between the two reads could otherwise
+        hand back a flag for one and a sandbox for the other.
+        """
+
+
+__all__ = ["Boundary", "Sandboxer"]

zettcode_tools-0.4.1/src/zettcode/tools/files.py

@@ -0,0 +1,279 @@
+"""The file tools: read, search, write and edit.
+
+Each tool is a typed function whose signature is its input schema and whose docstring is what the model is told:
+the first paragraph is the description, ``Args`` the parameters, ``Snippet`` the call shapes, ``Guidelines`` the rules
+that go in the prompt. ``zett_agent.tools.base.tool`` turns one into an :class:`~zett_agent.tools.base.AgentTool`;
+the work is done here, in this repository, and by nothing else.
+
+There are two readers and no more. ``read`` returns UTF-8 text as a line range and a PNG, JPEG, GIF or WebP file as an
+image, so one tool covers both. ``read_file`` is the same reader without the pictures, for a model that cannot take
+an image; the capability filter above swaps one for the other.
+There is no ``view_image``, because ``read`` already shows a picture, and no ``delete_file``: removing a file is a
+shell command and goes through the shell's permission like any other, with no second path around it.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Annotated
+
+from pydantic import Field
+from zett_agent.messages import ImageBytesSource, ImageContent, TextContent
+from zett_agent.tools.base import tool
+
+from .images import MAX_IMAGE_BYTES, SIGNATURE_BYTES, image_media_type
+from .output import MAX_MATCH_BYTES, MAX_OUTPUT_BYTES, utf8_prefix
+from .workspace import (
+    MAX_FILE_BYTES,
+    Located,
+    Pattern,
+    apply_edit,
+    read_lines,
+    read_text,
+    write_text,
+)
+
+# The parameter types, named once so that the tools that share one agree on its limits and its schema.
+FilePath = Annotated[str, Field(min_length=1)]
+Expression = Annotated[str, Field(min_length=1)]
+Offset = Annotated[int, Field(ge=1)]
+Limit = Annotated[int, Field(ge=1, le=2_000)]
+MaxResults = Annotated[int, Field(ge=1, le=5_000)]
+OldString = Annotated[str, Field(min_length=1)]
+
+#: What a reader hands back: text, or a text line and the picture it describes.
+ReadResult = str | list[TextContent | ImageContent]
+
+
+def _image_type_of(path: Path) -> str | None:
+    """Return the media type of a local image file, or ``None`` when it is not a PNG, JPEG, GIF or WebP."""
+    with path.open("rb") as source:
+        return image_media_type(source.read(SIGNATURE_BYTES))
+
+
+@tool
+def read(file_path: FilePath, offset: Offset = 1, limit: Limit = 200) -> ReadResult:
+    """Read a local file: UTF-8 text as a line range, or a PNG, JPEG, GIF, or WebP image as an image.
+
+    Args:
+        file_path: Absolute path or a path relative to the current working directory.
+        offset: One-based line number to start reading from; ignored for images.
+        limit: Maximum number of lines to return; ignored for images.
+
+    Returns:
+        The requested line range, or the image and a one-line description of it; a trailing hint says how to
+        continue when more lines remain.
+
+    Snippet:
+        read(file_path="docs/diagram.png")
+
+    Guidelines:
+        - Use for any local file when you do not know whether it is text or an image.
+        - A text file comes back as a line range; use line ranges for large files.
+        - A trailing hint says how to go on: the next offset when more lines remain, or the shell when one line is
+          longer than the output limit.
+        - An image comes back as an image with a one-line description; the range arguments do not apply to it.
+        - PDF and SVG files are not supported.
+    """
+    located = Located.of(file_path)
+    target = located.require_file()
+    media_type = _image_type_of(target)
+    if media_type is None:
+        return read_lines(located, offset, limit)
+    size = target.stat().st_size
+    if size > MAX_IMAGE_BYTES:
+        raise ValueError(f"Image exceeds the {MAX_IMAGE_BYTES}-byte limit: {file_path}")
+    return [
+        TextContent(text=f"Image {target.name}: {media_type}, {size} bytes"),
+        ImageContent(source=ImageBytesSource(data=target.read_bytes(), media_type=media_type), alt_text=target.name),
+    ]
+
+
+@tool
+def read_file(file_path: FilePath, offset: Offset = 1, limit: Limit = 200) -> str:
+    """Read a line range from a UTF-8 text file.
+
+    Args:
+        file_path: Absolute path or a path relative to the current working directory.
+        offset: One-based line number to start reading from.
+        limit: Maximum number of lines to return.
+
+    Returns:
+        The requested line range; a trailing hint says how to continue when more lines remain.
+
+    Snippet:
+        read_file(file_path="src/app.py", offset=1, limit=200)
+
+    Guidelines:
+        - Use line ranges for large files.
+        - Inspect the current content before editing a file.
+    """
+    located = Located.of(file_path)
+    located.require_file()
+    return read_lines(located, offset, limit)
+
+
+@tool
+def glob(pattern: Expression, max_results: MaxResults = 200) -> str:
+    """Find paths matching a relative or absolute glob pattern.
+
+    Args:
+        pattern: Glob pattern such as **/*.py, ../tests/*.py, or /work/src/**/*.py.
+        max_results: Maximum number of sorted paths to return.
+
+    Returns:
+        Matching paths, one per line and sorted, empty when nothing matches, with a trailing note when the result was
+        truncated.
+
+    Snippet:
+        glob(pattern="src/**/*.py", max_results=200)
+
+    Guidelines:
+        - Use glob to discover files before reading or searching them.
+        - Narrow the pattern when the result is truncated.
+    """
+    spec = Pattern.parse(pattern)
+    paths: list[str] = []
+    used = 0
+    truncated = False
+    for match in spec.matches():
+        rendered = spec.render(match)
+        size = len(rendered.encode("utf-8"))
+        if len(paths) == max_results or used + size > MAX_OUTPUT_BYTES:
+            truncated = True
+            break
+        paths.append(rendered)
+        used += size
+    result = "\n".join(paths)
+    if truncated:
+        result += "\n... [truncated; narrow the pattern or increase max_results]"
+    return result
+
+
+@tool
+def grep(
+    pattern: Expression,
+    file_pattern: Expression = "**/*",
+    case_sensitive: bool = True,
+    max_results: MaxResults = 200,
+) -> str:
+    """Search UTF-8 text files selected by a relative or absolute glob pattern.
+
+    Args:
+        pattern: Python regular expression to search for on each line.
+        file_pattern: Relative or absolute glob selecting files to search.
+        case_sensitive: Whether letter case must match exactly.
+        max_results: Maximum number of matching lines to return.
+
+    Returns:
+        Matching lines as ``path:line: text``, one per line, with a trailing note when the result was truncated.
+
+    Snippet:
+        grep(pattern="class\\s+Agent", file_pattern="src/**/*.py", max_results=200)
+
+    Guidelines:
+        - Use a narrow file_pattern to avoid scanning unrelated files.
+        - Escape regular-expression characters when searching for literal text.
+        - Use the read tool for surrounding context.
+    """
+    spec = Pattern.parse(file_pattern)
+    try:
+        expression = re.compile(pattern, 0 if case_sensitive else re.IGNORECASE)
+    except re.error as error:
+        raise ValueError(f"Invalid regular expression: {error}") from error
+    lines: list[str] = []
+    used = 0
+    truncated = False
+    for candidate in spec.matches():
+        if not candidate.is_file() or candidate.stat().st_size > MAX_FILE_BYTES:
+            continue
+        try:
+            content = candidate.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            continue
+        shown = spec.render(candidate)
+        for number, line in enumerate(content.splitlines(), start=1):
+            found = expression.search(line)
+            if found is None:
+                continue
+            if len(lines) == max_results:
+                truncated = True
+                break
+            first = max(0, found.start() - 100) if len(line.encode("utf-8")) > MAX_MATCH_BYTES else 0
+            rendered = f"{shown}:{number}: {utf8_prefix(line[first:], MAX_MATCH_BYTES)}"
+            size = len(rendered.encode("utf-8"))
+            if used + size > MAX_OUTPUT_BYTES:
+                truncated = True
+                break
+            lines.append(rendered)
+            used += size
+        if truncated:
+            break
+    result = "\n".join(lines)
+    if truncated:
+        result += "\n... [truncated; narrow the search or increase max_results]"
+    return result
+
+
+@tool
+def write_file(file_path: FilePath, content: str, overwrite: bool = True) -> str:
+    """Atomically write a UTF-8 text file.
+
+    Args:
+        file_path: Absolute path or a path relative to the current working directory.
+        content: Complete UTF-8 text to write.
+        overwrite: Whether an existing file may be replaced.
+
+    Returns:
+        One line saying whether the file was created or replaced, and its path.
+
+    Snippet:
+        write_file(file_path="notes/plan.md", content="# Plan\\n")
+
+    Guidelines:
+        - Use for new files or intentional full-file replacement.
+        - Prefer edit for a small change to an existing file.
+    """
+    located = Located.of(file_path)
+    exists = located.path.exists()
+    if exists and not located.path.is_file():
+        raise ValueError(f"Path is not a file: {file_path}")
+    if exists and not overwrite:
+        raise ValueError(f"File already exists: {file_path}")
+    write_text(located.path, content)
+    return f"{'Wrote' if exists else 'Created'} {located.shown}"
+
+
+@tool
+def edit(file_path: FilePath, old_string: OldString, new_string: str, replace_all: bool = False) -> str:
+    """Edit an existing UTF-8 file by replacing exact text; always use this tool to modify a file.
+
+    Args:
+        file_path: Absolute path or a path relative to the current working directory.
+        old_string: The exact text to replace, copied from the file as it is now.
+        new_string: The text to put in its place; an empty string deletes the match.
+        replace_all: Replace every occurrence of old_string instead of insisting it appears once.
+
+    Returns:
+        One line saying how many occurrences were replaced, and in which file.
+
+    Snippet:
+        edit(file_path="notes/plan.md", old_string="Draft", new_string="Final")
+        edit(file_path="notes/plan.md", old_string="Total", new_string="Sum", replace_all=True)
+
+    Guidelines:
+        - Always use this tool when you need to modify an existing file, instead of sed, awk, perl -i, python -c, tee, or shell redirection. Reserve those for bulk or mechanical rewrites this tool cannot express, such as renaming a symbol across many files.
+        - One change a call: call edit again for the next change, and nothing is written when the old text does not match.
+        - Avoid sed, cat, and similar shell commands for reading or changing files unless it is necessary; inspect files with the read tool and change them with this tool.
+        - Read the target lines first and copy old_string exactly: same indentation, quotes, and line breaks.
+        - Make old_string unique by including a neighbouring line or two; set replace_all only when every occurrence must change.
+        - When an edit fails, read the file again and retry with corrected old_string rather than switching to a shell command.
+    """
+    located = Located.of(file_path)
+    updated, replacements = apply_edit(read_text(located.require_file()), old_string, new_string, replace_all)
+    write_text(located.path, updated)
+    return f"Replaced {replacements} occurrence{'s' if replacements != 1 else ''} in {located.shown}"
+
+
+__all__ = ["edit", "glob", "grep", "read", "read_file", "write_file"]

zettcode_tools-0.4.1/src/zettcode/tools/images.py

@@ -0,0 +1,32 @@
+"""Recognising the image files ``read`` hands to a model: by their first bytes, never by their name."""
+
+from __future__ import annotations
+
+#: The largest image file ``read`` returns.
+MAX_IMAGE_BYTES = 10_000_000
+
+#: Bytes needed to recognise every signature below.
+SIGNATURE_BYTES = 12
+
+
+def image_media_type(header: bytes) -> str | None:
+    """Return the media type of a PNG, JPEG, GIF or WebP from the first bytes of a file, or ``None`` for anything else.
+
+    A file's name says what its author meant; its first bytes say what it is, and a model rejects a picture
+    announced as the wrong type. A file that is not one of the four is text as far as ``read`` is concerned.
+
+    Args:
+        header: At least :data:`SIGNATURE_BYTES` bytes from the start of the file (fewer for a short file).
+    """
+    if header.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image/png"
+    if header.startswith(b"\xff\xd8\xff"):
+        return "image/jpeg"
+    if header.startswith((b"GIF87a", b"GIF89a")):
+        return "image/gif"
+    if header.startswith(b"RIFF") and header[8:12] == b"WEBP":
+        return "image/webp"
+    return None
+
+
+__all__ = ["MAX_IMAGE_BYTES", "SIGNATURE_BYTES", "image_media_type"]

zettcode_tools-0.4.1/src/zettcode/tools/output.py

@@ -0,0 +1,81 @@
+"""Bounded UTF-8 previews: a long text is cut to a budget, and the full output stays in a file for later reads."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import NamedTuple
+
+#: The most bytes, lines and bytes-per-matching-line any one tool result may carry.
+MAX_OUTPUT_BYTES = 50 * 1024
+MAX_OUTPUT_LINES = 2000
+MAX_MATCH_BYTES = 2000
+
+#: What stands where the middle of a long output was left out.
+OMISSION = "\n... [middle omitted; read the output file for full content] ...\n"
+
+
+class Preview(NamedTuple):
+    """A bounded view of one output stream.
+
+    Attributes:
+        text: What fits the budget.
+        truncated: Whether the stream was longer than that, so the full text has to be kept.
+    """
+
+    text: str
+    truncated: bool
+
+
+def utf8_prefix(text: str, budget: int) -> str:
+    """Return the longest prefix of ``text`` whose UTF-8 encoding fits in ``budget`` bytes, never half a character."""
+    return text.encode("utf-8")[:budget].decode("utf-8", errors="ignore")
+
+
+def stream_preview(path: Path, marker: str = OMISSION) -> Preview:
+    """Return a bounded preview of a stream written to ``path``: its start and its end, with the middle left out.
+
+    A quarter of the byte budget goes to the start (what a command printed while it started up) and the rest to
+    the end (its final results). Only bounded windows are read from disk, so a gigabyte of output costs what fifty
+    kilobytes does, and a partial UTF-8 character at either edge of a window is dropped. The line limit applies to
+    the text returned, the marker included. The file is never changed, which is what lets the model read what was
+    omitted.
+
+    Args:
+        path: The stream that was written.
+        marker: What stands between the two windows. A caller that knows where the whole text is kept puts that in
+            the marker; it is written once, where the cut is, so a copy of the default marker inside the text cannot
+            be mistaken for the cut.
+    """
+    with path.open("rb") as source:
+        data = source.read(MAX_OUTPUT_BYTES + 1)
+        text = data.decode("utf-8", errors="replace")
+        if (
+            len(data) <= MAX_OUTPUT_BYTES
+            and len(text.encode("utf-8")) <= MAX_OUTPUT_BYTES
+            and len(text.splitlines()) <= MAX_OUTPUT_LINES
+        ):
+            return Preview(text, False)
+        budget = max(2, MAX_OUTPUT_BYTES - len(marker.encode("utf-8")))
+        head_budget = budget // 4
+        tail_budget = budget - head_budget
+        head = data[:head_budget].decode("utf-8", errors="ignore")
+        source.seek(max(0, path.stat().st_size - tail_budget))
+        tail = source.read(tail_budget).decode("utf-8", errors="ignore")
+    # What the marker takes is held back so the line limit covers the text that comes back: a caller's marker is a
+    # sentence or two, and the floor keeps a long one from eating the whole preview.
+    lines = max(2, MAX_OUTPUT_LINES - max(1, marker.count("\n")) - 1)
+    head_lines = max(1, lines // 4)
+    head = "".join(head.splitlines(keepends=True)[:head_lines])
+    tail = "".join(tail.splitlines(keepends=True)[-(lines - head_lines) :])
+    return Preview(head + marker + tail, True)
+
+
+__all__ = [
+    "MAX_MATCH_BYTES",
+    "MAX_OUTPUT_BYTES",
+    "MAX_OUTPUT_LINES",
+    "OMISSION",
+    "Preview",
+    "stream_preview",
+    "utf8_prefix",
+]

zettcode_tools-0.4.1/src/zettcode/tools/shell.py

@@ -0,0 +1,185 @@
+"""The shell tool: run one command in a directory it names, bounded in time and in what comes back.
+
+The command's two streams go to files and not into memory, so the volume of output costs nothing, and what is
+returned is a preview of each (see :func:`~zettcode.tools.output.stream_preview`). When a preview had to cut
+something, the files are kept and the result ends with a note naming them, which the model can read; otherwise they
+are removed. A command that fails or runs out of time is reported as an error that carries its output.
+
+At the ``safe`` permission level the command is rewritten to run inside the sandbox (see
+:class:`~zettcode.tools.boundary.Boundary`).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import signal
+import tempfile
+from pathlib import Path
+from typing import Annotated
+
+from pydantic import Field
+from zett_agent.tools.base import AgentTool, tool
+
+from .boundary import Boundary
+from .output import Preview, stream_preview
+from .workspace import Located
+
+#: Where a command's kept output goes, inside the working directory. The part that moves it out of the reader's
+#: project (`zettcode-tool-output`) names the same folder, and a test holds the two equal.
+OUTPUT_ROOT = ".zett-tool-output"
+
+#: The runtime a command is given unless the model asks for another, and the most it may ask for.
+DEFAULT_TIMEOUT_SECONDS = 60
+MAX_TIMEOUT_SECONDS = 600
+
+Command = Annotated[str, Field(min_length=1)]
+WorkDir = Annotated[str, Field(min_length=1)]
+TimeoutSeconds = Annotated[int, Field(ge=1, le=MAX_TIMEOUT_SECONDS)]
+
+
+class CommandFailed(RuntimeError):
+    """A command that exited with a code other than 0, or ran out of time. The message carries what it printed."""
+
+
+def _compose(stdout: Preview, stderr: Preview, kept: tuple[Path, Path] | None, directory: Path) -> str:
+    """Join the two previews into the text a result carries, with the note that names the kept files if there are any."""
+    sections: list[str] = []
+    if stdout.text:
+        sections.append(stdout.text)
+    if stderr.text:
+        sections.append(f"[stderr]\n{stderr.text}")
+    composed = "\n".join(sections)
+    if kept is not None:
+        first, second = (path.relative_to(directory).as_posix() for path in kept)
+        composed += f"\n... [output truncated; full streams kept at {first} and {second}]"
+    return composed
+
+
+async def _stop(process: asyncio.subprocess.Process) -> None:
+    """Kill a command and every process it started, and wait for it to be gone."""
+    try:
+        if os.name == "posix":
+            os.killpg(process.pid, signal.SIGKILL)
+        else:
+            process.kill()
+    except ProcessLookupError:
+        pass
+    await process.wait()
+
+
+async def run_command(command: str, timeout_seconds: int, directory: Path | None = None) -> str:
+    """Run ``command`` in ``directory``, or in the working directory, and return what it printed.
+
+    Raises:
+        ValueError: The command is blank.
+        CommandFailed: It exited with a code other than 0, or did not finish in ``timeout_seconds``; the message
+            carries its output either way.
+    """
+    if not command.strip():
+        raise ValueError("Shell command cannot be blank")
+    root = Located.of(OUTPUT_ROOT)
+    # No await between making the root and making the folder in it: the extension that tidies an empty root away
+    # runs on this loop, so it can never see the one without the other.
+    root.path.mkdir(parents=True, exist_ok=True)
+    folder = Path(tempfile.mkdtemp(prefix="shell-", dir=root.path))
+    stdout_file, stderr_file = folder / "stdout.txt", folder / "stderr.txt"
+    timed_out = False
+    # File descriptors rather than pipes keep memory bounded however much is printed. The two streams stay
+    # separate: how they interleaved is not kept.
+    with stdout_file.open("wb") as stdout_target, stderr_file.open("wb") as stderr_target:
+        process = await asyncio.create_subprocess_shell(
+            command,
+            cwd=directory if directory is not None else root.directory,
+            stdin=asyncio.subprocess.DEVNULL,
+            stdout=stdout_target,
+            stderr=stderr_target,
+            start_new_session=os.name == "posix",
+        )
+        try:
+            await asyncio.wait_for(process.wait(), timeout_seconds)
+        # ``asyncio.TimeoutError`` is the builtin from 3.11 on and a class of its own before.
+        except (TimeoutError, asyncio.TimeoutError):
+            await _stop(process)
+            timed_out = True
+        except asyncio.CancelledError:
+            await _stop(process)
+            raise
+    stdout, stderr = stream_preview(stdout_file), stream_preview(stderr_file)
+    truncated = stdout.truncated or stderr.truncated
+    composed = _compose(stdout, stderr, (stdout_file, stderr_file) if truncated else None, root.directory)
+    if not truncated:
+        stdout_file.unlink()
+        stderr_file.unlink()
+        folder.rmdir()
+    if timed_out:
+        plural = "" if timeout_seconds == 1 else "s"
+        raise CommandFailed(f"Command timed out after {timeout_seconds} second{plural}\n{composed}")
+    code = process.returncode if process.returncode is not None else -1
+    if code != 0:
+        raise CommandFailed(f"Command exited with code {code}\n{composed}")
+    return composed
+
+
+def shell_tool(boundary: Boundary | None = None) -> AgentTool:
+    """Return the ``bash`` tool.
+
+    Args:
+        boundary: What decides, at each command, whether it runs inside a sandbox. ``None`` runs every command
+            as it was given. A command that fails inside a sandbox has a note appended that names the boundary,
+            because "Operation not permitted" alone sends the model round the same wall again.
+    """
+
+    @tool
+    async def bash(
+        command: Command, work_dir: WorkDir | None = None, timeout_seconds: TimeoutSeconds = DEFAULT_TIMEOUT_SECONDS
+    ) -> str:
+        """Run a shell command with captured output, in the workspace or in ``work_dir``.
+
+        Args:
+            command: Shell command to execute.
+            work_dir: Directory to run in: absolute, a `~` path, or relative to the workspace. Omit it to run in the workspace.
+            timeout_seconds: Maximum command runtime in seconds; the default is 60.
+
+        Returns:
+            The command's captured output, with startup and final results kept within the byte and line limits; a
+            truncation hint names the retained files.
+
+        Snippet:
+            bash(command="git status --short", timeout_seconds=60)
+            bash(command="npm test", work_dir="clients/vscode")
+
+        Guidelines:
+            - Use for builds, tests, git, and searching the code: rg or grep for text, find or ls for paths. Prefer the read, write_file and edit tools for reading and changing a file.
+            - Set work_dir instead of cd: every call starts a new shell, so a cd lasts only that one command.
+            - Search with a narrow pattern and a path, and pipe through head when the result may be long.
+            - Use only for bounded commands in a trusted working directory.
+            - A non-zero exit code or a timeout is reported as an error carrying the output.
+            - Output previews keep startup and final results within byte and line limits.
+            - When output is truncated, read the retained files named in the hint.
+        """
+        if not command.strip():
+            raise ValueError("Shell command cannot be blank")
+        located = None if work_dir is None else Located.of(work_dir)
+        if located is not None and not located.path.is_dir():
+            raise ValueError(f"Not a directory: {work_dir}")
+        directory = located.path if located is not None else None
+        sandbox = boundary.sandbox_now() if boundary is not None else None
+        if sandbox is None:
+            return await run_command(command, timeout_seconds, directory)
+        try:
+            return await run_command(sandbox.wrap(command, directory), timeout_seconds, directory)
+        except CommandFailed as error:
+            raise CommandFailed(f"{error}\n{sandbox.hint()}") from error
+
+    return bash
+
+
+__all__ = [
+    "DEFAULT_TIMEOUT_SECONDS",
+    "MAX_TIMEOUT_SECONDS",
+    "OUTPUT_ROOT",
+    "CommandFailed",
+    "run_command",
+    "shell_tool",
+]

zettcode_tools-0.4.1/src/zettcode/tools/workspace.py

@@ -0,0 +1,271 @@
+"""The file system as the tools see it: paths taken from the working directory, text read in ranges, written whole."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TextIO
+
+from .output import MAX_OUTPUT_BYTES, utf8_prefix
+
+#: The largest text file a tool reads whole, writes, or searches.
+MAX_FILE_BYTES = 2 * 1024 * 1024
+
+#: How much of a line is read at a time, so one enormous line cannot use the memory of a whole file.
+FRAGMENT_CHARACTERS = 4096
+
+
+# -- where a path points ---------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class Located:
+    """A path a tool was asked about, resolved against the directory the process is in.
+
+    Attributes:
+        given: The path as the model wrote it, which is what an error message quotes back.
+        directory: The working directory, resolved.
+        path: Where ``given`` points: absolute paths and ``~`` are taken as they are, anything else is relative to
+            ``directory``; symbolic links are followed.
+    """
+
+    given: str
+    directory: Path
+    path: Path
+
+    @classmethod
+    def of(cls, given: str) -> Located:
+        """Resolve ``given`` against the working directory."""
+        directory = Path.cwd().resolve()
+        candidate = Path(given).expanduser()
+        target = candidate.resolve() if candidate.is_absolute() else (directory / candidate).resolve()
+        return cls(given, directory, target)
+
+    @property
+    def shown(self) -> str:
+        """Return the path as a result should name it: relative to the working directory for a relative ``given``."""
+        if Path(self.given).expanduser().is_absolute():
+            return self.path.as_posix()
+        return Path(os.path.relpath(self.path, self.directory)).as_posix()
+
+    def require_file(self) -> Path:
+        """Return the path, which has to be an existing regular file.
+
+        Raises:
+            ValueError: With the path as given, when there is no such file.
+        """
+        if not self.path.is_file():
+            raise ValueError(f"File does not exist: {self.given}")
+        return self.path
+
+
+@dataclass(frozen=True, slots=True)
+class Pattern:
+    """A glob pattern, split into the directory to search from and the expression to match below it.
+
+    Attributes:
+        root: Where the search starts: the filesystem root for an absolute pattern, else the working directory.
+        expression: What is matched below ``root``, as ``Path.glob`` takes it.
+        absolute: Whether matches are reported as absolute paths (the pattern was one) or relative to ``directory``.
+        directory: The working directory, resolved.
+    """
+
+    root: Path
+    expression: str
+    absolute: bool
+    directory: Path
+
+    @classmethod
+    def parse(cls, pattern: str) -> Pattern:
+        """Split ``pattern`` the way ``glob`` and ``grep`` take it."""
+        directory = Path.cwd().resolve()
+        candidate = Path(pattern).expanduser()
+        if candidate.is_absolute():
+            root = Path(candidate.anchor)
+            return cls(root, candidate.relative_to(root).as_posix(), True, directory)
+        return cls(directory, candidate.as_posix(), False, directory)
+
+    def matches(self) -> list[Path]:
+        """Return everything the pattern matches, in sorted order."""
+        return sorted(self.root.glob(self.expression))
+
+    def render(self, match: Path) -> str:
+        """Return a match in the style the pattern was written in."""
+        if self.absolute:
+            return match.absolute().as_posix()
+        return Path(os.path.relpath(match.absolute(), self.directory)).as_posix()
+
+
+# -- reading and writing text ------------------------------------------------------------------------------------
+
+
+def read_text(path: Path) -> str:
+    """Return a whole text file.
+
+    Raises:
+        ValueError: When the file is over :data:`MAX_FILE_BYTES`.
+        UnicodeDecodeError: When it is not UTF-8.
+    """
+    if path.stat().st_size > MAX_FILE_BYTES:
+        raise ValueError(f"File exceeds the {MAX_FILE_BYTES}-byte limit")
+    return path.read_text(encoding="utf-8")
+
+
+def write_text(path: Path, content: str) -> None:
+    """Replace a file with ``content`` atomically: it is written beside the file and renamed over it.
+
+    A reader of the file sees the old text or the new text, never half of it, and a failed write leaves the old
+    text and no stray file behind. Missing parent directories are made.
+
+    Raises:
+        ValueError: When the text is over :data:`MAX_FILE_BYTES`.
+    """
+    encoded = content.encode("utf-8")
+    if len(encoded) > MAX_FILE_BYTES:
+        raise ValueError(f"Content exceeds the {MAX_FILE_BYTES}-byte limit")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, delete=False) as handle:
+            handle.write(encoded)
+            temporary = Path(handle.name)
+        os.replace(temporary, path)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def read_lines(located: Located, offset: int, limit: int) -> str:
+    """Return ``limit`` lines of a UTF-8 file from line ``offset``, with a note on how to continue when more remains.
+
+    The result is held to :data:`MAX_OUTPUT_BYTES`. A line longer than that is cut, and the note says so: the
+    reader has no column to resume from, because one 2 MB line is not what the tool is for — the shell reads it.
+    A cut inside an ordinary line is not that: the budget was used up before it, and the note then names the line
+    to start again from. The file is read a fragment at a time and never whole.
+
+    Args:
+        located: The file; it has to exist.
+        offset: One-based first line.
+        limit: Most lines to return.
+
+    Raises:
+        ValueError: When the file is not UTF-8 text.
+    """
+    selected: list[str] = []
+    number, used, line_start = 1, 0, 0
+    next_offset: int | None = None
+    cut_at: int | None = None
+    oversized = False
+    try:
+        with located.path.open(encoding="utf-8") as source:
+            while fragment := source.readline(FRAGMENT_CHARACTERS):
+                if number >= offset + limit:
+                    next_offset = number
+                    break
+                if number >= offset:
+                    part = utf8_prefix(fragment, MAX_OUTPUT_BYTES - used)
+                    selected.append(part)
+                    used += len(part.encode("utf-8"))
+                    if len(part) < len(fragment):
+                        # Whether this line could fill a call of its own decides what the note says, and only
+                        # what is left of the line knows that. Reading on is bounded by one budget.
+                        cut_at = number
+                        oversized = _line_longer_than_budget(
+                            source, used - line_start, fragment[len(part) :], MAX_OUTPUT_BYTES
+                        )
+                        break
+                if fragment.endswith("\n"):
+                    number, line_start = number + 1, used
+    except UnicodeDecodeError as error:
+        raise ValueError(f"File is not UTF-8 text: {located.given}") from error
+    content = "".join(selected)
+    if cut_at is not None and oversized:
+        content += (
+            f"\n... [line truncated; line {cut_at} is longer than the {MAX_OUTPUT_BYTES // 1024} KB output limit "
+            "and was cut here; read it with the shell]"
+        )
+    elif cut_at is not None:
+        content += f"\n... [output truncated; continue with offset={cut_at}]"
+    elif next_offset is not None:
+        content += f"\n... [more lines; continue with offset={next_offset}]"
+    return content
+
+
+def _line_longer_than_budget(source: TextIO, seen: int, tail: str, budget: int) -> bool:
+    """Return whether the line continues for more than ``budget`` bytes in all.
+
+    Args:
+        source: The file, positioned just after the fragment the cut happened in.
+        seen: How many bytes of this line are already in the result.
+        tail: The rest of the fragment the cut happened in, which is not in the result yet.
+        budget: What one result may carry.
+
+    The line is read on to its end, but never past ``budget`` bytes: past that point the answer is known, and
+    the rest of a runaway line is what the shell is for.
+    """
+    fragment = tail
+    while True:
+        seen += len(fragment.encode("utf-8"))
+        if seen > budget:
+            return True
+        if not fragment or fragment.endswith("\n"):
+            return False
+        fragment = source.readline(FRAGMENT_CHARACTERS)
+
+
+# -- editing ------------------------------------------------------------------------------------------------------
+
+
+def apply_edit(content: str, old_string: str, new_string: str, replace_all: bool = False) -> tuple[str, int]:
+    """Replace ``old_string`` in ``content`` once, or everywhere when ``replace_all`` says so.
+
+    By default ``old_string`` must appear exactly once in the text, so include the surrounding characters that
+    make it unique; ``new_string`` may be empty, which deletes the match.
+
+    Returns:
+        The new text and how many occurrences were replaced.
+
+    Raises:
+        ValueError: Saying what to do, when the old text is not found or is not unique. Nothing has been written
+            by then, since the caller writes once, after the replacement has passed.
+    """
+    matches = content.count(old_string)
+    if matches == 0:
+        raise ValueError(f"Text to replace was not found: {_excerpt(old_string)}. {_miss_hint(content, old_string)}")
+    if matches > 1 and not replace_all:
+        raise ValueError(
+            f"Text to replace is not unique; found {matches} matches "
+            f"(include more surrounding lines, or set replace_all to change every one): {_excerpt(old_string)}"
+        )
+    updated = content.replace(old_string, new_string, -1 if replace_all else 1)
+    return updated, matches if replace_all else 1
+
+
+def _miss_hint(content: str, old_string: str) -> str:
+    """Say why an exact match failed, so the model fixes ``old_string`` instead of leaving the tool."""
+    if " ".join(content.split()).count(" ".join(old_string.split())) > 0:
+        return (
+            "A match exists that differs only in whitespace or indentation; "
+            "read the file again and copy the lines exactly. Nothing was written."
+        )
+    return "Read the file again and copy old_string from its current content. Nothing was written."
+
+
+def _excerpt(value: str, *, limit: int = 60) -> str:
+    """Shorten one matched snippet for an error message."""
+    collapsed = " ".join(value.split())
+    return repr(f"{collapsed[:limit]}\u2026" if len(collapsed) > limit else collapsed)
+
+
+__all__ = [
+    "FRAGMENT_CHARACTERS",
+    "MAX_FILE_BYTES",
+    "Located",
+    "Pattern",
+    "apply_edit",
+    "read_lines",
+    "read_text",
+    "write_text",
+]