zettcode-engine 0.3.4__py3-none-any.whl

zettcode/engine/tools/workspace.py

@@ -0,0 +1,271 @@
+"""The file system as the tools see it: paths taken from the working directory, text read in ranges, written whole."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TextIO
+
+from .output import MAX_OUTPUT_BYTES, utf8_prefix
+
+#: The largest text file a tool reads whole, writes, or searches.
+MAX_FILE_BYTES = 2 * 1024 * 1024
+
+#: How much of a line is read at a time, so one enormous line cannot use the memory of a whole file.
+FRAGMENT_CHARACTERS = 4096
+
+
+# -- where a path points ---------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class Located:
+    """A path a tool was asked about, resolved against the directory the process is in.
+
+    Attributes:
+        given: The path as the model wrote it, which is what an error message quotes back.
+        directory: The working directory, resolved.
+        path: Where ``given`` points: absolute paths and ``~`` are taken as they are, anything else is relative to
+            ``directory``; symbolic links are followed.
+    """
+
+    given: str
+    directory: Path
+    path: Path
+
+    @classmethod
+    def of(cls, given: str) -> Located:
+        """Resolve ``given`` against the working directory."""
+        directory = Path.cwd().resolve()
+        candidate = Path(given).expanduser()
+        target = candidate.resolve() if candidate.is_absolute() else (directory / candidate).resolve()
+        return cls(given, directory, target)
+
+    @property
+    def shown(self) -> str:
+        """Return the path as a result should name it: relative to the working directory for a relative ``given``."""
+        if Path(self.given).expanduser().is_absolute():
+            return self.path.as_posix()
+        return Path(os.path.relpath(self.path, self.directory)).as_posix()
+
+    def require_file(self) -> Path:
+        """Return the path, which has to be an existing regular file.
+
+        Raises:
+            ValueError: With the path as given, when there is no such file.
+        """
+        if not self.path.is_file():
+            raise ValueError(f"File does not exist: {self.given}")
+        return self.path
+
+
+@dataclass(frozen=True, slots=True)
+class Pattern:
+    """A glob pattern, split into the directory to search from and the expression to match below it.
+
+    Attributes:
+        root: Where the search starts: the filesystem root for an absolute pattern, else the working directory.
+        expression: What is matched below ``root``, as ``Path.glob`` takes it.
+        absolute: Whether matches are reported as absolute paths (the pattern was one) or relative to ``directory``.
+        directory: The working directory, resolved.
+    """
+
+    root: Path
+    expression: str
+    absolute: bool
+    directory: Path
+
+    @classmethod
+    def parse(cls, pattern: str) -> Pattern:
+        """Split ``pattern`` the way ``glob`` and ``grep`` take it."""
+        directory = Path.cwd().resolve()
+        candidate = Path(pattern).expanduser()
+        if candidate.is_absolute():
+            root = Path(candidate.anchor)
+            return cls(root, candidate.relative_to(root).as_posix(), True, directory)
+        return cls(directory, candidate.as_posix(), False, directory)
+
+    def matches(self) -> list[Path]:
+        """Return everything the pattern matches, in sorted order."""
+        return sorted(self.root.glob(self.expression))
+
+    def render(self, match: Path) -> str:
+        """Return a match in the style the pattern was written in."""
+        if self.absolute:
+            return match.absolute().as_posix()
+        return Path(os.path.relpath(match.absolute(), self.directory)).as_posix()
+
+
+# -- reading and writing text ------------------------------------------------------------------------------------
+
+
+def read_text(path: Path) -> str:
+    """Return a whole text file.
+
+    Raises:
+        ValueError: When the file is over :data:`MAX_FILE_BYTES`.
+        UnicodeDecodeError: When it is not UTF-8.
+    """
+    if path.stat().st_size > MAX_FILE_BYTES:
+        raise ValueError(f"File exceeds the {MAX_FILE_BYTES}-byte limit")
+    return path.read_text(encoding="utf-8")
+
+
+def write_text(path: Path, content: str) -> None:
+    """Replace a file with ``content`` atomically: it is written beside the file and renamed over it.
+
+    A reader of the file sees the old text or the new text, never half of it, and a failed write leaves the old
+    text and no stray file behind. Missing parent directories are made.
+
+    Raises:
+        ValueError: When the text is over :data:`MAX_FILE_BYTES`.
+    """
+    encoded = content.encode("utf-8")
+    if len(encoded) > MAX_FILE_BYTES:
+        raise ValueError(f"Content exceeds the {MAX_FILE_BYTES}-byte limit")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, delete=False) as handle:
+            handle.write(encoded)
+            temporary = Path(handle.name)
+        os.replace(temporary, path)
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def read_lines(located: Located, offset: int, limit: int) -> str:
+    """Return ``limit`` lines of a UTF-8 file from line ``offset``, with a note on how to continue when more remains.
+
+    The result is held to :data:`MAX_OUTPUT_BYTES`. A line longer than that is cut, and the note says so: the
+    reader has no column to resume from, because one 2 MB line is not what the tool is for — the shell reads it.
+    A cut inside an ordinary line is not that: the budget was used up before it, and the note then names the line
+    to start again from. The file is read a fragment at a time and never whole.
+
+    Args:
+        located: The file; it has to exist.
+        offset: One-based first line.
+        limit: Most lines to return.
+
+    Raises:
+        ValueError: When the file is not UTF-8 text.
+    """
+    selected: list[str] = []
+    number, used, line_start = 1, 0, 0
+    next_offset: int | None = None
+    cut_at: int | None = None
+    oversized = False
+    try:
+        with located.path.open(encoding="utf-8") as source:
+            while fragment := source.readline(FRAGMENT_CHARACTERS):
+                if number >= offset + limit:
+                    next_offset = number
+                    break
+                if number >= offset:
+                    part = utf8_prefix(fragment, MAX_OUTPUT_BYTES - used)
+                    selected.append(part)
+                    used += len(part.encode("utf-8"))
+                    if len(part) < len(fragment):
+                        # Whether this line could fill a call of its own decides what the note says, and only
+                        # what is left of the line knows that. Reading on is bounded by one budget.
+                        cut_at = number
+                        oversized = _line_longer_than_budget(
+                            source, used - line_start, fragment[len(part) :], MAX_OUTPUT_BYTES
+                        )
+                        break
+                if fragment.endswith("\n"):
+                    number, line_start = number + 1, used
+    except UnicodeDecodeError as error:
+        raise ValueError(f"File is not UTF-8 text: {located.given}") from error
+    content = "".join(selected)
+    if cut_at is not None and oversized:
+        content += (
+            f"\n... [line truncated; line {cut_at} is longer than the {MAX_OUTPUT_BYTES // 1024} KB output limit "
+            "and was cut here; read it with the shell]"
+        )
+    elif cut_at is not None:
+        content += f"\n... [output truncated; continue with offset={cut_at}]"
+    elif next_offset is not None:
+        content += f"\n... [more lines; continue with offset={next_offset}]"
+    return content
+
+
+def _line_longer_than_budget(source: TextIO, seen: int, tail: str, budget: int) -> bool:
+    """Return whether the line continues for more than ``budget`` bytes in all.
+
+    Args:
+        source: The file, positioned just after the fragment the cut happened in.
+        seen: How many bytes of this line are already in the result.
+        tail: The rest of the fragment the cut happened in, which is not in the result yet.
+        budget: What one result may carry.
+
+    The line is read on to its end, but never past ``budget`` bytes: past that point the answer is known, and
+    the rest of a runaway line is what the shell is for.
+    """
+    fragment = tail
+    while True:
+        seen += len(fragment.encode("utf-8"))
+        if seen > budget:
+            return True
+        if not fragment or fragment.endswith("\n"):
+            return False
+        fragment = source.readline(FRAGMENT_CHARACTERS)
+
+
+# -- editing ------------------------------------------------------------------------------------------------------
+
+
+def apply_edit(content: str, old_string: str, new_string: str, replace_all: bool = False) -> tuple[str, int]:
+    """Replace ``old_string`` in ``content`` once, or everywhere when ``replace_all`` says so.
+
+    By default ``old_string`` must appear exactly once in the text, so include the surrounding characters that
+    make it unique; ``new_string`` may be empty, which deletes the match.
+
+    Returns:
+        The new text and how many occurrences were replaced.
+
+    Raises:
+        ValueError: Saying what to do, when the old text is not found or is not unique. Nothing has been written
+            by then, since the caller writes once, after the replacement has passed.
+    """
+    matches = content.count(old_string)
+    if matches == 0:
+        raise ValueError(f"Text to replace was not found: {_excerpt(old_string)}. {_miss_hint(content, old_string)}")
+    if matches > 1 and not replace_all:
+        raise ValueError(
+            f"Text to replace is not unique; found {matches} matches "
+            f"(include more surrounding lines, or set replace_all to change every one): {_excerpt(old_string)}"
+        )
+    updated = content.replace(old_string, new_string, -1 if replace_all else 1)
+    return updated, matches if replace_all else 1
+
+
+def _miss_hint(content: str, old_string: str) -> str:
+    """Say why an exact match failed, so the model fixes ``old_string`` instead of leaving the tool."""
+    if " ".join(content.split()).count(" ".join(old_string.split())) > 0:
+        return (
+            "A match exists that differs only in whitespace or indentation; "
+            "read the file again and copy the lines exactly. Nothing was written."
+        )
+    return "Read the file again and copy old_string from its current content. Nothing was written."
+
+
+def _excerpt(value: str, *, limit: int = 60) -> str:
+    """Shorten one matched snippet for an error message."""
+    collapsed = " ".join(value.split())
+    return repr(f"{collapsed[:limit]}\u2026" if len(collapsed) > limit else collapsed)
+
+
+__all__ = [
+    "FRAGMENT_CHARACTERS",
+    "MAX_FILE_BYTES",
+    "Located",
+    "Pattern",
+    "apply_edit",
+    "read_lines",
+    "read_text",
+    "write_text",
+]

zettcode/error_text.py

@@ -0,0 +1,217 @@
+"""Say why a request failed, in words a row can show.
+
+A leaf the engine, the SDK and the views all use, so none of them has to import another to say it: the runtime composes
+the extensions, an extension reports its failures with this, and a function both of them need cannot live in either
+without the one importing the other.
+"""
+
+from __future__ import annotations
+
+import re
+
+from zett_agent.providers.base import ProviderTimeoutError
+
+from zettcode._compat import BaseExceptionGroup
+
+#: The most characters of what a server said that an error carries: enough for a sentence, not a page of HTML.
+MAX_DETAIL = 200
+
+#: What to try, by the status a server answered with. Only the ones where the fix is the reader's own to make.
+HINTS = {
+    401: "check the token with /config",
+    403: "check the token and what it may use, with /config",
+    404: "check the model id and the address with /config",
+}
+
+#: Something that looks like a credential in text a server sent back, which must not reach the screen:
+#: ``sk-proj-****abcd`` is how a provider says which key it refused, and the end of a key is still part of it.
+_SECRET = re.compile(r"\b(?:sk|pk|rk|key|tok|ghp|xox[a-z])[-_][A-Za-z0-9_*.\-]{4,}|Bearer\s+\S+", re.IGNORECASE)
+_TAGS = re.compile(r"<[^>]+>")
+
+
+#: The statuses that say the server was too busy or too broken to answer, and not that the request was wrong.
+TRANSIENT_STATUSES = frozenset({408, 409, 425, 429})
+
+#: What a failure that may pass says to do: the request is stored, and the library knows where it stopped.
+CONTINUE_HINT = "/continue tries the request again from where it stopped."
+
+
+def failure_text(error: BaseException) -> str:
+    """Return the line a failed request is shown as: what failed, and ``/continue`` when trying again may work."""
+    text = f"error: {describe_error(error)}"
+    if not may_pass(error):
+        return text
+    return f"{text.rstrip('.')}. {CONTINUE_HINT}"
+
+
+def may_pass(error: BaseException) -> bool:
+    """Return whether a failure is the provider's or the network's, so the same request may well succeed later.
+
+    A provider that sent nothing in time, a connection that dropped, a server that answered 5xx, 429 or 408: asking
+    again is what the reader would do. A wrong token, a model that is not there or a request the server refused
+    would fail again, so those say nothing of the kind.
+    """
+    pending: list[BaseException] = [error]
+    seen: set[int] = set()
+    while pending:
+        current = pending.pop()
+        if id(current) in seen:
+            continue
+        seen.add(id(current))
+        if isinstance(current, BaseExceptionGroup):
+            pending.extend(current.exceptions)
+            continue
+        status = getattr(current, "status_code", None)
+        if isinstance(status, int) and not isinstance(status, bool) and (status >= 500 or status in TRANSIENT_STATUSES):
+            return True
+        if isinstance(current, (ProviderTimeoutError, OSError, TimeoutError)):
+            return True
+        if type(current).__module__.startswith("httpx"):
+            return True
+        if current.__cause__ is not None:
+            pending.append(current.__cause__)
+    return False
+
+
+def describe_error(error: BaseException) -> str:
+    """Return the reasons an exception carries, not the group that wraps them, and what the server said.
+
+    A failure inside a task group arrives as an ``ExceptionGroup`` whose own
+    text says nothing — "unhandled errors in a TaskGroup (1 sub-exception)" —
+    and a server refusing a connection is reported exactly that way. The reasons
+    are the leaves, so that is what a row shows, each one once, however deeply
+    the groups nest.
+
+    A provider's own errors say what failed and not why: ``Provider stream request failed`` is the whole of
+    a 404 for a model that does not exist, a 429 for an account out of quota, and a 502 from a proxy. The why
+    is on the exception that caused it, so each leaf is followed down its causes to the HTTP status and what
+    the server said, and that is added in brackets — with what to try, where the fix is the reader's.
+
+    Args:
+        error: The exception a request failed with.
+
+    Returns:
+        The leaf reasons joined by ``"; "``, or the class name when a reason has
+        no text of its own.
+    """
+    reasons: list[str] = []
+    pending: list[BaseException] = [error]
+    while pending:
+        current = pending.pop(0)
+        if isinstance(current, BaseExceptionGroup):
+            pending[:0] = list(current.exceptions)
+            continue
+        text = _leaf_text(current)
+        if text not in reasons:
+            reasons.append(text)
+    return "; ".join(reasons)
+
+
+def _leaf_text(error: BaseException) -> str:
+    """Return one exception's text with the cause that explains it, when it has one."""
+    own = _server_said(error)
+    if own is not None:
+        return own  # an SDK error raised as it is: its own text is a dict printed after "Error code: 404 -"
+    text = _SECRET.sub("\u2026", str(error).strip()) or type(error).__name__
+    detail = _cause_detail(error)
+    if not detail or detail.lower() in text.lower():
+        return text
+    if text.lower() in detail.lower():
+        return detail  # the wrapper already said the first half of what the cause says in full
+    said = _nearest_status(error)
+    if said is not None and said[1] and said[1].lower() in text.lower():
+        # The library wrote the server's words into its own sentence ("openai returned HTTP 401: ..."), so they are
+        # not said twice; what it cannot know is what to try.
+        hint = HINTS.get(said[0])
+        return f"{text} \u2014 {hint}" if hint else text
+    return f"{text} ({detail})"
+
+
+def _nearest_status(error: BaseException) -> tuple[int, str] | None:
+    """Return the status and the server's words of the nearest cause that carries a status, or ``None``."""
+    seen = {id(error)}
+    cause = error.__cause__
+    while cause is not None and id(cause) not in seen:
+        seen.add(id(cause))
+        status = getattr(cause, "status_code", None)
+        if isinstance(status, int) and not isinstance(status, bool):
+            return status, _message_of(cause)
+        cause = cause.__cause__
+    return None
+
+
+def _cause_detail(error: BaseException) -> str | None:
+    """Return what the nearest cause says: what a server answered, or what the connection did."""
+    seen = {id(error)}
+    cause = error.__cause__
+    while cause is not None and id(cause) not in seen:
+        seen.add(id(cause))
+        said = _server_said(cause)
+        if said is not None:
+            return said
+        if isinstance(cause, (OSError, TimeoutError)) or type(cause).__module__.startswith("httpx"):
+            reason = str(cause).strip()
+            return f"{type(cause).__name__}: {reason}" if reason else type(cause).__name__
+        cause = cause.__cause__
+    return None
+
+
+def _server_said(error: BaseException) -> str | None:
+    """Return ``HTTP 404: the model does not exist — check ...`` for an error that carries a status, else ``None``."""
+    status = getattr(error, "status_code", None)
+    if isinstance(status, bool) or not isinstance(status, int):
+        return None
+    message = _message_of(error)
+    said = f"HTTP {status}: {message}" if message else f"HTTP {status}"
+    hint = HINTS.get(status)
+    return f"{said} \u2014 {hint}" if hint else said
+
+
+def _message_of(error: BaseException) -> str:
+    """Return the sentence a server gave for a failure, whatever shape its error body had, cleaned for one line."""
+    message = _library_said(error)
+    if message:
+        return _one_line(message)
+    body = getattr(error, "body", None)
+    if isinstance(body, dict):
+        inner = body.get("error")
+        source = inner if isinstance(inner, dict) else body
+        found = source.get("message") if isinstance(source, dict) else None
+        message = found if isinstance(found, str) else ""
+    elif isinstance(body, str):
+        message = body
+    if not message:
+        text = str(getattr(error, "message", "") or "")
+        message = re.sub(r"^Error code: \d+\s*-?\s*", "", text)
+        if message.lstrip().startswith(("{", "[")):
+            message = ""  # the dict printed back: nothing in it that was not already tried above
+    return _one_line(message)
+
+
+def _library_said(error: BaseException) -> str:
+    """Return what the library already read out of the response, when it is its own typed HTTP error.
+
+    ``ProviderHTTPError`` carries the provider's explanation (``reason``), the start of the body when it gave none
+    (``preview``), and whether it was a web page; that is the same reading this module does of an SDK error, done
+    once where the response is, so it is taken as it is. A web page is named as one and not shown as markup.
+    """
+    reason, preview = getattr(error, "reason", None), getattr(error, "preview", None)
+    if getattr(error, "is_html", False) is True:
+        return (
+            f"a web page, not an API response \u2014 {preview}"
+            if isinstance(preview, str) and preview
+            else "a web page, not an API response"
+        )
+    for found in (reason, preview):
+        if isinstance(found, str) and found.strip():
+            return found
+    return ""
+
+
+def _one_line(message: str) -> str:
+    """Return a server's words on one line, without markup or anything that looks like a key, and cut short."""
+    message = _TAGS.sub(" ", message)
+    message = _SECRET.sub("\u2026", " ".join(message.split()))
+    if len(message) > MAX_DETAIL:
+        message = message[: MAX_DETAIL - 1].rstrip() + "\u2026"
+    return message

zettcode/filelock.py

@@ -0,0 +1,125 @@
+"""An exclusive lock on a file, which the operating system lets go of when the holder dies.
+
+It makes the session broker a singleton and marks a session as open in one terminal. Starting the broker is a
+race — two sessions launched together both find nobody home and both start one — so the one that gets the lock
+keeps running and the other exits. A lock on a file is what
+makes that safe: unlike a pid written into a file, it cannot go stale. A holder that was killed
+releases it with the rest of its descriptors, and the next one takes it.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+from typing import IO
+
+#: How often a caller that is willing to wait looks at the lock again.
+POLL_SECONDS = 0.02
+
+
+class SingletonLock:
+    """An exclusive, non-blocking lock on one file, held until :meth:`release` or the process ends.
+
+    The platform's own call is imported where it is used: ``fcntl`` exists only on POSIX and
+    ``msvcrt`` only on Windows, and the module has to import on both.
+    """
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._file: IO[bytes] | None = None
+
+    @property
+    def held(self) -> bool:
+        """Return whether this object holds the lock now."""
+        return self._file is not None
+
+    def acquire(self, wait: float = 0.0) -> bool:
+        """Take the lock if nobody holds it, and say whether it was taken.
+
+        Args:
+            wait: Seconds to keep trying when somebody holds it. A client that is only *looking* (see
+                :func:`is_held`) takes the lock for an instant, and a holder that arrives at that
+                instant must not conclude that another holder exists; a short wait tells the two apart.
+        """
+        deadline = time.monotonic() + max(0.0, wait)
+        while True:
+            if self._try():
+                return True
+            if time.monotonic() >= deadline:
+                return False
+            time.sleep(POLL_SECONDS)
+
+    def _try(self) -> bool:
+        if self._file is not None:
+            return True
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        handle = open(self.path, "a+b")  # noqa: SIM115 - the handle is the lock, and outlives this call
+        try:
+            if os.name == "nt":
+                import msvcrt
+
+                handle.seek(0)
+                msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)  # type: ignore[attr-defined,unused-ignore]
+            else:
+                import fcntl
+
+                fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            handle.close()
+            return False
+        self._file = handle
+        return True
+
+    def still_ours(self) -> bool:
+        """Return whether the file at this path is still the one this lock is held on.
+
+        Somebody who deletes the directory by hand removes the lock file out from under a running holder, and
+        the next one to start would create a new file and take a lock on *that*: two holders, each the only one
+        holding its own lock. A holder that checks now and then can see it happened and step aside. Where a file that is open cannot be deleted (Windows), it is always true.
+        """
+        handle = self._file
+        if handle is None:
+            return False
+        if os.name == "nt":
+            return True
+        try:
+            here, there = os.stat(self.path), os.fstat(handle.fileno())
+        except OSError:
+            return False
+        return (here.st_ino, here.st_dev) == (there.st_ino, there.st_dev)
+
+    def release(self) -> None:
+        """Let go of the lock; safe to call when it is not held, and safe to call twice."""
+        handle, self._file = self._file, None
+        if handle is None:
+            return
+        try:
+            if os.name == "nt":
+                import msvcrt
+
+                handle.seek(0)
+                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)  # type: ignore[attr-defined,unused-ignore]
+            else:
+                import fcntl
+
+                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+        except OSError:
+            pass
+        finally:
+            handle.close()
+
+
+def is_held(path: Path) -> bool:
+    """Return whether some process holds the lock at ``path`` right now.
+
+    This is the one reliable way to ask whether a broker is alive: unlike a process id in a file (which
+    outlives the process, and may be reused by another) or a port number (which may belong to anything
+    by now), the lock is the operating system's own record, and it disappears with the holder however
+    the holder died. The answer is a snapshot, and the lock is taken for an instant to find it out.
+    """
+    probe = SingletonLock(path)
+    if probe.acquire():
+        probe.release()
+        return False
+    return True