turnloop 0.1.0__py3-none-any.whl

turnloop/tools/grep.py

@@ -0,0 +1,314 @@
+"""Grep — content search.
+
+Prefers ripgrep when it is on PATH (an order of magnitude faster on real repos,
+and it already honors .gitignore), with a pure-Python fallback so the tool is
+never simply unavailable. Both paths produce identical output formats, because a
+tool whose output shape depends on what happens to be installed is a tool the
+model cannot learn.
+
+The default output mode is `files_with_matches` on purpose: "which files mention
+this" is the question that actually starts most searches, and it costs a fraction
+of the context that matching lines do.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from turnloop.tools.base import Tool, ToolContext, ToolOutput
+from turnloop.tools.glob import ALWAYS_SKIP, _is_ignored
+
+OutputMode = Literal["files_with_matches", "content", "count"]
+DEFAULT_HEAD_LIMIT = 250
+
+
+class GrepArgs(BaseModel):
+    pattern: str = Field(description="Regular expression to search for.")
+    path: str | None = Field(default=None, description="File or directory to search. Default cwd.")
+    glob: str | None = Field(default=None, description="Restrict to files matching this glob.")
+    output_mode: OutputMode = Field(
+        default="files_with_matches",
+        description="'files_with_matches' (default), 'content', or 'count'.",
+    )
+    case_insensitive: bool = Field(default=False, description="Ignore case.")
+    show_line_numbers: bool = Field(default=True, description="Line numbers in content mode.")
+    context_lines: int = Field(default=0, description="Lines of context around each match.")
+    head_limit: int | None = Field(default=None, description=f"Cap results (default {DEFAULT_HEAD_LIMIT}).")
+    multiline: bool = Field(default=False, description="Let the pattern span line breaks.")
+
+
+class GrepTool(Tool):
+    name = "Grep"
+    Args = GrepArgs
+    read_only = True
+    parallel_safe = True
+    bulky = True
+    timeout_s = 60.0
+
+    descriptions = {
+        "terse": """
+Regex search across files. Modes: files_with_matches (default), content, count.
+Filter with `glob`. Prefer this over shelling out to grep.
+""",
+        "normal": """
+Search file contents with a regular expression.
+
+- `output_mode`: `files_with_matches` (default, just paths), `content` (matching
+  lines), or `count` (matches per file).
+- Filter the search with `glob`, e.g. `**/*.py`.
+- `context_lines` adds surrounding lines in content mode.
+- Ignored and vendored directories are skipped.
+- Start with the default mode to find where something lives, then Read those
+  files or re-run in content mode. Dumping every matching line first wastes
+  context you will want later.
+""",
+        "verbose": """
+Search file contents with a regular expression.
+
+Arguments:
+- `pattern`: a regular expression. Escape regex metacharacters when you mean them
+  literally — searching for `foo(bar)` needs `foo\\(bar\\)`.
+- `path`: a file or directory. Defaults to the working directory.
+- `glob`: restrict to matching filenames, e.g. `**/*.ts`, `*.md`.
+- `output_mode`: `files_with_matches` (default) lists paths only; `content` lists
+  matching lines; `count` lists match counts per file.
+- `case_insensitive`, `show_line_numbers`, `context_lines`, `multiline`,
+  `head_limit`.
+
+Behavior:
+- Uses ripgrep when available and a pure-Python search otherwise; output format is
+  identical either way.
+- Ignored files are excluded (.gitignore when in a repository, plus a skip list
+  for node_modules, __pycache__, .venv, dist, build, target, vendor).
+- Binary files are skipped.
+- Results are capped at 250 entries by default with an explicit truncation note.
+- Strategy that keeps context small: search in the default mode first to learn
+  which files are involved, then Read the interesting ones. Content mode on a
+  common term can return thousands of lines you will pay for all session.
+""",
+    }
+
+    async def run(self, args: GrepArgs, ctx: ToolContext) -> ToolOutput:
+        target = ctx.resolve(args.path) if args.path else ctx.cwd
+        if not target.exists():
+            return ToolOutput.error(f"{target} does not exist")
+        if not ctx.within_allowed(target):
+            return ToolOutput.error(f"{target} is outside the project root ({ctx.root}).")
+
+        try:
+            re.compile(args.pattern)
+        except re.error as exc:
+            return ToolOutput.error(f"invalid regular expression: {exc}")
+
+        limit = args.head_limit or DEFAULT_HEAD_LIMIT
+
+        if shutil.which("rg"):
+            result = _run_ripgrep(args, target, limit)
+            if result is not None:
+                return _format(result, args, target, ctx, engine="ripgrep", limit=limit)
+
+        result = _run_python(args, target, limit)
+        return _format(result, args, target, ctx, engine="python", limit=limit)
+
+    def summary(self, args: GrepArgs) -> str:  # type: ignore[override]
+        scope = f" in {args.glob}" if args.glob else ""
+        return f"Grep {args.pattern!r}{scope}"
+
+
+class _Result:
+    def __init__(self):
+        self.lines: list[str] = []
+        self.files: set[str] = set()
+        self.total: int = 0
+        self.truncated: bool = False
+
+
+def _run_ripgrep(args: GrepArgs, target: Path, limit: int) -> _Result | None:
+    cmd = ["rg", "--no-config"]
+    if target.is_dir() and _is_ignored(target):
+        # rg honors .gitignore by default, same as `git ls-files`, and has the
+        # same blind spot: if `target` itself sits inside an ignored tree, rg
+        # reports zero matches instead of searching it. --no-ignore-vcs is
+        # scoped to exactly this case -- passing it unconditionally would make
+        # Grep search genuinely ignored files inside a normal, non-ignored repo.
+        cmd.append("--no-ignore-vcs")
+    if args.output_mode == "files_with_matches":
+        cmd.append("--files-with-matches")
+    elif args.output_mode == "count":
+        cmd.append("--count-matches")
+    else:
+        if args.show_line_numbers:
+            cmd.append("--line-number")
+        if args.context_lines:
+            cmd += ["--context", str(args.context_lines)]
+    if args.case_insensitive:
+        cmd.append("--ignore-case")
+    if args.multiline:
+        cmd += ["--multiline", "--multiline-dotall"]
+    if args.glob:
+        cmd += ["--glob", args.glob]
+    cmd += ["--", args.pattern, str(target)]
+
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=45)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    # rg exits 1 for "no matches", which is not an error here.
+    if proc.returncode not in (0, 1):
+        return None
+
+    result = _Result()
+    for line in proc.stdout.splitlines():
+        if not line:
+            continue
+        result.total += 1
+        if len(result.lines) < limit:
+            result.lines.append(line)
+        else:
+            result.truncated = True
+    return result
+
+
+def _iter_candidate_files(target: Path, glob: str | None) -> list[Path]:
+    if target.is_file():
+        return [target]
+
+    from turnloop.permissions.rules import _glob_to_regex
+    from turnloop.tools.glob import _normalize_pattern
+
+    matcher = re.compile(_glob_to_regex(_normalize_pattern(glob))) if glob else None
+    out: list[Path] = []
+    for dirpath, dirnames, filenames in os.walk(target):
+        dirnames[:] = [d for d in dirnames if d not in ALWAYS_SKIP]
+        for name in filenames:
+            full = Path(dirpath) / name
+            if matcher is not None:
+                try:
+                    rel = full.relative_to(target).as_posix()
+                except ValueError:
+                    continue
+                if not matcher.fullmatch(rel):
+                    continue
+            out.append(full)
+    return out
+
+
+def _run_python(args: GrepArgs, target: Path, limit: int) -> _Result:
+    flags = re.IGNORECASE if args.case_insensitive else 0
+    if args.multiline:
+        flags |= re.DOTALL | re.MULTILINE
+    regex = re.compile(args.pattern, flags)
+    result = _Result()
+
+    for path in _iter_candidate_files(target, args.glob):
+        try:
+            raw = path.read_bytes()
+        except OSError:
+            continue
+        if b"\x00" in raw[:4_000]:
+            continue
+        text = raw.decode("utf-8", errors="replace")
+
+        if args.multiline:
+            hits = list(regex.finditer(text))
+            if not hits:
+                continue
+            result.files.add(str(path))
+            result.total += len(hits)
+            if args.output_mode == "content" and len(result.lines) < limit:
+                for hit in hits:
+                    line_no = text.count("\n", 0, hit.start()) + 1
+                    snippet = hit.group(0).replace("\n", "\\n")[:300]
+                    result.lines.append(f"{path}:{line_no}:{snippet}")
+            continue
+
+        lines = text.splitlines()
+        matched_indices = [i for i, line in enumerate(lines) if regex.search(line)]
+        if not matched_indices:
+            continue
+
+        result.files.add(str(path))
+        result.total += len(matched_indices)
+
+        if args.output_mode != "content":
+            continue
+
+        emitted: set[int] = set()
+        for i in matched_indices:
+            lo = max(0, i - args.context_lines)
+            hi = min(len(lines), i + args.context_lines + 1)
+            for j in range(lo, hi):
+                if j in emitted:
+                    continue
+                emitted.add(j)
+                if len(result.lines) >= limit:
+                    result.truncated = True
+                    break
+                prefix = f"{path}:{j + 1}:" if args.show_line_numbers else f"{path}:"
+                result.lines.append(prefix + lines[j])
+            if result.truncated:
+                break
+
+    if args.output_mode == "files_with_matches":
+        ordered = sorted(result.files)
+        result.truncated = len(ordered) > limit
+        result.lines = ordered[:limit]
+    elif args.output_mode == "count":
+        # Per-file counts need a second read: the first pass tracked which files
+        # matched, not how often. Only count mode pays for it.
+        result.lines = []
+        for name in sorted(result.files):
+            try:
+                text = Path(name).read_bytes().decode("utf-8", errors="replace")
+            except OSError:
+                continue
+            n = len(regex.findall(text))
+            if n:
+                result.lines.append(f"{name}:{n}")
+        result.truncated = len(result.lines) > limit
+        result.lines = result.lines[:limit]
+
+    return result
+
+
+def _format(result: _Result, args: GrepArgs, target: Path, ctx: ToolContext,
+            engine: str, limit: int) -> ToolOutput:
+    if not result.lines:
+        return ToolOutput(
+            content=f"No matches for {args.pattern!r} in {target}.",
+            display=f"Grep {args.pattern!r} → no matches",
+            metrics={"matches": 0, "engine": engine},
+        )
+
+    root = ctx.cwd.resolve()
+    rendered = []
+    for line in result.lines:
+        rendered.append(line.replace(str(root) + os.sep, "").replace(str(root) + "/", ""))
+
+    body = "\n".join(rendered)
+    if result.truncated:
+        body += f"\n\n[truncated at {limit} results; narrow the pattern or add a glob filter]"
+
+    label = {
+        "files_with_matches": "file(s)",
+        "content": "line(s)",
+        "count": "file(s)",
+    }[args.output_mode]
+
+    return ToolOutput(
+        content=body,
+        display=f"Grep {args.pattern!r} → {len(result.lines)} {label}",
+        metrics={
+            "matches": result.total,
+            "shown": len(result.lines),
+            "engine": engine,
+            "truncated": result.truncated,
+        },
+    )

turnloop/tools/read.py

@@ -0,0 +1,151 @@
+"""Read — the tool models use most, so its output format matters most."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from turnloop.tools.base import Tool, ToolContext, ToolOutput
+
+MAX_LINE_CHARS = 2_000
+BINARY_SNIFF_BYTES = 8_000
+
+
+class ReadArgs(BaseModel):
+    file_path: str = Field(description="Absolute or project-relative path to read.")
+    offset: int | None = Field(default=None, description="1-indexed first line to read.")
+    limit: int | None = Field(default=None, description="How many lines to read (default 2000).")
+
+
+class ReadTool(Tool):
+    name = "Read"
+    Args = ReadArgs
+    read_only = True
+    parallel_safe = True
+    bulky = True
+
+    descriptions = {
+        "terse": """
+Read a file. Returns lines numbered from 1, `offset`/`limit` for large files.
+Always Read before Edit or Write.
+""",
+        "normal": """
+Read a file from the filesystem.
+
+- `file_path` may be absolute or relative to the working directory.
+- Output is line-numbered starting at 1, so you can cite exact lines and build
+  Edit calls against them.
+- Reads up to 2000 lines by default. For a larger file, use `offset` and `limit`
+  to page through it rather than reading it whole — a big file consumes context
+  you will need later.
+- You must Read a file before you Edit or Write it.
+- Binary files are refused rather than dumped.
+""",
+        "verbose": """
+Read a file from the filesystem.
+
+Arguments:
+- `file_path`: absolute, or relative to the working directory.
+- `offset`: 1-indexed line to start from. Use with `limit` to page a large file.
+- `limit`: how many lines to return. Defaults to 2000.
+
+Behavior and constraints:
+- Output is `cat -n` style: right-aligned line numbers, a tab, then the line.
+  Cite these numbers when discussing code, and use them to target Edit calls.
+- Lines longer than 2000 characters are truncated with a marker; the file is not
+  modified.
+- Reading a file records it as seen. Write and Edit both refuse to touch a file
+  that has not been read in this session, and refuse again if it changed on disk
+  afterwards — re-read it in that case.
+- Binary files (detected by NUL bytes) are refused with their size and type
+  rather than being dumped into the conversation.
+- Directories are refused; use Glob or Grep to explore instead.
+- Paths outside the project root are refused unless the user configured
+  additional directories.
+""",
+    }
+
+    async def run(self, args: ReadArgs, ctx: ToolContext) -> ToolOutput:
+        path = ctx.resolve(args.file_path)
+
+        if not ctx.within_allowed(path):
+            return ToolOutput.error(
+                f"{path} is outside the project root ({ctx.root}). "
+                "Ask the user to add it to permissions.additional_directories if this is intended."
+            )
+        if path.is_dir():
+            return ToolOutput.error(f"{path} is a directory. Use Glob or Grep to explore it.")
+        if not path.exists():
+            hint = _nearby_suggestion(path)
+            return ToolOutput.error(f"{path} does not exist." + (f" {hint}" if hint else ""))
+
+        raw = path.read_bytes()
+        if b"\x00" in raw[:BINARY_SNIFF_BYTES]:
+            return ToolOutput.error(
+                f"{path.name} looks binary ({len(raw):,} bytes). Not reading it as text."
+            )
+
+        # utf-8-sig: a BOM must not appear as an invisible U+FEFF on line 1, or the
+        # model copies it into an Edit's old_string and the match fails invisibly.
+        text = raw.decode("utf-8-sig", errors="replace")
+        lines = text.splitlines()
+        total = len(lines)
+
+        start = max(1, args.offset or 1)
+        limit = args.limit or 2_000
+        window = lines[start - 1 : start - 1 + limit]
+
+        if not window:
+            return ToolOutput(
+                content=f"(no lines: file has {total} lines, offset {start} is past the end)"
+            )
+
+        numbered = []
+        for i, line in enumerate(window, start=start):
+            if len(line) > MAX_LINE_CHARS:
+                line = line[:MAX_LINE_CHARS] + f"… [{len(line) - MAX_LINE_CHARS} chars truncated]"
+            numbered.append(f"{i:6d}\t{line}")
+
+        body = "\n".join(numbered)
+        shown_end = start + len(window) - 1
+        if shown_end < total:
+            body += (
+                f"\n\n[showing lines {start}-{shown_end} of {total}. "
+                f"Use offset={shown_end + 1} to continue.]"
+            )
+
+        # Only a complete read establishes read-before-write: recording a partial
+        # read would let an Edit be built against content we never showed.
+        if start == 1 and shown_end >= total:
+            ctx.files.record_read(path, text)
+
+        return ToolOutput(
+            content=body,
+            display=f"Read {_rel(path, ctx.root)} ({len(window)} of {total} lines)",
+            metrics={"lines": len(window), "total_lines": total, "bytes": len(raw)},
+        )
+
+    def summary(self, args: ReadArgs) -> str:  # type: ignore[override]
+        return f"Read {args.file_path}"
+
+
+def _rel(path: Path, root: Path) -> str:
+    try:
+        return str(path.resolve().relative_to(root.resolve()))
+    except ValueError:
+        return str(path)
+
+
+def _nearby_suggestion(path: Path, limit: int = 4) -> str:
+    """Name the closest existing siblings — a typo'd path is the common case."""
+    parent = path.parent
+    if not parent.is_dir():
+        return f"Its parent directory {parent} does not exist either."
+    import difflib
+
+    names = [p.name for p in parent.iterdir()]
+    close = difflib.get_close_matches(path.name, names, n=limit, cutoff=0.6)
+    if close:
+        return "Did you mean: " + ", ".join(close) + "?"
+    return ""