pythinker-code 2.0.0__py3-none-any.whl

pythinker_code/tools/file/read.py

@@ -0,0 +1,300 @@
+from collections import deque
+from pathlib import Path
+from typing import override
+
+from pydantic import BaseModel, Field, model_validator
+from pythinker_core.tooling import CallableTool2, ToolError, ToolOk, ToolReturnValue
+from pythinker_host.path import HostPath
+
+from pythinker_code.soul.agent import Runtime
+from pythinker_code.tools.file.utils import MEDIA_SNIFF_BYTES, detect_file_type
+from pythinker_code.tools.utils import load_desc, truncate_line
+from pythinker_code.utils.logging import logger
+from pythinker_code.utils.path import is_within_workspace
+from pythinker_code.utils.sensitive import is_sensitive_file
+
+MAX_LINES = 1000
+MAX_LINE_LENGTH = 2000
+MAX_BYTES = 100 << 10  # 100KB
+
+
+class Params(BaseModel):
+    path: str = Field(
+        description=(
+            "The path to the file to read. Absolute paths are required when reading files "
+            "outside the working directory."
+        )
+    )
+    line_offset: int = Field(
+        description=(
+            "The line number to start reading from. "
+            "By default read from the beginning of the file. "
+            "Set this when the file is too large to read at once. "
+            "Negative values read from the end of the file (e.g. -100 reads the last 100 lines). "
+            f"The absolute value of negative offset cannot exceed {MAX_LINES}."
+        ),
+        default=1,
+    )
+    n_lines: int = Field(
+        description=(
+            "The number of lines to read. "
+            f"By default read up to {MAX_LINES} lines, which is the max allowed value. "
+            "Set this value when the file is too large to read at once."
+        ),
+        default=MAX_LINES,
+        ge=1,
+    )
+
+    @model_validator(mode="after")
+    def _validate_line_offset(self) -> "Params":
+        if self.line_offset == 0:
+            raise ValueError(
+                "line_offset cannot be 0; use 1 for the first line or -1 for the last line"
+            )
+        if self.line_offset < -MAX_LINES:
+            raise ValueError(
+                f"line_offset cannot be less than -{MAX_LINES}. "
+                "Use a positive line_offset with the total line count "
+                "to read from a specific position."
+            )
+        return self
+
+
+class ReadFile(CallableTool2[Params]):
+    name: str = "ReadFile"
+    params: type[Params] = Params
+
+    def __init__(self, runtime: Runtime) -> None:
+        description = load_desc(
+            Path(__file__).parent / "read.md",
+            {
+                "MAX_LINES": MAX_LINES,
+                "MAX_LINE_LENGTH": MAX_LINE_LENGTH,
+                "MAX_BYTES": MAX_BYTES,
+            },
+        )
+        super().__init__(description=description)
+        self._runtime = runtime
+        self._work_dir = runtime.builtin_args.PYTHINKER_WORK_DIR
+        self._additional_dirs = runtime.additional_dirs
+
+    async def _validate_path(self, path: HostPath) -> ToolError | None:
+        """Validate that the path is safe to read."""
+        resolved_path = path.canonical()
+
+        if (
+            not is_within_workspace(resolved_path, self._work_dir, self._additional_dirs)
+            and not path.is_absolute()
+        ):
+            # Outside files can only be read with absolute paths
+            return ToolError(
+                message=(
+                    f"`{path}` is not an absolute path. "
+                    "You must provide an absolute path to read a file "
+                    "outside the working directory."
+                ),
+                brief="Invalid path",
+            )
+        return None
+
+    @override
+    async def __call__(self, params: Params) -> ToolReturnValue:
+        if not params.path:
+            return ToolError(
+                message="File path cannot be empty.",
+                brief="Empty file path",
+            )
+
+        try:
+            p = HostPath(params.path).expanduser()
+            if err := await self._validate_path(p):
+                return err
+            p = p.canonical()
+
+            if is_sensitive_file(str(p)):
+                return ToolError(
+                    message=(
+                        f"`{params.path}` appears to contain secrets "
+                        "(matched sensitive file pattern). "
+                        "Reading this file is blocked to protect credentials."
+                    ),
+                    brief="Sensitive file",
+                )
+
+            if not await p.exists():
+                return ToolError(
+                    message=f"`{params.path}` does not exist.",
+                    brief="File not found",
+                )
+            if not await p.is_file():
+                return ToolError(
+                    message=f"`{params.path}` is not a file.",
+                    brief="Invalid path",
+                )
+
+            header = await p.read_bytes(MEDIA_SNIFF_BYTES)
+            file_type = detect_file_type(str(p), header=header)
+            if file_type.kind in ("image", "video"):
+                return ToolError(
+                    message=(
+                        f"`{params.path}` is a {file_type.kind} file. "
+                        "Use other appropriate tools to read image or video files."
+                    ),
+                    brief="Unsupported file type",
+                )
+
+            if file_type.kind == "unknown":
+                return ToolError(
+                    message=(
+                        f"`{params.path}` seems not readable. "
+                        "You may need to read it with proper shell commands, Python tools "
+                        "or MCP tools if available. "
+                        "If you read/operate it with Python, you MUST ensure that any "
+                        "third-party packages are installed in a virtual environment (venv)."
+                    ),
+                    brief="File not readable",
+                )
+
+            assert params.n_lines >= 1
+            assert params.line_offset != 0
+
+            if params.line_offset < 0:
+                return await self._read_tail(p, params)
+            else:
+                return await self._read_forward(p, params)
+        except Exception as e:
+            logger.warning("ReadFile failed: {path}: {error}", path=params.path, error=e)
+            return ToolError(
+                message=f"Failed to read {params.path}. Error: {e}",
+                brief="Failed to read file",
+            )
+
+    async def _read_forward(self, p: HostPath, params: Params) -> ToolReturnValue:
+        """Read file from a positive line_offset, counting total lines."""
+        lines: list[str] = []
+        n_bytes = 0
+        truncated_line_numbers: list[int] = []
+        max_lines_reached = False
+        max_bytes_reached = False
+        collecting = True  # False once we've collected enough lines
+        current_line_no = 0
+        async for line in p.read_lines(errors="replace"):
+            current_line_no += 1
+            if not collecting:
+                continue
+            if current_line_no < params.line_offset:
+                continue
+            truncated = truncate_line(line, MAX_LINE_LENGTH)
+            if truncated != line:
+                truncated_line_numbers.append(current_line_no)
+            lines.append(truncated)
+            n_bytes += len(truncated.encode("utf-8"))
+            if len(lines) >= params.n_lines:
+                collecting = False
+            elif len(lines) >= MAX_LINES:
+                max_lines_reached = True
+                collecting = False
+            elif n_bytes >= MAX_BYTES:
+                max_bytes_reached = True
+                collecting = False
+
+        total_lines = current_line_no
+
+        # Format output with line numbers like `cat -n`
+        start_line = params.line_offset
+        lines_with_no: list[str] = []
+        for line_num, line in zip(range(start_line, start_line + len(lines)), lines, strict=True):
+            lines_with_no.append(f"{line_num:6d}\t{line}")
+
+        message = (
+            f"{len(lines)} lines read from file starting from line {start_line}."
+            if len(lines) > 0
+            else "No lines read from file."
+        )
+        message += f" Total lines in file: {total_lines}."
+        if max_lines_reached:
+            message += f" Max {MAX_LINES} lines reached."
+        elif max_bytes_reached:
+            message += f" Max {MAX_BYTES} bytes reached."
+        elif len(lines) < params.n_lines:
+            message += " End of file reached."
+        if truncated_line_numbers:
+            message += f" Lines {truncated_line_numbers} were truncated."
+        return ToolOk(
+            output="".join(lines_with_no),
+            message=message,
+        )
+
+    async def _read_tail(self, p: HostPath, params: Params) -> ToolReturnValue:
+        """Read file from a negative line_offset (tail mode)."""
+        tail_count = abs(params.line_offset)
+
+        # Use a deque to keep the last `tail_count` lines with their line numbers
+        # Each entry: (line_no, truncated_line, was_truncated)
+        tail_buf: deque[tuple[int, str, bool]] = deque(maxlen=tail_count)
+        current_line_no = 0
+        async for line in p.read_lines(errors="replace"):
+            current_line_no += 1
+            truncated = truncate_line(line, MAX_LINE_LENGTH)
+            tail_buf.append((current_line_no, truncated, truncated != line))
+
+        total_lines = current_line_no
+
+        # Step 1: Apply n_lines / MAX_LINES from head of tail_buf.
+        # This preserves the user's requested start position.
+        all_entries = list(tail_buf)
+        line_limit = min(params.n_lines, MAX_LINES)
+        candidates = all_entries[:line_limit]
+        max_lines_reached = len(all_entries) > MAX_LINES and len(candidates) == MAX_LINES
+
+        # Step 2: Apply MAX_BYTES — if candidates exceed the byte budget,
+        # reverse-scan to keep the newest (closest to EOF) lines that fit.
+        total_candidate_bytes = sum(len(entry[1].encode("utf-8")) for entry in candidates)
+        if total_candidate_bytes > MAX_BYTES:
+            max_bytes_reached = True
+            kept = 0
+            n_bytes = 0
+            for entry in reversed(candidates):
+                n_bytes += len(entry[1].encode("utf-8"))
+                if n_bytes > MAX_BYTES:
+                    break
+                kept += 1
+            candidates = candidates[len(candidates) - kept :]
+        else:
+            max_bytes_reached = False
+
+        # Step 3: Collect results from candidates
+        lines: list[str] = []
+        line_numbers: list[int] = []
+        truncated_line_numbers: list[int] = []
+
+        for line_no, truncated, was_truncated in candidates:
+            if was_truncated:
+                truncated_line_numbers.append(line_no)
+            lines.append(truncated)
+            line_numbers.append(line_no)
+
+        # Format output with absolute line numbers
+        lines_with_no: list[str] = []
+        for line_num, line in zip(line_numbers, lines, strict=True):
+            lines_with_no.append(f"{line_num:6d}\t{line}")
+
+        start_line = line_numbers[0] if line_numbers else total_lines + 1
+        message = (
+            f"{len(lines)} lines read from file starting from line {start_line}."
+            if len(lines) > 0
+            else "No lines read from file."
+        )
+        message += f" Total lines in file: {total_lines}."
+        if max_lines_reached:
+            message += f" Max {MAX_LINES} lines reached."
+        elif max_bytes_reached:
+            message += f" Max {MAX_BYTES} bytes reached."
+        elif len(lines) < params.n_lines:
+            message += " End of file reached."
+        if truncated_line_numbers:
+            message += f" Lines {truncated_line_numbers} were truncated."
+        return ToolOk(
+            output="".join(lines_with_no),
+            message=message,
+        )

pythinker_code/tools/file/read_media.md

@@ -0,0 +1,24 @@
+Read media content from a file.
+
+**Tips:**
+- Make sure you follow the description of each tool parameter.
+- A `<system>` tag will be given before the read file content.
+- The system will notify you when there is anything wrong when reading the file.
+- This tool is a tool that you typically want to use in parallel. Always read multiple files in one response when possible.
+- This tool can only read image or video files. To read other types of files, use the ReadFile tool. To list directories, use the Glob tool or `ls` command via the Shell tool.
+- If the file doesn't exist or path is invalid, an error will be returned.
+- The maximum size that can be read is ${MAX_MEDIA_MEGABYTES}MB. An error will be returned if the file is larger than this limit.
+- The media content will be returned in a form that you can directly view and understand.
+
+**Capabilities**
+{% if "image_in" in capabilities and "video_in" in capabilities %}
+- This tool supports image and video files for the current model.
+{% elif "image_in" in capabilities %}
+- This tool supports image files for the current model.
+- Video files are not supported by the current model.
+{% elif "video_in" in capabilities %}
+- This tool supports video files for the current model.
+- Image files are not supported by the current model.
+{% else %}
+- The current model does not support image or video input.
+{% endif %}

pythinker_code/tools/file/read_media.py

@@ -0,0 +1,217 @@
+import base64
+from io import BytesIO
+from pathlib import Path
+from typing import override
+
+from pydantic import BaseModel, Field
+from pythinker_core.chat_provider.pythinker import Pythinker
+from pythinker_core.tooling import CallableTool2, ToolError, ToolOk, ToolReturnValue
+from pythinker_host.path import HostPath
+
+from pythinker_code.soul.agent import Runtime
+from pythinker_code.tools import SkipThisTool
+from pythinker_code.tools.file.utils import MEDIA_SNIFF_BYTES, FileType, detect_file_type
+from pythinker_code.tools.utils import load_desc
+from pythinker_code.utils.logging import logger
+from pythinker_code.utils.media_tags import wrap_media_part
+from pythinker_code.utils.path import is_within_workspace
+from pythinker_code.wire.types import ImageURLPart, VideoURLPart
+
+MAX_MEDIA_MEGABYTES = 100
+
+
+def _to_data_url(mime_type: str, data: bytes) -> str:
+    encoded = base64.b64encode(data).decode("utf-8", errors="replace")
+    return f"data:{mime_type};base64,{encoded}"
+
+
+def _extract_image_size(data: bytes) -> tuple[int, int] | None:
+    try:
+        from PIL import Image
+    except Exception:
+        return None
+    try:
+        with Image.open(BytesIO(data)) as image:
+            image.load()
+            return image.size
+    except Exception:
+        return None
+
+
+class Params(BaseModel):
+    path: str = Field(
+        description=(
+            "The path to the file to read. Absolute paths are required when reading files "
+            "outside the working directory."
+        )
+    )
+
+
+class ReadMediaFile(CallableTool2[Params]):
+    name: str = "ReadMediaFile"
+    params: type[Params] = Params
+
+    def __init__(self, runtime: Runtime) -> None:
+        capabilities = runtime.llm.capabilities if runtime.llm else set[str]()
+        if "image_in" not in capabilities and "video_in" not in capabilities:
+            raise SkipThisTool()
+
+        description = load_desc(
+            Path(__file__).parent / "read_media.md",
+            {
+                "MAX_MEDIA_MEGABYTES": MAX_MEDIA_MEGABYTES,
+                "capabilities": capabilities,
+            },
+        )
+        super().__init__(description=description)
+
+        self._runtime = runtime
+        self._work_dir = runtime.builtin_args.PYTHINKER_WORK_DIR
+        self._additional_dirs = runtime.additional_dirs
+        self._capabilities = capabilities
+
+    async def _validate_path(self, path: HostPath) -> ToolError | None:
+        """Validate that the path is safe to read."""
+        resolved_path = path.canonical()
+
+        if (
+            not is_within_workspace(resolved_path, self._work_dir, self._additional_dirs)
+            and not path.is_absolute()
+        ):
+            # Outside files can only be read with absolute paths
+            return ToolError(
+                message=(
+                    f"`{path}` is not an absolute path. "
+                    "You must provide an absolute path to read a file "
+                    "outside the working directory."
+                ),
+                brief="Invalid path",
+            )
+        return None
+
+    async def _read_media(self, path: HostPath, file_type: FileType) -> ToolReturnValue:
+        assert file_type.kind in ("image", "video")
+
+        media_path = str(path)
+        stat = await path.stat()
+        size = stat.st_size
+        if size == 0:
+            return ToolError(
+                message=f"`{path}` is empty.",
+                brief="Empty file",
+            )
+        if size > (MAX_MEDIA_MEGABYTES << 20):
+            return ToolError(
+                message=(
+                    f"`{path}` is {size} bytes, which exceeds the max "
+                    f"{MAX_MEDIA_MEGABYTES}MB bytes for media files."
+                ),
+                brief="File too large",
+            )
+
+        match file_type.kind:
+            case "image":
+                data = await path.read_bytes()
+                data_url = _to_data_url(file_type.mime_type, data)
+                part = ImageURLPart(image_url=ImageURLPart.ImageURL(url=data_url))
+                wrapped = wrap_media_part(part, tag="image", attrs={"path": media_path})
+                image_size = _extract_image_size(data)
+            case "video":
+                data = await path.read_bytes()
+                if (llm := self._runtime.llm) and isinstance(llm.chat_provider, Pythinker):
+                    part = await llm.chat_provider.files.upload_video(
+                        data=data,
+                        mime_type=file_type.mime_type,
+                    )
+                    wrapped = wrap_media_part(part, tag="video", attrs={"path": media_path})
+                else:
+                    data_url = _to_data_url(file_type.mime_type, data)
+                    part = VideoURLPart(video_url=VideoURLPart.VideoURL(url=data_url))
+                    wrapped = wrap_media_part(part, tag="video", attrs={"path": media_path})
+                image_size = None
+
+        size_hint = ""
+        if image_size:
+            size_hint = f", original size {image_size[0]}x{image_size[1]}px"
+        note = (
+            " If you need to output coordinates, output relative coordinates first and "
+            "compute absolute coordinates using the original image size; if you generate or "
+            "edit images/videos via commands or scripts, read the result back immediately "
+            "before continuing."
+        )
+        return ToolOk(
+            output=wrapped,
+            message=(
+                f"Loaded {file_type.kind} file `{path}` "
+                f"({file_type.mime_type}, {size} bytes{size_hint}).{note}"
+            ),
+        )
+
+    @override
+    async def __call__(self, params: Params) -> ToolReturnValue:
+        if not params.path:
+            return ToolError(
+                message="File path cannot be empty.",
+                brief="Empty file path",
+            )
+
+        try:
+            p = HostPath(params.path).expanduser()
+            if err := await self._validate_path(p):
+                return err
+            p = p.canonical()
+
+            if not await p.exists():
+                return ToolError(
+                    message=f"`{params.path}` does not exist.",
+                    brief="File not found",
+                )
+            if not await p.is_file():
+                return ToolError(
+                    message=f"`{params.path}` is not a file.",
+                    brief="Invalid path",
+                )
+
+            header = await p.read_bytes(MEDIA_SNIFF_BYTES)
+            file_type = detect_file_type(str(p), header=header)
+            if file_type.kind == "text":
+                return ToolError(
+                    message=f"`{params.path}` is a text file. Use ReadFile to read text files.",
+                    brief="Unsupported file type",
+                )
+            if file_type.kind == "unknown":
+                return ToolError(
+                    message=(
+                        f"`{params.path}` seems not readable as an image or video file. "
+                        "You may need to read it with proper shell commands, Python tools "
+                        "or MCP tools if available. "
+                        "If you read/operate it with Python, you MUST ensure that any "
+                        "third-party packages are installed in a virtual environment (venv)."
+                    ),
+                    brief="File not readable",
+                )
+
+            if file_type.kind == "image" and "image_in" not in self._capabilities:
+                return ToolError(
+                    message=(
+                        "The current model does not support image input. "
+                        "Tell the user to use a model with image input capability."
+                    ),
+                    brief="Unsupported media type",
+                )
+            if file_type.kind == "video" and "video_in" not in self._capabilities:
+                return ToolError(
+                    message=(
+                        "The current model does not support video input. "
+                        "Tell the user to use a model with video input capability."
+                    ),
+                    brief="Unsupported media type",
+                )
+
+            return await self._read_media(p, file_type)
+        except Exception as e:
+            logger.warning("ReadMediaFile failed: {path}: {error}", path=params.path, error=e)
+            return ToolError(
+                message=f"Failed to read {params.path}. Error: {e}",
+                brief="Failed to read file",
+            )

pythinker_code/tools/file/replace.md

@@ -0,0 +1,7 @@
+Replace specific strings within a specified file.
+
+**Tips:**
+- Only use this tool on text files.
+- Multi-line strings are supported.
+- Can specify a single edit or a list of edits in one call.
+- You should prefer this tool over WriteFile tool and Shell `sed` command.