pythinker-code 2.0.0__py3-none-any.whl

pythinker_code/tools/todo/__init__.py

@@ -0,0 +1,168 @@
+import json
+from pathlib import Path
+from typing import Any, Literal, cast, override
+
+from pydantic import BaseModel, Field
+from pythinker_core.tooling import CallableTool2, ToolReturnValue
+
+from pythinker_code.session_state import TodoItemState
+from pythinker_code.soul.agent import Runtime
+from pythinker_code.tools.display import TodoDisplayBlock, TodoDisplayItem
+from pythinker_code.tools.utils import load_desc
+from pythinker_code.utils.logging import logger
+
+
+class Todo(BaseModel):
+    title: str = Field(description="The title of the todo", min_length=1)
+    status: Literal["pending", "in_progress", "done"] = Field(description="The status of the todo")
+
+
+class Params(BaseModel):
+    todos: list[Todo] | None = Field(
+        default=None,
+        description=(
+            "The updated todo list. "
+            "If not provided, returns the current todo list without making changes."
+        ),
+    )
+
+
+class SetTodoList(CallableTool2[Params]):
+    name: str = "SetTodoList"
+    description: str = load_desc(Path(__file__).parent / "set_todo_list.md")
+    params: type[Params] = Params
+
+    def __init__(self, runtime: Runtime) -> None:
+        super().__init__()
+        self._runtime = runtime
+
+    @override
+    async def __call__(self, params: Params) -> ToolReturnValue:
+        if params.todos is None:
+            return self._read_todos()
+        return self._write_todos(params.todos)
+
+    # ---- Write mode --------------------------------------------------------
+
+    def _write_todos(self, todos: list[Todo]) -> ToolReturnValue:
+        """Persist the todo list and return confirmation."""
+        self._save_todos(todos)
+
+        items = [TodoDisplayItem(title=todo.title, status=todo.status) for todo in todos]
+        return ToolReturnValue(
+            is_error=False,
+            output="Todo list updated",
+            message="Todo list updated",
+            display=[TodoDisplayBlock(items=items)],
+        )
+
+    # ---- Read mode ---------------------------------------------------------
+
+    def _read_todos(self) -> ToolReturnValue:
+        """Return the current todo list as text output for the model."""
+        todos = self._load_todos()
+        if not todos:
+            return ToolReturnValue(
+                is_error=False,
+                output="Todo list is empty.",
+                message="",
+                display=[],
+            )
+
+        lines: list[str] = ["Current todo list:"]
+        for todo in todos:
+            lines.append(f"- [{todo.status}] {todo.title}")
+        return ToolReturnValue(
+            is_error=False,
+            output="\n".join(lines),
+            message="",
+            display=[],
+        )
+
+    # ---- Persistence -------------------------------------------------------
+
+    def _save_todos(self, todos: list[Todo]) -> None:
+        """Persist todos to the appropriate state file."""
+        items = [TodoItemState(title=t.title, status=t.status) for t in todos]
+
+        if self._runtime.role == "root":
+            self._save_root_todos(items)
+        else:
+            self._save_subagent_todos(items)
+
+    def _load_todos(self) -> list[Todo]:
+        """Load todos from the appropriate state file."""
+        if self._runtime.role == "root":
+            return self._load_root_todos()
+        else:
+            return self._load_subagent_todos()
+
+    def _save_root_todos(self, items: list[TodoItemState]) -> None:
+        session = self._runtime.session
+        session.state.todos = items
+        session.save_state()
+
+    def _load_root_todos(self) -> list[Todo]:
+        from pythinker_code.session_state import load_session_state
+
+        session = self._runtime.session
+        fresh = load_session_state(session.dir)
+        session.state.todos = fresh.todos
+        result: list[Todo] = []
+        for t in fresh.todos:
+            try:
+                result.append(Todo(title=t.title, status=t.status))
+            except Exception:
+                logger.warning("Skipping malformed todo item in root state: {t}", t=t)
+        return result
+
+    def _save_subagent_todos(self, items: list[TodoItemState]) -> None:
+        state_file = self._subagent_state_file()
+        if state_file is None:
+            return
+        data = self._read_subagent_state(state_file)
+        data["todos"] = [item.model_dump() for item in items]
+        self._write_subagent_state(state_file, data)
+
+    def _load_subagent_todos(self) -> list[Todo]:
+        state_file = self._subagent_state_file()
+        if state_file is None:
+            return []
+        data = self._read_subagent_state(state_file)
+        raw_todos_val = data.get("todos", [])
+        raw_todos = cast(list[Any], raw_todos_val) if isinstance(raw_todos_val, list) else []
+        result: list[Todo] = []
+        for item in raw_todos:
+            try:
+                result.append(Todo(**item))
+            except Exception:
+                logger.warning("Skipping malformed todo item in subagent state: {item}", item=item)
+        return result
+
+    def _subagent_state_file(self) -> Path | None:
+        store = self._runtime.subagent_store
+        agent_id = self._runtime.subagent_id
+        if store is None or agent_id is None:
+            return None
+        return store.instance_dir(agent_id) / "state.json"
+
+    @staticmethod
+    def _read_subagent_state(path: Path) -> dict[str, Any]:
+        if not path.exists():
+            return {}
+        try:
+            data = json.loads(path.read_text(encoding="utf-8", errors="replace"))
+        except (json.JSONDecodeError, OSError, UnicodeDecodeError):
+            logger.warning("Corrupted subagent todo state, using defaults: {path}", path=path)
+            return {}
+        if not isinstance(data, dict):
+            logger.warning("Invalid subagent todo state type, using defaults: {path}", path=path)
+            return {}
+        return cast(dict[str, Any], data)
+
+    @staticmethod
+    def _write_subagent_state(path: Path, data: dict[str, Any]) -> None:
+        from pythinker_code.utils.io import atomic_json_write
+
+        path.parent.mkdir(parents=True, exist_ok=True)
+        atomic_json_write(data, path)

pythinker_code/tools/todo/set_todo_list.md

@@ -0,0 +1,23 @@
+Manage your todo list for tracking task progress.
+
+Todo list is a simple yet powerful tool to help you get things done. You typically want to use this tool when the given task involves multiple subtasks/milestones, or, multiple tasks are given in a single request. This tool can help you to break down the task and track the progress.
+
+**Usage modes:**
+
+- **Update mode**: Pass `todos` to set the entire todo list. The previous list is replaced.
+- **Query mode**: Omit `todos` (or pass null) to retrieve the current todo list without changes.
+- **Clear mode**: Pass an empty array `[]` to clear all todos.
+
+This is the only todo list tool available to you. That said, each time you want to update the todo list, you need to provide the whole list. Make sure to maintain the todo items and their statuses properly.
+
+Once you finished a subtask/milestone, remember to update the todo list to reflect the progress. Also, you can give yourself a self-encouragement to keep you motivated.
+
+Abusing this tool to track too small steps will just waste your time and make your context messy. For example, here are some cases you should not use this tool:
+
+- When the user just simply ask you a question. E.g. "What language and framework is used in the project?", "What is the best practice for x?"
+- When it only takes a few steps/tool calls to complete the task. E.g. "Fix the unit test function 'test_xxx'", "Refactor the function 'xxx' to make it more solid."
+- When the user prompt is very specific and the only thing you need to do is brainlessly following the instructions. E.g. "Replace xxx to yyy in the file zzz", "Create a file xxx with content yyy."
+
+However, do not get stuck in a rut. Be flexible. Sometimes, you may try to use todo list at first, then realize the task is too simple and you can simply stop using it; or, sometimes, you may realize the task is complex after a few steps and then you can start using todo list to break it down.
+
+IMPORTANT: Do not call this tool repeatedly without making real progress on at least one task between calls. If you are unsure about the current state, use Query mode (omit `todos`) to check before updating. If you find yourself unable to advance any task with your available tools, inform the user about what is blocking you instead of replanning. Repeatedly updating the todo list without doing actual work is counterproductive.

pythinker_code/tools/utils.py

@@ -0,0 +1,199 @@
+import re
+from pathlib import Path
+
+from jinja2 import Environment, Undefined
+from pythinker_core.tooling import BriefDisplayBlock, DisplayBlock, ToolError, ToolReturnValue
+from pythinker_core.utils.typing import JsonType
+
+
+class _KeepPlaceholderUndefined(Undefined):
+    def __str__(self) -> str:
+        if self._undefined_name is None:
+            return ""
+        return f"${{{self._undefined_name}}}"
+
+    __repr__ = __str__
+
+
+def load_desc(path: Path, context: dict[str, object] | None = None) -> str:
+    """Load a tool description from a file, rendered via Jinja2."""
+    description = path.read_text(encoding="utf-8")
+    env = Environment(
+        keep_trailing_newline=True,
+        lstrip_blocks=True,
+        trim_blocks=True,
+        variable_start_string="${",
+        variable_end_string="}",
+        undefined=_KeepPlaceholderUndefined,
+    )
+    template = env.from_string(description)
+    return template.render(context or {})
+
+
+def truncate_line(line: str, max_length: int, marker: str = "...") -> str:
+    """
+    Truncate a line if it exceeds `max_length`, preserving the beginning and the line break.
+    The output may be longer than `max_length` if it is too short to fit the marker.
+    """
+    if len(line) <= max_length:
+        return line
+
+    # Find line breaks at the end of the line
+    m = re.search(r"[\r\n]+$", line)
+    linebreak = m.group(0) if m else ""
+    end = marker + linebreak
+    max_length = max(max_length, len(end))
+    return line[: max_length - len(end)] + end
+
+
+# Default output limits
+DEFAULT_MAX_CHARS = 50_000
+DEFAULT_MAX_LINE_LENGTH = 2000
+
+
+class ToolResultBuilder:
+    """
+    Builder for tool results with character and line limits.
+    """
+
+    def __init__(
+        self,
+        max_chars: int = DEFAULT_MAX_CHARS,
+        max_line_length: int | None = DEFAULT_MAX_LINE_LENGTH,
+    ):
+        self.max_chars = max_chars
+        self.max_line_length = max_line_length
+        self._marker = "[...truncated]"
+        if max_line_length is not None:
+            assert max_line_length > len(self._marker)
+        self._buffer: list[str] = []
+        self._n_chars = 0
+        self._n_lines = 0
+        self._truncation_happened = False
+        self._display: list[DisplayBlock] = []
+        self._extras: dict[str, JsonType] | None = None
+
+    @property
+    def is_full(self) -> bool:
+        """Check if output buffer is full due to character limit."""
+        return self._n_chars >= self.max_chars
+
+    @property
+    def n_chars(self) -> int:
+        """Get current character count."""
+        return self._n_chars
+
+    @property
+    def n_lines(self) -> int:
+        """Get current line count."""
+        return self._n_lines
+
+    def write(self, text: str) -> int:
+        """
+        Write text to the output buffer.
+
+        Returns:
+            int: Number of characters actually written
+        """
+        if self.is_full:
+            return 0
+
+        lines = text.splitlines(keepends=True)
+        if not lines:
+            return 0
+
+        chars_written = 0
+
+        for line in lines:
+            if self.is_full:
+                break
+
+            original_line = line
+            remaining_chars = self.max_chars - self._n_chars
+            limit = (
+                min(remaining_chars, self.max_line_length)
+                if self.max_line_length is not None
+                else remaining_chars
+            )
+            line = truncate_line(line, limit, self._marker)
+            if line != original_line:
+                self._truncation_happened = True
+
+            self._buffer.append(line)
+            chars_written += len(line)
+            self._n_chars += len(line)
+            if line.endswith("\n"):
+                self._n_lines += 1
+
+        return chars_written
+
+    def display(self, *blocks: DisplayBlock) -> None:
+        """Add display blocks to the tool result."""
+        self._display.extend(blocks)
+
+    def extras(self, **extras: JsonType) -> None:
+        """Add extra data to the tool result."""
+        if self._extras is None:
+            self._extras = {}
+        self._extras.update(extras)
+
+    def ok(self, message: str = "", *, brief: str = "") -> ToolReturnValue:
+        """Create a ToolReturnValue with is_error=False and the current output."""
+        output = "".join(self._buffer)
+
+        final_message = message
+        if final_message and not final_message.endswith("."):
+            final_message += "."
+        truncation_msg = "Output is truncated to fit in the message."
+        if self._truncation_happened:
+            if final_message:
+                final_message += f" {truncation_msg}"
+            else:
+                final_message = truncation_msg
+        return ToolReturnValue(
+            is_error=False,
+            output=output,
+            message=final_message,
+            display=([BriefDisplayBlock(text=brief)] if brief else []) + self._display,
+            extras=self._extras,
+        )
+
+    def error(self, message: str, *, brief: str) -> ToolReturnValue:
+        """Create a ToolReturnValue with is_error=True and the current output."""
+        output = "".join(self._buffer)
+
+        final_message = message
+        if self._truncation_happened:
+            truncation_msg = "Output is truncated to fit in the message."
+            if final_message:
+                final_message += f" {truncation_msg}"
+            else:
+                final_message = truncation_msg
+
+        return ToolReturnValue(
+            is_error=True,
+            output=output,
+            message=final_message,
+            display=([BriefDisplayBlock(text=brief)] if brief else []) + self._display,
+            extras=self._extras,
+        )
+
+
+class ToolRejectedError(ToolError):
+    has_feedback: bool = False
+
+    def __init__(
+        self,
+        message: str | None = None,
+        brief: str = "Rejected by user",
+        has_feedback: bool = False,
+    ):
+        super().__init__(
+            message=message
+            or (
+                "The tool call is rejected by the user. "
+                "Stop what you are doing and wait for the user to tell you how to proceed."
+            ),
+            brief=brief,
+        )
+        self.has_feedback = has_feedback

pythinker_code/tools/web/__init__.py

@@ -0,0 +1,4 @@
+from .fetch import FetchURL
+from .search import SearchWeb
+
+__all__ = ("SearchWeb", "FetchURL")

pythinker_code/tools/web/fetch.md

@@ -0,0 +1 @@
+Fetch a web page from a URL and extract main text content from it.

pythinker_code/tools/web/fetch.py

@@ -0,0 +1,189 @@
+from pathlib import Path
+from typing import override
+
+import aiohttp
+import trafilatura
+from pydantic import BaseModel, Field
+from pythinker_core.tooling import CallableTool2, ToolReturnValue
+
+from pythinker_code.config import Config
+from pythinker_code.constant import USER_AGENT
+from pythinker_code.soul.agent import Runtime
+from pythinker_code.soul.toolset import get_current_tool_call_or_none
+from pythinker_code.tools.utils import ToolResultBuilder, load_desc
+from pythinker_code.utils.aiohttp import new_client_session
+from pythinker_code.utils.logging import logger
+
+
+class Params(BaseModel):
+    url: str = Field(description="The URL to fetch content from.")
+
+
+class FetchURL(CallableTool2[Params]):
+    name: str = "FetchURL"
+    description: str = load_desc(Path(__file__).parent / "fetch.md", {})
+    params: type[Params] = Params
+
+    def __init__(self, config: Config, runtime: Runtime):
+        super().__init__()
+        self._runtime = runtime
+        self._service_config = config.services.pythinker_ai_fetch
+
+    @override
+    async def __call__(self, params: Params) -> ToolReturnValue:
+        if self._service_config:
+            ret = await self._fetch_with_service(params)
+            if not ret.is_error:
+                return ret
+            logger.warning("Failed to fetch URL via service: {error}", error=ret.message)
+            # fallback to local fetch if service fetch fails
+        return await self.fetch_with_http_get(params)
+
+    @staticmethod
+    async def fetch_with_http_get(params: Params) -> ToolReturnValue:
+        builder = ToolResultBuilder(max_line_length=None)
+        try:
+            # Fetching arbitrary web pages can take a while on large/slow sites.
+            fetch_timeout = aiohttp.ClientTimeout(total=180, sock_read=60, sock_connect=15)
+            async with (
+                new_client_session(timeout=fetch_timeout) as session,
+                session.get(
+                    params.url,
+                    headers={
+                        "User-Agent": (
+                            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                            "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+                        ),
+                    },
+                ) as response,
+            ):
+                if response.status >= 400:
+                    logger.warning(
+                        "FetchURL HTTP error: status={status}, url={url}",
+                        status=response.status,
+                        url=params.url,
+                    )
+                    return builder.error(
+                        (
+                            f"Failed to fetch URL. Status: {response.status}. "
+                            f"This may indicate the page is not accessible or the server is down."
+                        ),
+                        brief=f"HTTP {response.status} error",
+                    )
+
+                resp_text = (await response.read()).decode("utf-8", errors="replace")
+
+                content_type = response.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower()
+                if content_type.startswith(("text/plain", "text/markdown")):
+                    builder.write(resp_text)
+                    return builder.ok("The returned content is the full content of the page.")
+        except TimeoutError:
+            logger.warning("FetchURL timed out: url={url}", url=params.url)
+            return builder.error(
+                "Failed to fetch URL: request timed out. The server may be slow or unreachable.",
+                brief="Request timed out",
+            )
+        except aiohttp.ClientError as e:
+            logger.warning("FetchURL network error: {error}, url={url}", error=e, url=params.url)
+            return builder.error(
+                (
+                    f"Failed to fetch URL due to network error: {e}. "
+                    "This may indicate the URL is invalid or the server is unreachable."
+                ),
+                brief="Network error",
+            )
+
+        if not resp_text:
+            return builder.ok(
+                "The response body is empty.",
+                brief="Empty response body",
+            )
+
+        extracted_text = trafilatura.extract(
+            resp_text,
+            include_comments=True,
+            include_tables=True,
+            include_formatting=False,
+            output_format="txt",
+            with_metadata=True,
+        )
+
+        if not extracted_text:
+            return builder.error(
+                (
+                    "Failed to extract meaningful content from the page. "
+                    "This may indicate the page content is not suitable for text extraction, "
+                    "or the page requires JavaScript to render its content."
+                ),
+                brief="No content extracted",
+            )
+
+        builder.write(extracted_text)
+        return builder.ok("The returned content is the main text content extracted from the page.")
+
+    async def _fetch_with_service(self, params: Params) -> ToolReturnValue:
+        assert self._service_config is not None
+
+        tool_call = get_current_tool_call_or_none()
+        assert tool_call is not None, "Tool call is expected to be set"
+
+        builder = ToolResultBuilder(max_line_length=None)
+        api_key = self._runtime.oauth.resolve_api_key(
+            self._service_config.api_key, self._service_config.oauth
+        )
+        if not api_key:
+            return builder.error(
+                "Fetch service is not configured. You may want to try other methods to fetch.",
+                brief="Fetch service not configured",
+            )
+        headers = {
+            "User-Agent": USER_AGENT,
+            "Authorization": f"Bearer {api_key}",
+            "Accept": "text/markdown",
+            "X-Msh-Tool-Call-Id": tool_call.id,
+            **self._runtime.oauth.common_headers(),
+            **(self._service_config.custom_headers or {}),
+        }
+
+        try:
+            async with (
+                new_client_session() as session,
+                session.post(
+                    self._service_config.base_url,
+                    headers=headers,
+                    json={"url": params.url},
+                ) as response,
+            ):
+                if response.status != 200:
+                    logger.warning(
+                        "FetchURL service HTTP error: status={status}, url={url}",
+                        status=response.status,
+                        url=params.url,
+                    )
+                    return builder.error(
+                        f"Failed to fetch URL via service. Status: {response.status}.",
+                        brief="Failed to fetch URL via fetch service",
+                    )
+
+                content = (await response.read()).decode("utf-8", errors="replace")
+                builder.write(content)
+                return builder.ok(
+                    "The returned content is the main content extracted from the page."
+                )
+        except TimeoutError:
+            logger.warning("FetchURL service timed out: url={url}", url=params.url)
+            return builder.error(
+                "Failed to fetch URL via service: request timed out.",
+                brief="Service request timed out",
+            )
+        except aiohttp.ClientError as e:
+            logger.warning(
+                "FetchURL service network error: {error}, url={url}", error=e, url=params.url
+            )
+            return builder.error(
+                (
+                    f"Failed to fetch URL via service due to network error: {e}. "
+                    "This may indicate the service is unreachable."
+                ),
+                brief="Network error when calling fetch service",
+            )

pythinker_code/tools/web/search.md

@@ -0,0 +1 @@
+WebSearch tool allows you to search on the internet to get latest information, including news, documents, release notes, blog posts, papers, etc.