yycode 0.3.2__py3-none-any.whl

agent/lsp/manager.py

@@ -0,0 +1,234 @@
+"""High-level manager for read-only Python LSP navigation."""
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+from pathlib import Path
+from typing import Any
+
+from agent.lsp.client import LspClient, LspClientError
+from agent.lsp.types import (
+    Diagnostic,
+    Location,
+    Symbol,
+    path_to_uri,
+    range_start,
+    symbol_kind_name,
+    uri_to_path,
+)
+from tools.workspace import Workspace
+
+
+class LspUnavailable(RuntimeError):
+    """Raised when no supported language server is available."""
+
+
+def _python_server_command() -> list[str] | None:
+    if shutil.which("pyright-langserver"):
+        return ["pyright-langserver", "--stdio"]
+    if shutil.which("pylsp"):
+        return ["pylsp"]
+    return None
+
+
+class LspManager:
+    """Lazy Python LSP manager scoped to one workspace."""
+
+    def __init__(self, workdir: Path | str, timeout: float = 10.0):
+        self.workspace = Workspace(Path(workdir))
+        self.timeout = timeout
+        self._client: LspClient | None = None
+        self._opened: set[str] = set()
+
+    async def document_symbols(self, path: str) -> list[Symbol]:
+        file_path = self._safe_python_file(path)
+        client = await self._client_for_python()
+        await self._open(client, file_path)
+        result = await client.request(
+            "textDocument/documentSymbol",
+            {"textDocument": {"uri": path_to_uri(str(file_path))}},
+        )
+        return self._parse_document_symbols(result or [], file_path)
+
+    async def workspace_symbols(self, query: str) -> list[Symbol]:
+        client = await self._client_for_python()
+        result = await client.request("workspace/symbol", {"query": query})
+        return [self._parse_workspace_symbol(item) for item in (result or [])]
+
+    async def definition(self, path: str, line: int, character: int) -> list[Location]:
+        return await self._locations_request("textDocument/definition", path, line, character)
+
+    async def references(
+        self,
+        path: str,
+        line: int,
+        character: int,
+        include_declaration: bool = False,
+    ) -> list[Location]:
+        return await self._locations_request(
+            "textDocument/references",
+            path,
+            line,
+            character,
+            extra={"context": {"includeDeclaration": include_declaration}},
+        )
+
+    async def hover(self, path: str, line: int, character: int) -> str:
+        file_path = self._safe_python_file(path)
+        client = await self._client_for_python()
+        await self._open(client, file_path)
+        result = await client.request("textDocument/hover", self._position_params(file_path, line, character))
+        contents = (result or {}).get("contents") if isinstance(result, dict) else result
+        return self._format_hover(contents)
+
+    async def diagnostics(self, path: str | None = None) -> list[Diagnostic]:
+        if path:
+            file_path = self._safe_python_file(path)
+            client = await self._client_for_python()
+            await self._open(client, file_path)
+        elif self._client is None:
+            await self._client_for_python()
+        # Pull diagnostics are not universally supported. Return an empty list rather than guessing.
+        return []
+
+    async def shutdown(self) -> None:
+        if self._client:
+            await self._client.shutdown()
+        self._client = None
+        self._opened.clear()
+
+    async def _client_for_python(self) -> LspClient:
+        if self._client is not None and self._client.process is not None and self._client.process.returncode is None:
+            return self._client
+        command = _python_server_command()
+        if not command:
+            raise LspUnavailable("pyright-langserver and pylsp not found")
+        self._client = LspClient(command, self.workspace.root, timeout=self.timeout)
+        try:
+            await self._client.start()
+        except FileNotFoundError as exc:
+            raise LspUnavailable(f"language server not found: {command[0]}") from exc
+        except (OSError, LspClientError, asyncio.TimeoutError) as exc:
+            raise LspUnavailable(f"language server failed to start: {exc}") from exc
+        return self._client
+
+    async def _open(self, client: LspClient, file_path: Path) -> None:
+        uri = path_to_uri(str(file_path))
+        if uri in self._opened:
+            return
+        await client.did_open(uri, file_path.read_text(), language_id="python")
+        self._opened.add(uri)
+
+    async def _locations_request(
+        self,
+        method: str,
+        path: str,
+        line: int,
+        character: int,
+        extra: dict[str, Any] | None = None,
+    ) -> list[Location]:
+        file_path = self._safe_python_file(path)
+        client = await self._client_for_python()
+        await self._open(client, file_path)
+        params = self._position_params(file_path, line, character)
+        if extra:
+            params.update(extra)
+        result = await client.request(method, params)
+        if isinstance(result, dict):
+            result = [result]
+        locations = [self._parse_location(item) for item in (result or [])]
+        return [location for location in locations if location is not None]
+
+    def _position_params(self, file_path: Path, line: int, character: int) -> dict[str, Any]:
+        return {
+            "textDocument": {"uri": path_to_uri(str(file_path))},
+            "position": {"line": max(0, int(line)), "character": max(0, int(character))},
+        }
+
+    def _safe_python_file(self, path: str) -> Path:
+        file_path = self.workspace.safe_path(path)
+        if not file_path.exists():
+            raise ValueError(f"file does not exist: {path}")
+        if not file_path.is_file():
+            raise ValueError(f"path is not a file: {path}")
+        if file_path.suffix != ".py":
+            raise ValueError(f"only Python files are supported in LSP MVP: {path}")
+        return file_path
+
+    def _parse_document_symbols(self, items: list[dict[str, Any]], file_path: Path) -> list[Symbol]:
+        symbols: list[Symbol] = []
+        ignored_kinds = {"file", "module", "package", "namespace"}
+
+        def visit(item: dict[str, Any], container: str | None = None) -> None:
+            line, character = range_start(item)
+            name = str(item.get("name", "<unknown>"))
+            kind = symbol_kind_name(item.get("kind"))
+            if kind not in ignored_kinds:
+                symbols.append(
+                    Symbol(
+                        name=name,
+                        kind=kind,
+                        container_name=container,
+                        location=Location(self.workspace.relative_path(file_path), line, character, name),
+                    )
+                )
+            for child in item.get("children") or []:
+                visit(child, name)
+
+        for item in items:
+            visit(item)
+        return symbols
+
+    def _parse_workspace_symbol(self, item: dict[str, Any]) -> Symbol:
+        location = item.get("location") or {}
+        parsed_location = self._parse_location(location, name=item.get("name"))
+        return Symbol(
+            name=str(item.get("name", "<unknown>")),
+            kind=symbol_kind_name(item.get("kind")),
+            container_name=item.get("containerName"),
+            location=parsed_location,
+        )
+
+    def _parse_location(self, item: dict[str, Any], name: str | None = None) -> Location | None:
+        uri = item.get("uri") or item.get("targetUri") or ""
+        path = Path(uri_to_path(uri))
+        try:
+            relative = self.workspace.relative_path(path)
+        except ValueError:
+            return None
+        line, character = range_start(item.get("range") and item or item.get("targetSelectionRange", {}) or {})
+        return Location(relative, line, character, str(name) if name else None)
+
+    def _format_hover(self, contents: Any) -> str:
+        if not contents:
+            return ""
+        if isinstance(contents, str):
+            return contents
+        if isinstance(contents, dict):
+            value = contents.get("value") or contents.get("language") or str(contents)
+            return str(value)
+        if isinstance(contents, list):
+            return "\n".join(self._format_hover(item) for item in contents if item)
+        return str(contents)
+
+
+_MANAGERS: dict[Path, LspManager] = {}
+
+
+def get_lsp_manager(workdir: Path | str) -> LspManager:
+    """Return a cached LSP manager for a workspace."""
+    root = Workspace(Path(workdir)).root
+    manager = _MANAGERS.get(root)
+    if manager is None:
+        manager = LspManager(root)
+        _MANAGERS[root] = manager
+    return manager
+
+
+async def shutdown_lsp_managers() -> None:
+    """Shutdown all cached LSP managers."""
+    managers = list(_MANAGERS.values())
+    _MANAGERS.clear()
+    for manager in managers:
+        await manager.shutdown()

agent/lsp/types.py

@@ -0,0 +1,119 @@
+"""Small typed containers and formatters for LSP results."""
+
+from dataclasses import dataclass
+from typing import Any
+
+
+SYMBOL_KINDS = {
+    1: "file",
+    2: "module",
+    3: "namespace",
+    4: "package",
+    5: "class",
+    6: "method",
+    7: "property",
+    8: "field",
+    9: "constructor",
+    10: "enum",
+    11: "interface",
+    12: "function",
+    13: "variable",
+    14: "constant",
+    15: "string",
+    16: "number",
+    17: "boolean",
+    18: "array",
+    19: "object",
+    20: "key",
+    21: "null",
+    22: "enumMember",
+    23: "struct",
+    24: "event",
+    25: "operator",
+    26: "typeParameter",
+}
+
+SEVERITIES = {1: "error", 2: "warning", 3: "information", 4: "hint"}
+
+
+@dataclass(frozen=True)
+class Location:
+    """A workspace-relative source location."""
+
+    path: str
+    line: int
+    character: int
+    name: str | None = None
+
+    def format(self) -> str:
+        suffix = f" {self.name}" if self.name else ""
+        return f"{self.path}:{self.line + 1}:{self.character + 1}{suffix}"
+
+
+@dataclass(frozen=True)
+class Symbol:
+    """A document or workspace symbol."""
+
+    name: str
+    kind: str
+    location: Location
+    container_name: str | None = None
+
+    def format(self) -> str:
+        container = f" {self.container_name}." if self.container_name else " "
+        return f"{self.kind}{container}{self.name} {self.location.format()}"
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    """An LSP diagnostic."""
+
+    path: str
+    line: int
+    character: int
+    severity: str
+    message: str
+    code: str | None = None
+
+    def format(self) -> str:
+        code = f" {self.code}" if self.code else ""
+        return f"{self.path}:{self.line + 1}:{self.character + 1} {self.severity}{code} {self.message}"
+
+
+def uri_to_path(uri: str) -> str:
+    """Convert a file URI to a local path string."""
+    from urllib.parse import unquote, urlparse
+
+    parsed = urlparse(uri)
+    if parsed.scheme != "file":
+        return uri
+    return unquote(parsed.path)
+
+
+def path_to_uri(path: str) -> str:
+    """Convert a local path string to a file URI."""
+    from pathlib import Path
+
+    return Path(path).resolve().as_uri()
+
+
+def range_start(item: dict[str, Any]) -> tuple[int, int]:
+    """Return zero-based line/character from an LSP range-like item."""
+    location = item.get("location") if isinstance(item.get("location"), dict) else {}
+    start = (
+        item.get("range", {}).get("start")
+        or item.get("selectionRange", {}).get("start")
+        or location.get("range", {}).get("start")
+        or {}
+    )
+    return int(start.get("line", 0)), int(start.get("character", 0))
+
+
+def symbol_kind_name(kind: Any) -> str:
+    """Return a readable symbol kind."""
+    return SYMBOL_KINDS.get(int(kind or 0), "symbol")
+
+
+def diagnostic_severity(severity: Any) -> str:
+    """Return a readable diagnostic severity."""
+    return SEVERITIES.get(int(severity or 0), "diagnostic")

agent/message_context_manager.py

@@ -0,0 +1,322 @@
+"""Analyze and compact current session message token usage."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Literal
+
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage, ToolMessage
+
+from agent.context_compressor import MANUAL_COMPRESSION_REASON, compress_tool_message
+
+
+DEFAULT_KEEP_RECENT_MESSAGES = 20
+DEFAULT_MIN_TOOL_TOKENS = 500
+COMPACT_MARKER_TOKENS = 80
+
+RiskLevel = Literal["low", "medium", "high"]
+PressureLevel = Literal["low", "medium", "high", "critical"]
+TokenSource = Literal["exact", "estimated"]
+
+
+@dataclass(frozen=True)
+class MessageTokenStat:
+    index: int
+    role: str
+    message_type: str
+    estimated_tokens: int
+    percent: float
+    preview: str
+    protected: bool
+    compressible: bool
+    recommendation: str
+    risk: RiskLevel
+    context_policy: str
+    ephemeral_kind: str
+
+
+@dataclass(frozen=True)
+class ContextBlockStat:
+    name: str
+    estimated_tokens: int
+    protected: bool
+    preview: str
+
+
+@dataclass(frozen=True)
+class MessageContextSummary:
+    total_tokens: int
+    token_source: TokenSource
+    context_window_tokens: int
+    remaining_tokens: int
+    pressure: PressureLevel
+    by_role: dict[str, int]
+    by_type: dict[str, int]
+    largest_messages: list[int]
+    compression_savings_estimate: int
+
+
+@dataclass(frozen=True)
+class CompressionSuggestion:
+    message_indexes: list[int]
+    strategy: str
+    reason: str
+    original_tokens: int
+    estimated_after_tokens: int
+    saved_tokens: int
+    risk: RiskLevel
+
+
+class MessageContextManager:
+    """Read-only token analysis plus deterministic old-tool compression."""
+
+    def __init__(
+        self,
+        *,
+        keep_recent_messages: int = DEFAULT_KEEP_RECENT_MESSAGES,
+        min_tool_tokens: int = DEFAULT_MIN_TOOL_TOKENS,
+    ) -> None:
+        self.keep_recent_messages = keep_recent_messages
+        self.min_tool_tokens = min_tool_tokens
+
+    def analyze(
+        self,
+        messages: list[BaseMessage],
+        *,
+        system_prompt: str,
+        tools: list[dict],
+        context_window_tokens: int,
+        total_tokens: int | None = None,
+        token_source: TokenSource = "estimated",
+    ) -> MessageContextSummary:
+        """Return aggregate token pressure and breakdowns."""
+        blocks = self.context_blocks(system_prompt, tools)
+        stats = self.message_stats(messages)
+        estimated_total = sum(block.estimated_tokens for block in blocks) + sum(
+            stat.estimated_tokens for stat in stats
+        )
+        total = max(0, int(total_tokens if total_tokens is not None else estimated_total))
+        by_role: dict[str, int] = {block.name: block.estimated_tokens for block in blocks}
+        by_type: dict[str, int] = {block.name: block.estimated_tokens for block in blocks}
+        for stat in stats:
+            by_role[stat.role] = by_role.get(stat.role, 0) + stat.estimated_tokens
+            by_type[stat.message_type] = by_type.get(stat.message_type, 0) + stat.estimated_tokens
+        remaining = max(context_window_tokens - total, 0) if context_window_tokens > 0 else 0
+        suggestions = self.suggest_compression(messages)
+        return MessageContextSummary(
+            total_tokens=total,
+            token_source=token_source,
+            context_window_tokens=max(0, int(context_window_tokens or 0)),
+            remaining_tokens=remaining,
+            pressure=self._pressure(total, context_window_tokens),
+            by_role=by_role,
+            by_type=by_type,
+            largest_messages=[
+                stat.index
+                for stat in sorted(stats, key=lambda item: item.estimated_tokens, reverse=True)[:5]
+            ],
+            compression_savings_estimate=sum(item.saved_tokens for item in suggestions),
+        )
+
+    def context_blocks(self, system_prompt: str, tools: list[dict]) -> list[ContextBlockStat]:
+        """Return protected non-message context blocks."""
+        return [
+            ContextBlockStat(
+                name="system_prompt",
+                estimated_tokens=_estimate_text_tokens(system_prompt),
+                protected=True,
+                preview=_preview(system_prompt),
+            ),
+            ContextBlockStat(
+                name="tools_schema",
+                estimated_tokens=_estimate_text_tokens(str(tools or [])),
+                protected=True,
+                preview=f"{len(tools or [])} tool definitions",
+            ),
+        ]
+
+    def message_stats(self, messages: list[BaseMessage]) -> list[MessageTokenStat]:
+        """Return per-message estimated token stats."""
+        total = sum(_estimate_message_tokens(message) for message in messages)
+        latest_user = self._latest_user_index(messages)
+        return [
+            self._message_stat(index, message, total, latest_user, len(messages))
+            for index, message in enumerate(messages)
+        ]
+
+    def suggest_compression(self, messages: list[BaseMessage]) -> list[CompressionSuggestion]:
+        """Suggest deterministic compression for old large tool outputs."""
+        suggestions = []
+        for index, message in enumerate(messages):
+            if not self._is_compressible_tool(index, message, len(messages)):
+                continue
+            original = _estimate_message_tokens(message)
+            if original < self.min_tool_tokens:
+                continue
+            after = min(original, COMPACT_MARKER_TOKENS)
+            suggestions.append(
+                CompressionSuggestion(
+                    message_indexes=[index],
+                    strategy="old_tool_outputs",
+                    reason="old tool output outside recent message window",
+                    original_tokens=original,
+                    estimated_after_tokens=after,
+                    saved_tokens=max(original - after, 0),
+                    risk="low",
+                )
+            )
+        return suggestions
+
+    def compress_selected(
+        self,
+        messages: list[BaseMessage],
+        indexes: list[int],
+    ) -> list[BaseMessage]:
+        """Return messages with selected compressible ToolMessages compacted."""
+        selected = set(indexes)
+        compressed: list[BaseMessage] = []
+        for index, message in enumerate(messages):
+            if index in selected and self._is_compressible_tool(index, message, len(messages)):
+                compressed.append(
+                    compress_tool_message(
+                        message,
+                        reason=MANUAL_COMPRESSION_REASON,
+                        estimated_original_tokens=_estimate_message_tokens(message),
+                    )
+                )
+            else:
+                compressed.append(message)
+        return compressed
+
+    def _message_stat(
+        self,
+        index: int,
+        message: BaseMessage,
+        total_tokens: int,
+        latest_user_index: int | None,
+        message_count: int,
+    ) -> MessageTokenStat:
+        estimated = _estimate_message_tokens(message)
+        compressible = self._is_compressible_tool(index, message, message_count)
+        protected = index == latest_user_index or self._is_recent(index, message_count)
+        if _is_compressed(message):
+            recommendation = "keep compressed"
+        elif compressible and estimated >= self.min_tool_tokens:
+            recommendation = "compress"
+        elif protected:
+            recommendation = "protected"
+        else:
+            recommendation = "keep"
+        return MessageTokenStat(
+            index=index,
+            role=_message_role(message),
+            message_type=type(message).__name__,
+            estimated_tokens=estimated,
+            percent=(estimated / total_tokens * 100) if total_tokens else 0.0,
+            preview=_preview(_message_content_text(message)),
+            protected=protected,
+            compressible=compressible,
+            recommendation=recommendation,
+            risk="low" if compressible else "medium",
+            context_policy=_context_policy(message),
+            ephemeral_kind=_ephemeral_kind(message),
+        )
+
+    def _latest_user_index(self, messages: list[BaseMessage]) -> int | None:
+        for index in range(len(messages) - 1, -1, -1):
+            if isinstance(messages[index], HumanMessage):
+                return index
+        return None
+
+    def _is_recent(self, index: int, message_count: int) -> bool:
+        return index >= max(message_count - self.keep_recent_messages, 0)
+
+    def _is_compressible_tool(self, index: int, message: BaseMessage, message_count: int) -> bool:
+        if self._is_recent(index, message_count):
+            return False
+        if not isinstance(message, ToolMessage):
+            return False
+        if _is_compressed(message):
+            return False
+        content = message.content
+        return isinstance(content, str) and bool(content.strip())
+
+    def _pressure(self, total_tokens: int, context_window_tokens: int) -> PressureLevel:
+        if context_window_tokens <= 0:
+            return "low"
+        percent = total_tokens / context_window_tokens * 100
+        if percent >= 90:
+            return "critical"
+        if percent >= 75:
+            return "high"
+        if percent >= 50:
+            return "medium"
+        return "low"
+
+
+def _estimate_message_tokens(message: BaseMessage) -> int:
+    total_chars = len(_message_content_text(message))
+    name = getattr(message, "name", None)
+    if name:
+        total_chars += len(str(name))
+    tool_call_id = getattr(message, "tool_call_id", None)
+    if tool_call_id:
+        total_chars += len(str(tool_call_id))
+    additional_kwargs = getattr(message, "additional_kwargs", None)
+    if additional_kwargs:
+        total_chars += len(str(additional_kwargs))
+    return _estimate_chars_as_tokens(total_chars)
+
+
+def _estimate_text_tokens(text: object) -> int:
+    return _estimate_chars_as_tokens(len(str(text or "")))
+
+
+def _estimate_chars_as_tokens(chars: int) -> int:
+    return math.ceil(chars / 4) if chars > 0 else 0
+
+
+def _message_content_text(message: BaseMessage) -> str:
+    content = getattr(message, "content", "")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(str(item) for item in content)
+    return str(content)
+
+
+def _message_role(message: BaseMessage) -> str:
+    if isinstance(message, HumanMessage):
+        return "user"
+    if isinstance(message, AIMessage):
+        return "assistant"
+    if isinstance(message, ToolMessage):
+        return "tool"
+    if isinstance(message, SystemMessage):
+        return "system"
+    return "other"
+
+
+def _preview(text: object, limit: int = 120) -> str:
+    value = " ".join(str(text or "").split())
+    if len(value) <= limit:
+        return value
+    return value[: max(0, limit - 3)] + "..."
+
+
+def _is_compressed(message: BaseMessage) -> bool:
+    kwargs = getattr(message, "additional_kwargs", {}) or {}
+    return bool(kwargs.get("context_compressed"))
+
+
+def _context_policy(message: BaseMessage) -> str:
+    kwargs = getattr(message, "additional_kwargs", {}) or {}
+    return str(kwargs.get("context_policy") or "full")
+
+
+def _ephemeral_kind(message: BaseMessage) -> str:
+    kwargs = getattr(message, "additional_kwargs", {}) or {}
+    if not kwargs.get("context_ephemeral"):
+        return ""
+    return str(kwargs.get("ephemeral_kind") or "ephemeral")