pythonclaw 0.2.0__py3-none-any.whl

pythonclaw/core/compaction.py

@@ -0,0 +1,220 @@
+"""
+Context compaction for pythonclaw.
+
+Compaction summarises older conversation history into a compact summary entry
+and keeps recent messages intact — preventing context-window overflows in long
+sessions while preserving important information.
+
+Inspired by openclaw's compaction model:
+  https://docs.openclaw.ai/concepts/compaction
+
+How it works
+------------
+1. Split chat history into "old" (to summarise) + "recent" (to keep verbatim).
+2. Memory flush — silently ask the LLM to extract key facts from the old
+   messages and persist them via the agent's `remember` tool, so nothing
+   important is lost permanently.
+3. Summarise — call the LLM with a summarisation prompt to produce a concise
+   paragraph covering decisions, facts, tasks, and open questions.
+4. Persist — append the summary to `context/compaction/history.jsonl` as an
+   audit trail.
+5. Replace — swap the old messages with a single [Compaction Summary] system
+   message and increment the compaction counter.
+
+Token estimation
+----------------
+We don't bundle a tokeniser, so we use a conservative character-based
+approximation: 1 token ≈ 4 characters.  This is good enough for triggering
+auto-compaction; the actual model enforces the hard limit.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .llm.base import LLMProvider
+    from .memory.manager import MemoryManager
+
+logger = logging.getLogger(__name__)
+
+CHARS_PER_TOKEN = 4
+DEFAULT_AUTO_THRESHOLD_TOKENS = 6000   # trigger auto-compaction at ~6k tokens
+DEFAULT_RECENT_KEEP = 6                # keep last N chat messages verbatim
+COMPACTION_LOG_FILE = os.path.join("context", "compaction", "history.jsonl")
+
+
+# ── Token estimation ──────────────────────────────────────────────────────────
+
+def estimate_tokens(messages: list[dict]) -> int:
+    """Rough character-based token estimate for a list of messages."""
+    total_chars = sum(len(str(m.get("content") or "")) for m in messages)
+    return total_chars // CHARS_PER_TOKEN
+
+
+# ── JSONL persistence ─────────────────────────────────────────────────────────
+
+def persist_compaction(summary: str, message_count: int, log_path: str = COMPACTION_LOG_FILE) -> None:
+    """Append one compaction entry to the JSONL audit log."""
+    os.makedirs(os.path.dirname(log_path), exist_ok=True)
+    entry = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "summarised_messages": message_count,
+        "summary": summary,
+    }
+    with open(log_path, "a", encoding="utf-8") as f:
+        f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+    logger.debug("[Compaction] Persisted to %s", log_path)
+
+
+# ── Message → plain text ──────────────────────────────────────────────────────
+
+def messages_to_text(messages: list[dict]) -> str:
+    """Convert a message list to a readable transcript for summarisation."""
+    lines = []
+    for m in messages:
+        role = m.get("role", "?")
+        content = m.get("content") or ""
+        if role == "assistant" and not content and m.get("tool_calls"):
+            content = f"[called tools: {[tc.get('function', {}).get('name') if isinstance(tc, dict) else tc.function.name for tc in m.get('tool_calls', [])]}]"
+        if role == "tool":
+            content = f"[tool result]: {content[:300]}{'...' if len(content) > 300 else ''}"
+        if content:
+            lines.append(f"{role.upper()}: {content}")
+    return "\n".join(lines)
+
+
+# ── Memory flush ──────────────────────────────────────────────────────────────
+
+def memory_flush(
+    messages_to_flush: list[dict],
+    provider: "LLMProvider",
+    memory: "MemoryManager",
+) -> int:
+    """
+    Silent LLM call that extracts key facts from old messages and saves them
+    to long-term memory before those messages are discarded.
+
+    Returns the number of facts saved.
+    """
+    if not messages_to_flush:
+        return 0
+
+    history_text = messages_to_text(messages_to_flush)
+    prompt = (
+        "You are a memory extraction assistant. "
+        "Given the following conversation transcript, identify ALL important facts, "
+        "decisions, preferences, and context that should be remembered long-term. "
+        "Return a JSON array of objects with 'key' and 'value' fields. "
+        "If nothing important, return [].\n\n"
+        f"TRANSCRIPT:\n{history_text}\n\n"
+        "Return ONLY valid JSON, no explanation."
+    )
+    try:
+        response = provider.chat(
+            messages=[{"role": "user", "content": prompt}],
+            tools=[],
+            tool_choice="none",
+        )
+        raw = response.choices[0].message.content or "[]"
+        # Strip markdown fences if present
+        raw = raw.strip()
+        if raw.startswith("```"):
+            raw = "\n".join(raw.split("\n")[1:])
+        if raw.endswith("```"):
+            raw = raw[: raw.rfind("```")]
+        facts: list[dict] = json.loads(raw)
+        saved = 0
+        for fact in facts:
+            key = str(fact.get("key", "")).strip()
+            value = str(fact.get("value", "")).strip()
+            if key and value:
+                memory.remember(value, key)
+                saved += 1
+        logger.info("[Compaction] Memory flush saved %d fact(s).", saved)
+        return saved
+    except Exception as exc:
+        logger.warning("[Compaction] Memory flush failed (non-fatal): %s", exc)
+        return 0
+
+
+# ── Core compact function ─────────────────────────────────────────────────────
+
+def compact(
+    messages: list[dict],
+    provider: "LLMProvider",
+    memory: "MemoryManager | None" = None,
+    recent_keep: int = DEFAULT_RECENT_KEEP,
+    instruction: str | None = None,
+    log_path: str = COMPACTION_LOG_FILE,
+) -> tuple[list[dict], str]:
+    """
+    Compact conversation history.
+
+    Parameters
+    ----------
+    messages      : full message list (system + chat)
+    provider      : LLM provider used for summarisation
+    memory        : MemoryManager for pre-compaction memory flush (optional)
+    recent_keep   : number of recent chat messages to keep verbatim
+    instruction   : optional extra focus hint for the summarisation prompt
+    log_path      : where to persist the compaction JSONL entry
+
+    Returns
+    -------
+    (new_messages, summary_text)
+    """
+    system_msgs = [m for m in messages if m.get("role") == "system"]
+    chat_msgs   = [m for m in messages if m.get("role") != "system"]
+
+    if len(chat_msgs) <= recent_keep:
+        logger.info("[Compaction] Not enough history to compact (%d messages).", len(chat_msgs))
+        return messages, ""
+
+    to_summarise = chat_msgs[:-recent_keep]
+    to_keep      = chat_msgs[-recent_keep:]
+
+    logger.info(
+        "[Compaction] Summarising %d message(s), keeping %d recent.",
+        len(to_summarise), len(to_keep),
+    )
+
+    # 1. Memory flush — save important facts before discarding old messages
+    if memory is not None:
+        memory_flush(to_summarise, provider, memory)
+
+    # 2. Summarise
+    history_text = messages_to_text(to_summarise)
+    focus = f"\nAdditional focus: {instruction}" if instruction else ""
+    summarise_prompt = (
+        f"Summarise the following conversation history concisely. "
+        f"Focus on: decisions made, key facts learned, tasks completed, open questions.{focus}\n\n"
+        f"CONVERSATION:\n{history_text}\n\n"
+        f"Provide a compact summary (3–8 sentences or bullet points):"
+    )
+    try:
+        response = provider.chat(
+            messages=[{"role": "user", "content": summarise_prompt}],
+            tools=[],
+            tool_choice="none",
+        )
+        summary = (response.choices[0].message.content or "").strip()
+    except Exception as exc:
+        logger.error("[Compaction] Summarisation failed: %s", exc)
+        raise
+
+    # 3. Persist
+    persist_compaction(summary, len(to_summarise), log_path=log_path)
+
+    # 4. Build new message list
+    summary_system_msg = {
+        "role": "system",
+        "content": f"[Compaction Summary — {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}]\n{summary}",
+    }
+    new_messages = system_msgs + [summary_system_msg] + to_keep
+
+    return new_messages, summary

pythonclaw/core/knowledge/rag.py

@@ -0,0 +1,93 @@
+"""
+Knowledge-base RAG wrapper.
+
+Replaces the old SimpleRAG (keyword-only) with a HybridRetriever that
+combines BM25 sparse retrieval, dense embedding retrieval, RRF fusion,
+and an optional LLM re-ranker.
+
+Backwards-compatible API:
+    rag = KnowledgeRAG(knowledge_dir, provider=llm_provider)
+    hits = rag.retrieve("my query", top_k=5)
+    # hits = [{"source": "filename.txt", "content": "..."}, ...]
+
+Old SimpleRAG is kept as an alias for scripts that import it directly.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from ..retrieval.chunker import load_corpus_from_directory
+from ..retrieval.retriever import HybridRetriever
+
+if TYPE_CHECKING:
+    from ..llm.base import LLMProvider
+
+logger = logging.getLogger(__name__)
+
+
+class KnowledgeRAG:
+    """
+    Loads .txt / .md files from a directory and retrieves relevant chunks
+    using hybrid sparse + dense retrieval with optional LLM re-ranking.
+
+    Parameters
+    ----------
+    knowledge_dir : path to the directory containing knowledge files.
+    provider      : LLMProvider (enables LLM re-ranker when provided).
+    use_sparse    : enable BM25 retrieval (default True).
+    use_dense     : enable embedding retrieval (default True).
+    use_reranker  : enable LLM re-ranking (default True; requires provider).
+    dense_model   : sentence-transformers model name.
+    """
+
+    def __init__(
+        self,
+        knowledge_dir: str,
+        provider: "LLMProvider | None" = None,
+        use_sparse: bool = True,
+        use_dense: bool = True,
+        use_reranker: bool = True,
+        dense_model: str = "all-MiniLM-L6-v2",
+    ) -> None:
+        self.knowledge_dir = knowledge_dir
+
+        self._retriever = HybridRetriever(
+            provider=provider,
+            use_sparse=use_sparse,
+            use_dense=use_dense,
+            use_reranker=use_reranker,
+            dense_model=dense_model,
+        )
+
+        corpus = load_corpus_from_directory(knowledge_dir)
+        self._retriever.fit(corpus)
+        logger.info(
+            "[KnowledgeRAG] Loaded %d chunks from '%s'", len(corpus), knowledge_dir
+        )
+
+    def retrieve(self, query: str, top_k: int = 5) -> list[dict]:
+        """
+        Return up to *top_k* relevant chunks for *query*.
+
+        Each result dict has at least:
+            {"source": str, "content": str}
+        """
+        return self._retriever.retrieve(query, top_k=top_k)
+
+    def reload(self) -> None:
+        """Re-scan the knowledge directory and re-index (hot reload)."""
+        corpus = load_corpus_from_directory(self.knowledge_dir)
+        self._retriever.fit(corpus)
+        logger.info("[KnowledgeRAG] Reloaded %d chunks.", len(corpus))
+
+    def __len__(self) -> int:
+        return len(self._retriever)
+
+    def __bool__(self) -> bool:
+        return bool(self._retriever)
+
+
+# Backwards-compatibility alias
+SimpleRAG = KnowledgeRAG

pythonclaw/core/llm/anthropic_client.py

@@ -0,0 +1,107 @@
+"""
+Anthropic (Claude) provider — adapts the Anthropic API to the OpenAI-compatible
+response format used by Agent.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional
+
+import anthropic
+
+from .base import LLMProvider
+from .response import MockChoice, MockFunction, MockMessage, MockResponse, MockToolCall
+
+
+class AnthropicProvider(LLMProvider):
+    def __init__(self, api_key: str, model_name: str = "claude-3-5-sonnet-20241022"):
+        self.client = anthropic.Anthropic(api_key=api_key)
+        self.model_name = model_name
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        tool_choice: Any = "auto",
+    ) -> Any:
+        system_prompt = ""
+        filtered_messages: list[dict] = []
+
+        for msg in messages:
+            if msg["role"] == "system":
+                system_prompt += msg["content"] + "\n"
+
+            elif msg["role"] == "tool":
+                filtered_messages.append({
+                    "role": "user",
+                    "content": [{
+                        "type": "tool_result",
+                        "tool_use_id": msg["tool_call_id"],
+                        "content": msg["content"],
+                    }],
+                })
+
+            elif msg["role"] == "assistant" and "tool_calls" in msg:
+                content_block: list[dict] = []
+                if msg.get("content"):
+                    content_block.append({"type": "text", "text": msg["content"]})
+                for tc in msg["tool_calls"]:
+                    tc_id = tc["id"] if isinstance(tc, dict) else tc.id
+                    func = tc["function"] if isinstance(tc, dict) else tc.function
+                    fname = func["name"] if isinstance(func, dict) else func.name
+                    fargs = json.loads(func["arguments"] if isinstance(func, dict) else func.arguments)
+                    content_block.append({
+                        "type": "tool_use",
+                        "id": tc_id,
+                        "name": fname,
+                        "input": fargs,
+                    })
+                filtered_messages.append({"role": "assistant", "content": content_block})
+
+            else:
+                filtered_messages.append(msg)
+
+        # Convert tool schemas
+        anthropic_tools = []
+        if tools:
+            for t in tools:
+                if t["type"] == "function":
+                    anthropic_tools.append({
+                        "name": t["function"]["name"],
+                        "description": t["function"]["description"],
+                        "input_schema": t["function"]["parameters"],
+                    })
+
+        kwargs: dict = {
+            "model": self.model_name,
+            "messages": filtered_messages,
+            "max_tokens": 4096,
+        }
+        if system_prompt:
+            kwargs["system"] = system_prompt
+        if anthropic_tools:
+            kwargs["tools"] = anthropic_tools
+            if tool_choice == "required":
+                kwargs["tool_choice"] = {"type": "any"}
+
+        response = self.client.messages.create(**kwargs)
+
+        # Convert to OpenAI-compatible format
+        content_text = ""
+        tool_calls: list[MockToolCall] = []
+        for block in response.content:
+            if block.type == "text":
+                content_text += block.text
+            elif block.type == "tool_use":
+                tool_calls.append(MockToolCall(
+                    id=block.id,
+                    function=MockFunction(name=block.name, arguments=json.dumps(block.input)),
+                ))
+
+        return MockResponse(choices=[
+            MockChoice(message=MockMessage(
+                content=content_text or None,
+                tool_calls=tool_calls or None,
+            ))
+        ])

pythonclaw/core/llm/base.py

@@ -0,0 +1,26 @@
+"""Abstract base class for LLM providers."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class LLMProvider(ABC):
+    @abstractmethod
+    def chat(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None = None,
+        tool_choice: Any = "auto",
+    ) -> Any:
+        """
+        Send a chat request to the LLM.
+
+        All providers must return an object compatible with OpenAI's response
+        structure: ``response.choices[0].message.content`` and
+        ``response.choices[0].message.tool_calls``.
+
+        Non-OpenAI providers should use the Mock* dataclasses from
+        ``llm.response`` to build compatible return values.
+        """

pythonclaw/core/llm/gemini_client.py

@@ -0,0 +1,139 @@
+"""
+Google Gemini provider — adapts the Gemini API to the OpenAI-compatible
+response format used by Agent.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Any, Dict, List, Optional
+
+import google.generativeai as genai
+from google.generativeai.types import content_types
+
+from .base import LLMProvider
+from .response import MockChoice, MockFunction, MockMessage, MockResponse, MockToolCall
+
+
+class GeminiProvider(LLMProvider):
+    def __init__(self, api_key: str, model_name: str = "gemini-2.0-flash-exp"):
+        genai.configure(api_key=api_key)
+        self.model = genai.GenerativeModel(model_name)
+
+    def _convert_tool_calls_to_parts(self, tool_calls_data: list) -> list:
+        parts = []
+        for tc in tool_calls_data:
+            func = tc["function"] if isinstance(tc, dict) else tc.function
+            name = func["name"] if isinstance(func, dict) else func.name
+            args = json.loads(func["arguments"] if isinstance(func, dict) else func.arguments)
+            parts.append(content_types.FunctionCall(name=name, args=args))
+        return parts
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        tool_choice: Any = "auto",
+    ) -> Any:
+        gemini_history: list[dict] = []
+        system_instruction: str | None = None
+
+        for msg in messages:
+            role = msg["role"]
+            content = msg.get("content")
+
+            if role == "system":
+                system_instruction = (
+                    content if system_instruction is None
+                    else system_instruction + "\n" + content
+                )
+
+            elif role == "user":
+                gemini_history.append({"role": "user", "parts": [content]})
+
+            elif role == "assistant":
+                parts: list = []
+                if content:
+                    parts.append(content)
+                if msg.get("tool_calls"):
+                    parts.extend(self._convert_tool_calls_to_parts(msg["tool_calls"]))
+                gemini_history.append({"role": "model", "parts": parts})
+
+            elif role == "tool":
+                func_name = self._find_tool_name(messages, msg["tool_call_id"])
+                try:
+                    resp_dict = json.loads(msg["content"])
+                except (json.JSONDecodeError, TypeError):
+                    resp_dict = {"result": msg["content"]}
+                gemini_history.append({
+                    "role": "user",
+                    "parts": [content_types.FunctionResponse(name=func_name, response=resp_dict)],
+                })
+
+        # Convert tool schemas
+        gemini_tools = None
+        if tools:
+            declarations = [
+                {
+                    "name": t["function"]["name"],
+                    "description": t["function"].get("description"),
+                    "parameters": t["function"].get("parameters"),
+                }
+                for t in tools if t["type"] == "function"
+            ]
+            if declarations:
+                gemini_tools = [declarations]
+
+        # Inject system instruction into the first user message
+        if gemini_history and gemini_history[0]["role"] == "model":
+            gemini_history.insert(0, {"role": "user", "parts": ["Hi"]})
+        if system_instruction and gemini_history:
+            first_parts = gemini_history[0]["parts"]
+            if isinstance(first_parts, list):
+                first_parts.insert(0, f"System Instruction: {system_instruction}")
+
+        response = self.model.generate_content(
+            contents=gemini_history,
+            tools=gemini_tools,
+        )
+
+        # Convert to OpenAI-compatible format
+        try:
+            _ = response.parts[0]
+        except (IndexError, AttributeError, ValueError):
+            return MockResponse(choices=[
+                MockChoice(message=MockMessage(content="Error: empty response from Gemini", tool_calls=None))
+            ])
+
+        content_text: str | None = None
+        tool_calls: list[MockToolCall] = []
+        for part in response.parts:
+            if part.text:
+                content_text = (content_text or "") + part.text
+            if part.function_call:
+                tool_calls.append(MockToolCall(
+                    id=f"call_{uuid.uuid4().hex[:8]}",
+                    function=MockFunction(
+                        name=part.function_call.name,
+                        arguments=json.dumps(dict(part.function_call.args)),
+                    ),
+                ))
+
+        return MockResponse(choices=[
+            MockChoice(message=MockMessage(
+                content=content_text,
+                tool_calls=tool_calls or None,
+            ))
+        ])
+
+    @staticmethod
+    def _find_tool_name(messages: list[dict], tool_call_id: str) -> str:
+        """Walk backwards through messages to find the function name for a tool_call_id."""
+        for prev in reversed(messages):
+            for tc in prev.get("tool_calls") or []:
+                tc_id = tc["id"] if isinstance(tc, dict) else tc.id
+                if tc_id == tool_call_id:
+                    func = tc["function"] if isinstance(tc, dict) else tc.function
+                    return func["name"] if isinstance(func, dict) else func.name
+        return "unknown_tool"

pythonclaw/core/llm/openai_compatible.py

@@ -0,0 +1,39 @@
+"""OpenAI-compatible LLM provider.
+
+Works with any API that follows the OpenAI chat-completions contract:
+DeepSeek, Grok (xAI), Kimi (Moonshot), GLM (Zhipu), and others.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from openai import OpenAI
+
+from .base import LLMProvider
+
+
+class OpenAICompatibleProvider(LLMProvider):
+    """Thin wrapper around the OpenAI SDK for chat completions."""
+
+    def __init__(self, api_key: str, base_url: str, model_name: str) -> None:
+        self.client = OpenAI(api_key=api_key, base_url=base_url)
+        self.model_name = model_name
+
+    def chat(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None = None,
+        tool_choice: Any = "auto",
+        **kwargs: Any,
+    ) -> Any:
+        req: dict[str, Any] = {
+            "model": self.model_name,
+            "messages": messages,
+            **kwargs,
+        }
+        if tools:
+            req["tools"] = tools
+            req["tool_choice"] = tool_choice
+
+        return self.client.chat.completions.create(**req)

pythonclaw/core/llm/response.py

@@ -0,0 +1,57 @@
+"""
+OpenAI-compatible response dataclasses.
+
+Non-OpenAI providers (Anthropic, Gemini) convert their native responses into
+these dataclasses so the Agent can use a single code path regardless of which
+LLM backend is active.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class MockFunction:
+    name: str
+    arguments: str
+
+
+@dataclass
+class MockToolCall:
+    id: str
+    function: MockFunction
+    type: str = "function"
+
+
+@dataclass
+class MockMessage:
+    content: Optional[str]
+    tool_calls: Optional[list[MockToolCall]]
+
+    def model_dump(self) -> dict:
+        d: dict = {"role": "assistant", "content": self.content}
+        if self.tool_calls:
+            d["tool_calls"] = [
+                {
+                    "id": tc.id,
+                    "type": "function",
+                    "function": {
+                        "name": tc.function.name,
+                        "arguments": tc.function.arguments,
+                    },
+                }
+                for tc in self.tool_calls
+            ]
+        return d
+
+
+@dataclass
+class MockChoice:
+    message: MockMessage
+
+
+@dataclass
+class MockResponse:
+    choices: list[MockChoice]