agentslim 0.1.0__py3-none-any.whl

agentslim/__init__.py

@@ -0,0 +1,27 @@
+"""
+agentslim — Make your AI agents leaner, faster, and cheaper.
+
+Core modules:
+  - memory:     Smart context window with auto-summarization
+  - compressor: Text / HTML / JSON compressor before sending to LLM
+  - tools:      Tool/function-call schema minifier
+  - code:       Code-aware context extractor (send only relevant chunks)
+"""
+
+from agentslim.memory import AgentMemory
+from agentslim.compressor import Compressor
+from agentslim.tools import ToolMinifier
+from agentslim.code import CodeContext
+from agentslim.utils import count_tokens, estimate_cost
+
+__version__ = "0.1.0"
+__author__ = "agentslim contributors"
+
+__all__ = [
+    "AgentMemory",
+    "Compressor",
+    "ToolMinifier",
+    "CodeContext",
+    "count_tokens",
+    "estimate_cost",
+]

agentslim/code.py

@@ -0,0 +1,307 @@
+"""
+code.py — Code-aware context extractor for coding agents.
+
+Instead of sending an entire file (hundreds of tokens), send only the
+relevant "chunk" — the function or class the agent needs, plus configurable
+surrounding context lines.
+
+Supports Python, JavaScript / TypeScript, and generic line-based extraction.
+
+Quickstart::
+
+    from agentslim import CodeContext
+
+    ctx = CodeContext.extract_function("my_file.py", "process_payment")
+    print(ctx)
+    # → def process_payment(order_id: str) -> dict:
+    #       ...
+
+    outline = CodeContext.outline("my_file.py")
+    # → ['class PaymentService (L12)', 'def process_payment (L34)', ...]
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+from typing import Optional, Union
+
+
+class CodeContext:
+    """
+    Code-aware context extractor — send only what the agent needs.
+
+    All methods are static; no instantiation required.
+    """
+
+    # ──────────────────────────────────────────────────────────────────────────
+    # Public API
+    # ──────────────────────────────────────────────────────────────────────────
+
+    @staticmethod
+    def extract_function(
+        source: Union[str, Path],
+        function_name: str,
+        context_lines: int = 3,
+    ) -> Optional[str]:
+        """
+        Extract a single function or method from a Python source file.
+
+        Args:
+            source:        File path or raw source code string.
+            function_name: Name of the function / method to find.
+            context_lines: Lines of surrounding context to include.
+
+        Returns:
+            Source of the function with surrounding context, or ``None``
+            if not found.
+
+        Example::
+
+            snippet = CodeContext.extract_function("app.py", "handle_request")
+        """
+        code = _read_source(source)
+        try:
+            tree = ast.parse(code)
+        except SyntaxError:
+            return _fallback_extract(code, function_name, context_lines)
+
+        lines = code.splitlines()
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                if node.name == function_name:
+                    start = max(0, node.lineno - 1 - context_lines)
+                    end = node.end_lineno  # type: ignore[attr-defined]
+                    end = min(len(lines), end + context_lines)
+                    snippet = "\n".join(lines[start:end])
+                    return f"# File: {source}  (L{start + 1}–L{end})\n{snippet}"
+        return None
+
+    @staticmethod
+    def extract_class(
+        source: Union[str, Path],
+        class_name: str,
+        methods_only: bool = False,
+    ) -> Optional[str]:
+        """
+        Extract a class definition from Python source.
+
+        Args:
+            source:       File path or raw source code string.
+            class_name:   Name of the class to find.
+            methods_only: If ``True``, return only method signatures (no bodies).
+
+        Returns:
+            Class source or skeleton, or ``None`` if not found.
+        """
+        code = _read_source(source)
+        try:
+            tree = ast.parse(code)
+        except SyntaxError:
+            return None
+
+        lines = code.splitlines()
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef) and node.name == class_name:
+                start = node.lineno - 1
+                end = node.end_lineno  # type: ignore[attr-defined]
+                if methods_only:
+                    return _class_skeleton(node, lines)
+                snippet = "\n".join(lines[start:end])
+                return f"# File: {source}  (L{start + 1}–L{end})\n{snippet}"
+        return None
+
+    @staticmethod
+    def extract_lines(
+        source: Union[str, Path],
+        start_line: int,
+        end_line: int,
+        context_lines: int = 0,
+    ) -> str:
+        """
+        Extract specific line range from a file (1-indexed).
+
+        Args:
+            source:        File path or raw source code string.
+            start_line:    First line to include (1-indexed).
+            end_line:      Last line to include (1-indexed, inclusive).
+            context_lines: Extra lines of context around the range.
+        """
+        code = _read_source(source)
+        lines = code.splitlines()
+        start = max(0, start_line - 1 - context_lines)
+        end = min(len(lines), end_line + context_lines)
+        snippet = "\n".join(lines[start:end])
+        return f"# File: {source}  (L{start + 1}–L{end})\n{snippet}"
+
+    @staticmethod
+    def outline(source: Union[str, Path], language: str = "auto") -> list[str]:
+        """
+        Return a high-level outline of the file: class and function names
+        with line numbers.
+
+        Useful for giving an agent an overview without sending full source.
+
+        Args:
+            source:   File path or raw source code string.
+            language: ``"python"``, ``"js"``/``"ts"``, or ``"auto"`` (detect).
+
+        Returns:
+            List of strings like ``['class PaymentService (L12)', ...]``.
+
+        Example::
+
+            for item in CodeContext.outline("api/routes.py"):
+                print(item)
+        """
+        code = _read_source(source)
+        lang = _detect_language(source, language)
+
+        if lang == "python":
+            return _python_outline(code)
+        if lang in ("js", "ts"):
+            return _js_outline(code)
+        return _generic_outline(code)
+
+    @staticmethod
+    def folded(source: Union[str, Path], language: str = "auto") -> str:
+        """
+        Return the source with function/method bodies replaced by ``...``.
+
+        Dramatically reduces token count while preserving structure.
+
+        Example::
+
+            folded = CodeContext.folded("large_module.py")
+            # class MyClass:
+            #     def method_one(self): ...
+            #     def method_two(self, x: int) -> str: ...
+        """
+        code = _read_source(source)
+        lang = _detect_language(source, language)
+
+        if lang == "python":
+            return _python_folded(code)
+        # For other languages, return outline as string
+        items = CodeContext.outline(source, language)
+        return "\n".join(items)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Internal helpers
+# ──────────────────────────────────────────────────────────────────────────────
+
+def _read_source(source: Union[str, Path]) -> str:
+    path = Path(source)
+    if path.exists():
+        return path.read_text(encoding="utf-8", errors="replace")
+    return str(source)  # Treat as raw source code
+
+
+def _detect_language(source: Union[str, Path], hint: str) -> str:
+    if hint != "auto":
+        return hint.lower()
+    suffix = Path(str(source)).suffix.lower()
+    return {"py": "python", ".py": "python", ".js": "js", ".ts": "ts", ".jsx": "js", ".tsx": "ts"}.get(
+        suffix, "generic"
+    )
+
+
+def _fallback_extract(code: str, name: str, context_lines: int) -> Optional[str]:
+    """Regex-based fallback for non-parsable files."""
+    lines = code.splitlines()
+    for i, line in enumerate(lines):
+        if re.match(rf"\s*def\s+{re.escape(name)}\s*\(", line):
+            start = max(0, i - context_lines)
+            # Find end heuristically: next line at same or lower indent
+            indent = len(line) - len(line.lstrip())
+            end = i + 1
+            while end < len(lines):
+                l = lines[end]
+                if l.strip() == "":
+                    end += 1
+                    continue
+                if len(l) - len(l.lstrip()) <= indent and end > i + 1:
+                    break
+                end += 1
+            end = min(len(lines), end + context_lines)
+            return "\n".join(lines[start:end])
+    return None
+
+
+def _python_outline(code: str) -> list[str]:
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return _generic_outline(code)
+    items = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            items.append((node.lineno, f"class {node.name} (L{node.lineno})"))
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def"
+            items.append((node.lineno, f"{prefix} {node.name} (L{node.lineno})"))
+    items.sort()
+    return [label for _, label in items]
+
+
+def _js_outline(code: str) -> list[str]:
+    items = []
+    patterns = [
+        (r"^(export\s+)?(default\s+)?class\s+(\w+)", "class"),
+        (r"^(export\s+)?(async\s+)?function\s+(\w+)", "function"),
+        (r"^\s*(const|let|var)\s+(\w+)\s*=\s*(async\s+)?\(", "arrow fn"),
+    ]
+    for i, line in enumerate(code.splitlines(), 1):
+        for pattern, kind in patterns:
+            m = re.match(pattern, line.strip())
+            if m:
+                name = m.group(m.lastindex or 1) if m.lastindex else line[:40]
+                items.append(f"{kind} {name} (L{i})")
+                break
+    return items
+
+
+def _generic_outline(code: str) -> list[str]:
+    items = []
+    for i, line in enumerate(code.splitlines(), 1):
+        if re.match(r"^\s*(def |class |function |const |export )", line):
+            items.append(f"L{i}: {line.strip()[:80]}")
+    return items
+
+
+def _class_skeleton(node: ast.ClassDef, lines: list[str]) -> str:
+    """Return class header + method signatures only, bodies as '...'."""
+    parts = [f"class {node.name}:"]
+    for item in node.body:
+        if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            sig_line = lines[item.lineno - 1].strip()
+            parts.append(f"    {sig_line} ...")
+    return "\n".join(parts)
+
+
+def _python_folded(code: str) -> str:
+    """Replace function bodies with '...' using AST."""
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return code
+
+    lines = code.splitlines()
+    replacements: list[tuple[int, int, str]] = []  # (start_line, end_line, replacement)
+
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            sig_line = node.lineno - 1
+            body_start = node.body[0].lineno - 1
+            body_end = node.end_lineno - 1  # type: ignore[attr-defined]
+            indent = " " * (len(lines[sig_line]) - len(lines[sig_line].lstrip()) + 4)
+            replacements.append((body_start, body_end, f"{indent}..."))
+
+    # Apply replacements from bottom to top to preserve line numbers
+    replacements.sort(key=lambda x: x[0], reverse=True)
+    for start, end, replacement in replacements:
+        lines[start : end + 1] = [replacement]
+
+    return "\n".join(lines)

agentslim/compressor.py

@@ -0,0 +1,242 @@
+"""
+compressor.py — Text, HTML, JSON, and Markdown compressor for LLM prompts.
+
+Strategy:
+  1. Strip HTML tags / attributes that carry zero semantic value.
+  2. Collapse redundant whitespace and blank lines.
+  3. Remove code comments (optional — preserves intent by default).
+  4. Compact JSON / YAML to minimal representation.
+  5. Remove filler phrases from natural language ("As an AI language model…").
+
+All steps are individually togglable so you control the trade-off between
+compression ratio and information fidelity.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Optional
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Filler phrases common in LLM outputs / web scrapes (English + Russian)
+# ──────────────────────────────────────────────────────────────────────────────
+_FILLER_PHRASES: list[str] = [
+    r"as an ai language model[,\s]*",
+    r"i('m| am) just an ai[,\s]*",
+    r"certainly[,!]?\s*",
+    r"of course[,!]?\s*",
+    r"sure[,!]?\s*here('s| is)[,\s]*",
+    r"great question[,!]?\s*",
+    r"absolutely[,!]?\s*",
+    r"как языковая модель[,\s]*",
+    r"конечно[,!]?\s*",
+    r"разумеется[,!]?\s*",
+    r"отличный вопрос[,!]?\s*",
+]
+_FILLER_RE = re.compile(
+    "|".join(_FILLER_PHRASES),
+    flags=re.IGNORECASE,
+)
+
+# HTML tags that are purely structural/decorative — safe to remove entirely
+_DECORATIVE_TAGS = {
+    "script", "style", "nav", "footer", "header",
+    "aside", "advertisement", "noscript", "svg",
+}
+
+
+@dataclass
+class CompressorConfig:
+    """Fine-grained control over which compression steps are applied."""
+
+    strip_html: bool = True
+    """Remove HTML tags, keeping only inner text."""
+
+    remove_decorative_html: bool = True
+    """Drop entire <script>, <style>, <nav>, etc. blocks before stripping."""
+
+    collapse_whitespace: bool = True
+    """Merge multiple blank lines / spaces into one."""
+
+    remove_filler_phrases: bool = True
+    """Strip common LLM filler phrases from text."""
+
+    compact_json: bool = True
+    """Re-serialize JSON with no indentation / extra spaces."""
+
+    remove_python_comments: bool = False
+    """Strip ``# …`` comment lines from Python code (off by default)."""
+
+    remove_js_comments: bool = False
+    """Strip ``// …`` and ``/* … */`` comments from JS/TS code."""
+
+    max_consecutive_newlines: int = 2
+    """Maximum number of consecutive newlines kept after collapsing."""
+
+
+class Compressor:
+    """
+    Multi-format text compressor that reduces token usage before sending
+    content to an LLM.
+
+    Quickstart::
+
+        from agentslim import Compressor
+
+        c = Compressor()
+        slim = c.compress(raw_html_page)
+        print(slim)
+
+    You can also use format-specific helpers::
+
+        c.compress_json(big_json_string)
+        c.compress_html(html_string)
+        c.compress_code(python_source, language="python")
+    """
+
+    def __init__(self, config: Optional[CompressorConfig] = None) -> None:
+        self.config = config or CompressorConfig()
+
+    # ──────────────────────────────────────────────────────────────────────────
+    # Public API
+    # ──────────────────────────────────────────────────────────────────────────
+
+    def compress(self, text: str) -> str:
+        """
+        Auto-detect format and compress accordingly.
+
+        Detects JSON, HTML, or treats input as plain text / markdown.
+        """
+        stripped = text.strip()
+        if self._looks_like_json(stripped):
+            return self.compress_json(stripped)
+        if self._looks_like_html(stripped):
+            return self.compress_html(stripped)
+        return self.compress_text(stripped)
+
+    def compress_text(self, text: str) -> str:
+        """Compress natural-language text (plain text / Markdown)."""
+        if self.config.remove_filler_phrases:
+            text = _FILLER_RE.sub("", text)
+        if self.config.collapse_whitespace:
+            text = self._collapse_whitespace(text)
+        return text.strip()
+
+    def compress_html(self, html: str) -> str:
+        """Strip HTML tags and decorative blocks, returning clean text."""
+        if self.config.remove_decorative_html:
+            html = self._remove_decorative_blocks(html)
+        if self.config.strip_html:
+            html = self._strip_html_tags(html)
+        return self.compress_text(html)
+
+    def compress_json(self, json_str: str) -> str:
+        """
+        Compact JSON to minimal representation (no indentation, no spaces).
+
+        Falls back gracefully if the input is not valid JSON.
+        """
+        if not self.config.compact_json:
+            return json_str
+        try:
+            obj = json.loads(json_str)
+            return json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
+        except json.JSONDecodeError:
+            # Not valid JSON — treat as plain text
+            return self.compress_text(json_str)
+
+    def compress_code(self, source: str, language: str = "python") -> str:
+        """
+        Compress source code by optionally removing comments.
+
+        Args:
+            source:   Raw source code string.
+            language: ``"python"``, ``"js"``, or ``"ts"`` (TypeScript).
+        """
+        lang = language.lower()
+        if lang == "python" and self.config.remove_python_comments:
+            source = self._strip_python_comments(source)
+        elif lang in ("js", "ts", "javascript", "typescript") and self.config.remove_js_comments:
+            source = self._strip_js_comments(source)
+        if self.config.collapse_whitespace:
+            source = self._collapse_whitespace(source)
+        return source.strip()
+
+    # ──────────────────────────────────────────────────────────────────────────
+    # Private helpers
+    # ──────────────────────────────────────────────────────────────────────────
+
+    @staticmethod
+    def _looks_like_json(text: str) -> bool:
+        return (text.startswith("{") and text.endswith("}")) or (
+            text.startswith("[") and text.endswith("]")
+        )
+
+    @staticmethod
+    def _looks_like_html(text: str) -> bool:
+        return bool(re.search(r"<\s*(html|body|div|p|span|a|head)", text, re.IGNORECASE))
+
+    @staticmethod
+    def _remove_decorative_blocks(html: str) -> str:
+        for tag in _DECORATIVE_TAGS:
+            pattern = re.compile(
+                rf"<{tag}[^>]*>.*?</{tag}>",
+                flags=re.IGNORECASE | re.DOTALL,
+            )
+            html = pattern.sub("", html)
+        return html
+
+    @staticmethod
+    def _strip_html_tags(html: str) -> str:
+        # Replace block-level tags with newlines for readability
+        block_tags = re.compile(
+            r"<(br|p|div|li|tr|h[1-6]|blockquote)[^>]*>",
+            flags=re.IGNORECASE,
+        )
+        html = block_tags.sub("\n", html)
+        # Strip remaining tags
+        html = re.sub(r"<[^>]+>", "", html)
+        # Decode common HTML entities
+        html = (
+            html.replace("&amp;", "&")
+            .replace("&lt;", "<")
+            .replace("&gt;", ">")
+            .replace("&quot;", '"')
+            .replace("&#39;", "'")
+            .replace("&nbsp;", " ")
+        )
+        return html
+
+    def _collapse_whitespace(self, text: str) -> str:
+        # Replace tabs and non-breaking spaces with regular space
+        text = re.sub(r"[ \t]+", " ", text)
+        # Collapse more than N consecutive newlines
+        limit = self.config.max_consecutive_newlines
+        text = re.sub(rf"\n{{{limit + 1},}}", "\n" * limit, text)
+        # Strip trailing space on each line
+        text = "\n".join(line.rstrip() for line in text.splitlines())
+        return text
+
+    @staticmethod
+    def _strip_python_comments(source: str) -> str:
+        lines = []
+        for line in source.splitlines():
+            stripped = line.lstrip()
+            if stripped.startswith("#"):
+                continue  # Drop full-line comments
+            # Inline comment — keep code, drop comment
+            # (naive: doesn't handle # inside strings)
+            if " #" in line:
+                line = line[: line.index(" #")]
+            lines.append(line)
+        return "\n".join(lines)
+
+    @staticmethod
+    def _strip_js_comments(source: str) -> str:
+        # Remove /* … */ block comments
+        source = re.sub(r"/\*.*?\*/", "", source, flags=re.DOTALL)
+        # Remove // … line comments (not inside strings — simplified)
+        source = re.sub(r"//[^\n]*", "", source)
+        return source