code-muse 0.0.1__py3-none-any.whl

code_muse/plugins/filter_engine/content_detector.py

@@ -0,0 +1,184 @@
+"""Content type detector for shell command output.
+
+Sniffs stdout to classify output as JSON, diff, log, HTML, code, or unknown.
+Uses fast heuristics — no LLM calls.
+"""
+
+import enum
+import json
+import re
+from typing import ClassVar
+
+
+class ContentType(enum.Enum):
+    """Output content types."""
+
+    JSON = "json"
+    DIFF = "diff"
+    LOG = "log"
+    HTML = "html"
+    CODE = "code"
+    SEARCH = "search"
+    UNKNOWN = "unknown"
+
+
+class ContentTypeDetector:
+    """Detect content type from shell command stdout."""
+
+    # Pre-compiled patterns
+    DIFF_HEADER: ClassVar[re.Pattern] = re.compile(
+        r"^@@\s+-(\d+),?\d*\s+\+(\d+),?\d*\s+@@", re.MULTILINE
+    )
+    LOG_PATTERNS: ClassVar[list[re.Pattern]] = [
+        re.compile(r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}"),  # ISO timestamp
+        re.compile(r"^\d{2}:\d{2}:\d{2}\.\d{3}"),  # HH:MM:SS.mmm
+        re.compile(r"^\[?\d{4}-\d{2}-\d{2}\]?"),  # [YYYY-MM-DD]
+        re.compile(r"^\w+\s+\d+\s+\d{2}:\d{2}:\d{2}"),  # syslog style
+        re.compile(r"^(ERROR|WARN|INFO|DEBUG|TRACE|FATAL)\b", re.IGNORECASE),
+    ]
+    HTML_PATTERNS: ClassVar[list[re.Pattern]] = [
+        re.compile(r"<html[\s>]", re.IGNORECASE),
+        re.compile(r"<!DOCTYPE\s+html", re.IGNORECASE),
+        re.compile(r"<head[\s>]", re.IGNORECASE),
+        re.compile(r"<body[\s>]", re.IGNORECASE),
+        re.compile(r"<div[\s>]", re.IGNORECASE),
+        re.compile(
+            r"</?(?:html|head|body|div|span|p|a|table|tr|td)[\s>]", re.IGNORECASE
+        ),
+    ]
+    CODE_KEYWORDS: ClassVar[set[str]] = {
+        "def",
+        "class",
+        "import",
+        "from",
+        "function",
+        "const",
+        "let",
+        "var",
+        "return",
+        "if",
+        "else",
+        "for",
+        "while",
+        "try",
+        "except",
+        "catch",
+        "public",
+        "private",
+        "protected",
+        "static",
+        "void",
+        "int",
+        "string",
+        "package",
+        "export",
+        "require",
+        "module",
+    }
+    SEARCH_PATTERNS: ClassVar[list[re.Pattern]] = [
+        re.compile(r"^Found\s+\d+\s+results?", re.IGNORECASE),
+        re.compile(r"^\d+\s+matches?", re.IGNORECASE),
+        re.compile(r"^---\s+search\s+results?\s+---", re.IGNORECASE),
+        re.compile(r"^\d+:\d+:.*\x1b", re.MULTILINE),  # grep -n output
+    ]
+
+    @classmethod
+    def detect(cls, stdout: str) -> ContentType:
+        """Detect content type from stdout string.
+
+        Args:
+            stdout: The full stdout from the shell command.
+
+        Returns:
+            ContentType enum value.
+        """
+        if not stdout or not stdout.strip():
+            return ContentType.UNKNOWN
+
+        text = stdout.strip()
+
+        # 1. JSON detection — try to parse
+        if cls._is_json(text):
+            return ContentType.JSON
+
+        # 2. Diff detection — unified diff headers
+        if cls.DIFF_HEADER.search(text):
+            return ContentType.DIFF
+
+        # 3. Log detection — timestamp patterns or log levels
+        if cls._is_log(text):
+            return ContentType.LOG
+
+        # 4. HTML detection
+        if cls._is_html(text):
+            return ContentType.HTML
+
+        # 5. Search results detection
+        if cls._is_search(text):
+            return ContentType.SEARCH
+
+        # 6. Code detection — keyword density
+        if cls._is_code(text):
+            return ContentType.CODE
+
+        return ContentType.UNKNOWN
+
+    # ------------------------------------------------------------------
+    # Private heuristics
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def _is_json(cls, text: str) -> bool:
+        """Test if text is valid JSON (object or array)."""
+        # Fast check: first non-whitespace char is { or [
+        first = text.lstrip()[0] if text else ""
+        if first not in ("{", "["):
+            return False
+        try:
+            json.loads(text)
+            return True
+        except ValueError:
+            return False
+
+    @classmethod
+    def _is_log(cls, text: str) -> bool:
+        """Test if text looks like log output."""
+        lines = text.splitlines()
+        if not lines:
+            return False
+        # Count lines matching log patterns
+        log_lines = 0
+        for line in lines[:50]:  # sample first 50 lines
+            for pat in cls.LOG_PATTERNS:
+                if pat.search(line):
+                    log_lines += 1
+                    break
+        # If > 30% of sampled lines match, treat as log
+        return log_lines > max(1, min(len(lines), 50) * 0.3)
+
+    @classmethod
+    def _is_html(cls, text: str) -> bool:
+        """Test if text looks like HTML."""
+        return any(pat.search(text[:2000]) for pat in cls.HTML_PATTERNS)
+
+    @classmethod
+    def _is_code(cls, text: str) -> bool:
+        """Test if text looks like source code by keyword density."""
+        lines = text.splitlines()
+        if not lines:
+            return False
+        words = text.lower().split()
+        if not words:
+            return False
+        code_words = sum(1 for w in words[:500] if w in cls.CODE_KEYWORDS)
+        # If > 5% of words are code keywords, treat as code
+        return code_words > max(3, min(len(words), 500) * 0.05)
+
+    @classmethod
+    def _is_search(cls, text: str) -> bool:
+        """Test if text looks like search results (grep/rg/find output)."""
+        for pat in cls.SEARCH_PATTERNS:
+            if pat.search(text[:2000]):
+                return True
+        # Also check for filename:line: pattern (grep -n)
+        return re.search(r"^\S+\.\w+:\d+:", text[:2000], re.MULTILINE) is not None

code_muse/plugins/filter_engine/dispatcher.py

@@ -0,0 +1,244 @@
+"""Filter dispatcher for the filter engine.
+
+Orchestrates classification → strategy lookup → execution → compression.
+"""
+
+import logging
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from code_muse.plugins.filter_engine.classifier import CommandClassifier
+from code_muse.plugins.filter_engine.registry import get_registry
+from code_muse.plugins.filter_engine.verbosity import get_verbosity
+from code_muse.tools.command_runner import ShellCommandOutput, _execute_shell_command
+
+logger = logging.getLogger(__name__)
+
+
+class FilterDispatcher:
+    """Singleton dispatcher that handles the full filter pipeline."""
+
+    _instance: FilterDispatcher | None = None
+
+    def __init__(self) -> None:
+        """Initialise the dispatcher with its classifier."""
+        self.classifier = CommandClassifier()
+
+    @classmethod
+    def get_instance(cls) -> FilterDispatcher:
+        """Return the module-level singleton."""
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @staticmethod
+    def _normalize_command(command: str) -> str:
+        """Rewrite plain ``git status`` to force porcelain format.
+
+        Appends ``--porcelain -b`` when the command is ``git status`` (or
+        variants like ``git -C <dir> status``) and does not already request
+        porcelain or short output.
+
+        Args:
+            command: The raw shell command.
+
+        Returns:
+            The original command, or a rewritten version with porcelain flags.
+        """
+        stripped = command.strip()
+        # Must contain git + status as separate words
+        if not re.search(r"\bgit\b.*\bstatus\b", stripped):
+            return command
+
+        tokens = stripped.split()
+        # Already porcelain/short — leave alone
+        if any(t in {"--porcelain", "--short", "-s"} for t in tokens):
+            return command
+
+        return f"{stripped} --porcelain -b"
+
+    async def handle(
+        self,
+        context: Any,
+        command: str,
+        cwd: str | None,
+        timeout: int,
+    ) -> dict[str, Any] | None:
+        """Run the full filter pipeline for a shell command.
+
+        Steps:
+
+        1. Classify the command.
+        2. Look up the strategy for the category.
+        3. If the strategy is ``None`` or the ``unknown`` passthrough,
+           return ``None`` so normal execution proceeds.
+        4. Execute the command via :func:`_execute_shell_command`.
+        5. Apply the strategy to the resulting :class:`ShellCommandOutput`.
+        6. Return ``{"pre_executed": True, "output": filtered_output}``.
+
+        On any strategy exception the error is logged and ``None`` is returned
+        so the raw output is preserved as a fallback.
+
+        Args:
+            context: pydantic-ai RunContext (unused).
+            command: The shell command to execute.
+            cwd: Working directory.
+            timeout: Timeout in seconds.
+
+        Returns:
+            A pre-executed result dict or ``None`` for passthrough.
+        """
+        category = self.classifier.classify(command)
+        registry = get_registry()
+
+        # Passthrough strategies return None
+        if category == "unknown":
+            return None
+
+        verbosity = get_verbosity()
+
+        # If verbosity is RAW, skip filtering entirely
+        if verbosity.value >= 4:
+            return None
+
+        try:
+            # Execute the command (sub-agents run silently)
+            from code_muse.tools.subagent_context import is_subagent
+
+            silent = is_subagent()
+            group_id = f"filter_engine_{id(command)}"
+
+            # Force porcelain for plain git status commands so the strategy
+            # parser always receives machine-readable output.
+            effective_command = self._normalize_command(command)
+            if effective_command != command:
+                logger.debug(
+                    "FilterDispatcher: rewritten %r → %r", command, effective_command
+                )
+
+            # _execute_shell_command is async, but we are already in async context
+            output = await _execute_shell_command(
+                command=effective_command,
+                cwd=cwd,
+                timeout=timeout,
+                group_id=group_id,
+                silent=silent,
+            )
+
+            # -----------------------------------------------------------------
+            # Content-type routing (Epic 019)
+            # -----------------------------------------------------------------
+            from code_muse.plugins.filter_engine.content_detector import (
+                ContentType,
+                ContentTypeDetector,
+            )
+
+            content_type = ContentTypeDetector.detect(output.stdout or "")
+            logger.debug(
+                "Detected content type: %s for command: %s", content_type.value, command
+            )
+
+            content_strategy_map = {
+                ContentType.JSON: "json",
+                ContentType.DIFF: "diff",
+                ContentType.LOG: "log",
+                ContentType.HTML: "html",
+                ContentType.SEARCH: "search",
+                ContentType.CODE: category,
+                ContentType.UNKNOWN: category,
+            }
+
+            effective_category = content_strategy_map.get(content_type, category)
+
+            strategy = registry.get_strategy(effective_category)
+            if strategy is None and effective_category != category:
+                logger.debug(
+                    "No strategy for %s, falling back to command category %s",
+                    effective_category,
+                    category,
+                )
+                effective_category = category
+                strategy = registry.get_strategy(effective_category)
+
+            if strategy is None:
+                return None
+
+            # Apply strategy
+            filtered = strategy(
+                command,
+                output.stdout or "",
+                output.stderr or "",
+                output.exit_code or 0,
+                verbosity,
+            )
+
+            if filtered is None:
+                return None
+
+            # Track execution metrics (best-effort, never blocks)
+            try:
+                from code_muse.plugins.token_tracking.record import record_command
+
+                record_command(
+                    command=command,
+                    raw_stdout=output.stdout or "",
+                    raw_stderr=output.stderr or "",
+                    compressed_stdout=filtered.stdout or "",
+                    compressed_stderr=filtered.stderr or "",
+                    category=effective_category,
+                    strategy=strategy.__name__
+                    if hasattr(strategy, "__name__")
+                    else str(strategy),
+                    exit_code=output.exit_code or 0,
+                )
+            except Exception:
+                pass
+
+            return {"pre_executed": True, "output": filtered}
+        except Exception:
+            logger.exception("FilterDispatcher: strategy failed for %r", command)
+            # Tee recovery: save raw output so user can recover
+            try:
+                import tempfile
+
+                raw_stdout = (
+                    getattr(locals().get("output"), "stdout", "(unavailable)")
+                    or "(empty)"
+                )
+                raw_stderr = (
+                    getattr(locals().get("output"), "stderr", "(unavailable)")
+                    or "(empty)"
+                )
+
+                tee_dir = Path(tempfile.gettempdir()) / "muse_tee"
+                tee_dir.mkdir(exist_ok=True)
+                tee_path = (
+                    tee_dir
+                    / f"tee_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}_{hash(command) & 0xFFFF:04x}.txt"
+                )
+                tee_path.write_text(
+                    f"# Raw output saved after filter error\n"
+                    f"# Command: {command}\n"
+                    f"# Timestamp: {datetime.now(UTC).isoformat()}\n\n"
+                    f"STDOUT:\n{raw_stdout}\n\n"
+                    f"STDERR:\n{raw_stderr}\n"
+                )
+                tee_path.chmod(0o600)  # user-only readable
+                logger.info("Tee recovery: raw output saved to %s", tee_path)
+                # Return hint that includes the tee path
+                return {
+                    "pre_executed": True,
+                    "output": ShellCommandOutput(
+                        success=False,
+                        command=command,
+                        stdout=f"⚠ Filter error — raw output saved to {tee_path}",
+                        stderr="",
+                        exit_code=-1,
+                        execution_time=0.0,
+                    ),
+                }
+            except Exception as tee_exc:
+                logger.error("Tee recovery itself failed: %s", tee_exc)
+            return None

code_muse/plugins/filter_engine/register_callbacks.py

@@ -0,0 +1,188 @@
+"""Filter Engine plugin for Muse — callback registration.
+
+Registers:
+    - ``run_shell_command`` callback that intercepts shell commands
+    - ``/init`` custom command for one-command project setup
+    - Startup hook for tee-file cleanup
+"""
+
+import logging
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+from code_muse.callbacks import register_callback
+from code_muse.messaging import emit_info, emit_success, emit_warning
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Cython JIT hooks — safe fallback to pure Python
+# ---------------------------------------------------------------------------
+
+
+def _setup_cython_hooks() -> None:
+    """Enable pyximport so .pyx modules compile on-the-fly.
+
+    Falls back gracefully when Cython is missing or compilation fails.
+    """
+    try:
+        import pyximport
+
+        pyximport.install(
+            language_level=3,
+            build_in_temp=True,
+            inplace=True,
+        )
+        import code_muse
+
+        if code_muse.CYTHON_ENABLED:
+            emit_success(
+                f"✅ Cython enabled — {code_muse.PYX_MODULE_COUNT} modules compiled"
+            )
+        else:
+            emit_warning("⚠️ Cython not available — running in pure Python mode")
+    except ImportError:
+        emit_warning("⚠️ Cython not available — running in pure Python mode")
+    except Exception:  # noqa: BLE001
+        emit_warning("⚠️ Cython not available — running in pure Python mode")
+
+
+# ---------------------------------------------------------------------------
+# Startup hook — tee file cleanup
+# ---------------------------------------------------------------------------
+
+
+def _on_startup() -> None:
+    """Delete tee files older than 24 hours on startup."""
+    try:
+        tee_dir = Path(tempfile.gettempdir()) / "muse_tee"
+        if tee_dir.exists():
+            now = time.time()
+            for f in tee_dir.iterdir():
+                if f.is_file() and now - f.stat().st_mtime > 86400:
+                    f.unlink(missing_ok=True)
+    except Exception:
+        pass
+
+
+# ---------------------------------------------------------------------------
+# Run-shell-command callback
+# ---------------------------------------------------------------------------
+
+
+async def filter_engine_callback(
+    context: Any,
+    command: str,
+    cwd: str | None = None,
+    timeout: int = 60,
+) -> dict[str, Any] | None:
+    """Run-shell-command callback for the filter engine.
+
+    Args:
+        context: The pydantic-ai RunContext (unused).
+        command: The shell command to potentially intercept.
+        cwd: Working directory for the command.
+        timeout: Timeout in seconds.
+
+    Returns:
+        ``{"pre_executed": True, "output": ShellCommandOutput(...)}`` when the
+        filter engine handles the command, or ``None`` to let normal execution
+        proceed.
+    """
+    from code_muse.plugins.filter_engine.dispatcher import FilterDispatcher
+
+    dispatcher = FilterDispatcher.get_instance()
+    return await dispatcher.handle(context, command, cwd, timeout)
+
+
+# ---------------------------------------------------------------------------
+# /init custom command
+# ---------------------------------------------------------------------------
+
+
+_INIT_MARKDOWN = """# Fast-Puppy Token Saving
+
+This project uses Fast-Puppy to compress shell command output, reducing token usage by 60-90%.
+
+## Enabled Strategies
+- Git output compression (status, log, diff)
+- Test runner failure focus
+- Lint output grouping
+- Code-aware read filtering
+
+## Configuration
+See `~/.muse/config.toml` for global settings.
+
+## Commands
+Run `/tracking gain` to see token savings.
+"""
+
+_PROJECT_FILES = [
+    "pyproject.toml",
+    "package.json",
+    "Cargo.toml",
+    "go.mod",
+    "Gemfile",
+]
+
+
+def _detect_project_type(cwd: Path) -> str | None:
+    """Detect project type from known manifest files."""
+    for manifest in _PROJECT_FILES:
+        if (cwd / manifest).exists():
+            return manifest
+    return None
+
+
+def _on_custom_command(command: str, name: str) -> bool | None:  # noqa: ARG001
+    """Handle ``/init`` — one-command Fast-Puppy setup."""
+    if name != "init":
+        return None
+
+    cwd = Path.cwd()
+    manifest = _detect_project_type(cwd)
+
+    md_path = cwd / "FAST_PUPPY.md"
+    if md_path.exists():
+        emit_info("FAST_PUPPY.md already exists — nothing to do.")
+        return True
+
+    md_path.write_text(_INIT_MARKDOWN, encoding="utf-8")
+    if manifest:
+        emit_success(
+            f"✅ Fast-Puppy initialized. Detected {manifest}. "
+            f"Created FAST_PUPPY.md. Filter engine is active."
+        )
+    else:
+        emit_success(
+            "✅ Fast-Puppy initialized. Created FAST_PUPPY.md. Filter engine is active."
+        )
+    return True
+
+
+# ---------------------------------------------------------------------------
+# Help entries
+# ---------------------------------------------------------------------------
+
+
+def _on_custom_command_help() -> list[tuple[str, str]]:
+    return [
+        ("/init", "Initialize Fast-Puppy in the current project"),
+    ]
+
+
+# ---------------------------------------------------------------------------
+# Register all callbacks
+# ---------------------------------------------------------------------------
+
+register_callback("startup", _on_startup)
+register_callback("startup", _setup_cython_hooks)
+# Priority: runs first (before policy_engine=50). Performs content-type detection and routing.
+register_callback("run_shell_command", filter_engine_callback, priority=10)
+register_callback("custom_command", _on_custom_command)
+register_callback("custom_command_help", _on_custom_command_help)
+
+logger.debug("Filter Engine plugin callbacks registered")