ripperdoc 0.2.6__py3-none-any.whl

ripperdoc/utils/session_usage.py

@@ -0,0 +1,117 @@
+"""Session-level usage tracking for model calls."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+
+@dataclass
+class ModelUsage:
+    """Aggregate token and duration stats for a single model."""
+
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_input_tokens: int = 0
+    cache_creation_input_tokens: int = 0
+    requests: int = 0
+    duration_ms: float = 0.0
+    cost_usd: float = 0.0
+
+
+@dataclass
+class SessionUsage:
+    """In-memory snapshot of usage for the current session."""
+
+    models: Dict[str, ModelUsage] = field(default_factory=dict)
+
+    @property
+    def total_input_tokens(self) -> int:
+        return sum(usage.input_tokens for usage in self.models.values())
+
+    @property
+    def total_output_tokens(self) -> int:
+        return sum(usage.output_tokens for usage in self.models.values())
+
+    @property
+    def total_cache_read_tokens(self) -> int:
+        return sum(usage.cache_read_input_tokens for usage in self.models.values())
+
+    @property
+    def total_cache_creation_tokens(self) -> int:
+        return sum(usage.cache_creation_input_tokens for usage in self.models.values())
+
+    @property
+    def total_requests(self) -> int:
+        return sum(usage.requests for usage in self.models.values())
+
+    @property
+    def total_duration_ms(self) -> float:
+        return sum(usage.duration_ms for usage in self.models.values())
+
+    @property
+    def total_cost_usd(self) -> float:
+        return sum(usage.cost_usd for usage in self.models.values())
+
+
+_SESSION_USAGE = SessionUsage()
+
+
+def _as_int(value: Any) -> int:
+    """Best-effort integer conversion."""
+    try:
+        if value is None:
+            return 0
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _model_key(model: str) -> str:
+    """Normalize model names for use as dictionary keys."""
+    return model or "unknown"
+
+
+def record_usage(
+    model: str,
+    *,
+    input_tokens: int = 0,
+    output_tokens: int = 0,
+    cache_read_input_tokens: int = 0,
+    cache_creation_input_tokens: int = 0,
+    duration_ms: float = 0.0,
+    cost_usd: float = 0.0,
+) -> None:
+    """Record a single model invocation."""
+    global _SESSION_USAGE
+    key = _model_key(model)
+    usage = _SESSION_USAGE.models.setdefault(key, ModelUsage())
+
+    usage.input_tokens += _as_int(input_tokens)
+    usage.output_tokens += _as_int(output_tokens)
+    usage.cache_read_input_tokens += _as_int(cache_read_input_tokens)
+    usage.cache_creation_input_tokens += _as_int(cache_creation_input_tokens)
+    usage.duration_ms += float(duration_ms) if duration_ms and duration_ms > 0 else 0.0
+    usage.requests += 1
+    usage.cost_usd += float(cost_usd) if cost_usd and cost_usd > 0 else 0.0
+
+
+def get_session_usage() -> SessionUsage:
+    """Return a copy of the current session usage."""
+    return deepcopy(_SESSION_USAGE)
+
+
+def reset_session_usage() -> None:
+    """Clear all recorded usage (primarily for tests)."""
+    global _SESSION_USAGE
+    _SESSION_USAGE = SessionUsage()
+
+
+__all__ = [
+    "ModelUsage",
+    "SessionUsage",
+    "get_session_usage",
+    "record_usage",
+    "reset_session_usage",
+]

ripperdoc/utils/shell_token_utils.py

@@ -0,0 +1,95 @@
+"""Shell token parsing utilities."""
+
+from __future__ import annotations
+
+import re
+import shlex
+from typing import Iterable, List
+
+# Operators and redirections that should not be treated as executable tokens.
+SHELL_OPERATORS_WITH_REDIRECTION: set[str] = {
+    "|",
+    "||",
+    "&&",
+    ";",
+    ">",
+    ">>",
+    "<",
+    "<<",
+    "2>",
+    "&>",
+    "2>&1",
+    "|&",
+}
+
+_REDIRECTION_PATTERNS = (
+    re.compile(r"^\d?>?&\d+$"),  # 2>&1, >&2, etc.
+    re.compile(r"^\d?>/dev/null$"),  # 2>/dev/null, >/dev/null
+    re.compile(r"^/dev/null$"),
+)
+
+
+def parse_shell_tokens(shell_command: str) -> List[str]:
+    """Parse a shell command into tokens, preserving operators for inspection."""
+    if not shell_command:
+        return []
+
+    lexer = shlex.shlex(shell_command, posix=True)
+    lexer.whitespace_split = True
+    lexer.commenters = ""
+
+    try:
+        return list(lexer)
+    except ValueError:
+        # Fall back to a coarse split to avoid hard failures.
+        return shell_command.split()
+
+
+def filter_valid_tokens(tokens: Iterable[str]) -> list[str]:
+    """Remove shell control operators and redirection tokens."""
+    return [token for token in tokens if token not in SHELL_OPERATORS_WITH_REDIRECTION]
+
+
+def _is_redirection_token(token: str) -> bool:
+    return any(pattern.match(token) for pattern in _REDIRECTION_PATTERNS)
+
+
+def parse_and_clean_shell_tokens(raw_shell_string: str) -> List[str]:
+    """Parse tokens and strip benign redirections to mirror reference cleaning."""
+    tokens = parse_shell_tokens(raw_shell_string)
+    if not tokens:
+        return []
+
+    cleaned: list[str] = []
+    skip_next = False
+
+    for idx, token in enumerate(tokens):
+        if skip_next:
+            skip_next = False
+            continue
+
+        # Handle explicit redirection operators that are followed by a target.
+        if token in {">&", ">", "1>", "2>", ">>"}:
+            if idx + 1 < len(tokens):
+                next_token = tokens[idx + 1]
+                if _is_redirection_token(next_token):
+                    skip_next = True
+                    continue
+            cleaned.append(token)
+            continue
+
+        # Skip inlined redirection tokens to /dev/null or file descriptors.
+        if _is_redirection_token(token):
+            continue
+
+        cleaned.append(token)
+
+    return filter_valid_tokens(cleaned)
+
+
+__all__ = [
+    "parse_shell_tokens",
+    "parse_and_clean_shell_tokens",
+    "filter_valid_tokens",
+    "SHELL_OPERATORS_WITH_REDIRECTION",
+]

ripperdoc/utils/shell_utils.py

@@ -0,0 +1,159 @@
+"""Shell detection helpers.
+
+Selects a suitable interactive shell for running commands, preferring bash/zsh
+over the system's /bin/sh default to ensure features like brace expansion.
+On Windows, prefers Git Bash and falls back to cmd.exe if no bash is available.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from typing import Iterable, List
+
+from ripperdoc.utils.log import get_logger
+
+logger = get_logger()
+
+# Common locations to probe if shutil.which misses an otherwise standard path.
+_COMMON_BIN_DIRS: tuple[str, ...] = ("/bin", "/usr/bin", "/usr/local/bin", "/opt/homebrew/bin")
+_IS_WINDOWS = os.name == "nt"
+
+
+def _is_executable(path: str) -> bool:
+    return bool(path) and os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def _dedupe_preserve_order(items: Iterable[str]) -> list[str]:
+    seen = set()
+    ordered: list[str] = []
+    for item in items:
+        if item and item not in seen:
+            ordered.append(item)
+            seen.add(item)
+    return ordered
+
+
+def _find_git_bash_windows() -> str | None:
+    env_path = os.environ.get("GIT_BASH_PATH") or os.environ.get("GITBASH")
+    if env_path and _is_executable(env_path):
+        return env_path
+
+    bash_in_path = shutil.which("bash")
+    if bash_in_path and "git" in bash_in_path.lower():
+        return bash_in_path
+
+    common = [
+        r"C:\Program Files\Git\bin\bash.exe",
+        r"C:\Program Files\Git\usr\bin\bash.exe",
+        r"C:\Program Files (x86)\Git\bin\bash.exe",
+        r"C:\Program Files (x86)\Git\usr\bin\bash.exe",
+    ]
+    for path in common:
+        if _is_executable(path):
+            return path
+    return None
+
+
+def _windows_cmd_path() -> str | None:
+    comspec = os.environ.get("ComSpec")
+    if _is_executable(comspec or ""):
+        return comspec
+    which_cmd = shutil.which("cmd.exe") or shutil.which("cmd")
+    if which_cmd and _is_executable(which_cmd):
+        return which_cmd
+    system32 = os.path.join(os.environ.get("SystemRoot", r"C:\Windows"), "System32", "cmd.exe")
+    if _is_executable(system32):
+        return system32
+    return None
+
+
+def find_suitable_shell() -> str:
+    """Return a best-effort shell path, preferring bash/zsh (Git Bash on Windows).
+
+    Priority on Unix:
+      1) $SHELL if it's bash/zsh and executable
+      2) bash/zsh from PATH
+      3) bash/zsh in common bin directories
+
+    Priority on Windows:
+      1) Git Bash (env override or known locations / PATH)
+      2) cmd.exe as a last resort
+
+    Raises:
+        RuntimeError: if no suitable shell is found.
+    """
+
+    env_override = os.environ.get("RIPPERDOC_SHELL") or os.environ.get("RIPPERDOC_SHELL_PATH")
+    if env_override and _is_executable(env_override):
+        logger.debug("Using shell from RIPPERDOC_SHELL*: %s", env_override)
+        return env_override
+
+    current_shell = os.environ.get("SHELL", "")
+    current_is_bash = "bash" in current_shell
+    current_is_zsh = "zsh" in current_shell
+
+    if not _IS_WINDOWS:
+        if (current_is_bash or current_is_zsh) and _is_executable(current_shell):
+            logger.debug("Using SHELL from environment: %s", current_shell)
+            return current_shell
+
+        bash_path = shutil.which("bash") or ""
+        zsh_path = shutil.which("zsh") or ""
+        preferred_order = ["bash", "zsh"] if current_is_bash else ["zsh", "bash"]
+
+        candidates: list[str] = []
+        for name in preferred_order:
+            if name == "bash" and bash_path:
+                candidates.append(bash_path)
+            if name == "zsh" and zsh_path:
+                candidates.append(zsh_path)
+
+        for bin_dir in _COMMON_BIN_DIRS:
+            candidates.append(os.path.join(bin_dir, "bash"))
+            candidates.append(os.path.join(bin_dir, "zsh"))
+
+        for candidate in _dedupe_preserve_order(candidates):
+            if _is_executable(candidate):
+                logger.debug("Selected shell: %s", candidate)
+                return candidate
+
+        error_message = (
+            "No suitable shell found. Please install bash or zsh and ensure $SHELL is set. "
+            "Tried bash/zsh in PATH and common locations."
+        )
+        logger.error(error_message)
+        raise RuntimeError(error_message)
+
+    git_bash = _find_git_bash_windows()
+    if git_bash:
+        logger.debug("Using Git Bash: %s", git_bash)
+        return git_bash
+
+    cmd_path = _windows_cmd_path()
+    if cmd_path:
+        logger.warning("Falling back to cmd.exe; bash not found. Using: %s", cmd_path)
+        return cmd_path
+
+    error_message = (
+        "No suitable shell found on Windows. Install Git for Windows to provide bash "
+        "or ensure cmd.exe is available."
+    )
+    logger.error(error_message)
+    raise RuntimeError(error_message)
+
+
+def build_shell_command(shell_path: str, command: str) -> List[str]:
+    """Build argv for running a command with the selected shell.
+
+    For bash/zsh (including Git Bash), use -lc to run as login shell.
+    For cmd.exe fallback, use /d /s /c.
+    """
+
+    lower = shell_path.lower()
+    if lower.endswith("cmd.exe") or lower.endswith("\\cmd"):
+        return [shell_path, "/d", "/s", "/c", command]
+    return [shell_path, "-lc", command]
+
+
+__all__ = ["find_suitable_shell", "build_shell_command"]

ripperdoc/utils/todo.py

@@ -0,0 +1,203 @@
+"""Todo storage and utilities for Ripperdoc.
+
+This module provides simple, file-based todo management so tools can
+persist and query tasks between turns. Todos are stored under the user's
+home directory at `~/.ripperdoc/todos/<project>/todos.json`, where
+`<project>` is a sanitized form of the project path.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import List, Literal, Optional, Sequence, Tuple
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from ripperdoc.utils.log import get_logger
+from ripperdoc.utils.path_utils import project_storage_dir
+
+
+logger = get_logger()
+
+TodoStatus = Literal["pending", "in_progress", "completed"]
+TodoPriority = Literal["high", "medium", "low"]
+
+
+class TodoItem(BaseModel):
+    """Represents a single todo entry."""
+
+    id: str = Field(description="Unique identifier for the todo item")
+    content: str = Field(description="Task description")
+    status: TodoStatus = Field(
+        default="pending", description="Current state: pending, in_progress, completed"
+    )
+    priority: TodoPriority = Field(default="medium", description="Priority: high|medium|low")
+    created_at: Optional[float] = Field(default=None, description="Unix timestamp when created")
+    updated_at: Optional[float] = Field(default=None, description="Unix timestamp when updated")
+    previous_status: Optional[TodoStatus] = Field(
+        default=None, description="Previous status, used for audits"
+    )
+    model_config = ConfigDict(extra="ignore")
+
+
+MAX_TODOS = 200
+
+
+def _storage_path(project_root: Optional[Path], ensure_dir: bool) -> Path:
+    """Return the todo storage path, optionally ensuring the directory exists."""
+    root = project_root or Path.cwd()
+    base_dir = Path.home() / ".ripperdoc" / "todos"
+    storage_dir = project_storage_dir(base_dir, root, ensure=ensure_dir)
+    return storage_dir / "todos.json"
+
+
+def validate_todos(
+    todos: Sequence[TodoItem], max_items: int = MAX_TODOS
+) -> Tuple[bool, str | None]:
+    """Basic validation for a todo list."""
+    if len(todos) > max_items:
+        return False, f"Too many todos; limit is {max_items}."
+
+    ids = [todo.id for todo in todos]
+    duplicate_ids = {id_ for id_ in ids if ids.count(id_) > 1}
+    if duplicate_ids:
+        return False, f"Duplicate todo IDs found: {sorted(duplicate_ids)}"
+
+    in_progress = [todo for todo in todos if todo.status == "in_progress"]
+    if len(in_progress) > 1:
+        return False, "Only one todo can be marked in_progress at a time."
+
+    empty_contents = [todo.id for todo in todos if not todo.content.strip()]
+    if empty_contents:
+        return False, f"Todos require content. Empty content for IDs: {sorted(empty_contents)}"
+
+    return True, None
+
+
+def load_todos(project_root: Optional[Path] = None) -> List[TodoItem]:
+    """Load todos from disk."""
+    path = _storage_path(project_root, ensure_dir=False)
+    if not path.exists():
+        return []
+
+    try:
+        raw = json.loads(path.read_text())
+    except (json.JSONDecodeError, OSError, IOError, UnicodeDecodeError) as exc:
+        logger.warning(
+            "Failed to load todos from disk: %s: %s",
+            type(exc).__name__, exc,
+            extra={"path": str(path)},
+        )
+        return []
+
+    todos: List[TodoItem] = []
+    for item in raw:
+        try:
+            todos.append(TodoItem(**item))
+        except ValidationError as exc:
+            logger.error(f"Failed to parse todo item: {exc}")
+            continue
+
+    # Preserve stored order; do not reorder based on status/priority.
+    return todos
+
+
+def save_todos(todos: Sequence[TodoItem], project_root: Optional[Path] = None) -> None:
+    """Persist todos to disk."""
+    path = _storage_path(project_root, ensure_dir=True)
+    path.write_text(json.dumps([todo.model_dump() for todo in todos], indent=2))
+
+
+def set_todos(
+    todos: Sequence[TodoItem],
+    project_root: Optional[Path] = None,
+) -> List[TodoItem]:
+    """Validate, normalize, and persist the provided todos."""
+    ok, message = validate_todos(todos)
+    if not ok:
+        raise ValueError(message or "Invalid todos.")
+
+    existing = {todo.id: todo for todo in load_todos(project_root)}
+    now = time.time()
+
+    normalized: List[TodoItem] = []
+    for todo in todos:
+        previous = existing.get(todo.id)
+        normalized.append(
+            todo.model_copy(
+                update={
+                    "created_at": previous.created_at if previous else todo.created_at or now,
+                    "updated_at": now,
+                    "previous_status": (
+                        previous.status
+                        if previous and previous.status != todo.status
+                        else todo.previous_status
+                    ),
+                }
+            )
+        )
+
+    # Keep the caller-provided order; do not resort.
+    save_todos(normalized, project_root)
+    return list(normalized)
+
+
+def clear_todos(project_root: Optional[Path] = None) -> None:
+    """Remove all todos."""
+    save_todos([], project_root)
+
+
+def get_next_actionable(todos: Sequence[TodoItem]) -> Optional[TodoItem]:
+    """Return the next todo to work on (in_progress first, then pending)."""
+    for status in ("in_progress", "pending"):
+        for todo in todos:
+            if todo.status == status:
+                return todo
+    return None
+
+
+def summarize_todos(todos: Sequence[TodoItem]) -> dict:
+    """Return simple statistics for a todo collection."""
+    return {
+        "total": len(todos),
+        "by_status": {
+            "pending": len([t for t in todos if t.status == "pending"]),
+            "in_progress": len([t for t in todos if t.status == "in_progress"]),
+            "completed": len([t for t in todos if t.status == "completed"]),
+        },
+        "by_priority": {
+            "high": len([t for t in todos if t.priority == "high"]),
+            "medium": len([t for t in todos if t.priority == "medium"]),
+            "low": len([t for t in todos if t.priority == "low"]),
+        },
+    }
+
+
+def format_todo_summary(todos: Sequence[TodoItem]) -> str:
+    """Create a concise summary string for use in tool outputs."""
+    stats = summarize_todos(todos)
+    summary = (
+        f"Todos updated (total {stats['total']}; "
+        f"{stats['by_status']['pending']} pending, "
+        f"{stats['by_status']['in_progress']} in progress, "
+        f"{stats['by_status']['completed']} completed)."
+    )
+
+    next_item = get_next_actionable(todos)
+    if next_item:
+        summary += f" Next to tackle: {next_item.content} (id: {next_item.id}, status: {next_item.status})."
+    elif stats["total"] == 0:
+        summary += " No todos stored yet."
+
+    return summary
+
+
+def format_todo_lines(todos: Sequence[TodoItem]) -> List[str]:
+    """Return human-readable todo lines."""
+    status_marker = {
+        "completed": "●",
+        "in_progress": "◐",
+        "pending": "○",
+    }
+    return [f"{status_marker.get(todo.status, '○')} {todo.content}" for todo in todos]

ripperdoc/utils/token_estimation.py

@@ -0,0 +1,34 @@
+"""Shared token estimation utilities."""
+
+from __future__ import annotations
+
+import math
+
+from ripperdoc.utils.log import get_logger
+
+logger = get_logger()
+
+# Optional: use tiktoken for accurate counts when available.
+_TIKTOKEN_ENCODING: tiktoken.Encoding | None = None
+try:  # pragma: no cover - optional dependency
+    import tiktoken  # type: ignore
+
+    _TIKTOKEN_ENCODING = tiktoken.get_encoding("cl100k_base")
+except (ImportError, ModuleNotFoundError, OSError, RuntimeError):  # pragma: no cover - runtime fallback
+    pass
+
+
+def estimate_tokens(text: str) -> int:
+    """Estimate token count, preferring tiktoken when available."""
+    if not text:
+        return 0
+    if _TIKTOKEN_ENCODING:
+        try:
+            return len(_TIKTOKEN_ENCODING.encode(text))
+        except (UnicodeDecodeError, ValueError, RuntimeError):
+            logger.debug("[token_estimation] tiktoken encode failed; falling back to heuristic")
+    # Heuristic: ~4 characters per token
+    return max(1, math.ceil(len(text) / 4))
+
+
+__all__ = ["estimate_tokens"]