ripperdoc 0.1.0__py3-none-any.whl

ripperdoc/utils/session_history.py

@@ -0,0 +1,223 @@
+"""Session log storage and retrieval."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+from ripperdoc.utils.log import get_logger
+from ripperdoc.utils.messages import (
+    AssistantMessage,
+    ProgressMessage,
+    UserMessage,
+)
+from ripperdoc.utils.path_utils import project_storage_dir
+
+
+logger = get_logger()
+
+ConversationMessage = UserMessage | AssistantMessage | ProgressMessage
+
+
+@dataclass
+class SessionSummary:
+    session_id: str
+    path: Path
+    message_count: int
+    created_at: datetime
+    updated_at: datetime
+    first_prompt: str
+
+
+def _sessions_root() -> Path:
+    return Path.home() / ".ripperdoc" / "sessions"
+
+
+def _session_file(project_path: Path, session_id: str) -> Path:
+    directory = project_storage_dir(_sessions_root(), project_path, ensure=True)
+    return directory / f"{session_id}.jsonl"
+
+
+def _now_iso() -> str:
+    return datetime.utcnow().isoformat() + "Z"
+
+
+def _extract_prompt(payload: dict) -> str:
+    """Pull a short preview of the first user message."""
+    if payload.get("type") != "user":
+        return ""
+    message = payload.get("message") or {}
+    content = message.get("content")
+    preview = ""
+    if isinstance(content, str):
+        preview = content
+    elif isinstance(content, list):
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "text" and block.get("text"):
+                preview = str(block["text"])
+                break
+    preview = (preview or "").replace("\n", " ").strip()
+    if len(preview) > 80:
+        preview = preview[:77] + "..."
+    return preview
+
+
+def _deserialize_message(payload: dict) -> Optional[ConversationMessage]:
+    """Rebuild a message model from a stored payload."""
+    msg_type = payload.get("type")
+    if msg_type == "user":
+        return UserMessage(**payload)
+    if msg_type == "assistant":
+        return AssistantMessage(**payload)
+    if msg_type == "progress":
+        return ProgressMessage(**payload)
+    return None
+
+
+class SessionHistory:
+    """Append-only session log for a single session id."""
+
+    def __init__(self, project_path: Path, session_id: str):
+        self.project_path = project_path
+        self.session_id = session_id
+        self.path = _session_file(project_path, session_id)
+        self._seen_ids: set[str] = set()
+        self._load_seen_ids()
+
+    def _load_seen_ids(self) -> None:
+        if not self.path.exists():
+            return
+        try:
+            with self.path.open("r", encoding="utf-8") as fh:
+                for line in fh:
+                    try:
+                        data = json.loads(line)
+                        payload = data.get("payload") or {}
+                        msg_uuid = payload.get("uuid")
+                        if isinstance(msg_uuid, str):
+                            self._seen_ids.add(msg_uuid)
+                    except Exception as exc:
+                        logger.debug(f"Failed to parse session history line: {exc}")
+                        continue
+        except Exception as exc:
+            logger.error(f"Failed to load seen IDs from session {self.session_id}: {exc}")
+            return
+
+    def append(self, message: ConversationMessage) -> None:
+        """Persist a single message to the session log."""
+        # Skip progress noise
+        if getattr(message, "type", None) == "progress":
+            return
+        msg_uuid = getattr(message, "uuid", None)
+        if isinstance(msg_uuid, str) and msg_uuid in self._seen_ids:
+            return
+
+        payload = message.model_dump(mode="json")
+        entry = {
+            "logged_at": _now_iso(),
+            "payload": payload,
+        }
+        try:
+            with self.path.open("a", encoding="utf-8") as fh:
+                json.dump(entry, fh)
+                fh.write("\n")
+            if isinstance(msg_uuid, str):
+                self._seen_ids.add(msg_uuid)
+        except Exception as exc:
+            # Avoid crashing the UI if logging fails
+            logger.error(f"Failed to append message to session {self.session_id}: {exc}")
+            return
+
+
+def list_session_summaries(project_path: Path) -> List[SessionSummary]:
+    """Return available sessions for the project ordered by last update desc."""
+    directory = project_storage_dir(_sessions_root(), project_path)
+    if not directory.exists():
+        return []
+
+    summaries: List[SessionSummary] = []
+    for jsonl_path in directory.glob("*.jsonl"):
+        try:
+            with jsonl_path.open("r", encoding="utf-8") as fh:
+                messages = [json.loads(line) for line in fh if line.strip()]
+        except Exception as exc:
+            logger.error(f"Failed to load session summary from {jsonl_path}: {exc}")
+            continue
+
+        payloads = [entry.get("payload") or {} for entry in messages]
+        conversation_payloads = [
+            payload for payload in payloads if payload.get("type") in ("user", "assistant")
+        ]
+        if not conversation_payloads:
+            continue
+
+        created_raw = messages[0].get("logged_at")
+        updated_raw = messages[-1].get("logged_at")
+        created_at = (
+            datetime.fromisoformat(created_raw.replace("Z", "+00:00"))
+            if isinstance(created_raw, str)
+            else datetime.fromtimestamp(jsonl_path.stat().st_ctime)
+        )
+        updated_at = (
+            datetime.fromisoformat(updated_raw.replace("Z", "+00:00"))
+            if isinstance(updated_raw, str)
+            else datetime.fromtimestamp(jsonl_path.stat().st_mtime)
+        )
+        first_prompt = ""
+        for payload in conversation_payloads:
+            first_prompt = _extract_prompt(payload)
+            if first_prompt:
+                break
+        summaries.append(
+            SessionSummary(
+                session_id=jsonl_path.stem,
+                path=jsonl_path,
+                message_count=len(conversation_payloads),
+                created_at=created_at,
+                updated_at=updated_at,
+                first_prompt=first_prompt or "(no prompt)",
+            )
+        )
+
+    return sorted(summaries, key=lambda s: s.updated_at, reverse=True)
+
+
+def load_session_messages(project_path: Path, session_id: str) -> List[ConversationMessage]:
+    """Load messages for a stored session."""
+    path = _session_file(project_path, session_id)
+    if not path.exists():
+        return []
+
+    messages: List[ConversationMessage] = []
+    try:
+        with path.open("r", encoding="utf-8") as fh:
+            for line in fh:
+                if not line.strip():
+                    continue
+                try:
+                    data = json.loads(line)
+                    payload = data.get("payload") or {}
+                    msg = _deserialize_message(payload)
+                    if msg is not None and getattr(msg, "type", None) != "progress":
+                        messages.append(msg)
+                except Exception as exc:
+                    logger.debug(f"Failed to deserialize message in session {session_id}: {exc}")
+                    continue
+    except Exception as exc:
+        logger.error(f"Failed to load session messages for {session_id}: {exc}")
+        return []
+
+    return messages
+
+
+__all__ = [
+    "SessionHistory",
+    "SessionSummary",
+    "list_session_summaries",
+    "load_session_messages",
+]

ripperdoc/utils/session_usage.py

@@ -0,0 +1,110 @@
+"""Session-level usage tracking for model calls."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+
+@dataclass
+class ModelUsage:
+    """Aggregate token and duration stats for a single model."""
+
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_input_tokens: int = 0
+    cache_creation_input_tokens: int = 0
+    requests: int = 0
+    duration_ms: float = 0.0
+
+
+@dataclass
+class SessionUsage:
+    """In-memory snapshot of usage for the current session."""
+
+    models: Dict[str, ModelUsage] = field(default_factory=dict)
+
+    @property
+    def total_input_tokens(self) -> int:
+        return sum(usage.input_tokens for usage in self.models.values())
+
+    @property
+    def total_output_tokens(self) -> int:
+        return sum(usage.output_tokens for usage in self.models.values())
+
+    @property
+    def total_cache_read_tokens(self) -> int:
+        return sum(usage.cache_read_input_tokens for usage in self.models.values())
+
+    @property
+    def total_cache_creation_tokens(self) -> int:
+        return sum(usage.cache_creation_input_tokens for usage in self.models.values())
+
+    @property
+    def total_requests(self) -> int:
+        return sum(usage.requests for usage in self.models.values())
+
+    @property
+    def total_duration_ms(self) -> float:
+        return sum(usage.duration_ms for usage in self.models.values())
+
+
+_SESSION_USAGE = SessionUsage()
+
+
+def _as_int(value: Any) -> int:
+    """Best-effort integer conversion."""
+    try:
+        if value is None:
+            return 0
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _model_key(model: str) -> str:
+    """Normalize model names for use as dictionary keys."""
+    return model or "unknown"
+
+
+def record_usage(
+    model: str,
+    *,
+    input_tokens: int = 0,
+    output_tokens: int = 0,
+    cache_read_input_tokens: int = 0,
+    cache_creation_input_tokens: int = 0,
+    duration_ms: float = 0.0,
+) -> None:
+    """Record a single model invocation."""
+    global _SESSION_USAGE
+    key = _model_key(model)
+    usage = _SESSION_USAGE.models.setdefault(key, ModelUsage())
+
+    usage.input_tokens += _as_int(input_tokens)
+    usage.output_tokens += _as_int(output_tokens)
+    usage.cache_read_input_tokens += _as_int(cache_read_input_tokens)
+    usage.cache_creation_input_tokens += _as_int(cache_creation_input_tokens)
+    usage.duration_ms += float(duration_ms) if duration_ms and duration_ms > 0 else 0.0
+    usage.requests += 1
+
+
+def get_session_usage() -> SessionUsage:
+    """Return a copy of the current session usage."""
+    return deepcopy(_SESSION_USAGE)
+
+
+def reset_session_usage() -> None:
+    """Clear all recorded usage (primarily for tests)."""
+    global _SESSION_USAGE
+    _SESSION_USAGE = SessionUsage()
+
+
+__all__ = [
+    "ModelUsage",
+    "SessionUsage",
+    "get_session_usage",
+    "record_usage",
+    "reset_session_usage",
+]

ripperdoc/utils/shell_token_utils.py

@@ -0,0 +1,95 @@
+"""Shell token parsing utilities."""
+
+from __future__ import annotations
+
+import re
+import shlex
+from typing import Iterable, List
+
+# Operators and redirections that should not be treated as executable tokens.
+SHELL_OPERATORS_WITH_REDIRECTION: set[str] = {
+    "|",
+    "||",
+    "&&",
+    ";",
+    ">",
+    ">>",
+    "<",
+    "<<",
+    "2>",
+    "&>",
+    "2>&1",
+    "|&",
+}
+
+_REDIRECTION_PATTERNS = (
+    re.compile(r"^\d?>?&\d+$"),  # 2>&1, >&2, etc.
+    re.compile(r"^\d?>/dev/null$"),  # 2>/dev/null, >/dev/null
+    re.compile(r"^/dev/null$"),
+)
+
+
+def parse_shell_tokens(shell_command: str) -> List[str]:
+    """Parse a shell command into tokens, preserving operators for inspection."""
+    if not shell_command:
+        return []
+
+    lexer = shlex.shlex(shell_command, posix=True)
+    lexer.whitespace_split = True
+    lexer.commenters = ""
+
+    try:
+        return list(lexer)
+    except ValueError:
+        # Fall back to a coarse split to avoid hard failures.
+        return shell_command.split()
+
+
+def filter_valid_tokens(tokens: Iterable[str]) -> list[str]:
+    """Remove shell control operators and redirection tokens."""
+    return [token for token in tokens if token not in SHELL_OPERATORS_WITH_REDIRECTION]
+
+
+def _is_redirection_token(token: str) -> bool:
+    return any(pattern.match(token) for pattern in _REDIRECTION_PATTERNS)
+
+
+def parse_and_clean_shell_tokens(raw_shell_string: str) -> List[str]:
+    """Parse tokens and strip benign redirections to mirror reference cleaning."""
+    tokens = parse_shell_tokens(raw_shell_string)
+    if not tokens:
+        return []
+
+    cleaned: list[str] = []
+    skip_next = False
+
+    for idx, token in enumerate(tokens):
+        if skip_next:
+            skip_next = False
+            continue
+
+        # Handle explicit redirection operators that are followed by a target.
+        if token in {">&", ">", "1>", "2>", ">>"}:
+            if idx + 1 < len(tokens):
+                next_token = tokens[idx + 1]
+                if _is_redirection_token(next_token):
+                    skip_next = True
+                    continue
+            cleaned.append(token)
+            continue
+
+        # Skip inlined redirection tokens to /dev/null or file descriptors.
+        if _is_redirection_token(token):
+            continue
+
+        cleaned.append(token)
+
+    return filter_valid_tokens(cleaned)
+
+
+__all__ = [
+    "parse_shell_tokens",
+    "parse_and_clean_shell_tokens",
+    "filter_valid_tokens",
+    "SHELL_OPERATORS_WITH_REDIRECTION",
+]

ripperdoc/utils/todo.py

@@ -0,0 +1,199 @@
+"""Todo storage and utilities for Ripperdoc.
+
+This module provides simple, file-based todo management so tools can
+persist and query tasks between turns. Todos are stored under the user's
+home directory at `~/.ripperdoc/todos/<project>/todos.json`, where
+`<project>` is a sanitized form of the project path.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import List, Literal, Optional, Sequence, Tuple
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from ripperdoc.utils.log import get_logger
+from ripperdoc.utils.path_utils import project_storage_dir
+
+
+logger = get_logger()
+
+TodoStatus = Literal["pending", "in_progress", "completed"]
+TodoPriority = Literal["high", "medium", "low"]
+
+
+class TodoItem(BaseModel):
+    """Represents a single todo entry."""
+
+    id: str = Field(description="Unique identifier for the todo item")
+    content: str = Field(description="Task description")
+    status: TodoStatus = Field(
+        default="pending", description="Current state: pending, in_progress, completed"
+    )
+    priority: TodoPriority = Field(default="medium", description="Priority: high|medium|low")
+    created_at: Optional[float] = Field(default=None, description="Unix timestamp when created")
+    updated_at: Optional[float] = Field(default=None, description="Unix timestamp when updated")
+    previous_status: Optional[TodoStatus] = Field(
+        default=None, description="Previous status, used for audits"
+    )
+    model_config = ConfigDict(extra="ignore")
+
+
+MAX_TODOS = 200
+
+
+def _storage_path(project_root: Optional[Path], ensure_dir: bool) -> Path:
+    """Return the todo storage path, optionally ensuring the directory exists."""
+    root = project_root or Path.cwd()
+    base_dir = Path.home() / ".ripperdoc" / "todos"
+    storage_dir = project_storage_dir(base_dir, root, ensure=ensure_dir)
+    return storage_dir / "todos.json"
+
+
+def validate_todos(
+    todos: Sequence[TodoItem], max_items: int = MAX_TODOS
+) -> Tuple[bool, str | None]:
+    """Basic validation for a todo list."""
+    if len(todos) > max_items:
+        return False, f"Too many todos; limit is {max_items}."
+
+    ids = [todo.id for todo in todos]
+    duplicate_ids = {id_ for id_ in ids if ids.count(id_) > 1}
+    if duplicate_ids:
+        return False, f"Duplicate todo IDs found: {sorted(duplicate_ids)}"
+
+    in_progress = [todo for todo in todos if todo.status == "in_progress"]
+    if len(in_progress) > 1:
+        return False, "Only one todo can be marked in_progress at a time."
+
+    empty_contents = [todo.id for todo in todos if not todo.content.strip()]
+    if empty_contents:
+        return False, f"Todos require content. Empty content for IDs: {sorted(empty_contents)}"
+
+    return True, None
+
+
+def load_todos(project_root: Optional[Path] = None) -> List[TodoItem]:
+    """Load todos from disk."""
+    path = _storage_path(project_root, ensure_dir=False)
+    if not path.exists():
+        return []
+
+    try:
+        raw = json.loads(path.read_text())
+    except Exception as exc:
+        logger.error(f"Failed to load todos from {path}: {exc}")
+        return []
+
+    todos: List[TodoItem] = []
+    for item in raw:
+        try:
+            todos.append(TodoItem(**item))
+        except ValidationError as exc:
+            logger.error(f"Failed to parse todo item: {exc}")
+            continue
+
+    # Preserve stored order; do not reorder based on status/priority.
+    return todos
+
+
+def save_todos(todos: Sequence[TodoItem], project_root: Optional[Path] = None) -> None:
+    """Persist todos to disk."""
+    path = _storage_path(project_root, ensure_dir=True)
+    path.write_text(json.dumps([todo.model_dump() for todo in todos], indent=2))
+
+
+def set_todos(
+    todos: Sequence[TodoItem],
+    project_root: Optional[Path] = None,
+) -> List[TodoItem]:
+    """Validate, normalize, and persist the provided todos."""
+    ok, message = validate_todos(todos)
+    if not ok:
+        raise ValueError(message or "Invalid todos.")
+
+    existing = {todo.id: todo for todo in load_todos(project_root)}
+    now = time.time()
+
+    normalized: List[TodoItem] = []
+    for todo in todos:
+        previous = existing.get(todo.id)
+        normalized.append(
+            todo.model_copy(
+                update={
+                    "created_at": previous.created_at if previous else todo.created_at or now,
+                    "updated_at": now,
+                    "previous_status": (
+                        previous.status
+                        if previous and previous.status != todo.status
+                        else todo.previous_status
+                    ),
+                }
+            )
+        )
+
+    # Keep the caller-provided order; do not resort.
+    save_todos(normalized, project_root)
+    return list(normalized)
+
+
+def clear_todos(project_root: Optional[Path] = None) -> None:
+    """Remove all todos."""
+    save_todos([], project_root)
+
+
+def get_next_actionable(todos: Sequence[TodoItem]) -> Optional[TodoItem]:
+    """Return the next todo to work on (in_progress first, then pending)."""
+    for status in ("in_progress", "pending"):
+        for todo in todos:
+            if todo.status == status:
+                return todo
+    return None
+
+
+def summarize_todos(todos: Sequence[TodoItem]) -> dict:
+    """Return simple statistics for a todo collection."""
+    return {
+        "total": len(todos),
+        "by_status": {
+            "pending": len([t for t in todos if t.status == "pending"]),
+            "in_progress": len([t for t in todos if t.status == "in_progress"]),
+            "completed": len([t for t in todos if t.status == "completed"]),
+        },
+        "by_priority": {
+            "high": len([t for t in todos if t.priority == "high"]),
+            "medium": len([t for t in todos if t.priority == "medium"]),
+            "low": len([t for t in todos if t.priority == "low"]),
+        },
+    }
+
+
+def format_todo_summary(todos: Sequence[TodoItem]) -> str:
+    """Create a concise summary string for use in tool outputs."""
+    stats = summarize_todos(todos)
+    summary = (
+        f"Todos updated (total {stats['total']}; "
+        f"{stats['by_status']['pending']} pending, "
+        f"{stats['by_status']['in_progress']} in progress, "
+        f"{stats['by_status']['completed']} completed)."
+    )
+
+    next_item = get_next_actionable(todos)
+    if next_item:
+        summary += f" Next to tackle: {next_item.content} (id: {next_item.id}, status: {next_item.status})."
+    elif stats["total"] == 0:
+        summary += " No todos stored yet."
+
+    return summary
+
+
+def format_todo_lines(todos: Sequence[TodoItem]) -> List[str]:
+    """Return human-readable todo lines."""
+    status_marker = {
+        "completed": "●",
+        "in_progress": "◐",
+        "pending": "○",
+    }
+    return [f"{status_marker.get(todo.status, '○')} {todo.content}" for todo in todos]