tsugite-cli 0.3.3__py3-none-any.whl

tsugite/history/__init__.py

@@ -0,0 +1,29 @@
+"""Conversation history management."""
+
+from .index import (
+    get_conversation_metadata,
+    query_index,
+    rebuild_index,
+    update_index,
+)
+from .models import ConversationMetadata, IndexEntry, Turn
+from .storage import (
+    generate_conversation_id,
+    get_history_dir,
+    load_conversation,
+    save_turn_to_history,
+)
+
+__all__ = [
+    "ConversationMetadata",
+    "IndexEntry",
+    "Turn",
+    "generate_conversation_id",
+    "get_conversation_metadata",
+    "get_history_dir",
+    "load_conversation",
+    "query_index",
+    "rebuild_index",
+    "save_turn_to_history",
+    "update_index",
+]

tsugite/history/index.py

@@ -0,0 +1,210 @@
+"""JSON index for fast conversation metadata lookup."""
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from .models import ConversationMetadata, IndexEntry, Turn
+from .storage import get_history_dir, list_conversation_files, load_conversation
+
+
+def _get_index_path() -> Path:
+    """Get path to index.json file.
+
+    Returns:
+        Path to index file
+    """
+    return get_history_dir() / "index.json"
+
+
+def load_index() -> Dict[str, IndexEntry]:
+    """Load conversation index from JSON file.
+
+    Returns:
+        Dictionary mapping conversation IDs to IndexEntry models
+        Empty dict if index doesn't exist
+    """
+    index_path = _get_index_path()
+
+    if not index_path.exists():
+        return {}
+
+    try:
+        with open(index_path, "r", encoding="utf-8") as f:
+            raw_index = json.load(f)
+
+        return {conv_id: IndexEntry.model_validate(metadata) for conv_id, metadata in raw_index.items()}
+
+    except (json.JSONDecodeError, IOError):
+        # Index corrupted, return empty dict
+        # Will be rebuilt on next update
+        return {}
+
+
+def save_index(index: Dict[str, IndexEntry]) -> None:
+    """Save conversation index to JSON file.
+
+    Args:
+        index: Index data to save (IndexEntry models)
+
+    Raises:
+        RuntimeError: If save fails
+    """
+    index_path = _get_index_path()
+
+    # Ensure directory exists
+    index_path.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        serializable_index = {
+            conv_id: entry.model_dump(mode="json", exclude_none=True) for conv_id, entry in index.items()
+        }
+
+        with open(index_path, "w", encoding="utf-8") as f:
+            json.dump(serializable_index, f, indent=2, ensure_ascii=False)
+    except IOError as e:
+        raise RuntimeError(f"Failed to save index to {index_path}: {e}")
+
+
+def update_index(conversation_id: str, metadata: IndexEntry) -> None:
+    """Update index entry for a conversation.
+
+    Creates or updates the index entry with provided metadata.
+    Preserves created_at timestamp for existing entries.
+
+    Args:
+        conversation_id: Conversation ID
+        metadata: IndexEntry model with metadata
+
+    Raises:
+        RuntimeError: If save fails
+    """
+    index = load_index()
+
+    metadata_dict = metadata.model_dump(mode="json")
+
+    # Preserve created_at if entry exists
+    if conversation_id in index:
+        metadata_dict["created_at"] = index[conversation_id].created_at.isoformat()
+
+    # Ensure updated_at is set
+    if "updated_at" not in metadata_dict:
+        metadata_dict["updated_at"] = datetime.now(timezone.utc).isoformat()
+
+    index[conversation_id] = IndexEntry.model_validate(metadata_dict)
+    save_index(index)
+
+
+def rebuild_index() -> int:
+    """Rebuild index from all conversation files.
+
+    Scans all JSONL files and rebuilds the index from scratch.
+    Useful for recovering from index corruption or manual file changes.
+
+    Returns:
+        Number of conversations indexed
+
+    Raises:
+        RuntimeError: If rebuild fails
+    """
+    conversation_files = list_conversation_files()
+    new_index = {}
+
+    for file_path in conversation_files:
+        conversation_id = file_path.stem  # Filename without .jsonl extension
+
+        try:
+            # Load conversation and extract metadata
+            records = load_conversation(conversation_id)
+
+            if not records:
+                continue
+
+            first_record = records[0]
+            last_record = records[-1]
+
+            # Build metadata from records
+            turns = [r for r in records if isinstance(r, Turn)]
+            metadata_dict = {
+                "agent": first_record.agent if isinstance(first_record, ConversationMetadata) else "unknown",
+                "model": first_record.model if isinstance(first_record, ConversationMetadata) else "unknown",
+                "machine": first_record.machine if isinstance(first_record, ConversationMetadata) else "unknown",
+                "created_at": (
+                    first_record.timestamp.isoformat()
+                    if hasattr(first_record, "timestamp")
+                    else datetime.now(timezone.utc).isoformat()
+                ),
+                "updated_at": (
+                    last_record.timestamp.isoformat()
+                    if hasattr(last_record, "timestamp")
+                    else datetime.now(timezone.utc).isoformat()
+                ),
+                "turn_count": len(turns),
+                "total_tokens": sum(r.tokens or 0 for r in turns),
+                "total_cost": sum(r.cost or 0.0 for r in turns),
+            }
+
+            new_index[conversation_id] = IndexEntry.model_validate(metadata_dict)
+
+        except Exception as e:
+            # Skip files that can't be read
+            print(f"Warning: Failed to index {conversation_id}: {e}")
+            continue
+
+    save_index(new_index)
+    return len(new_index)
+
+
+def query_index(
+    machine: Optional[str] = None,
+    agent: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> List[Dict[str, str]]:
+    """Query conversation index with filters.
+
+    Args:
+        machine: Filter by machine name
+        agent: Filter by agent name
+        limit: Maximum number of results
+
+    Returns:
+        List of conversation metadata dicts (sorted by updated_at, newest first)
+    """
+    index = load_index()
+
+    # Convert to list of dicts with conversation_id included
+    results = []
+    for conv_id, entry in index.items():
+        entry_dict = entry.model_dump(mode="json")
+        entry_dict["conversation_id"] = conv_id
+        results.append(entry_dict)
+
+    # Apply filters
+    if machine:
+        results = [r for r in results if r.get("machine") == machine]
+
+    if agent:
+        results = [r for r in results if r.get("agent") == agent]
+
+    # Sort by updated_at (newest first)
+    results.sort(key=lambda r: r.get("updated_at", ""), reverse=True)
+
+    # Apply limit
+    if limit:
+        results = results[:limit]
+
+    return results
+
+
+def get_conversation_metadata(conversation_id: str) -> Optional[IndexEntry]:
+    """Get metadata for a specific conversation from index.
+
+    Args:
+        conversation_id: Conversation ID
+
+    Returns:
+        IndexEntry model if found, None otherwise
+    """
+    index = load_index()
+    return index.get(conversation_id)

tsugite/history/models.py

@@ -0,0 +1,106 @@
+"""Pydantic models for history and conversation data structures."""
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+class ConversationMetadata(BaseModel):
+    """Metadata for a conversation session.
+
+    This is the first record in each conversation JSONL file, containing
+    basic information about the conversation.
+    """
+
+    model_config = ConfigDict(
+        extra="allow",  # Allow extra fields for backward/forward compatibility
+        str_strip_whitespace=True,  # Auto-strip whitespace from strings
+    )
+
+    type: str = Field(default="metadata", description="Record type identifier")
+    id: str = Field(..., description="Unique conversation identifier")
+    agent: str = Field(..., description="Agent name used in this conversation")
+    model: str = Field(..., description="Model identifier (provider:model format)")
+    machine: str = Field(..., description="Hostname/machine name where conversation occurred")
+    created_at: datetime = Field(..., description="Conversation creation timestamp")
+    timestamp: Optional[datetime] = Field(default=None, description="Alias for created_at (for backward compatibility)")
+
+    @field_validator("created_at", "timestamp", mode="before")
+    @classmethod
+    def parse_datetime(cls, v):
+        """Parse ISO format strings to datetime objects."""
+        if v is None:
+            return None
+        if isinstance(v, str):
+            return datetime.fromisoformat(v.replace("Z", "+00:00"))
+        return v
+
+    @model_validator(mode="after")
+    def set_timestamp_default(self):
+        """Set timestamp to created_at if not provided."""
+        if self.timestamp is None:
+            self.timestamp = self.created_at
+        return self
+
+
+class Turn(BaseModel):
+    """A single conversation turn (user message + assistant response).
+
+    Represents one complete interaction in the conversation, including
+    the user's input, assistant's response, and metadata about tool usage
+    and cost.
+    """
+
+    model_config = ConfigDict(
+        extra="allow",  # Allow extra fields for future extensions
+        str_strip_whitespace=True,
+    )
+
+    type: str = Field(default="turn", description="Record type identifier")
+    timestamp: datetime = Field(..., description="When this turn occurred")
+    user: str = Field(..., description="User's input message")
+    assistant: str = Field(..., description="Assistant's response")
+    tools: list[str] = Field(default_factory=list, description="Tools used in this turn")
+    tokens: Optional[int] = Field(default=None, description="Total tokens used in this turn")
+    cost: Optional[float] = Field(default=None, description="Estimated cost for this turn")
+    steps: Optional[list[dict]] = Field(default=None, description="Detailed execution steps (thought/code/output)")
+    messages: Optional[list[dict]] = Field(default=None, description="Full LiteLLM message history for this turn")
+
+    @field_validator("timestamp", mode="before")
+    @classmethod
+    def parse_datetime(cls, v):
+        """Parse ISO format strings to datetime objects."""
+        if isinstance(v, str):
+            return datetime.fromisoformat(v.replace("Z", "+00:00"))
+        return v
+
+
+class IndexEntry(BaseModel):
+    """Entry in the conversation index for fast lookups.
+
+    The index maps conversation IDs to summary metadata, allowing quick
+    queries without reading entire JSONL files.
+    """
+
+    model_config = ConfigDict(
+        extra="allow",  # Allow extra fields
+        str_strip_whitespace=True,
+    )
+
+    agent: str = Field(..., description="Agent name")
+    model: str = Field(..., description="Model identifier")
+    machine: str = Field(..., description="Hostname/machine name")
+    created_at: datetime = Field(..., description="When conversation was created")
+    updated_at: datetime = Field(..., description="Last update timestamp")
+    turn_count: int = Field(default=0, description="Total number of turns in conversation")
+    total_tokens: Optional[int] = Field(default=None, description="Cumulative token count")
+    total_cost: Optional[float] = Field(default=None, description="Cumulative cost")
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def parse_datetime(cls, v):
+        """Parse ISO format strings to datetime objects."""
+        if isinstance(v, str):
+            return datetime.fromisoformat(v.replace("Z", "+00:00"))
+        return v

tsugite/history/storage.py

@@ -0,0 +1,157 @@
+"""JSONL-based conversation history storage."""
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Optional, Union
+
+from tsugite.xdg import get_xdg_data_path
+
+from .models import ConversationMetadata, Turn
+
+
+def get_history_dir() -> Path:
+    """Get path to conversation history directory.
+
+    Returns:
+        Path to history directory in XDG data location
+        (~/.local/share/tsugite/history/)
+    """
+    return get_xdg_data_path("history")
+
+
+def generate_conversation_id(agent_name: str, timestamp: Optional[datetime] = None) -> str:
+    """Generate unique conversation ID.
+
+    Format: YYYYMMDD_HHMMSS_{agent}_{hash}
+    Example: 20251024_103000_chat_abc123
+
+    Args:
+        agent_name: Name of the agent
+        timestamp: Optional timestamp (defaults to now)
+
+    Returns:
+        Unique conversation ID
+    """
+    if timestamp is None:
+        timestamp = datetime.now(timezone.utc)
+
+    # Format: YYYYMMDD_HHMMSS
+    date_str = timestamp.strftime("%Y%m%d_%H%M%S")
+
+    # Clean agent name (remove special chars)
+    clean_agent = "".join(c if c.isalnum() or c == "-" else "_" for c in agent_name)
+    clean_agent = clean_agent[:20]  # Limit length
+
+    # Generate short hash from timestamp + agent name
+    hash_input = f"{timestamp.isoformat()}_{agent_name}".encode()
+    hash_str = hashlib.sha256(hash_input).hexdigest()[:6]
+
+    return f"{date_str}_{clean_agent}_{hash_str}"
+
+
+def _get_conversation_path(conversation_id: str) -> Path:
+    """Get path to conversation JSONL file.
+
+    Args:
+        conversation_id: Conversation ID
+
+    Returns:
+        Path to JSONL file
+    """
+    history_dir = get_history_dir()
+    return history_dir / f"{conversation_id}.jsonl"
+
+
+def save_turn_to_history(conversation_id: str, turn_data: Union[Turn, ConversationMetadata]) -> None:
+    """Append a turn to conversation history.
+
+    Creates the history directory and file if they don't exist.
+    Appends turn as JSONL line.
+
+    Args:
+        conversation_id: Conversation ID
+        turn_data: Turn or ConversationMetadata model to append
+
+    Raises:
+        RuntimeError: If write fails
+    """
+    conversation_path = _get_conversation_path(conversation_id)
+
+    # Ensure directory exists
+    conversation_path.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        with open(conversation_path, "a", encoding="utf-8") as f:
+            f.write(turn_data.model_dump_json(exclude_none=True))
+            f.write("\n")
+    except IOError as e:
+        raise RuntimeError(f"Failed to save turn to {conversation_path}: {e}")
+
+
+def load_conversation(conversation_id: str) -> List[Union[ConversationMetadata, Turn]]:
+    """Load full conversation from JSONL file.
+
+    Args:
+        conversation_id: Conversation ID
+
+    Returns:
+        List of ConversationMetadata/Turn models
+
+    Raises:
+        FileNotFoundError: If conversation doesn't exist
+        RuntimeError: If read fails
+    """
+    conversation_path = _get_conversation_path(conversation_id)
+
+    if not conversation_path.exists():
+        raise FileNotFoundError(f"Conversation not found: {conversation_id}")
+
+    records = []
+    try:
+        with open(conversation_path, "r", encoding="utf-8") as f:
+            for line_num, line in enumerate(f, 1):
+                line = line.strip()
+                if not line:
+                    continue  # Skip empty lines
+
+                try:
+                    data = json.loads(line)
+                    record_type = data.get("type")
+
+                    if record_type == "metadata":
+                        records.append(ConversationMetadata.model_validate(data))
+                    elif record_type == "turn":
+                        records.append(Turn.model_validate(data))
+                    else:
+                        print(f"Warning: Unknown record type '{record_type}' at line {line_num}")
+
+                except json.JSONDecodeError as e:
+                    # Skip malformed lines but log warning
+                    print(f"Warning: Skipping malformed JSON at line {line_num} in {conversation_id}: {e}")
+                    continue
+
+        return records
+    except IOError as e:
+        raise RuntimeError(f"Failed to load conversation {conversation_id}: {e}")
+
+
+def list_conversation_files() -> List[Path]:
+    """List all conversation JSONL files.
+
+    Returns:
+        List of conversation file paths (sorted by modification time, newest first)
+    """
+    history_dir = get_history_dir()
+
+    if not history_dir.exists():
+        return []
+
+    try:
+        files = list(history_dir.glob("*.jsonl"))
+        # Sort by modification time (newest first)
+        files.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+        return files
+    except OSError:
+        return []