emdash-core 0.1.7__py3-none-any.whl → 0.1.25__py3-none-any.whl

emdash_core/checkpoint/manager.py

@@ -0,0 +1,231 @@
+"""Checkpoint manager for orchestrating git-based checkpoints."""
+
+import hashlib
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+from uuid import uuid4
+
+from ..utils.logger import log
+from .models import CheckpointMetadata, ConversationState
+from .git_operations import GitCheckpointOperations
+from .storage import CheckpointStorage
+
+
+class CheckpointManager:
+    """Manages git-based checkpoints for agent sessions.
+
+    Creates checkpoint commits after each agentic loop completes,
+    storing file changes in git and conversation state in files.
+
+    Example:
+        manager = CheckpointManager(repo_root=Path("."))
+
+        # After agent completes a run
+        checkpoint = manager.create_checkpoint(
+            messages=runner._messages,
+            model=runner.model,
+            system_prompt=runner.system_prompt,
+            tools_used=["read_file", "write_to_file"],
+            token_usage={"input": 1000, "output": 500},
+        )
+
+        # List checkpoints
+        for cp in manager.list_checkpoints():
+            print(f"{cp.id}: {cp.summary}")
+
+        # Restore to checkpoint
+        conv = manager.restore_checkpoint("cp_abc123_001")
+    """
+
+    def __init__(
+        self,
+        repo_root: Path,
+        session_id: Optional[str] = None,
+        enabled: bool = True,
+    ):
+        """Initialize checkpoint manager.
+
+        Args:
+            repo_root: Root of the git repository
+            session_id: Optional session ID (auto-generated if None)
+            enabled: Whether checkpointing is enabled
+        """
+        self.repo_root = repo_root.resolve()
+        self.session_id = session_id or str(uuid4())[:8]
+        self.enabled = enabled
+        self.iteration = 0
+
+        self.git_ops = GitCheckpointOperations(repo_root)
+        self.storage = CheckpointStorage(repo_root)
+
+    def create_checkpoint(
+        self,
+        messages: list[dict],
+        model: str,
+        system_prompt: str,
+        tools_used: list[str],
+        token_usage: dict[str, int],
+        summary: Optional[str] = None,
+    ) -> Optional[CheckpointMetadata]:
+        """Create a checkpoint after successful agent run.
+
+        Creates a git commit with all file changes and saves
+        conversation state for later restoration.
+
+        Args:
+            messages: Full conversation message history
+            model: Model used for the run
+            system_prompt: System prompt used
+            tools_used: List of tool names used during the run
+            token_usage: Token usage stats (input, output, thinking)
+            summary: Optional human-readable summary
+
+        Returns:
+            CheckpointMetadata if checkpoint was created, None if no changes
+        """
+        if not self.enabled:
+            log.debug("Checkpointing disabled, skipping")
+            return None
+
+        # Check for changes
+        if not self.git_ops.has_changes():
+            log.debug("No changes to checkpoint")
+            return None
+
+        self.iteration += 1
+        checkpoint_id = f"cp_{self.session_id}_{self.iteration:03d}"
+        timestamp = datetime.now().isoformat()
+
+        # Generate summary if not provided
+        if not summary:
+            summary = self._generate_summary(messages)
+
+        # Get list of modified files
+        files_modified = self.git_ops.get_modified_files()
+
+        # Build metadata
+        metadata = CheckpointMetadata(
+            id=checkpoint_id,
+            session_id=self.session_id,
+            iteration=self.iteration,
+            timestamp=timestamp,
+            summary=summary,
+            tools_used=tools_used,
+            files_modified=files_modified,
+            token_usage=token_usage,
+        )
+
+        # Save conversation state first (so it's included in the commit)
+        conv_state = ConversationState(
+            messages=messages,
+            model=model,
+            system_prompt_hash=hashlib.sha256(
+                system_prompt.encode()
+            ).hexdigest()[:16],
+            token_usage=token_usage,
+        )
+        self.storage.save_conversation(checkpoint_id, self.session_id, conv_state)
+
+        # Create git commit (includes the conversation file)
+        commit_sha = self.git_ops.create_checkpoint_commit(metadata, summary)
+        metadata.commit_sha = commit_sha
+
+        # Update index
+        self.storage.update_index(metadata)
+
+        log.info(f"Created checkpoint {checkpoint_id} at {commit_sha[:8]}")
+        return metadata
+
+    def _generate_summary(self, messages: list[dict]) -> str:
+        """Generate a brief summary from messages.
+
+        Extracts the last user query and summarizes the response.
+
+        Args:
+            messages: Conversation messages
+
+        Returns:
+            Brief summary string
+        """
+        # Find last user message
+        user_query = ""
+        for msg in reversed(messages):
+            if msg.get("role") == "user":
+                content = msg.get("content", "")
+                if isinstance(content, str) and not content.startswith("[SYSTEM"):
+                    user_query = content[:100]
+                    if len(content) > 100:
+                        user_query += "..."
+                    break
+
+        if user_query:
+            return f"Response to: {user_query}"
+        return "Agent checkpoint"
+
+    def restore_checkpoint(
+        self,
+        checkpoint_id: str,
+        restore_conversation: bool = True,
+        create_branch: bool = True,
+    ) -> Optional[ConversationState]:
+        """Restore to a checkpoint.
+
+        Checks out the git commit and optionally loads conversation state.
+
+        Args:
+            checkpoint_id: ID of checkpoint to restore
+            restore_conversation: Whether to load conversation state
+            create_branch: Whether to create a branch at the restored state
+
+        Returns:
+            ConversationState if restore_conversation=True, None otherwise
+
+        Raises:
+            ValueError: If checkpoint not found
+        """
+        # Find checkpoint in index
+        metadata = self.storage.find_checkpoint(checkpoint_id)
+        if not metadata:
+            raise ValueError(f"Checkpoint not found: {checkpoint_id}")
+
+        if not metadata.commit_sha:
+            raise ValueError(f"Checkpoint {checkpoint_id} has no commit SHA")
+
+        # Git checkout
+        self.git_ops.restore_to_commit(metadata.commit_sha, create_branch)
+
+        # Load conversation if requested
+        if restore_conversation:
+            return self.storage.load_conversation(
+                checkpoint_id,
+                metadata.session_id,
+            )
+        return None
+
+    def list_checkpoints(
+        self,
+        session_id: Optional[str] = None,
+        limit: int = 50,
+    ) -> list[CheckpointMetadata]:
+        """List checkpoints.
+
+        Args:
+            session_id: Filter by session ID (optional)
+            limit: Maximum number of checkpoints
+
+        Returns:
+            List of CheckpointMetadata, most recent first
+        """
+        return self.storage.get_checkpoints(session_id, limit)
+
+    def get_checkpoint(self, checkpoint_id: str) -> Optional[CheckpointMetadata]:
+        """Get a specific checkpoint by ID.
+
+        Args:
+            checkpoint_id: Checkpoint ID
+
+        Returns:
+            CheckpointMetadata if found, None otherwise
+        """
+        return self.storage.find_checkpoint(checkpoint_id)

emdash_core/checkpoint/models.py

@@ -0,0 +1,107 @@
+"""Data models for checkpoint system."""
+
+from dataclasses import dataclass, field, asdict
+from typing import Any, Optional
+
+
+@dataclass
+class CheckpointMetadata:
+    """Metadata stored in commit message and index.
+
+    This is the lightweight metadata that can be parsed from
+    commit messages and stored in the index for fast lookup.
+    """
+    id: str
+    session_id: str
+    iteration: int
+    timestamp: str
+    commit_sha: Optional[str] = None
+    summary: str = ""
+    tools_used: list[str] = field(default_factory=list)
+    files_modified: list[str] = field(default_factory=list)
+    token_usage: dict[str, int] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "CheckpointMetadata":
+        """Create from dictionary."""
+        return cls(**data)
+
+
+@dataclass
+class ConversationState:
+    """Full conversation state for restoration.
+
+    This is the complete state needed to restore an agent
+    session, including all messages and context.
+    """
+    messages: list[dict]
+    model: str
+    system_prompt_hash: str
+    token_usage: dict[str, int] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "version": "1.0",
+            "messages": self.messages,
+            "model": self.model,
+            "system_prompt_hash": self.system_prompt_hash,
+            "token_usage": self.token_usage,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ConversationState":
+        """Create from dictionary."""
+        return cls(
+            messages=data.get("messages", []),
+            model=data.get("model", ""),
+            system_prompt_hash=data.get("system_prompt_hash", ""),
+            token_usage=data.get("token_usage", {}),
+        )
+
+
+@dataclass
+class CheckpointIndex:
+    """Index of all checkpoints for fast lookup.
+
+    Stored at .emdash/checkpoints/index.json
+    """
+    version: str = "1.0"
+    checkpoints: list[CheckpointMetadata] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "version": self.version,
+            "checkpoints": [cp.to_dict() for cp in self.checkpoints],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "CheckpointIndex":
+        """Create from dictionary."""
+        return cls(
+            version=data.get("version", "1.0"),
+            checkpoints=[
+                CheckpointMetadata.from_dict(cp)
+                for cp in data.get("checkpoints", [])
+            ],
+        )
+
+    def add(self, metadata: CheckpointMetadata) -> None:
+        """Add a checkpoint to the index."""
+        self.checkpoints.insert(0, metadata)  # Most recent first
+
+    def find(self, checkpoint_id: str) -> Optional[CheckpointMetadata]:
+        """Find a checkpoint by ID."""
+        for cp in self.checkpoints:
+            if cp.id == checkpoint_id:
+                return cp
+        return None
+
+    def find_by_session(self, session_id: str) -> list[CheckpointMetadata]:
+        """Find all checkpoints for a session."""
+        return [cp for cp in self.checkpoints if cp.session_id == session_id]

emdash_core/checkpoint/storage.py

@@ -0,0 +1,201 @@
+"""File storage for checkpoint data."""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from ..utils.logger import log
+from .models import CheckpointIndex, CheckpointMetadata, ConversationState
+
+
+class CheckpointStorage:
+    """Handles checkpoint file storage.
+
+    Stores conversation state and checkpoint index in:
+        .emdash/checkpoints/
+            index.json                           # Checkpoint index
+            {session_id}/{checkpoint_id}/
+                conversation.json                # Full conversation state
+    """
+
+    CHECKPOINT_DIR = ".emdash/checkpoints"
+    INDEX_FILE = "index.json"
+
+    def __init__(self, repo_root: Path):
+        """Initialize checkpoint storage.
+
+        Args:
+            repo_root: Root of the repository
+        """
+        self.repo_root = repo_root.resolve()
+        self.checkpoint_dir = self.repo_root / self.CHECKPOINT_DIR
+
+    def _ensure_dir(self, path: Path) -> None:
+        """Ensure directory exists."""
+        path.mkdir(parents=True, exist_ok=True)
+
+    def _get_checkpoint_path(self, session_id: str, checkpoint_id: str) -> Path:
+        """Get path for a checkpoint's data directory."""
+        return self.checkpoint_dir / session_id / checkpoint_id
+
+    def save_conversation(
+        self,
+        checkpoint_id: str,
+        session_id: str,
+        state: ConversationState,
+    ) -> Path:
+        """Save conversation state to checkpoint directory.
+
+        Args:
+            checkpoint_id: Unique checkpoint ID
+            session_id: Session ID
+            state: Conversation state to save
+
+        Returns:
+            Path to saved file
+        """
+        checkpoint_path = self._get_checkpoint_path(session_id, checkpoint_id)
+        self._ensure_dir(checkpoint_path)
+
+        file_path = checkpoint_path / "conversation.json"
+        with open(file_path, "w") as f:
+            json.dump(state.to_dict(), f, indent=2)
+
+        log.debug(f"Saved conversation to {file_path}")
+        return file_path
+
+    def load_conversation(
+        self,
+        checkpoint_id: str,
+        session_id: str,
+    ) -> Optional[ConversationState]:
+        """Load conversation state from checkpoint.
+
+        Args:
+            checkpoint_id: Unique checkpoint ID
+            session_id: Session ID
+
+        Returns:
+            ConversationState if found, None otherwise
+        """
+        checkpoint_path = self._get_checkpoint_path(session_id, checkpoint_id)
+        file_path = checkpoint_path / "conversation.json"
+
+        if not file_path.exists():
+            log.warning(f"Conversation file not found: {file_path}")
+            return None
+
+        with open(file_path) as f:
+            data = json.load(f)
+            return ConversationState.from_dict(data)
+
+    def get_index_path(self) -> Path:
+        """Get path to index file."""
+        return self.checkpoint_dir / self.INDEX_FILE
+
+    def load_index(self) -> CheckpointIndex:
+        """Load checkpoint index.
+
+        Returns:
+            CheckpointIndex, empty if file doesn't exist
+        """
+        index_path = self.get_index_path()
+        if not index_path.exists():
+            return CheckpointIndex()
+
+        try:
+            with open(index_path) as f:
+                data = json.load(f)
+                return CheckpointIndex.from_dict(data)
+        except (json.JSONDecodeError, KeyError) as e:
+            log.warning(f"Failed to load checkpoint index: {e}")
+            return CheckpointIndex()
+
+    def save_index(self, index: CheckpointIndex) -> None:
+        """Save checkpoint index.
+
+        Args:
+            index: CheckpointIndex to save
+        """
+        self._ensure_dir(self.checkpoint_dir)
+        index_path = self.get_index_path()
+
+        with open(index_path, "w") as f:
+            json.dump(index.to_dict(), f, indent=2)
+
+        log.debug(f"Saved checkpoint index to {index_path}")
+
+    def update_index(self, metadata: CheckpointMetadata) -> None:
+        """Add a checkpoint to the index.
+
+        Args:
+            metadata: Checkpoint metadata to add
+        """
+        index = self.load_index()
+        index.add(metadata)
+        self.save_index(index)
+
+    def get_checkpoints(
+        self,
+        session_id: Optional[str] = None,
+        limit: int = 50,
+    ) -> list[CheckpointMetadata]:
+        """Get checkpoints from index.
+
+        Args:
+            session_id: Filter by session ID (optional)
+            limit: Maximum number of checkpoints to return
+
+        Returns:
+            List of CheckpointMetadata, most recent first
+        """
+        index = self.load_index()
+
+        if session_id:
+            checkpoints = index.find_by_session(session_id)
+        else:
+            checkpoints = index.checkpoints
+
+        return checkpoints[:limit]
+
+    def find_checkpoint(self, checkpoint_id: str) -> Optional[CheckpointMetadata]:
+        """Find a checkpoint by ID.
+
+        Args:
+            checkpoint_id: Checkpoint ID to find
+
+        Returns:
+            CheckpointMetadata if found, None otherwise
+        """
+        index = self.load_index()
+        return index.find(checkpoint_id)
+
+    def delete_checkpoint(self, checkpoint_id: str, session_id: str) -> bool:
+        """Delete a checkpoint's data.
+
+        Note: This only deletes the conversation data, not the git commit.
+
+        Args:
+            checkpoint_id: Checkpoint ID to delete
+            session_id: Session ID
+
+        Returns:
+            True if deleted, False if not found
+        """
+        checkpoint_path = self._get_checkpoint_path(session_id, checkpoint_id)
+
+        if not checkpoint_path.exists():
+            return False
+
+        import shutil
+        shutil.rmtree(checkpoint_path)
+
+        # Update index
+        index = self.load_index()
+        index.checkpoints = [
+            cp for cp in index.checkpoints if cp.id != checkpoint_id
+        ]
+        self.save_index(index)
+
+        log.info(f"Deleted checkpoint {checkpoint_id}")
+        return True

emdash_core/config.py

@@ -36,7 +36,7 @@ class ServerConfig(BaseSettings):
 
     # Agent settings
     default_model: str = Field(default_factory=_get_default_model, description="Default LLM model")
-    max_iterations: int = Field(default=50, description="Max agent iterations")
+    max_iterations: int = Field(default=100, description="Max agent iterations")
     context_threshold: float = Field(default=0.6, description="Context window threshold for summarization")
 
     # SSE settings

emdash_core/core/config.py

@@ -268,7 +268,7 @@ class AgentConfig(BaseModel):
     )
 
     max_iterations: int = Field(
-        default=50,
+        default=100,
         ge=10,
         le=200,
         description="Maximum tool call iterations before stopping",
@@ -281,13 +281,29 @@ class AgentConfig(BaseModel):
         description="Maximum tokens for tool output (estimated at ~4 chars/token)",
     )
 
+    context_compact_threshold: float = Field(
+        default=0.8,
+        ge=0.5,
+        le=0.95,
+        description="Trigger context compaction at this % of model's context limit",
+    )
+
+    context_compact_target: float = Field(
+        default=0.5,
+        ge=0.3,
+        le=0.7,
+        description="Target context size after compaction (% of model's limit)",
+    )
+
     @classmethod
     def from_env(cls) -> "AgentConfig":
         """Load configuration from environment variables."""
         return cls(
             max_context_messages=int(os.getenv("EMDASH_MAX_CONTEXT_MESSAGES", "25")),
-            max_iterations=int(os.getenv("EMDASH_MAX_ITERATIONS", "50")),
+            max_iterations=int(os.getenv("EMDASH_MAX_ITERATIONS", "100")),
             tool_max_output_tokens=int(os.getenv("EMDASH_TOOL_MAX_OUTPUT", "25000")),
+            context_compact_threshold=float(os.getenv("EMDASH_CONTEXT_COMPACT_THRESHOLD", "0.8")),
+            context_compact_target=float(os.getenv("EMDASH_CONTEXT_COMPACT_TARGET", "0.5")),
         )
 
 

emdash_core/graph/schema.py

@@ -101,8 +101,8 @@ class SchemaManager:
                 PRIMARY KEY (url)
             )
         """,
-        "GitCommit": """
-            CREATE NODE TABLE GitCommit (
+        "Commit": """
+            CREATE NODE TABLE Commit (
                 sha STRING,
                 message STRING,
                 timestamp TIMESTAMP,
@@ -202,12 +202,12 @@ class SchemaManager:
         """,
         "AUTHORED_BY": """
             CREATE REL TABLE AUTHORED_BY (
-                FROM GitCommit TO Author
+                FROM Commit TO Author
             )
         """,
         "COMMIT_MODIFIES": """
             CREATE REL TABLE COMMIT_MODIFIES (
-                FROM GitCommit TO File,
+                FROM Commit TO File,
                 change_type STRING,
                 insertions INT64,
                 deletions INT64,
@@ -216,7 +216,7 @@ class SchemaManager:
         """,
         "PR_CONTAINS": """
             CREATE REL TABLE PR_CONTAINS (
-                FROM PullRequest TO GitCommit
+                FROM PullRequest TO Commit
             )
         """,
         "PR_MODIFIES": """