gobby 0.2.8__py3-none-any.whl → 0.2.11__py3-none-any.whl

gobby/storage/artifacts.py

@@ -283,3 +283,22 @@ class LocalArtifactManager:
 
         rows = self.db.fetchall(sql, tuple(params))
         return [Artifact.from_row(row) for row in rows]
+
+    def count_artifacts(self, session_id: str | None = None) -> int:
+        """Count total artifacts, optionally filtered by session.
+
+        Args:
+            session_id: Optional session ID to filter by
+
+        Returns:
+            Total artifact count
+        """
+        if session_id:
+            row = self.db.fetchone(
+                "SELECT COUNT(*) FROM session_artifacts WHERE session_id = ?",
+                (session_id,),
+            )
+        else:
+            row = self.db.fetchone("SELECT COUNT(*) FROM session_artifacts")
+
+        return row[0] if row else 0

gobby/storage/memories.py

@@ -145,11 +145,11 @@ class LocalMemoryManager:
 
         now = datetime.now(UTC).isoformat()
         # Normalize content for consistent ID generation (avoid duplicates from
-        # whitespace differences or project_id inconsistency)
+        # whitespace differences)
         normalized_content = content.strip()
-        project_str = project_id if project_id else ""
-        # Use delimiter to prevent collisions (e.g., "abc" + "def" vs "abcd" + "ef")
-        memory_id = generate_prefixed_id("mm", f"{normalized_content}||{project_str}")
+        # Global dedup: ID based on content only (project_id stored but not in ID)
+        # This aligns with content_exists() which checks globally
+        memory_id = generate_prefixed_id("mm", normalized_content)
 
         # Check if memory already exists to avoid duplicate insert errors
         existing_row = self.db.fetchone("SELECT * FROM memories WHERE id = ?", (memory_id,))

gobby/storage/migrations.py

@@ -43,11 +43,12 @@ class MigrationUnsupportedError(Exception):
 # Migration can be SQL string or a callable that takes LocalDatabase
 MigrationAction = str | Callable[[LocalDatabase], None]
 
-# Baseline version - the schema state at v78 (flattened)
+# Baseline version - the schema state at v81 (flattened)
 # This is applied for new databases directly
-BASELINE_VERSION = 78
+# Note: Migrations >= BASELINE_VERSION still run for existing databases
+BASELINE_VERSION = 81
 
-# Baseline schema - flattened from v78 production state, includes hub tracking fields
+# Baseline schema - flattened from v81 production state, includes all migrations
 # This is applied for new databases directly
 # Generated by: sqlite3 ~/.gobby/gobby-hub.db .schema
 BASELINE_SCHEMA = """
@@ -220,6 +221,7 @@ CREATE TABLE sessions (
     agent_depth INTEGER DEFAULT 0,
     spawned_by_agent_id TEXT,
     workflow_name TEXT,
+    step_variables TEXT,
     agent_run_id TEXT REFERENCES agent_runs(id) ON DELETE SET NULL,
     context_injected INTEGER DEFAULT 0,
     original_prompt TEXT,
@@ -587,6 +589,8 @@ CREATE TABLE skills (
     hub_slug TEXT,
     hub_version TEXT,
     enabled INTEGER DEFAULT 1,
+    always_apply INTEGER DEFAULT 0,
+    injection_format TEXT DEFAULT 'summary',
     project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
     created_at TEXT NOT NULL,
     updated_at TEXT NOT NULL
@@ -594,6 +598,7 @@ CREATE TABLE skills (
 CREATE INDEX idx_skills_name ON skills(name);
 CREATE INDEX idx_skills_project_id ON skills(project_id);
 CREATE INDEX idx_skills_enabled ON skills(enabled);
+CREATE INDEX idx_skills_always_apply ON skills(always_apply);
 CREATE UNIQUE INDEX idx_skills_name_project ON skills(name, project_id);
 CREATE UNIQUE INDEX idx_skills_name_global ON skills(name) WHERE project_id IS NULL;
 
@@ -617,6 +622,42 @@ CREATE INDEX idx_clones_status ON clones(status);
 CREATE INDEX idx_clones_task ON clones(task_id);
 CREATE INDEX idx_clones_session ON clones(agent_session_id);
 CREATE UNIQUE INDEX idx_clones_path ON clones(clone_path);
+
+CREATE TABLE pipeline_executions (
+    id TEXT PRIMARY KEY,
+    pipeline_name TEXT NOT NULL,
+    project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+    status TEXT NOT NULL DEFAULT 'pending',
+    inputs_json TEXT,
+    outputs_json TEXT,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+    completed_at TEXT,
+    resume_token TEXT UNIQUE,
+    session_id TEXT REFERENCES sessions(id) ON DELETE SET NULL,
+    parent_execution_id TEXT REFERENCES pipeline_executions(id) ON DELETE CASCADE
+);
+CREATE INDEX idx_pipeline_executions_project ON pipeline_executions(project_id);
+CREATE INDEX idx_pipeline_executions_status ON pipeline_executions(status);
+CREATE INDEX idx_pipeline_executions_resume_token ON pipeline_executions(resume_token);
+
+CREATE TABLE step_executions (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    execution_id TEXT NOT NULL REFERENCES pipeline_executions(id) ON DELETE CASCADE,
+    step_id TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'pending',
+    started_at TEXT,
+    completed_at TEXT,
+    input_json TEXT,
+    output_json TEXT,
+    error TEXT,
+    approval_token TEXT UNIQUE,
+    approved_by TEXT,
+    approved_at TEXT,
+    UNIQUE(execution_id, step_id)
+);
+CREATE INDEX idx_step_executions_execution ON step_executions(execution_id);
+CREATE INDEX idx_step_executions_approval_token ON step_executions(approval_token);
 """
 
 # Future migrations (v61+)
@@ -709,6 +750,24 @@ def _migrate_add_hub_tracking_to_skills(db: LocalDatabase) -> None:
     logger.info("Added hub tracking fields to skills table")
 
 
+def _migrate_add_skill_injection_columns(db: LocalDatabase) -> None:
+    """Add always_apply and injection_format columns to skills table.
+
+    These columns enable per-skill control over:
+    - always_apply: Whether skill should always be injected at session start
+    - injection_format: How to inject the skill (summary, full, content)
+
+    The values are extracted from SKILL.md frontmatter during sync and stored
+    as columns for efficient querying.
+    """
+    with db.transaction() as conn:
+        conn.execute("ALTER TABLE skills ADD COLUMN always_apply INTEGER DEFAULT 0")
+        conn.execute("ALTER TABLE skills ADD COLUMN injection_format TEXT DEFAULT 'summary'")
+        conn.execute("CREATE INDEX idx_skills_always_apply ON skills(always_apply)")
+
+    logger.info("Added always_apply and injection_format columns to skills table")
+
+
 MIGRATIONS: list[tuple[int, str, MigrationAction]] = [
     # Project-scoped session refs: Change seq_num index from global to project-scoped
     (76, "Make sessions.seq_num project-scoped", _migrate_session_seq_num_project_scoped),
@@ -716,6 +775,62 @@ MIGRATIONS: list[tuple[int, str, MigrationAction]] = [
     (77, "Backfill sessions.seq_num per project", _migrate_backfill_session_seq_num_per_project),
     # Hub tracking: Add hub_name, hub_slug, hub_version to skills table
     (78, "Add hub tracking fields to skills", _migrate_add_hub_tracking_to_skills),
+    # Skill injection: Add always_apply and injection_format columns
+    (79, "Add skill injection columns", _migrate_add_skill_injection_columns),
+    # Pipeline system: Add pipeline_executions and step_executions tables
+    (
+        80,
+        "Add pipeline execution tables",
+        """
+        CREATE TABLE IF NOT EXISTS pipeline_executions (
+            id TEXT PRIMARY KEY,
+            pipeline_name TEXT NOT NULL,
+            project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+            status TEXT NOT NULL DEFAULT 'pending',
+            inputs_json TEXT,
+            outputs_json TEXT,
+            created_at TEXT NOT NULL DEFAULT (datetime('now')),
+            updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+            completed_at TEXT,
+            resume_token TEXT UNIQUE,
+            session_id TEXT REFERENCES sessions(id) ON DELETE SET NULL,
+            parent_execution_id TEXT REFERENCES pipeline_executions(id) ON DELETE CASCADE
+        );
+        CREATE INDEX IF NOT EXISTS idx_pipeline_executions_project ON pipeline_executions(project_id);
+        CREATE INDEX IF NOT EXISTS idx_pipeline_executions_status ON pipeline_executions(status);
+        CREATE INDEX IF NOT EXISTS idx_pipeline_executions_resume_token ON pipeline_executions(resume_token);
+
+        CREATE TABLE IF NOT EXISTS step_executions (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            execution_id TEXT NOT NULL REFERENCES pipeline_executions(id) ON DELETE CASCADE,
+            step_id TEXT NOT NULL,
+            status TEXT NOT NULL DEFAULT 'pending',
+            started_at TEXT,
+            completed_at TEXT,
+            input_json TEXT,
+            output_json TEXT,
+            error TEXT,
+            approval_token TEXT UNIQUE,
+            approved_by TEXT,
+            approved_at TEXT,
+            UNIQUE(execution_id, step_id)
+        );
+        CREATE INDEX IF NOT EXISTS idx_step_executions_execution ON step_executions(execution_id);
+        CREATE INDEX IF NOT EXISTS idx_step_executions_approval_token ON step_executions(approval_token);
+        """,
+    ),
+    # Add step_variables JSON column to sessions for spawn-time variable passing
+    (
+        81,
+        "Add step_variables to sessions",
+        "ALTER TABLE sessions ADD COLUMN step_variables TEXT",
+    ),
+    # Rename task status 'review' to 'needs_review' for clarity
+    (
+        82,
+        "Rename task status 'review' to 'needs_review'",
+        "UPDATE tasks SET status = 'needs_review' WHERE status = 'review'",
+    ),
 ]
 
 

gobby/storage/pipelines.py

@@ -0,0 +1,367 @@
+"""Local pipeline execution storage manager."""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from gobby.storage.database import DatabaseProtocol
+from gobby.utils.id import generate_prefixed_id
+from gobby.workflows.pipeline_state import (
+    ExecutionStatus,
+    PipelineExecution,
+    StepExecution,
+    StepStatus,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class LocalPipelineExecutionManager:
+    """Manager for local pipeline execution storage."""
+
+    def __init__(self, db: DatabaseProtocol, project_id: str):
+        """Initialize with database connection and project context.
+
+        Args:
+            db: Database connection
+            project_id: Project ID for scoping executions
+        """
+        self.db = db
+        self.project_id = project_id
+
+    def create_execution(
+        self,
+        pipeline_name: str,
+        inputs_json: str | None = None,
+        session_id: str | None = None,
+        parent_execution_id: str | None = None,
+    ) -> PipelineExecution:
+        """Create a new pipeline execution.
+
+        Args:
+            pipeline_name: Name of the pipeline being executed
+            inputs_json: JSON string of input parameters
+            session_id: Session that triggered the execution
+            parent_execution_id: Parent execution for nested pipelines
+
+        Returns:
+            Created PipelineExecution instance
+        """
+        execution_id = generate_prefixed_id("pe")
+        now = datetime.now(UTC).isoformat()
+
+        with self.db.transaction():
+            self.db.execute(
+                """
+                INSERT INTO pipeline_executions (
+                    id, pipeline_name, project_id, status, inputs_json,
+                    session_id, parent_execution_id, created_at, updated_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    execution_id,
+                    pipeline_name,
+                    self.project_id,
+                    ExecutionStatus.PENDING.value,
+                    inputs_json,
+                    session_id,
+                    parent_execution_id,
+                    now,
+                    now,
+                ),
+            )
+
+        execution = self.get_execution(execution_id)
+        if execution is None:
+            raise RuntimeError(f"Execution {execution_id} not found after creation")
+        return execution
+
+    def get_execution(self, execution_id: str) -> PipelineExecution | None:
+        """Get execution by ID.
+
+        Args:
+            execution_id: Execution UUID
+
+        Returns:
+            PipelineExecution or None if not found
+        """
+        row = self.db.fetchone(
+            "SELECT * FROM pipeline_executions WHERE id = ?",
+            (execution_id,),
+        )
+        return PipelineExecution.from_row(row) if row else None
+
+    def update_execution_status(
+        self,
+        execution_id: str,
+        status: ExecutionStatus,
+        resume_token: str | None = None,
+        outputs_json: str | None = None,
+    ) -> PipelineExecution | None:
+        """Update execution status.
+
+        Args:
+            execution_id: Execution UUID
+            status: New status
+            resume_token: Resume token for approval gates
+            outputs_json: JSON string of outputs (for completed status)
+
+        Returns:
+            Updated PipelineExecution or None if not found
+        """
+        now = datetime.now(UTC).isoformat()
+        completed_at = now if status == ExecutionStatus.COMPLETED else None
+
+        self.db.execute(
+            """
+            UPDATE pipeline_executions
+            SET status = ?, resume_token = ?, outputs_json = ?,
+                completed_at = COALESCE(?, completed_at), updated_at = ?
+            WHERE id = ?
+            """,
+            (
+                status.value,
+                resume_token,
+                outputs_json,
+                completed_at,
+                now,
+                execution_id,
+            ),
+        )
+
+        return self.get_execution(execution_id)
+
+    def list_executions(
+        self,
+        status: ExecutionStatus | None = None,
+        pipeline_name: str | None = None,
+        limit: int = 50,
+    ) -> list[PipelineExecution]:
+        """List executions for the project.
+
+        Args:
+            status: Filter by status
+            pipeline_name: Filter by pipeline name
+            limit: Maximum number of results
+
+        Returns:
+            List of PipelineExecution instances
+        """
+        query = "SELECT * FROM pipeline_executions WHERE project_id = ?"
+        params: list[Any] = [self.project_id]
+
+        if status is not None:
+            query += " AND status = ?"
+            params.append(status.value)
+
+        if pipeline_name is not None:
+            query += " AND pipeline_name = ?"
+            params.append(pipeline_name)
+
+        query += " ORDER BY created_at DESC LIMIT ?"
+        params.append(limit)
+
+        rows = self.db.fetchall(query, tuple(params))
+        return [PipelineExecution.from_row(row) for row in rows]
+
+    def get_execution_by_resume_token(self, token: str) -> PipelineExecution | None:
+        """Get execution by resume token.
+
+        Args:
+            token: Resume token
+
+        Returns:
+            PipelineExecution or None if not found
+        """
+        row = self.db.fetchone(
+            "SELECT * FROM pipeline_executions WHERE resume_token = ?",
+            (token,),
+        )
+        return PipelineExecution.from_row(row) if row else None
+
+    def resolve_execution_reference(self, ref: str) -> str:
+        """Resolve an execution reference to a UUID.
+
+        Supports:
+        - Full UUID: pe-abc123... or UUID format
+        - UUID prefix: pe-abc1 (matches by prefix)
+
+        Args:
+            ref: Execution reference
+
+        Returns:
+            Execution UUID
+
+        Raises:
+            ValueError: If reference cannot be resolved
+        """
+        # Try exact match first
+        execution = self.get_execution(ref)
+        if execution:
+            return execution.id
+
+        # Try prefix match
+        row = self.db.fetchone(
+            "SELECT id FROM pipeline_executions WHERE id LIKE ? AND project_id = ?",
+            (f"{ref}%", self.project_id),
+        )
+        if row:
+            result: str = row["id"]
+            return result
+
+        raise ValueError(f"Cannot resolve execution reference: {ref}")
+
+    # Step execution methods
+
+    def create_step_execution(
+        self,
+        execution_id: str,
+        step_id: str,
+        input_json: str | None = None,
+    ) -> StepExecution:
+        """Create a new step execution.
+
+        Args:
+            execution_id: Parent pipeline execution ID
+            step_id: Step ID from pipeline definition
+            input_json: JSON string of step input
+
+        Returns:
+            Created StepExecution instance
+        """
+        self.db.execute(
+            """
+            INSERT INTO step_executions (
+                execution_id, step_id, status, input_json
+            )
+            VALUES (?, ?, ?, ?)
+            """,
+            (
+                execution_id,
+                step_id,
+                StepStatus.PENDING.value,
+                input_json,
+            ),
+        )
+
+        # Get the created step by execution_id and step_id (unique combination)
+        row = self.db.fetchone(
+            "SELECT * FROM step_executions WHERE execution_id = ? AND step_id = ?",
+            (execution_id, step_id),
+        )
+        if row is None:
+            raise RuntimeError(f"Step {step_id} not found after creation")
+        return StepExecution.from_row(row)
+
+    def update_step_execution(
+        self,
+        step_execution_id: int,
+        status: StepStatus | None = None,
+        output_json: str | None = None,
+        error: str | None = None,
+        approval_token: str | None = None,
+        approved_by: str | None = None,
+    ) -> StepExecution | None:
+        """Update a step execution.
+
+        Args:
+            step_execution_id: Step execution ID (integer)
+            status: New status
+            output_json: JSON string of step output
+            error: Error message (for failed status)
+            approval_token: Token for approval gate
+            approved_by: Who approved the step
+
+        Returns:
+            Updated StepExecution or None if not found
+        """
+        now = datetime.now(UTC).isoformat()
+
+        # Build update parts dynamically (step_executions has no updated_at column)
+        updates: list[str] = []
+        params: list[Any] = []
+
+        if status is not None:
+            updates.append("status = ?")
+            params.append(status.value)
+            # Set timestamps based on status
+            if status == StepStatus.RUNNING:
+                updates.append("started_at = COALESCE(started_at, ?)")
+                params.append(now)
+            elif status in (StepStatus.COMPLETED, StepStatus.FAILED, StepStatus.SKIPPED):
+                updates.append("completed_at = COALESCE(completed_at, ?)")
+                params.append(now)
+
+        if output_json is not None:
+            updates.append("output_json = ?")
+            params.append(output_json)
+
+        if error is not None:
+            updates.append("error = ?")
+            params.append(error)
+
+        if approval_token is not None:
+            updates.append("approval_token = ?")
+            params.append(approval_token)
+
+        if approved_by is not None:
+            updates.append("approved_by = ?")
+            params.append(approved_by)
+            updates.append("approved_at = ?")
+            params.append(now)
+
+        if not updates:
+            # Nothing to update
+            row = self.db.fetchone(
+                "SELECT * FROM step_executions WHERE id = ?",
+                (step_execution_id,),
+            )
+            return StepExecution.from_row(row) if row else None
+
+        # Append step_execution_id for WHERE clause
+        params.append(step_execution_id)
+
+        # updates list contains only hardcoded column names, values are parameterized
+        self.db.execute(
+            f"UPDATE step_executions SET {', '.join(updates)} WHERE id = ?",  # nosec B608
+            tuple(params),
+        )
+
+        row = self.db.fetchone(
+            "SELECT * FROM step_executions WHERE id = ?",
+            (step_execution_id,),
+        )
+        return StepExecution.from_row(row) if row else None
+
+    def get_step_by_approval_token(self, token: str) -> StepExecution | None:
+        """Get step execution by approval token.
+
+        Args:
+            token: Approval token
+
+        Returns:
+            StepExecution or None if not found
+        """
+        row = self.db.fetchone(
+            "SELECT * FROM step_executions WHERE approval_token = ?",
+            (token,),
+        )
+        return StepExecution.from_row(row) if row else None
+
+    def get_steps_for_execution(self, execution_id: str) -> list[StepExecution]:
+        """Get all steps for an execution.
+
+        Args:
+            execution_id: Pipeline execution ID
+
+        Returns:
+            List of StepExecution instances
+        """
+        rows = self.db.fetchall(
+            "SELECT * FROM step_executions WHERE execution_id = ? ORDER BY id",
+            (execution_id,),
+        )
+        return [StepExecution.from_row(row) for row in rows]

gobby/storage/sessions.py

@@ -38,6 +38,7 @@ class Session:
     spawned_by_agent_id: str | None = None  # ID of agent that spawned this session
     # Terminal pickup metadata fields
     workflow_name: str | None = None  # Workflow to activate on terminal pickup
+    step_variables: dict[str, Any] | None = None  # Variables for workflow activation
     agent_run_id: str | None = None  # Link back to agent run record
     context_injected: bool = False  # Whether context was injected into prompt
     original_prompt: str | None = None  # Original prompt for terminal mode
@@ -77,6 +78,7 @@ class Session:
             agent_depth=row["agent_depth"] or 0,
             spawned_by_agent_id=row["spawned_by_agent_id"],
             workflow_name=row["workflow_name"],
+            step_variables=cls._parse_json_field(row, "step_variables"),
             agent_run_id=row["agent_run_id"],
             context_injected=bool(row["context_injected"]),
             original_prompt=row["original_prompt"],
@@ -110,6 +112,21 @@ class Session:
             logger.warning("Failed to parse terminal_context JSON, returning None")
             return None
 
+    @classmethod
+    def _parse_json_field(cls, row: Any, field_name: str) -> dict[str, Any] | None:
+        """Parse a JSON field from a database row, returning None on missing/malformed data."""
+        if field_name not in row.keys():
+            return None
+        raw = row[field_name]
+        if not raw:
+            return None
+        try:
+            result: dict[str, Any] = json.loads(raw)
+            return result
+        except json.JSONDecodeError:
+            logger.warning(f"Failed to parse {field_name} JSON, returning None")
+            return None
+
     def to_dict(self) -> dict[str, Any]:
         """Convert to dictionary."""
         return {
@@ -167,6 +184,7 @@ class LocalSessionManager:
         spawned_by_agent_id: str | None = None,
         terminal_context: dict[str, Any] | None = None,
         workflow_name: str | None = None,
+        step_variables: dict[str, Any] | None = None,
     ) -> Session:
         """
         Register a new session or return existing one.
@@ -176,9 +194,9 @@ class LocalSessionManager:
         returns the existing session. Otherwise creates a new one.
 
         Args:
-            external_id: External session identifier (e.g., Claude Code's session ID)
+            external_id: External session identifier (e.g., Claude Code session ID)
             machine_id: Machine identifier
-            source: CLI source (claude_code, codex, gemini)
+            source: CLI source (claude, gemini, codex, cursor, windsurf, copilot)
             project_id: Project ID (required - sessions must belong to a project)
             title: Optional session title
             jsonl_path: Path to transcript file
@@ -242,9 +260,9 @@ class LocalSessionManager:
                         id, external_id, machine_id, source, project_id, title,
                         jsonl_path, git_branch, parent_session_id,
                         agent_depth, spawned_by_agent_id, terminal_context,
-                        workflow_name, status, created_at, updated_at, seq_num, had_edits
+                        workflow_name, step_variables, status, created_at, updated_at, seq_num, had_edits
                     )
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, 0)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, 0)
                     """,
                     (
                         session_id,
@@ -260,6 +278,7 @@ class LocalSessionManager:
                         spawned_by_agent_id,
                         json.dumps(terminal_context) if terminal_context else None,
                         workflow_name,
+                        json.dumps(step_variables) if step_variables else None,
                         now,
                         now,
                         next_seq_num,