ralphx 0.3.4__py3-none-any.whl → 0.3.5__py3-none-any.whl

ralphx/core/project_db.py

@@ -14,7 +14,8 @@ This makes projects portable - clone a repo with .ralphx/ and all data comes wit
 """
 
 import json
-import re
+import logging
+import shutil
 import sqlite3
 import threading
 from contextlib import contextmanager
@@ -22,30 +23,11 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Iterator, Optional
 
+logger = logging.getLogger(__name__)
 
-# Schema version for project DB
-PROJECT_SCHEMA_VERSION = 15
-
-# Namespace validation pattern: lowercase, alphanumeric, underscores, dashes, max 64 chars
-# Must start with a letter
-NAMESPACE_PATTERN = re.compile(r'^[a-z][a-z0-9_-]{0,63}$')
-
-
-def validate_namespace(namespace: str) -> bool:
-    """Validate namespace format.
-
-    Namespaces must:
-    - Start with a lowercase letter
-    - Contain only lowercase letters, digits, underscores, and dashes
-    - Be 1-64 characters long
-
-    Args:
-        namespace: The namespace string to validate.
 
-    Returns:
-        True if valid, False otherwise.
-    """
-    return bool(NAMESPACE_PATTERN.match(namespace))
+# Schema version for project DB
+PROJECT_SCHEMA_VERSION = 16
 
 # Project database schema - all project-specific data
 PROJECT_SCHEMA_SQL = """
@@ -297,7 +279,6 @@ CREATE TABLE IF NOT EXISTS workflows (
     id TEXT PRIMARY KEY,
     template_id TEXT,                       -- Optional reference to template
     name TEXT NOT NULL,
-    namespace TEXT NOT NULL,                -- Workflow identifier
     status TEXT DEFAULT 'draft',            -- draft, active, paused, completed
     current_step INTEGER DEFAULT 1,
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -467,7 +448,6 @@ CREATE INDEX IF NOT EXISTS idx_loop_resources_type ON loop_resources(resource_ty
 
 -- Workflow indexes
 CREATE INDEX IF NOT EXISTS idx_workflows_status ON workflows(status);
-CREATE INDEX IF NOT EXISTS idx_workflows_namespace ON workflows(namespace);
 CREATE INDEX IF NOT EXISTS idx_workflow_steps_workflow ON workflow_steps(workflow_id, step_number);
 CREATE INDEX IF NOT EXISTS idx_workflow_steps_status ON workflow_steps(status);
 CREATE INDEX IF NOT EXISTS idx_planning_sessions_workflow ON planning_sessions(workflow_id);
@@ -598,9 +578,29 @@ class ProjectDatabase:
                     (PROJECT_SCHEMA_VERSION,),
                 )
             elif current_version < PROJECT_SCHEMA_VERSION:
+                # Create backup before running migrations
+                self._backup_before_migration(current_version)
                 # Run migrations (for future versions > 6)
                 self._run_migrations(conn, current_version)
 
+    def _backup_before_migration(self, from_version: int) -> None:
+        """Create a backup of the database before running migrations.
+
+        Creates a timestamped backup file in the same directory as the database.
+        This allows recovery if a migration fails or causes data loss.
+
+        Args:
+            from_version: Current schema version before migration.
+        """
+        try:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            backup_path = self.db_path.with_suffix(f".v{from_version}.{timestamp}.bak")
+            shutil.copy2(self.db_path, backup_path)
+            logger.info(f"Created database backup before migration: {backup_path}")
+        except Exception as e:
+            logger.warning(f"Failed to create backup before migration: {e}")
+            # Don't fail the migration if backup fails - just warn
+
     def _run_migrations(self, conn: sqlite3.Connection, from_version: int) -> None:
         """Run schema migrations from a version to the latest.
 
@@ -658,6 +658,11 @@ class ProjectDatabase:
         # Migration from v14 to v15: Add workflow_resource_versions table
         if from_version == 14:
             self._migrate_v14_to_v15(conn)
+            from_version = 15  # Continue to next migration
+
+        # Migration from v15 to v16: Remove namespace from workflows table
+        if from_version == 15:
+            self._migrate_v15_to_v16(conn)
 
         # Seed workflow templates for fresh databases
         self._seed_workflow_templates(conn)
@@ -960,6 +965,55 @@ class ProjectDatabase:
             ON workflow_resource_versions(workflow_resource_id, version_number DESC)
         """)
 
+    def _migrate_v15_to_v16(self, conn: sqlite3.Connection) -> None:
+        """Migrate from schema v15 to v16.
+
+        Removes:
+        - namespace column from workflows table (deprecated, replaced by workflow_id)
+        - idx_workflows_namespace index
+
+        SQLite doesn't support DROP COLUMN directly, so we recreate the table.
+
+        IMPORTANT: We must disable foreign keys before dropping the old table,
+        otherwise the ON DELETE CASCADE on workflow_steps will delete all steps!
+        """
+        # 0. Disable foreign keys to prevent CASCADE deletes during table swap
+        conn.execute("PRAGMA foreign_keys=OFF")
+
+        # 1. Create new table without namespace
+        conn.execute("""
+            CREATE TABLE workflows_new (
+                id TEXT PRIMARY KEY,
+                template_id TEXT,
+                name TEXT NOT NULL,
+                status TEXT DEFAULT 'draft',
+                current_step INTEGER DEFAULT 1,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                archived_at TIMESTAMP
+            )
+        """)
+
+        # 2. Copy data (excluding namespace)
+        conn.execute("""
+            INSERT INTO workflows_new (id, template_id, name, status, current_step, created_at, updated_at, archived_at)
+            SELECT id, template_id, name, status, current_step, created_at, updated_at, archived_at
+            FROM workflows
+        """)
+
+        # 3. Drop old table and index
+        conn.execute("DROP INDEX IF EXISTS idx_workflows_namespace")
+        conn.execute("DROP TABLE workflows")
+
+        # 4. Rename new table
+        conn.execute("ALTER TABLE workflows_new RENAME TO workflows")
+
+        # 5. Recreate the status index on the new table
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_workflows_status ON workflows(status)")
+
+        # 6. Re-enable foreign keys
+        conn.execute("PRAGMA foreign_keys=ON")
+
     # ========== Loops ==========
 
     def create_loop(
@@ -1808,9 +1862,8 @@ class ProjectDatabase:
         """Release all claims held by a specific loop.
 
         Used when deleting a loop to prevent orphaned claims.
-        Items with a namespace set are restored to 'completed' status so they
-        can be picked up by consumer loops again.
-        Items without a namespace are restored to 'pending'.
+        Released items are restored to 'pending' status so they can be
+        picked up by other loops.
 
         Args:
             loop_name: Name of the loop whose claims should be released.
@@ -1826,7 +1879,7 @@ class ProjectDatabase:
                 UPDATE work_items
                 SET claimed_by = NULL,
                     claimed_at = NULL,
-                    status = CASE WHEN namespace IS NOT NULL THEN 'completed' ELSE 'pending' END,
+                    status = 'pending',
                     updated_at = ?
                 WHERE claimed_by = ? AND status = 'claimed'
                 """,
@@ -1840,9 +1893,8 @@ class ProjectDatabase:
         This is an atomic operation that checks ownership and releases in one step
         to prevent TOCTOU race conditions.
 
-        Items with a namespace set are restored to 'completed' status so they
-        can be picked up by consumer loops again.
-        Items without a namespace are restored to 'pending'.
+        Released items are restored to 'pending' status so they can be
+        picked up by other loops.
 
         Args:
             id: Work item ID.
@@ -1858,7 +1910,7 @@ class ProjectDatabase:
                 UPDATE work_items
                 SET claimed_by = NULL,
                     claimed_at = NULL,
-                    status = CASE WHEN namespace IS NOT NULL THEN 'completed' ELSE 'pending' END,
+                    status = 'pending',
                     updated_at = ?
                 WHERE id = ? AND claimed_by = ? AND status = 'claimed'
                 """,
@@ -3093,11 +3145,12 @@ class ProjectDatabase:
         now = datetime.utcnow().isoformat()
 
         # Build Product workflow template
+        # Uses processing_type to reference PROCESSING_TYPES in mcp/tools/workflows.py
         build_product_phases = json.dumps([
             {
                 "number": 1,
-                "name": "Planning",
-                "type": "interactive",
+                "name": "Design Document",
+                "processing_type": "design_doc",
                 "description": "Describe what you want to build. Claude will help create a design document.",
                 "outputs": ["design_doc", "guardrails"],
                 "skippable": True,
@@ -3105,10 +3158,9 @@ class ProjectDatabase:
             },
             {
                 "number": 2,
-                "name": "Story Generation",
-                "type": "autonomous",
-                "loopType": "generator",
-                "description": "Claude generates detailed user stories from the design document.",
+                "name": "Story Generation (Extract)",
+                "processing_type": "extractgen_requirements",
+                "description": "Claude extracts user stories from the design document.",
                 "inputs": ["design_doc", "guardrails"],
                 "outputs": ["stories"],
                 "skippable": True,
@@ -3116,9 +3168,18 @@ class ProjectDatabase:
             },
             {
                 "number": 3,
+                "name": "Story Generation (Web)",
+                "processing_type": "webgen_requirements",
+                "description": "Claude discovers additional requirements via web research.",
+                "inputs": ["design_doc", "guardrails", "stories"],
+                "outputs": ["stories"],
+                "skippable": True,
+                "skipCondition": "Skip web research"
+            },
+            {
+                "number": 4,
                 "name": "Implementation",
-                "type": "autonomous",
-                "loopType": "consumer",
+                "processing_type": "implementation",
                 "description": "Claude implements each story, committing code to git.",
                 "inputs": ["stories", "design_doc", "guardrails"],
                 "outputs": ["code"],
@@ -3138,23 +3199,31 @@ class ProjectDatabase:
             ),
         )
 
-        # From Design Doc workflow - skips planning, starts with story generation
+        # From Design Doc workflow - skips design doc, starts with story generation
         from_design_doc_phases = json.dumps([
             {
                 "number": 1,
-                "name": "Story Generation",
-                "type": "autonomous",
-                "loopType": "generator",
-                "description": "Claude generates detailed user stories from your design document.",
+                "name": "Story Generation (Extract)",
+                "processing_type": "extractgen_requirements",
+                "description": "Claude extracts user stories from your design document.",
                 "inputs": ["design_doc"],
                 "outputs": ["stories"],
                 "skippable": False
             },
             {
                 "number": 2,
+                "name": "Story Generation (Web)",
+                "processing_type": "webgen_requirements",
+                "description": "Claude discovers additional requirements via web research.",
+                "inputs": ["design_doc", "stories"],
+                "outputs": ["stories"],
+                "skippable": True,
+                "skipCondition": "Skip web research"
+            },
+            {
+                "number": 3,
                 "name": "Implementation",
-                "type": "autonomous",
-                "loopType": "consumer",
+                "processing_type": "implementation",
                 "description": "Claude implements each story, committing code to git.",
                 "inputs": ["stories", "design_doc"],
                 "outputs": ["code"],
@@ -3179,8 +3248,7 @@ class ProjectDatabase:
             {
                 "number": 1,
                 "name": "Implementation",
-                "type": "autonomous",
-                "loopType": "consumer",
+                "processing_type": "implementation",
                 "description": "Claude implements each story, committing code to git.",
                 "inputs": ["stories"],
                 "outputs": ["code"],
@@ -3200,12 +3268,12 @@ class ProjectDatabase:
             ),
         )
 
-        # Planning Only workflow - just the interactive planning step
+        # Design Doc Only workflow - just the interactive design doc step
         planning_only_phases = json.dumps([
             {
                 "number": 1,
-                "name": "Planning",
-                "type": "interactive",
+                "name": "Design Document",
+                "processing_type": "design_doc",
                 "description": "Collaborate with Claude to create a comprehensive design document.",
                 "outputs": ["design_doc", "guardrails"],
                 "skippable": False
@@ -3273,7 +3341,6 @@ class ProjectDatabase:
         self,
         id: str,
         name: str,
-        namespace: str,
         template_id: Optional[str] = None,
         status: str = "draft",
     ) -> dict:
@@ -3282,29 +3349,19 @@ class ProjectDatabase:
         Args:
             id: Unique workflow identifier.
             name: User-facing workflow name.
-            namespace: Namespace to link all phases.
             template_id: Optional template ID this workflow is based on.
             status: Initial status (default: draft).
 
         Returns:
             The created workflow dict.
-
-        Raises:
-            ValueError: If namespace is invalid.
         """
-        if not validate_namespace(namespace):
-            raise ValueError(
-                f"Invalid namespace '{namespace}'. Must match pattern: "
-                "lowercase letter followed by up to 63 lowercase letters, digits, underscores, or dashes."
-            )
-
         with self._writer() as conn:
             now = datetime.utcnow().isoformat()
             conn.execute(
                 """INSERT INTO workflows
-                   (id, template_id, name, namespace, status, current_step, created_at, updated_at)
-                   VALUES (?, ?, ?, ?, ?, 1, ?, ?)""",
-                (id, template_id, name, namespace, status, now, now),
+                   (id, template_id, name, status, current_step, created_at, updated_at)
+                   VALUES (?, ?, ?, ?, 1, ?, ?)""",
+                (id, template_id, name, status, now, now),
             )
         return self.get_workflow(id)
 
@@ -3318,7 +3375,6 @@ class ProjectDatabase:
     def list_workflows(
         self,
         status: Optional[str] = None,
-        namespace: Optional[str] = None,
         include_archived: bool = False,
         archived_only: bool = False,
     ) -> list[dict]:
@@ -3326,7 +3382,6 @@ class ProjectDatabase:
 
         Args:
             status: Filter by workflow status.
-            namespace: Filter by namespace.
             include_archived: If True, include archived workflows.
             archived_only: If True, only return archived workflows.
         """
@@ -3337,9 +3392,6 @@ class ProjectDatabase:
             if status:
                 conditions.append("status = ?")
                 params.append(status)
-            if namespace:
-                conditions.append("namespace = ?")
-                params.append(namespace)
 
             # Handle archived filtering
             if archived_only:

ralphx/core/project_export.py

@@ -34,7 +34,6 @@ class WorkflowSummary:
     """Summary of a workflow in the project."""
     id: str
     name: str
-    namespace: str
     steps_count: int
     items_count: int
     resources_count: int
@@ -137,7 +136,6 @@ class ProjectExporter:
             summaries.append(WorkflowSummary(
                 id=wf['id'],
                 name=wf['name'],
-                namespace=wf['namespace'],
                 steps_count=wf_preview.steps_count,
                 items_count=wf_preview.items_total,
                 resources_count=wf_preview.resources_count,
@@ -231,7 +229,7 @@ class ProjectExporter:
             )
 
             for wf in workflows:
-                wf_prefix = f"workflows/{wf['namespace']}/"
+                wf_prefix = f"workflows/{wf['id']}/"
 
                 # Get workflow data
                 steps = self.db.list_workflow_steps(wf['id'])
@@ -261,7 +259,6 @@ class ProjectExporter:
                         'id': wf['id'],
                         'template_id': wf.get('template_id'),
                         'name': wf['name'],
-                        'namespace': wf['namespace'],
                         'status': 'draft',
                         'current_step': 1,
                         'created_at': wf.get('created_at'),
@@ -432,7 +429,6 @@ class ProjectExporter:
                     {
                         'id': w['id'],
                         'name': w['name'],
-                        'namespace': w['namespace'],
                     }
                     for w in workflows
                 ],

ralphx/core/project_import.py

@@ -30,7 +30,6 @@ class WorkflowPreviewInfo:
     """Preview info for a workflow in the project export."""
     id: str
     name: str
-    namespace: str
     steps_count: int
     items_count: int
     resources_count: int
@@ -263,8 +262,13 @@ class ProjectImporter:
 
         # Get workflow info from manifest
         for wf_info in manifest.get('contents', {}).get('workflows', []):
-            wf_namespace = wf_info['namespace']
-            wf_prefix = f"workflows/{wf_namespace}/"
+            wf_id = wf_info['id']
+            # Support both new (workflow_id) and old (namespace) path formats
+            wf_prefix = f"workflows/{wf_id}/"
+            # Check for old namespace-based paths for backward compatibility
+            old_namespace = wf_info.get('namespace')
+            if old_namespace and f"workflows/{old_namespace}/workflow.json" in zf.namelist():
+                wf_prefix = f"workflows/{old_namespace}/"
 
             # Count items
             items_count = 0
@@ -304,7 +308,6 @@ class ProjectImporter:
             workflows_info.append(WorkflowPreviewInfo(
                 id=wf_info['id'],
                 name=wf_info['name'],
-                namespace=wf_namespace,
                 steps_count=steps_count,
                 items_count=items_count,
                 resources_count=resources_count,
@@ -354,7 +357,6 @@ class ProjectImporter:
             workflows=[WorkflowPreviewInfo(
                 id=wf_info.get('id', ''),
                 name=wf_info.get('name', ''),
-                namespace=wf_info.get('namespace', ''),
                 steps_count=contents.get('steps', 0),
                 items_count=contents.get('items_total', 0),
                 resources_count=contents.get('resources', 0),
@@ -484,8 +486,13 @@ class ProjectImporter:
         import hashlib
         import uuid
 
-        wf_namespace = wf_info['namespace']
-        wf_prefix = f"workflows/{wf_namespace}/"
+        wf_id = wf_info['id']
+        # Support both new (workflow_id) and old (namespace) path formats
+        wf_prefix = f"workflows/{wf_id}/"
+        # Check for old namespace-based paths for backward compatibility
+        old_namespace = wf_info.get('namespace')
+        if old_namespace and f"workflows/{old_namespace}/workflow.json" in zf.namelist():
+            wf_prefix = f"workflows/{old_namespace}/"
 
         # Read workflow data
         workflow_data = json.loads(zf.read(f"{wf_prefix}workflow.json").decode('utf-8'))
@@ -522,15 +529,10 @@ class ProjectImporter:
             hash_suffix = hashlib.md5(f"{old_id}-{uuid.uuid4().hex}".encode()).hexdigest()[:8]
             id_mapping[old_id] = f"{old_id}-{hash_suffix}"
 
-        # Generate unique namespace
-        base_namespace = workflow_data['workflow']['namespace']
-        namespace = self._generate_unique_namespace(base_namespace)
-
         # Create workflow
         workflow = self.db.create_workflow(
             id=new_wf_id,
             name=workflow_data['workflow']['name'],
-            namespace=namespace,
             template_id=workflow_data['workflow'].get('template_id'),
             status='draft',
         )
@@ -666,20 +668,3 @@ class ProjectImporter:
             id_mapping=id_mapping,
             warnings=warnings,
         )
-
-    def _generate_unique_namespace(self, base_namespace: str) -> str:
-        """Generate a unique namespace."""
-        import uuid
-
-        existing_workflows = self.db.list_workflows()
-        existing_namespaces = {w['namespace'] for w in existing_workflows}
-
-        if base_namespace not in existing_namespaces:
-            return base_namespace
-
-        for i in range(1, 1000):
-            candidate = f"{base_namespace[:56]}-{i}"
-            if candidate not in existing_namespaces:
-                return candidate
-
-        return f"{base_namespace[:50]}-{uuid.uuid4().hex[:8]}"

ralphx/core/sample_project.py

@@ -124,14 +124,10 @@ def _create_workflow_with_stories(
             logger.warning(f"Template '{template_id}' not found")
             return None
 
-        # Generate namespace
-        namespace = f"excuse-gen-{uuid.uuid4().hex[:7]}"
-
-        # Create workflow
+        # Create workflow (namespace parameter removed in schema v16)
         workflow = project_db.create_workflow(
             id=workflow_id,
             name="Build Excuse Generator",
-            namespace=namespace,
             template_id=template_id,
             status="draft",
         )

ralphx/core/templates.py

@@ -9,15 +9,15 @@ from typing import Optional
 
 # Base loop templates
 TEMPLATES: dict[str, dict] = {
-    "research": {
-        "name": "research",
-        "display_name": "Research Loop",
+    "extractgen_requirements": {
+        "name": "extractgen_requirements",
+        "display_name": "Extract Requirements Loop",
         "description": "Discover and document user stories from design documents or web research",
         "type": "generator",
         "category": "discovery",
         "config": {
-            "name": "research",
-            "display_name": "Research Loop",
+            "name": "extractgen_requirements",
+            "display_name": "Extract Requirements Loop",
             "type": "generator",
             "description": "Discover and document user stories from design documents",
             "item_types": {
@@ -34,7 +34,7 @@ TEMPLATES: dict[str, dict] = {
                     "model": "claude-sonnet-4-20250514",
                     "timeout": 180,
                     "tools": [],
-                    "prompt_template": "prompts/research_turbo.md",
+                    "prompt_template": "prompts/extractgen_requirements_turbo.md",
                 },
                 {
                     "name": "deep",
@@ -42,7 +42,7 @@ TEMPLATES: dict[str, dict] = {
                     "model": "claude-sonnet-4-20250514",
                     "timeout": 900,
                     "tools": ["web_search"],
-                    "prompt_template": "prompts/research_deep.md",
+                    "prompt_template": "prompts/extractgen_requirements_deep.md",
                 },
             ],
             "mode_selection": {
@@ -109,7 +109,7 @@ TEMPLATES: dict[str, dict] = {
                 "input": {
                     "singular": "story",
                     "plural": "stories",
-                    "source": "research",
+                    "source": "extractgen_requirements",
                     "description": "Stories to implement",
                 },
                 "output": {
@@ -219,7 +219,7 @@ def get_template(name: str) -> Optional[dict]:
     """Get a template by name.
 
     Args:
-        name: Template name (e.g., 'research', 'implementation')
+        name: Template name (e.g., 'extractgen_requirements', 'implementation')
 
     Returns:
         Template dict or None if not found

ralphx/core/workflow_export.py

@@ -63,7 +63,7 @@ class SecretMatch:
 class ExportPreview:
     """Preview of what will be exported."""
     workflow_name: str
-    workflow_namespace: str
+    workflow_id: str
     steps_count: int
     items_total: int
     items_by_step: dict[int, int]  # step_id -> count
@@ -155,7 +155,7 @@ class WorkflowExporter:
 
         return ExportPreview(
             workflow_name=workflow['name'],
-            workflow_namespace=workflow['namespace'],
+            workflow_id=workflow['id'],
             steps_count=len(steps),
             items_total=total_items,  # Show real count, not truncated count
             items_by_step=items_by_step,
@@ -247,8 +247,7 @@ class WorkflowExporter:
 
         # Generate filename
         timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
-        namespace = workflow['namespace']
-        filename = f"workflow-{namespace}-{timestamp}.ralphx.zip"
+        filename = f"workflow-{workflow['id']}-{timestamp}.ralphx.zip"
 
         zip_bytes = zip_buffer.getvalue()
 
@@ -342,7 +341,7 @@ class WorkflowExporter:
                         snippet=redacted[:50] + '...' if len(redacted) > 50 else redacted,
                     ))
 
-        # Scan workflow name and namespace (unlikely but check)
+        # Scan workflow name (unlikely but check)
         scan_text(workflow.get('name', ''), 'workflow.name')
 
         # Scan resources
@@ -383,7 +382,6 @@ class WorkflowExporter:
             'workflow': {
                 'id': workflow['id'],
                 'name': workflow['name'],
-                'namespace': workflow['namespace'],
                 'template_id': workflow.get('template_id'),
             },
             'contents': {
@@ -447,7 +445,6 @@ class WorkflowExporter:
                 'id': workflow['id'],
                 'template_id': workflow.get('template_id'),
                 'name': workflow['name'],
-                'namespace': workflow['namespace'],
                 'status': 'draft',  # Reset status on export
                 'current_step': 1,  # Reset to beginning
                 'created_at': workflow.get('created_at'),