specweave 0.1.8 → 0.3.0

package/src/skills/spec-driven-debugging/test-cases/TC-002-race-condition-ultrathink.yaml

@@ -1,461 +0,0 @@
----
-name: "Race Condition Bug Requiring Ultrathink"
-description: "Debug intermittent race condition in distributed task queue with 3+ failed fixes requiring deep analysis"
-skill: spec-driven-debugging
-priority: P1
-estimated_time: "45-60 minutes (includes ultrathink)"
-
-input:
-  prompt: "Task queue occasionally processes the same task twice, violating exactly-once delivery guarantee"
-  context:
-    project_type: "Node.js + TypeScript + Redis + PostgreSQL"
-    increment: ".specweave/increments/0005-task-queue/"
-    complexity: "High (distributed systems, concurrency)"
-    files_exist:
-      - "spec.md"
-      - "plan.md"
-      - "tests.md"
-      - "src/queue/task-processor.ts"
-      - "src/queue/outbox-poller.ts"
-      - "src/db/task-repository.ts"
-    error_pattern: "Intermittent - 1-2% of tasks processed twice"
-    reproduction: "Inconsistent - happens under load, not in dev"
-
-expected_output:
-  phase_0_context_loading:
-    announcement: "Loading SpecWeave context for distributed systems bug."
-
-    files_loaded:
-      - path: ".specweave/increments/0005-task-queue/spec.md"
-        key_findings:
-          - "NFR-001: Exactly-once delivery guarantee for all tasks"
-          - "NFR-002: Handle 10,000 tasks/sec at peak"
-          - "FR-005: Tasks must be idempotent"
-          - "Acceptance: TC-003: No task processed more than once"
-
-      - path: ".specweave/increments/0005-task-queue/plan.md"
-        key_findings:
-          - "Event sourcing with outbox pattern"
-          - "PostgreSQL for transactional outbox"
-          - "Redis Streams for queue"
-          - "Idempotency keys (UUID v4)"
-          - "Distributed locks via Redis"
-
-      - path: ".specweave/docs/internal/architecture/adr/0007-exactly-once-semantics.md"
-        key_findings:
-          - "Decision: Use transactional outbox + idempotency keys"
-          - "At-least-once delivery + idempotent consumers = exactly-once effect"
-          - "Lock timeout: 30 seconds"
-          - "Idempotency key stored in tasks table (unique constraint)"
-
-    bug_classification: "Likely architectural issue - race condition in distributed system"
-    ultrathink_suggestion: "This is a distributed systems bug with race conditions. Should I **ultrathink** this to analyze all concurrency scenarios?"
-
-  phase_1_root_cause:
-    error_analysis:
-      symptom: "Task processed twice (duplicate charge, duplicate email)"
-      frequency: "1-2% under load (>5000 tasks/sec)"
-      environment: "Production only, not reproducible in dev"
-      logs:
-        - "[Worker-1] Processing task abc123 with lock"
-        - "[Worker-2] Processing task abc123 with lock" # BOTH got lock!
-
-    reproduction_attempts:
-      attempt_1:
-        approach: "Manual testing in dev"
-        result: "Cannot reproduce (too slow)"
-
-      attempt_2:
-        approach: "Load testing with 100 concurrent workers"
-        result: "Reproduced! 15 duplicates out of 1000 tasks"
-
-      attempt_3:
-        approach: "Added detailed logging at every step"
-        result: "Both workers believe they have exclusive lock"
-
-    spec_comparison:
-      spec_says: "NFR-001: Exactly-once delivery (no duplicates)"
-      actual_behavior: "1-2% duplicate processing under load"
-      discrepancy: "Race condition in lock acquisition or idempotency check"
-
-    multi_component_instrumentation:
-      component_1_outbox_poller:
-        log: "console.log('[Outbox] Task ${id} marked as processing by ${workerId}')"
-        finding: "✓ Correctly marks task in DB"
-
-      component_2_redis_lock:
-        log: "console.log('[Lock] Acquired lock ${taskId} for ${workerId}: ${acquired}')"
-        finding: "❌ Both workers report acquiring lock successfully!"
-
-      component_3_task_processing:
-        log: "console.log('[Processor] Starting task ${id} by ${workerId}')"
-        finding: "❌ Two workers process same task"
-
-      component_4_idempotency_check:
-        log: "console.log('[Idempotency] Key ${key} exists: ${exists}')"
-        finding: "✓ Idempotency key checked, but AFTER lock acquired"
-
-    data_flow_trace:
-      failing_scenario:
-        - step: "Worker-1 reads task from outbox"
-          timestamp: "t=100ms"
-        - step: "Worker-2 reads task from outbox"
-          timestamp: "t=102ms" # race window!
-        - step: "Worker-1 tries to acquire Redis lock"
-          timestamp: "t=110ms"
-          result: "SUCCESS"
-        - step: "Worker-2 tries to acquire Redis lock"
-          timestamp: "t=111ms"
-          result: "SUCCESS" # BOTH succeeded!
-        - step: "Worker-1 checks idempotency key (doesn't exist)"
-          timestamp: "t=115ms"
-        - step: "Worker-2 checks idempotency key (doesn't exist)"
-          timestamp: "t=115ms" # same time!
-        - step: "Worker-1 inserts idempotency key"
-          timestamp: "t=120ms"
-        - step: "Worker-2 inserts idempotency key"
-          timestamp: "t=121ms"
-          result: "Unique constraint violation (ignored)"
-        - step: "Both workers process task"
-          timestamp: "t=130ms"
-
-    root_cause_hypothesis_1: "Redis lock implementation has race condition"
-
-  phase_2_pattern_analysis:
-    working_examples: "No working examples - this is the only distributed lock usage"
-
-    redis_lock_code:
-      file: "src/queue/redis-lock.ts"
-      implementation: |
-        async acquireLock(taskId: string): Promise<boolean> {
-          const lockKey = `lock:${taskId}`;
-          const result = await redis.set(lockKey, workerId, {
-            NX: true,    // Only set if not exists
-            EX: 30       // Expire in 30 seconds
-          });
-          return result === 'OK';
-        }
-
-    idempotency_check_code:
-      file: "src/queue/task-processor.ts"
-      implementation: |
-        async processTask(task: Task) {
-          const locked = await acquireLock(task.id);
-          if (!locked) return; // already processing
-
-          // Check idempotency (AFTER lock)
-          const exists = await db.idempotencyKeys.findUnique({
-            where: { key: task.idempotencyKey }
-          });
-          if (exists) return; // already processed
-
-          // Process task
-          await doWork(task);
-
-          // Insert idempotency key
-          await db.idempotencyKeys.create({
-            data: { key: task.idempotencyKey, taskId: task.id }
-          });
-        }
-
-    problem_identified:
-      issue: "Lock and idempotency check are separate, creating race window"
-      sequence_problem:
-        - "Worker-1 acquires lock"
-        - "Worker-2 acquires lock (shouldn't be possible but happens)"
-        - "Both check idempotency (both see 'not exists')"
-        - "Both process task"
-
-  phase_3_hypothesis:
-    hypothesis_1:
-      statement: "Redis SET NX command has race condition under load"
-      test: "Add logging before and after SET NX call"
-      result: "FAILED - SET NX is atomic, not the issue"
-      attempts: 1
-
-    hypothesis_2:
-      statement: "Lock timeout (30s) expires before task completes, allowing second worker to acquire lock"
-      test: "Reduce task processing time to 5 seconds, check if duplicates persist"
-      result: "FAILED - Duplicates still occur even with 5s tasks"
-      attempts: 2
-
-    hypothesis_3:
-      statement: "Idempotency check should be BEFORE lock, not after"
-      test: "Move idempotency check before lock acquisition"
-      result: "FAILED - Still get duplicates, now with unique constraint violations"
-      attempts: 3
-
-    ultrathink_triggered: true
-    ultrathink_reason: "3 failed hypotheses - this is an architectural issue, not a simple code bug"
-
-    ultrathink_mode_activated:
-      thinking_tokens: 31999
-      analysis_depth: "Deep concurrency analysis across all components"
-
-      ultrathink_analysis:
-        problem_space:
-          - "Distributed system with multiple concurrent workers"
-          - "Two independent systems for consistency: Redis locks + DB idempotency keys"
-          - "Race window exists between these two systems"
-
-        edge_cases_explored:
-          - "Network partition between Redis and PostgreSQL"
-          - "Redis failover during lock acquisition"
-          - "Clock skew between workers"
-          - "Transaction isolation levels in PostgreSQL"
-          - "Redis persistence settings (AOF vs RDB)"
-
-        architectural_flaw_identified:
-          root_cause: "Fundamental design issue: Two separate consistency mechanisms (Redis lock + DB idempotency) are not atomic"
-          why_it_fails: |
-            Under high concurrency:
-            1. Redis lock prevents MOST concurrent processing
-            2. But Redis and PostgreSQL are separate systems
-            3. Tiny race window exists between lock acquisition and idempotency insert
-            4. Under load (5000+ tasks/sec), workers hit this window
-
-        alternative_architectures_evaluated:
-          approach_1:
-            name: "Single Source of Truth (DB-only locking)"
-            implementation: "PostgreSQL advisory locks + idempotency key in same transaction"
-            pros:
-              - "Single atomic operation (no race window)"
-              - "Simpler (one system, not two)"
-            cons:
-              - "PostgreSQL becomes bottleneck (can't scale horizontally)"
-              - "Might not support 10K tasks/sec requirement"
-            verdict: "Correct for consistency, but may not meet performance NFR"
-
-          approach_2:
-            name: "Idempotency Key as Primary Lock"
-            implementation: "Skip Redis lock entirely, use DB unique constraint as lock"
-            pros:
-              - "Single atomic operation (INSERT with UNIQUE constraint)"
-              - "Exactly-once guaranteed by DB"
-            cons:
-              - "Every worker attempts INSERT (thundering herd on DB)"
-              - "Failed inserts waste DB connections"
-            verdict: "Correct but inefficient"
-
-          approach_3:
-            name: "Hybrid: Redis pre-filter + DB transaction"
-            implementation: |
-              1. Redis lock as fast pre-filter (optimistic)
-              2. DB transaction with idempotency INSERT (pessimistic, source of truth)
-              3. If DB INSERT fails (unique violation) → already processed by another worker
-            pros:
-              - "Best of both: Redis speed + DB consistency"
-              - "Redis filters out MOST duplicate attempts"
-              - "DB guarantees exactly-once"
-            cons:
-              - "Still two systems, but DB is source of truth"
-              - "Graceful handling of unique violations needed"
-            verdict: "Best approach - maintains performance AND correctness"
-
-        recommended_solution:
-          approach: "Approach 3: Hybrid with DB as source of truth"
-          changes_needed:
-            - "Keep Redis lock (performance optimization)"
-            - "Move idempotency INSERT to BEGINNING of transaction"
-            - "Wrap task processing in transaction with idempotency INSERT"
-            - "Gracefully handle unique constraint violations (not errors, expected)"
-
-  phase_4_implementation:
-    fix_level: "Architecture-level fix"
-    reason: "Fundamental design flaw: two separate consistency mechanisms not atomic"
-
-    adr_creation:
-      needed: true
-      file: ".specweave/docs/internal/architecture/adr/0012-fix-exactly-once-with-hybrid-lock.md"
-      content: |
-        # ADR-0012: Fix Exactly-Once Delivery with Hybrid Lock
-
-        ## Context
-        Current implementation uses Redis locks + DB idempotency keys as two
-        separate systems. Race window exists between lock acquisition and
-        idempotency check, causing 1-2% duplicate processing under load.
-
-        ## Decision
-        Use hybrid approach: Redis lock as optimistic pre-filter, DB
-        idempotency key INSERT as pessimistic source of truth within
-        transaction.
-
-        ## Consequences
-        - Exactly-once guarantee restored
-        - Performance maintained (Redis filters most duplicates)
-        - DB is source of truth (no race window)
-        - Graceful handling of unique violations (expected, not errors)
-
-    test_creation:
-      level: "Level 2: Feature Tests + Level 3: Code Tests"
-      files:
-        - path: ".specweave/increments/0005-task-queue/tests.md"
-          test_case: "TC-008: Concurrency test with 100 workers processing same task → exactly one processes"
-
-        - path: "tests/integration/task-queue.test.ts"
-          test_case: |
-            describe('Task Queue - Exactly-Once', () => {
-              it('should process task exactly once with 100 concurrent workers', async () => {
-                const task = await createTask();
-                const workers = Array.from({ length: 100 }, (_, i) =>
-                  processTask(task.id, `worker-${i}`)
-                );
-
-                const results = await Promise.all(workers);
-                const processed = results.filter(r => r.success).length;
-
-                expect(processed).toBe(1); // exactly one succeeded
-                expect(results.filter(r => r.alreadyProcessed).length).toBe(99);
-              });
-            });
-
-      test_fails_before_fix: true
-
-    spec_update_needed: false # spec was correct (NFR-001)
-
-    code_fix:
-      file: "src/queue/task-processor.ts"
-      before: |
-        async processTask(task: Task) {
-          const locked = await acquireLock(task.id);
-          if (!locked) return;
-
-          const exists = await db.idempotencyKeys.findUnique({
-            where: { key: task.idempotencyKey }
-          });
-          if (exists) return;
-
-          await doWork(task);
-          await db.idempotencyKeys.create({
-            data: { key: task.idempotencyKey }
-          });
-        }
-
-      after: |
-        async processTask(task: Task) {
-          // Redis lock as optimistic pre-filter (performance)
-          const locked = await acquireLock(task.id);
-          if (!locked) return { success: false, reason: 'lock-failed' };
-
-          try {
-            // DB transaction with idempotency INSERT (source of truth)
-            await db.$transaction(async (tx) => {
-              // Insert idempotency key FIRST (atomic lock)
-              await tx.idempotencyKeys.create({
-                data: {
-                  key: task.idempotencyKey,
-                  taskId: task.id,
-                  processedAt: new Date()
-                }
-              });
-
-              // Process task (only if INSERT succeeded)
-              await doWork(task);
-            });
-
-            return { success: true };
-          } catch (error) {
-            // Unique constraint violation = already processed (expected!)
-            if (error.code === 'P2002') { // Prisma unique violation
-              return { success: false, reason: 'already-processed' };
-            }
-            throw error; // unexpected error
-          } finally {
-            await releaseLock(task.id);
-          }
-        }
-
-    verification:
-      test_passes: true
-      load_test_results:
-        tasks: 10000
-        concurrent_workers: 100
-        duplicates: 0 # ✓ Fixed!
-        duration: "45 seconds"
-      spec_aligned: true
-
-    attempts_count: 4 # 3 failed + 1 ultrathink success
-    ultrathink_used: true
-
-  phase_5_documentation:
-    increment_updates:
-      - file: ".specweave/increments/0005-task-queue/tests.md"
-        change: "Added TC-008: Concurrency test for exactly-once guarantee"
-
-      - file: ".specweave/increments/0005-task-queue/plan.md"
-        change: "Updated architecture section to reflect hybrid lock approach"
-
-    architecture_updates:
-      - file: ".specweave/docs/internal/architecture/adr/0012-fix-exactly-once-with-hybrid-lock.md"
-        change: "Created ADR documenting architectural decision"
-
-      - file: ".specweave/docs/internal/architecture/system-design.md"
-        change: "Updated task queue component diagram"
-
-    strategy_updates: [] # requirements unchanged
-
-    new_increment_needed: false # fixed within existing increment
-
-    commit_message: |
-      fix(queue): guarantee exactly-once delivery with hybrid lock
-
-      Root cause: Race condition between Redis lock and DB idempotency
-      check. Two separate systems (Redis + PostgreSQL) created race
-      window under high concurrency (>5000 tasks/sec).
-
-      After 3 failed hypotheses, ultrathinked the architecture:
-      - Analyzed edge cases: network partitions, clock skew, failover
-      - Evaluated 3 alternative architectures
-      - Chose hybrid approach: Redis pre-filter + DB source of truth
-
-      Solution: Moved idempotency INSERT to beginning of DB transaction.
-      DB unique constraint is now atomic lock (source of truth).
-      Redis lock remains as performance optimization (filters most duplicates).
-
-      Spec alignment: Restores NFR-001 (exactly-once delivery)
-
-      Test coverage:
-      - Added: TC-008 concurrency test (100 workers, 0 duplicates)
-      - Load tested: 10K tasks, 100 workers, 0 duplicates
-      - Verified: All existing tests pass
-
-      Documentation updated:
-      - ADR-0012: Documents architectural decision and trade-offs
-      - plan.md: Updated with hybrid lock architecture
-      - tests.md: Added TC-008 for regression prevention
-      - system-design.md: Updated task queue component diagram
-
-      Ultrathink analysis (31,999 tokens) identified fundamental
-      architectural issue after 3 failed fix attempts.
-
-      Fixes: #89
-
-      🤖 Generated with Claude Code
-      Co-Authored-By: Claude <noreply@anthropic.com>
-
-validation:
-  - "Phase 0: Loaded spec, plan, ADR for distributed system context"
-  - "Phase 0: Suggested ultrathink for complex distributed bug"
-  - "Phase 1: Reproduced under load (load testing)"
-  - "Phase 1: Added instrumentation across all components"
-  - "Phase 1: Traced data flow to identify race window"
-  - "Phase 2: Analyzed lock implementation and identified architectural flaw"
-  - "Phase 3: Tested 3 hypotheses (all failed)"
-  - "Phase 3: Triggered ultrathink after 3 failures (MANDATORY)"
-  - "Ultrathink: Analyzed edge cases, evaluated 3 architectures"
-  - "Ultrathink: Recommended hybrid approach with DB as source of truth"
-  - "Phase 4: Created ADR documenting architectural decision"
-  - "Phase 4: Fixed with architectural refactor (not symptom patch)"
-  - "Phase 4: Load tested to verify (0 duplicates out of 10K tasks)"
-  - "Phase 5: Updated plan.md, tests.md, ADR, system-design.md"
-
-expected_errors: []
-
-success_criteria:
-  - "Ultrathink activated after 3 failed fixes"
-  - "Architectural flaw identified (not just code bug)"
-  - "ADR created documenting decision"
-  - "Exactly-once guarantee restored (0 duplicates in load test)"
-  - "Performance maintained (Redis pre-filter)"
-  - "Living documentation comprehensive (ADR, plan, tests, design)"
----