valent-pipeline 0.2.15 → 0.2.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "valent-pipeline",
-  "version": "0.2.15",
+  "version": "0.2.17",
   "description": "v3 multi-agent AI pipeline for software development lifecycle",
   "type": "module",
   "bin": {

package/pipeline/docs/lean-spawn-human-tasks.md

@@ -135,7 +135,7 @@ Then test other commands:
 ```bash
 valent-pipeline config validate    # should exit 0
 valent-pipeline upgrade --dry-run  # should show no changes (just installed)
-valent-pipeline db init            # should create .valent-pipeline/pipeline.db
+valent-pipeline db rebuild          # indexes story artifacts (auto-creates DB if missing)
 valent-pipeline db rebuild         # should complete (no stories to index yet)
 ```
 

package/pipeline/prompts/embed.md

@@ -62,6 +62,6 @@ Send inbox message to lead: `[EMBED-COMPLETE] Indexed {count} items.` (or `[EMBE
 ## Error Handling
 
 - If embed-instructions.md is missing: send `[BLOCKER]` to lead, terminate.
-- If SQLite database is missing: run `valent-pipeline db init` first, or skip DB instructions and index curated files only.
+- If SQLite database is missing: it will be auto-created on first write. If the DB file cannot be created, skip DB instructions and index curated files only.
 - If ChromaDB connection fails (legacy mode): skip ChromaDB instructions, index curated files only, report partial completion.
 - If a curated file write fails: log the failure, continue to next instruction, report in completion message.

package/pipeline/scripts/db-bootstrap.ts

@@ -0,0 +1,153 @@
+/**
+ * db-bootstrap.ts — SQLite schema utilities for pipeline TS scripts.
+ *
+ * This file is the TypeScript-side copy of the schema defined in
+ * src/lib/db.js. Keep both files in sync when modifying the schema
+ * (see docs/design/refactor-checklist.md).
+ *
+ * Imported by embed-sqlite.ts and query-kb.ts to self-bootstrap the
+ * database — tables are created automatically if they don't exist.
+ */
+
+import Database from 'better-sqlite3';
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+
+// ---------------------------------------------------------------------------
+// Schema DDL — must match src/lib/db.js SCHEMA_DDL exactly.
+// ---------------------------------------------------------------------------
+
+export const SCHEMA_DDL = `
+  CREATE TABLE IF NOT EXISTS artifacts (
+    id TEXT PRIMARY KEY,
+    story_id TEXT NOT NULL,
+    agent TEXT NOT NULL,
+    artifact_type TEXT NOT NULL,
+    content TEXT NOT NULL,
+    metadata TEXT,
+    created_at TEXT DEFAULT (datetime('now')),
+    UNIQUE(story_id, artifact_type)
+  );
+
+  CREATE TABLE IF NOT EXISTS correction_directives (
+    id TEXT PRIMARY KEY,
+    target_agent TEXT NOT NULL,
+    directive TEXT NOT NULL,
+    reason TEXT,
+    status TEXT DEFAULT 'active',
+    created_batch INTEGER,
+    metadata TEXT
+  );
+
+  CREATE TABLE IF NOT EXISTS calibration (
+    story_id TEXT PRIMARY KEY,
+    story_points INTEGER,
+    ac_count INTEGER,
+    surface TEXT,
+    estimated_points INTEGER,
+    actual_mins REAL,
+    rework_cycles INTEGER DEFAULT 0,
+    sprint_id TEXT,
+    created_at TEXT DEFAULT (datetime('now'))
+  );
+
+  CREATE TABLE IF NOT EXISTS artifacts_working (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    story_id TEXT NOT NULL,
+    agent TEXT NOT NULL,
+    artifact_type TEXT NOT NULL,
+    content TEXT,
+    metadata TEXT,
+    created_at TEXT DEFAULT (datetime('now')),
+    UNIQUE(story_id, artifact_type)
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_artifacts_story ON artifacts(story_id);
+  CREATE INDEX IF NOT EXISTS idx_artifacts_type ON artifacts(artifact_type);
+  CREATE INDEX IF NOT EXISTS idx_artifacts_agent ON artifacts(agent);
+  CREATE INDEX IF NOT EXISTS idx_cd_target ON correction_directives(target_agent);
+  CREATE INDEX IF NOT EXISTS idx_cd_status ON correction_directives(status);
+  CREATE INDEX IF NOT EXISTS idx_calibration_sprint ON calibration(sprint_id);
+`;
+
+export const FTS_DDL = `
+  CREATE VIRTUAL TABLE IF NOT EXISTS artifacts_fts USING fts5(
+    content,
+    story_id UNINDEXED,
+    agent UNINDEXED,
+    artifact_type UNINDEXED,
+    content=artifacts,
+    content_rowid=rowid
+  );
+`;
+
+export const TRIGGERS_DDL = `
+  CREATE TRIGGER IF NOT EXISTS artifacts_ai AFTER INSERT ON artifacts BEGIN
+    INSERT INTO artifacts_fts(rowid, content, story_id, agent, artifact_type)
+    VALUES (new.rowid, new.content, new.story_id, new.agent, new.artifact_type);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS artifacts_ad AFTER DELETE ON artifacts BEGIN
+    INSERT INTO artifacts_fts(artifacts_fts, rowid, content, story_id, agent, artifact_type)
+    VALUES ('delete', old.rowid, old.content, old.story_id, old.agent, old.artifact_type);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS artifacts_au AFTER UPDATE ON artifacts BEGIN
+    INSERT INTO artifacts_fts(artifacts_fts, rowid, content, story_id, agent, artifact_type)
+    VALUES ('delete', old.rowid, old.content, old.story_id, old.agent, old.artifact_type);
+    INSERT INTO artifacts_fts(rowid, content, story_id, agent, artifact_type)
+    VALUES (new.rowid, new.content, new.story_id, new.agent, new.artifact_type);
+  END;
+`;
+
+// ---------------------------------------------------------------------------
+// Schema management
+// ---------------------------------------------------------------------------
+
+export function ensureSchema(db: InstanceType<typeof Database>): void {
+  db.exec(SCHEMA_DDL);
+  db.exec(FTS_DDL);
+  db.exec(TRIGGERS_DDL);
+}
+
+// ---------------------------------------------------------------------------
+// DB path resolution
+// ---------------------------------------------------------------------------
+
+export function resolveDbPath(flagDbPath?: string): string {
+  if (flagDbPath) return flagDbPath;
+  const configPath = join(process.cwd(), '.valent-pipeline', 'pipeline-config.yaml');
+  if (existsSync(configPath)) {
+    const content = readFileSync(configPath, 'utf-8');
+    const match = content.match(/sqlite_db_path:\s*"?([^"\n]+)"?/);
+    if (match) return join(process.cwd(), match[1].trim());
+  }
+  return join(process.cwd(), '.valent-pipeline', 'pipeline.db');
+}
+
+// ---------------------------------------------------------------------------
+// High-level open helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Open a read-write database connection with auto-schema creation.
+ * Creates the DB file and all tables if they don't exist.
+ */
+export function openDb(dbPath: string): InstanceType<typeof Database> {
+  const db = new Database(dbPath);
+  ensureSchema(db);
+  return db;
+}
+
+/**
+ * Open a read-only database connection. Does NOT run schema DDL.
+ * Errors with a helpful message if the DB file does not exist.
+ */
+export function openReadonlyDb(dbPath: string): InstanceType<typeof Database> {
+  if (!existsSync(dbPath)) {
+    console.error(`Database not found: ${dbPath}`);
+    console.error('Run "valent-pipeline init" or "valent-pipeline db init" to create it.');
+    process.exit(1);
+  }
+  return new Database(dbPath, { readonly: true });
+}

package/pipeline/scripts/embed-sqlite.ts

@@ -18,7 +18,7 @@
 
 import { readFileSync, existsSync, appendFileSync, mkdirSync, readdirSync, statSync } from 'fs';
 import { join, dirname, basename } from 'path';
-import Database from 'better-sqlite3';
+import { openDb } from './db-bootstrap.js';
 
 // Parse CLI args
 const args = process.argv.slice(2);
@@ -99,7 +99,7 @@ async function indexSingleHandoff(
     return;
   }
 
-  const db = new Database(dbPath);
+  const db = openDb(dbPath);
   const id = `${storyId}:${artifactType}`;
 
   db.prepare(`
@@ -133,7 +133,7 @@ async function rebuildAll(dbPath: string, storiesDir: string) {
     'story-report.md': { type: 'story-report', agent: 'JUDGE' },
   };
 
-  const db = new Database(dbPath);
+  const db = openDb(dbPath);
   const insert = db.prepare(`
     INSERT OR REPLACE INTO artifacts (id, story_id, agent, artifact_type, content, metadata)
     VALUES (?, ?, ?, ?, ?, ?)
@@ -181,7 +181,7 @@ async function processEmbedInstructions(instructionsPath: string, dbPath: string
 
   console.log(`Parsed ${items.length} embed instructions`);
 
-  const db = dryRun ? null : new Database(dbPath);
+  const db = dryRun ? null : openDb(dbPath);
   let indexed = 0;
   let curatedCount = 0;
 

package/pipeline/scripts/query-kb.ts

@@ -25,8 +25,7 @@
  *   # DB path defaults to .valent-pipeline/pipeline.db, override with --db-path
  */
 
-import Database from 'better-sqlite3';
-import { existsSync } from 'fs';
+import { resolveDbPath, openReadonlyDb } from './db-bootstrap.js';
 
 const args = process.argv.slice(2);
 const flags: Record<string, string> = {};
@@ -48,15 +47,8 @@ for (let i = 0; i < args.length; i++) {
   }
 }
 
-const dbPath = flags['db-path'] || '.valent-pipeline/pipeline.db';
-
-if (!existsSync(dbPath)) {
-  console.error(`Database not found: ${dbPath}`);
-  console.error('Run "valent-pipeline db init" to create it.');
-  process.exit(1);
-}
-
-const db = new Database(dbPath, { readonly: true });
+const dbPath = resolveDbPath(flags['db-path']);
+const db = openReadonlyDb(dbPath);
 const mode = modes[0];
 
 if (!mode) {

package/skills/valent-configure/SKILL.md

@@ -104,7 +104,7 @@ Ask the user to pick one:
 | `local-docker` | Local ChromaDB via Docker (legacy). Pipeline provides a docker-compose file. |
 | `connect-to-existing` | Remote/hosted ChromaDB instance for shared team knowledge (legacy). |
 
-**If `sqlite`:** Set `sqlite_db_path` to `./.valent-pipeline/pipeline.db` (default). Inform the user to run `valent-pipeline db init` to create the database, or it will be created automatically on first story run.
+**If `sqlite`:** Set `sqlite_db_path` to `./.valent-pipeline/pipeline.db` (default). The database is auto-created during `valent-pipeline init` and on first use — no separate `db init` step is needed.
 
 Also show the `knowledge_base_path` setting:
 - Default: `"./knowledge"`

package/skills/valent-help/SKILL.md

@@ -71,7 +71,7 @@ Read these as needed to answer questions:
 → `npx valent-pipeline upgrade` — replaces infrastructure files, never touches your config or knowledge files.
 
 **"How do I initialize the SQLite database?"**
-→ `npx valent-pipeline db init` to create it, `npx valent-pipeline db rebuild` to re-index from existing story artifacts.
+→ The database is auto-created during `init` and on first use. Run `npx valent-pipeline db rebuild` to re-index from existing story artifacts. Run `npx valent-pipeline db init` to manually reset it.
 
 ## Answer Format
 

package/skills/valent-setup-backlog/SKILL.md

@@ -42,7 +42,21 @@ From the source document, identify each story and extract:
 | Dependencies | No | Which stories must ship before this one can start (`depends_on`) |
 | Optional inputs | No | UX spec, architecture notes, scenario outlines if mentioned |
 
-**Priority assignment:** First story in first epic = 1, second story = 2, etc. Stories within the same epic are ordered by their natural sequence. Cross-epic dependencies use `depends_on`.
+**Priority assignment — E2E-first ordering:**
+
+The goal is to get working end-to-end slices as early as possible, not to finish one epic before starting the next.
+
+1. **Identify vertical slices.** Scan all stories across all epics and group them into vertical slices — sets of stories that together produce an end-to-end testable path through the system. What counts as "end-to-end" depends on project type:
+   - **backend-api**: request entry point → business logic → data persistence → response
+   - **fullstack-web**: UI interaction → API call → backend processing → UI display
+   - **data-pipeline**: data ingestion → transformation → output/storage
+   Each slice should be the smallest set of stories that delivers a complete path. Prefer simpler stories (fewest ACs, fewest dependencies) for the first slice.
+
+2. **Order slices by value.** Rank slices so the most foundational or highest-value E2E path comes first. Later slices may share stories with earlier ones — don't duplicate, just note the dependency.
+
+3. **Assign priorities slice-first.** Assign priorities 1 through K to the first slice (ordered by dependency chain), K+1 through M to the second slice, and so on. Stories that appear in multiple slices keep the priority from their earliest slice.
+
+4. **Fill in the rest.** Assign remaining priorities to stories that don't belong to any vertical slice, in normal epic order (story sequence within epic, epics in document order). Cross-epic dependencies use `depends_on`.
 
 **Dependency detection:** Look for:
 - Explicit "depends on" or "requires" references
@@ -52,18 +66,27 @@ From the source document, identify each story and extract:
 
 ## Step 4: Present Summary for Approval
 
-Before writing any files, show the user:
+Before writing any files, show the user with the vertical slice separated:
 
 ```
-Epic: {EPIC-NAME}
+Vertical Slice 1 (E2E path: {description}):
+  {STORY-ID}: {title} [priority: 1] {depends_on if any}
+  {STORY-ID}: {title} [priority: 2] {depends_on if any}
+
+Vertical Slice 2 (E2E path: {description}):
   {STORY-ID}: {title} [priority: {N}] {depends_on if any}
-  {STORY-ID}: {title} [priority: {N}]
   ...
 
-Epic: {EPIC-NAME}
-  ...
+Remaining Backlog:
+  Epic: {EPIC-NAME}
+    {STORY-ID}: {title} [priority: {N}] {depends_on if any}
+    {STORY-ID}: {title} [priority: {N}]
+    ...
+
+  Epic: {EPIC-NAME}
+    ...
 
-Total: {N} stories across {M} epics
+Total: {N} stories across {M} epics ({K} in {S} vertical slices)
 Dependencies: {list any depends_on relationships}
 ```
 
@@ -173,9 +196,8 @@ If the repo is empty or brand new (no source code yet), skip the subagents and c
 ### Step 7b: Initialize and Populate Knowledge Database
 
 After writing the curated knowledge files:
-1. Run `node .valent-pipeline/bin/cli.js db init` to create the SQLite database if it doesn't exist
-2. Run `node .valent-pipeline/bin/cli.js db rebuild` to index any existing story artifacts
-3. The database is now ready for the Knowledge Agent to query during story execution
+1. Run `npx valent-pipeline db rebuild` to index any existing story artifacts (the database is auto-created if it doesn't exist)
+2. The database is now ready for the Knowledge Agent to query during story execution
 
 ## Step 8: Report
 

package/src/commands/db-index.js

@@ -1,5 +1,6 @@
 import { join, dirname } from 'path';
 import { readFileSync, existsSync, appendFileSync, mkdirSync, readdirSync, statSync } from 'fs';
+import { resolveDbPath, openDb } from '../lib/db.js';
 
 const ARTIFACT_MAP = {
   'reqs-brief.md': { type: 'reqs-brief', agent: 'REQS' },
@@ -19,22 +20,6 @@ const ARTIFACT_MAP = {
   'visual-validation-checklist.md': { type: 'visual-validation-checklist', agent: 'QA-A' },
 };
 
-function getDb(dbPath) {
-  const Database = require(join(process.cwd(), '.valent-pipeline', 'node_modules', 'better-sqlite3'));
-  return new Database(dbPath);
-}
-
-function resolveDbPath(options) {
-  if (options.dbPath) return options.dbPath;
-  const configPath = join(process.cwd(), '.valent-pipeline', 'pipeline-config.yaml');
-  if (existsSync(configPath)) {
-    const content = readFileSync(configPath, 'utf-8');
-    const match = content.match(/sqlite_db_path:\s*"?([^"\n]+)"?/);
-    if (match) return join(process.cwd(), match[1].trim());
-  }
-  return join(process.cwd(), '.valent-pipeline', 'pipeline.db');
-}
-
 export async function dbIndexHandoff(options) {
   const dbPath = resolveDbPath(options);
   const { file, storyId, agent, artifactType } = options;
@@ -50,7 +35,7 @@ export async function dbIndexHandoff(options) {
     return;
   }
 
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
   const id = `${storyId}:${artifactType}`;
   db.prepare(`
     INSERT OR REPLACE INTO artifacts (id, story_id, agent, artifact_type, content, metadata)
@@ -70,7 +55,7 @@ export async function dbIndexWorking(options) {
   }
 
   const content = readFileSync(file, 'utf-8');
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
   db.prepare(`
     INSERT OR REPLACE INTO artifacts_working (story_id, agent, artifact_type, content, metadata)
     VALUES (?, ?, ?, ?, ?)
@@ -83,7 +68,7 @@ export async function dbFlushWorking(options) {
   const dbPath = resolveDbPath(options);
   const { sprintId } = options;
 
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
   const rows = db.prepare('SELECT * FROM artifacts_working').all();
 
   const insert = db.prepare(`
@@ -108,7 +93,7 @@ export async function dbQueryWorking(options) {
   const dbPath = resolveDbPath(options);
   const { sprintId, excludeStory } = options;
 
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
   let rows;
   if (excludeStory) {
     rows = db.prepare(
@@ -133,7 +118,7 @@ export async function dbRecordCalibration(options) {
   const dbPath = resolveDbPath(options);
   const { storyId, storyPoints, acCount, surface, estimatedPoints, actualMins, reworkCycles, sprintId } = options;
 
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
   db.prepare(`
     INSERT OR REPLACE INTO calibration (story_id, story_points, ac_count, surface, estimated_points, actual_mins, rework_cycles, sprint_id)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?)
@@ -144,7 +129,7 @@ export async function dbRecordCalibration(options) {
 
 export async function dbQueryVelocity(options) {
   const dbPath = resolveDbPath(options);
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
 
   const rows = db.prepare(`
     SELECT sprint_id, SUM(story_points) as points_shipped
@@ -182,7 +167,7 @@ export async function dbEmbed(options) {
   const items = parseEmbedInstructions(content);
   console.log(`Parsed ${items.length} embed instructions`);
 
-  const db = getDb(dbPath);
+  const db = await openDb({ dbPath });
   let indexed = 0;
   let curatedCount = 0;
 

package/src/commands/db-init.js

@@ -1,3 +1,4 @@
+import { openDb, resolveDbPath } from '../lib/db.js';
 import { join } from 'path';
 import { existsSync, readFileSync } from 'fs';
 
@@ -16,113 +17,7 @@ export async function dbInit() {
   const dbPath = dbPathMatch ? dbPathMatch[1].trim() : './.valent-pipeline/pipeline.db';
   const fullDbPath = join(projectRoot, dbPath);
 
-  let Database;
-  let sqliteVec;
-  try {
-    Database = (await import('better-sqlite3')).default;
-    sqliteVec = await import('sqlite-vec');
-  } catch (err) {
-    console.error('Error: better-sqlite3 or sqlite-vec not installed.');
-    console.error('Run: npm install better-sqlite3 sqlite-vec');
-    process.exit(1);
-  }
-
-  const db = new Database(fullDbPath);
-
-  // Load sqlite-vec extension
-  try {
-    sqliteVec.load(db);
-    console.log('Loaded sqlite-vec extension');
-  } catch (err) {
-    console.warn(`Warning: Could not load sqlite-vec: ${err.message}`);
-    console.warn('Vector search will not be available. FTS5 will still work.');
-  }
-
-  // Create tables
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS artifacts (
-      id TEXT PRIMARY KEY,
-      story_id TEXT NOT NULL,
-      agent TEXT NOT NULL,
-      artifact_type TEXT NOT NULL,
-      content TEXT NOT NULL,
-      metadata TEXT,
-      created_at TEXT DEFAULT (datetime('now')),
-      UNIQUE(story_id, artifact_type)
-    );
-
-    CREATE TABLE IF NOT EXISTS correction_directives (
-      id TEXT PRIMARY KEY,
-      target_agent TEXT NOT NULL,
-      directive TEXT NOT NULL,
-      reason TEXT,
-      status TEXT DEFAULT 'active',
-      created_batch INTEGER,
-      metadata TEXT
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_artifacts_story ON artifacts(story_id);
-    CREATE INDEX IF NOT EXISTS idx_artifacts_type ON artifacts(artifact_type);
-    CREATE INDEX IF NOT EXISTS idx_artifacts_agent ON artifacts(agent);
-    CREATE INDEX IF NOT EXISTS idx_cd_target ON correction_directives(target_agent);
-    CREATE INDEX IF NOT EXISTS idx_cd_status ON correction_directives(status);
-
-    CREATE TABLE IF NOT EXISTS calibration (
-      story_id TEXT PRIMARY KEY,
-      story_points INTEGER,
-      ac_count INTEGER,
-      surface TEXT,
-      estimated_points INTEGER,
-      actual_mins REAL,
-      rework_cycles INTEGER DEFAULT 0,
-      sprint_id TEXT,
-      created_at TEXT DEFAULT (datetime('now'))
-    );
-    CREATE INDEX IF NOT EXISTS idx_calibration_sprint ON calibration(sprint_id);
-
-    CREATE TABLE IF NOT EXISTS artifacts_working (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      story_id TEXT NOT NULL,
-      agent TEXT NOT NULL,
-      artifact_type TEXT NOT NULL,
-      content TEXT,
-      metadata TEXT,
-      created_at TEXT DEFAULT (datetime('now')),
-      UNIQUE(story_id, artifact_type)
-    );
-  `);
-
-  // Create FTS5 virtual table
-  db.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS artifacts_fts USING fts5(
-      content,
-      story_id UNINDEXED,
-      agent UNINDEXED,
-      artifact_type UNINDEXED,
-      content=artifacts,
-      content_rowid=rowid
-    );
-  `);
-
-  // Create FTS triggers for auto-sync
-  db.exec(`
-    CREATE TRIGGER IF NOT EXISTS artifacts_ai AFTER INSERT ON artifacts BEGIN
-      INSERT INTO artifacts_fts(rowid, content, story_id, agent, artifact_type)
-      VALUES (new.rowid, new.content, new.story_id, new.agent, new.artifact_type);
-    END;
-
-    CREATE TRIGGER IF NOT EXISTS artifacts_ad AFTER DELETE ON artifacts BEGIN
-      INSERT INTO artifacts_fts(artifacts_fts, rowid, content, story_id, agent, artifact_type)
-      VALUES ('delete', old.rowid, old.content, old.story_id, old.agent, old.artifact_type);
-    END;
-
-    CREATE TRIGGER IF NOT EXISTS artifacts_au AFTER UPDATE ON artifacts BEGIN
-      INSERT INTO artifacts_fts(artifacts_fts, rowid, content, story_id, agent, artifact_type)
-      VALUES ('delete', old.rowid, old.content, old.story_id, old.agent, old.artifact_type);
-      INSERT INTO artifacts_fts(rowid, content, story_id, agent, artifact_type)
-      VALUES (new.rowid, new.content, new.story_id, new.agent, new.artifact_type);
-    END;
-  `);
+  const db = await openDb({ dbPath: fullDbPath, loadVec: true });
 
   console.log(`Database initialized at ${fullDbPath}`);
   console.log('Tables: artifacts, artifacts_fts, correction_directives, calibration, artifacts_working');

package/src/commands/db-query.js

@@ -1,27 +1,10 @@
-import { join } from 'path';
-import { readFileSync, existsSync } from 'fs';
-
-function resolveDbPath(options) {
-  if (options.dbPath) return options.dbPath;
-  const configPath = join(process.cwd(), '.valent-pipeline', 'pipeline-config.yaml');
-  if (existsSync(configPath)) {
-    const content = readFileSync(configPath, 'utf-8');
-    const match = content.match(/sqlite_db_path:\s*"?([^"\n]+)"?/);
-    if (match) return join(process.cwd(), match[1].trim());
-  }
-  return join(process.cwd(), '.valent-pipeline', 'pipeline.db');
-}
-
-function getDb(dbPath) {
-  const Database = require(join(process.cwd(), '.valent-pipeline', 'node_modules', 'better-sqlite3'));
-  return new Database(dbPath, { readonly: true });
-}
+import { resolveDbPath, openReadonlyDb } from '../lib/db.js';
 
 export async function dbQueryArtifact(options) {
   const dbPath = resolveDbPath(options);
+  const db = await openReadonlyDb({ dbPath });
   const { story, type } = options;
 
-  const db = getDb(dbPath);
   const row = db.prepare(
     'SELECT content, agent, created_at FROM artifacts WHERE story_id = ? AND artifact_type = ?'
   ).get(story, type);
@@ -37,9 +20,9 @@ export async function dbQueryArtifact(options) {
 
 export async function dbQueryDirectives(options) {
   const dbPath = resolveDbPath(options);
+  const db = await openReadonlyDb({ dbPath });
   const { agent } = options;
 
-  const db = getDb(dbPath);
   let rows;
   if (agent) {
     rows = db.prepare(
@@ -65,9 +48,9 @@ export async function dbQueryDirectives(options) {
 
 export async function dbQuerySearch(options) {
   const dbPath = resolveDbPath(options);
+  const db = await openReadonlyDb({ dbPath });
   const { query } = options;
 
-  const db = getDb(dbPath);
   const rows = db.prepare(
     "SELECT story_id, agent, artifact_type, snippet(artifacts_fts, 0, '>>>', '<<<', '...', 40) as snippet FROM artifacts_fts WHERE artifacts_fts MATCH ? ORDER BY rank LIMIT 10"
   ).all(query);
@@ -84,9 +67,9 @@ export async function dbQuerySearch(options) {
 
 export async function dbQueryList(options) {
   const dbPath = resolveDbPath(options);
+  const db = await openReadonlyDb({ dbPath });
   const { story } = options;
 
-  const db = getDb(dbPath);
   const rows = db.prepare(
     'SELECT artifact_type, agent, length(content) as size, created_at FROM artifacts WHERE story_id = ? ORDER BY created_at'
   ).all(story);
@@ -104,7 +87,8 @@ export async function dbQueryList(options) {
 
 export async function dbQueryStories(options) {
   const dbPath = resolveDbPath(options);
-  const db = getDb(dbPath);
+  const db = await openReadonlyDb({ dbPath });
+
   const rows = db.prepare(
     'SELECT DISTINCT story_id, COUNT(*) as artifact_count FROM artifacts GROUP BY story_id ORDER BY story_id'
   ).all();
@@ -121,9 +105,9 @@ export async function dbQueryStories(options) {
 
 export async function dbQueryBugsSince(options) {
   const dbPath = resolveDbPath(options);
+  const db = await openReadonlyDb({ dbPath });
   const { since } = options;
 
-  const db = getDb(dbPath);
   const rows = db.prepare(
     "SELECT story_id, content FROM artifacts WHERE artifact_type = 'bugs' AND created_at > ? ORDER BY created_at"
   ).all(since);

package/src/commands/db-rebuild.js

@@ -1,5 +1,6 @@
 import { join } from 'path';
 import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
+import { openDb, resolveDbPath } from '../lib/db.js';
 
 // Artifact files we index (filename -> artifact_type mapping)
 const ARTIFACT_MAP = {
@@ -29,53 +30,14 @@ export async function dbRebuild() {
     process.exit(1);
   }
 
-  // Read db path from config
+  // Read config for db path and stories directory
   const content = readFileSync(configPath, 'utf-8');
   const dbPathMatch = content.match(/sqlite_db_path:\s*"?([^"\n]+)"?/);
   const dbPath = dbPathMatch ? dbPathMatch[1].trim() : './.valent-pipeline/pipeline.db';
   const fullDbPath = join(projectRoot, dbPath);
 
-  if (!existsSync(fullDbPath)) {
-    console.error('Error: Database not found. Run "valent-pipeline db init" first.');
-    process.exit(1);
-  }
-
-  let Database;
-  try {
-    Database = (await import('better-sqlite3')).default;
-  } catch (err) {
-    console.error('Error: better-sqlite3 not installed. Run: npm install better-sqlite3');
-    process.exit(1);
-  }
-
-  const db = new Database(fullDbPath);
-
-  // Ensure calibration and working tables exist
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS calibration (
-      story_id TEXT PRIMARY KEY,
-      story_points INTEGER,
-      ac_count INTEGER,
-      surface TEXT,
-      estimated_points INTEGER,
-      actual_mins REAL,
-      rework_cycles INTEGER DEFAULT 0,
-      sprint_id TEXT,
-      created_at TEXT DEFAULT (datetime('now'))
-    );
-    CREATE INDEX IF NOT EXISTS idx_calibration_sprint ON calibration(sprint_id);
-
-    CREATE TABLE IF NOT EXISTS artifacts_working (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      story_id TEXT NOT NULL,
-      agent TEXT NOT NULL,
-      artifact_type TEXT NOT NULL,
-      content TEXT,
-      metadata TEXT,
-      created_at TEXT DEFAULT (datetime('now')),
-      UNIQUE(story_id, artifact_type)
-    );
-  `);
+  // openDb auto-creates schema if DB is new
+  const db = await openDb({ dbPath: fullDbPath });
 
   // Find stories directory
   const storyDirMatch = content.match(/story_directory:\s*"?([^"\n]+)"?/);

package/src/commands/init.js

@@ -89,10 +89,12 @@ export async function init(options = {}) {
       writeFileSafe(vpPkgPath, JSON.stringify({
         "name": "valent-pipeline-runtime",
         "private": true,
+        "type": "module",
         "description": "Runtime dependencies for valent-pipeline scripts. Do not edit.",
         "dependencies": {
           "better-sqlite3": "^11.0.0",
-          "sqlite-vec": "^0.1.0"
+          "sqlite-vec": "^0.1.0",
+          "tsx": "^4.0.0"
         }
       }, null, 2) + '\n');
     }
@@ -100,11 +102,23 @@ export async function init(options = {}) {
     const { execSync } = await import('child_process');
     try {
       execSync('npm install --production', { cwd: pipelineDest, stdio: 'pipe' });
-      console.log('  Installed better-sqlite3 + sqlite-vec');
+      console.log('  Installed better-sqlite3 + sqlite-vec + tsx');
     } catch (err) {
       console.warn('  Warning: Failed to install SQLite dependencies. Run "cd .valent-pipeline && npm install" manually.');
     }
-    console.log('  Run "valent-pipeline db init" to create the database.');
+
+    // Auto-create the SQLite database (no separate "db init" step needed)
+    try {
+      const { openDb } = await import('../lib/db.js');
+      const dbPathMatch = config.knowledge?.sqlite_db_path;
+      const fullDbPath = join(projectRoot, dbPathMatch || './.valent-pipeline/pipeline.db');
+      const db = await openDb({ dbPath: fullDbPath });
+      db.close();
+      console.log('  Created SQLite knowledge database');
+    } catch (err) {
+      console.warn(`  Warning: Could not auto-create database: ${err.message}`);
+      console.warn('  Run "valent-pipeline db init" manually.');
+    }
   }
 
   // 7b. Install browser automation MCP for UI projects

package/src/commands/upgrade.js

@@ -91,6 +91,50 @@ export async function upgrade(options = {}) {
     }
   }
 
+  // Migrate runtime package.json: add "type": "module" and tsx if missing
+  const runtimePkgPath = join(pipelineDest, 'package.json');
+  if (fileExists(runtimePkgPath)) {
+    try {
+      const runtimePkg = JSON.parse(readFile(runtimePkgPath));
+      let needsInstall = false;
+
+      if (!runtimePkg.type) {
+        runtimePkg.type = 'module';
+        console.log('Migrated .valent-pipeline/package.json: added "type": "module"');
+      }
+
+      if (!runtimePkg.dependencies?.tsx) {
+        runtimePkg.dependencies = runtimePkg.dependencies || {};
+        runtimePkg.dependencies.tsx = '^4.0.0';
+        needsInstall = true;
+        console.log('Migrated .valent-pipeline/package.json: added tsx dependency');
+      }
+
+      writeFileSafe(runtimePkgPath, JSON.stringify(runtimePkg, null, 2) + '\n');
+
+      if (needsInstall) {
+        const { execSync } = await import('child_process');
+        try {
+          execSync('npm install --production', { cwd: pipelineDest, stdio: 'pipe' });
+          console.log('Installed new dependencies in .valent-pipeline/');
+        } catch (err) {
+          console.warn('Warning: Failed to install new dependencies. Run "cd .valent-pipeline && npm install" manually.');
+        }
+      }
+    } catch { /* skip if parse fails */ }
+  }
+
+  // Ensure database schema is up to date (adds any new tables from this version)
+  try {
+    const { openDb, resolveDbPath } = await import('../lib/db.js');
+    const dbPath = resolveDbPath();
+    const db = await openDb({ dbPath });
+    db.close();
+    console.log('Ensured database schema is up to date');
+  } catch {
+    // DB may not be configured — skip silently
+  }
+
   // Update version file
   writeFileSafe(versionFile, packageVersion);
   console.log(`Updated .valent-pipeline/.valent-version to ${packageVersion}`);

package/src/lib/db.js

@@ -0,0 +1,200 @@
+/**
+ * db.js — Shared SQLite database utilities for valent-pipeline.
+ *
+ * Single source of truth for:
+ *   - Schema DDL (all tables, indexes, FTS5, triggers)
+ *   - DB path resolution from config
+ *   - Database open/close with auto-schema creation
+ */
+
+import { join } from 'path';
+import { existsSync, readFileSync } from 'fs';
+
+// ---------------------------------------------------------------------------
+// Schema DDL — authoritative source. Keep pipeline/scripts/db-bootstrap.ts
+// in sync when modifying (see docs/design/refactor-checklist.md).
+// ---------------------------------------------------------------------------
+
+export const SCHEMA_DDL = `
+  CREATE TABLE IF NOT EXISTS artifacts (
+    id TEXT PRIMARY KEY,
+    story_id TEXT NOT NULL,
+    agent TEXT NOT NULL,
+    artifact_type TEXT NOT NULL,
+    content TEXT NOT NULL,
+    metadata TEXT,
+    created_at TEXT DEFAULT (datetime('now')),
+    UNIQUE(story_id, artifact_type)
+  );
+
+  CREATE TABLE IF NOT EXISTS correction_directives (
+    id TEXT PRIMARY KEY,
+    target_agent TEXT NOT NULL,
+    directive TEXT NOT NULL,
+    reason TEXT,
+    status TEXT DEFAULT 'active',
+    created_batch INTEGER,
+    metadata TEXT
+  );
+
+  CREATE TABLE IF NOT EXISTS calibration (
+    story_id TEXT PRIMARY KEY,
+    story_points INTEGER,
+    ac_count INTEGER,
+    surface TEXT,
+    estimated_points INTEGER,
+    actual_mins REAL,
+    rework_cycles INTEGER DEFAULT 0,
+    sprint_id TEXT,
+    created_at TEXT DEFAULT (datetime('now'))
+  );
+
+  CREATE TABLE IF NOT EXISTS artifacts_working (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    story_id TEXT NOT NULL,
+    agent TEXT NOT NULL,
+    artifact_type TEXT NOT NULL,
+    content TEXT,
+    metadata TEXT,
+    created_at TEXT DEFAULT (datetime('now')),
+    UNIQUE(story_id, artifact_type)
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_artifacts_story ON artifacts(story_id);
+  CREATE INDEX IF NOT EXISTS idx_artifacts_type ON artifacts(artifact_type);
+  CREATE INDEX IF NOT EXISTS idx_artifacts_agent ON artifacts(agent);
+  CREATE INDEX IF NOT EXISTS idx_cd_target ON correction_directives(target_agent);
+  CREATE INDEX IF NOT EXISTS idx_cd_status ON correction_directives(status);
+  CREATE INDEX IF NOT EXISTS idx_calibration_sprint ON calibration(sprint_id);
+`;
+
+export const FTS_DDL = `
+  CREATE VIRTUAL TABLE IF NOT EXISTS artifacts_fts USING fts5(
+    content,
+    story_id UNINDEXED,
+    agent UNINDEXED,
+    artifact_type UNINDEXED,
+    content=artifacts,
+    content_rowid=rowid
+  );
+`;
+
+export const TRIGGERS_DDL = `
+  CREATE TRIGGER IF NOT EXISTS artifacts_ai AFTER INSERT ON artifacts BEGIN
+    INSERT INTO artifacts_fts(rowid, content, story_id, agent, artifact_type)
+    VALUES (new.rowid, new.content, new.story_id, new.agent, new.artifact_type);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS artifacts_ad AFTER DELETE ON artifacts BEGIN
+    INSERT INTO artifacts_fts(artifacts_fts, rowid, content, story_id, agent, artifact_type)
+    VALUES ('delete', old.rowid, old.content, old.story_id, old.agent, old.artifact_type);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS artifacts_au AFTER UPDATE ON artifacts BEGIN
+    INSERT INTO artifacts_fts(artifacts_fts, rowid, content, story_id, agent, artifact_type)
+    VALUES ('delete', old.rowid, old.content, old.story_id, old.agent, old.artifact_type);
+    INSERT INTO artifacts_fts(rowid, content, story_id, agent, artifact_type)
+    VALUES (new.rowid, new.content, new.story_id, new.agent, new.artifact_type);
+  END;
+`;
+
+// ---------------------------------------------------------------------------
+// DB path resolution
+// ---------------------------------------------------------------------------
+
+export function resolveDbPath(options = {}) {
+  if (options.dbPath) return options.dbPath;
+  const configPath = join(process.cwd(), '.valent-pipeline', 'pipeline-config.yaml');
+  if (existsSync(configPath)) {
+    const content = readFileSync(configPath, 'utf-8');
+    const match = content.match(/sqlite_db_path:\s*"?([^"\n]+)"?/);
+    if (match) return join(process.cwd(), match[1].trim());
+  }
+  return join(process.cwd(), '.valent-pipeline', 'pipeline.db');
+}
+
+// ---------------------------------------------------------------------------
+// Database loading
+// ---------------------------------------------------------------------------
+
+export async function loadDatabase() {
+  try {
+    return (await import('better-sqlite3')).default;
+  } catch (err) {
+    console.error('Error: better-sqlite3 not installed.');
+    console.error('Run: npm install better-sqlite3');
+    process.exit(1);
+  }
+}
+
+export async function loadSqliteVec() {
+  try {
+    return await import('sqlite-vec');
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Schema management
+// ---------------------------------------------------------------------------
+
+export function ensureSchema(db) {
+  db.exec(SCHEMA_DDL);
+  db.exec(FTS_DDL);
+  db.exec(TRIGGERS_DDL);
+}
+
+// ---------------------------------------------------------------------------
+// High-level open helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Open a read-write database connection with auto-schema creation.
+ * Creates the DB file and all tables if they don't exist.
+ *
+ * @param {object} [options]
+ * @param {string} [options.dbPath] — explicit path, overrides config
+ * @param {boolean} [options.loadVec] — attempt to load sqlite-vec extension
+ * @returns {Promise<import('better-sqlite3').Database>}
+ */
+export async function openDb(options = {}) {
+  const dbPath = typeof options === 'string' ? options : resolveDbPath(options);
+  const Database = await loadDatabase();
+  const db = new Database(dbPath);
+
+  if (options.loadVec !== false) {
+    const sqliteVec = await loadSqliteVec();
+    if (sqliteVec) {
+      try {
+        sqliteVec.load(db);
+      } catch {
+        // sqlite-vec not available — FTS5 still works
+      }
+    }
+  }
+
+  ensureSchema(db);
+  return db;
+}
+
+/**
+ * Open a read-only database connection. Does NOT run schema DDL.
+ * Errors with a helpful message if the DB file does not exist.
+ *
+ * @param {object} [options]
+ * @param {string} [options.dbPath] — explicit path, overrides config
+ * @returns {Promise<import('better-sqlite3').Database>}
+ */
+export async function openReadonlyDb(options = {}) {
+  const dbPath = typeof options === 'string' ? options : resolveDbPath(options);
+
+  if (!existsSync(dbPath)) {
+    console.error(`Database not found: ${dbPath}`);
+    console.error('Run "valent-pipeline init" or "valent-pipeline db init" to create it.');
+    process.exit(1);
+  }
+
+  const Database = await loadDatabase();
+  return new Database(dbPath, { readonly: true });
+}