seo-intel 1.5.27 → 1.5.29

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## 1.5.29 (2026-05-17)
+
+### MCP — `ingest_insight` closes the loop (agents become collaborators, not consumers)
+The MCP server now accepts write-back. An agent can read your raw data, do its own analysis with its own flagship LLM, and persist findings into the Intelligence Ledger — surviving across sessions, surfacing in the dashboard, deduplicating against future runs.
+
+- **`ingest_insight(project, type, data, agent_name?)`** — **free tier**. The agent's LLM did the analysis; we just provide storage. Allowed types mirror what `analyze` writes: `keyword_gap`, `long_tail`, `quick_win`, `new_page`, `content_gap`, `technical_gap`, `positioning`.
+- **Dedup contract**: same `(project, type, fingerprint)` returns the existing row with `deduped: true` and bumps `last_seen` — no duplicate accumulation across sessions.
+- **Provenance**: source is stored as `agent:<name>` (e.g. `agent:claude-opus-4-7`) when `agent_name` is supplied, else just `agent`. Also stamped into the `data` JSON blob as `_source` for downstream consumers that only read `data`.
+- **Schema**: idempotent `ALTER TABLE insights ADD COLUMN source TEXT DEFAULT 'cli'` — existing rows backfill to `'cli'`; analyze-time writes stay as `'cli'`; agent writes flip to `'agent:*'`. Safe on existing DBs.
+
+### Logo
+- Updated product logo to the sharp / soft-corners v1 variant. Size dropped 1.46 MB → 953 KB. Dashboard favicon + npm package both pick up the new asset.
+
+## 1.5.28 (2026-05-17)
+
+### MCP — agents can now trigger crawls and watch progress
+The MCP server gains its first **active** tools — agents move from read-only to actually doing work on the user's machine.
+
+- **`run_crawl(project, stealth?, max_pages?)`** — spawn a crawl as a detached subprocess. Returns immediately with `{ started, pid, command, hint }`. Free tier — crawl page limits still apply (Solo unlocks unlimited). Refuses to start if any seo-intel job is already running (conflict guard mirrors the existing HTTP `/api/crawl` behaviour).
+- **`get_crawl_status()`** — read the most recent job's progress: status (`running` / `completed` / `crashed` / `stopped` / `idle`), command, project, pid, timestamps. PID liveness is verified — a "running" job whose process died gets re-tagged as `crashed`.
+
+A natural session now looks like: agent calls `run_crawl(carbium)` → polls `get_crawl_status()` every minute → once `completed`, calls `get_intel(carbium, for=raw)` and `get_pages(carbium)` to see new data. Free tier, end to end.
+
+### Internal — shared progress reader
+`server.js` and `mcp/server.js` now both read job state from `lib/progress.js` (the canonical implementation, with PID liveness detection). Eliminates a duplicate `readProgress()` and ensures any future progress-file schema changes propagate automatically.
+
 ## 1.5.27 (2026-05-16)
 
 ### MCP — three new free-tier read tools

package/db/db.js

@@ -29,6 +29,7 @@ export function getDb(dbPath = './seo-intel.db') {
   try { _db.exec('ALTER TABLE pages ADD COLUMN x_robots_tag TEXT'); } catch { /* already exists */ }
   try { _db.exec('ALTER TABLE analyses ADD COLUMN technical_gaps TEXT'); } catch { /* already exists */ }
   try { _db.exec('ALTER TABLE extractions ADD COLUMN intent_scores TEXT'); } catch { /* already exists */ }
+  try { _db.exec("ALTER TABLE insights ADD COLUMN source TEXT DEFAULT 'cli'"); } catch { /* already exists */ }
 
   // Backfill first_seen_at from crawled_at for existing rows
   _db.exec('UPDATE pages SET first_seen_at = crawled_at WHERE first_seen_at IS NULL');
@@ -225,6 +226,59 @@ export function upsertInsightsFromKeywords(db, project, keywordsReport) {
   }
 }
 
+// ── Agent-ingested insight (write-back from MCP) ────────────────────────────
+
+export const AGENT_INSIGHT_TYPES = ['keyword_gap', 'long_tail', 'quick_win', 'new_page', 'content_gap', 'technical_gap', 'positioning'];
+
+/**
+ * Insert a single insight on behalf of an external agent (e.g. via MCP).
+ * Uses the same dedup contract as analyze-time inserts (UNIQUE on
+ * project + type + fingerprint), so an agent repeating the same finding
+ * across sessions updates `last_seen` instead of duplicating rows.
+ *
+ * Returns { ok, id, fingerprint, deduped } — `deduped: true` when the row
+ * already existed and we only refreshed last_seen.
+ */
+export function insertAgentInsight(db, { project, type, data, agentName }) {
+  if (!AGENT_INSIGHT_TYPES.includes(type)) {
+    return { ok: false, error: `Unsupported type "${type}". Allowed: ${AGENT_INSIGHT_TYPES.join(', ')}` };
+  }
+  if (!project) return { ok: false, error: 'project is required' };
+  if (!data || typeof data !== 'object') return { ok: false, error: 'data must be an object' };
+
+  const fingerprint = _insightFingerprint(type, data);
+  if (!fingerprint) {
+    return { ok: false, error: `data is missing the identifier field this type needs (see _insightFingerprint in db/db.js for the per-type contract)` };
+  }
+
+  const source = agentName ? `agent:${agentName}` : 'agent';
+  const ts = Date.now();
+
+  // Stash provenance inside the data blob too — survives if/when the source
+  // column is ever queried separately, but also keeps it visible to consumers
+  // that only read `data`.
+  const enriched = { ...data, _source: source, _ingested_at: new Date(ts).toISOString() };
+
+  const existing = db.prepare(
+    'SELECT id FROM insights WHERE project = ? AND type = ? AND fingerprint = ?'
+  ).get(project, type, fingerprint);
+
+  db.prepare(`
+    INSERT INTO insights (project, type, status, fingerprint, first_seen, last_seen, source_analysis_id, data, source)
+    VALUES (?, ?, 'active', ?, ?, ?, NULL, ?, ?)
+    ON CONFLICT(project, type, fingerprint) DO UPDATE SET
+      last_seen = excluded.last_seen,
+      data = excluded.data,
+      source = excluded.source
+  `).run(project, type, fingerprint, ts, ts, JSON.stringify(enriched), source);
+
+  const row = db.prepare(
+    'SELECT id FROM insights WHERE project = ? AND type = ? AND fingerprint = ?'
+  ).get(project, type, fingerprint);
+
+  return { ok: true, id: row.id, fingerprint, deduped: !!existing, source, last_seen: ts };
+}
+
 // ── Read active insights (accumulated across all runs) ──────────────────────
 
 export function getActiveInsights(db, project) {

package/lib/progress.js

@@ -0,0 +1,37 @@
+/**
+ * lib/progress.js — Single source of truth for the seo-intel job progress file.
+ *
+ * The CLI's crawl/extract/analyze/aeo/... commands all write their state to
+ * `.extraction-progress.json` in the project root. Server.js, mcp/server.js,
+ * and any future consumer can read job status from here without spawning a
+ * subprocess.
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const PROGRESS_FILE = join(__dirname, '..', '.extraction-progress.json');
+
+/**
+ * Read the current job progress, with PID liveness detection so a "running"
+ * job whose process died gets re-tagged as "crashed".
+ *
+ * @returns {object|null}
+ */
+export function readProgress() {
+  try {
+    if (!existsSync(PROGRESS_FILE)) return null;
+    const data = JSON.parse(readFileSync(PROGRESS_FILE, 'utf8'));
+    if (data.status === 'running' && data.pid) {
+      try { process.kill(data.pid, 0); } catch (e) {
+        if (e.code === 'ESRCH') {
+          data.status = 'crashed';
+          data.crashed_at = data.updated_at;
+        }
+      }
+    }
+    return data;
+  } catch { return null; }
+}

package/mcp/server.js

@@ -21,12 +21,14 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import * as z from 'zod/v4';
 import { readFileSync, readdirSync, existsSync } from 'fs';
+import { spawn } from 'child_process';
 import { dirname, join } from 'path';
 import { fileURLToPath } from 'url';
 
-import { getDb } from '../db/db.js';
+import { getDb, insertAgentInsight, AGENT_INSIGHT_TYPES } from '../db/db.js';
 import { getIntel, INTEL_SLICES, FREE_SLICES } from '../lib/intel.js';
 import { isPro } from '../lib/license.js';
+import { readProgress } from '../lib/progress.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const ROOT = join(__dirname, '..');
@@ -233,11 +235,140 @@ server.registerTool(
   }
 );
 
+// ── Tool: run_crawl (free) ────────────────────────────────────────────────
+server.registerTool(
+  'run_crawl',
+  {
+    description: [
+      'Trigger a background crawl for an existing project. Spawns the crawl as a detached subprocess and returns immediately — the crawl will keep running even if this MCP server exits. Use get_crawl_status to monitor progress, or call get_intel/get_pages once the crawl completes to see results.',
+      '',
+      'Conflict guard: refuses to start if any seo-intel job is already running. Free tier — crawl page limits still apply (configurable via setup / Solo license unlocks unlimited).',
+    ].join('\n'),
+    inputSchema: {
+      project: z.string().describe('Existing project slug. Use list_projects to discover.'),
+      stealth: z.boolean().optional().describe('Enable stealth browser mode for JS-heavy or anti-bot sites'),
+      max_pages: z.number().int().positive().optional().describe('Override max pages per domain'),
+    },
+  },
+  async ({ project, stealth, max_pages }) => {
+    const configPath = join(CONFIG_DIR, `${project}.json`);
+    if (!existsSync(configPath)) {
+      const available = listConfigProjects().map(p => p.project).join(', ') || '(none configured)';
+      return {
+        content: [{ type: 'text', text: `Project "${project}" not found. Available: ${available}. Use list_projects to discover, or run \`seo-intel setup\` to add a new project.` }],
+        isError: true,
+      };
+    }
+    const progress = readProgress();
+    if (progress?.status === 'running') {
+      return {
+        content: [{ type: 'text', text: `A seo-intel job is already running (command="${progress.command}", project="${progress.project}", pid=${progress.pid}). Call get_crawl_status to monitor, or wait for it to finish before starting another.` }],
+        isError: true,
+      };
+    }
+
+    const args = ['cli.js', 'crawl', project];
+    if (stealth) args.push('--stealth');
+    if (max_pages) args.push('--max-pages', String(max_pages));
+
+    const child = spawn(process.execPath, args, {
+      cwd: ROOT,
+      detached: true,
+      stdio: 'ignore',
+    });
+    child.unref();
+
+    const result = {
+      started: true,
+      pid: child.pid,
+      project,
+      command: `node ${args.join(' ')}`,
+      hint: 'Crawl is running detached. Call get_crawl_status to check progress (updates every few seconds), or call get_intel(project, for=raw) in a minute or two to see new data.',
+    };
+    return {
+      content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+      structuredContent: result,
+    };
+  }
+);
+
+// ── Tool: get_crawl_status (free) ─────────────────────────────────────────
+server.registerTool(
+  'get_crawl_status',
+  {
+    description: 'Read the current state of the most recent seo-intel job (crawl/extract/analyze/etc). Returns status: running | completed | crashed | stopped | idle, plus project/command/pid/timestamps when available. Use this after run_crawl to monitor progress. Free tier.',
+  },
+  async () => {
+    const progress = readProgress() || { status: 'idle', note: 'No seo-intel job has been recorded since startup. Use run_crawl to start one.' };
+    return {
+      content: [{ type: 'text', text: JSON.stringify(progress, null, 2) }],
+      structuredContent: progress,
+    };
+  }
+);
+
+// ── Tool: ingest_insight (free — write-back closes the loop) ──────────────
+server.registerTool(
+  'ingest_insight',
+  {
+    description: [
+      'Persist an agent-generated insight into the SEO Intel Intelligence Ledger so it shows up in the dashboard and survives across sessions. Free tier — the agent\'s own LLM did the analysis; we just provide storage.',
+      '',
+      'Dedup contract: same (project, type, fingerprint) updates `last_seen` instead of creating a duplicate row. So an agent rediscovering the same finding across sessions cleanly bumps the timestamp.',
+      '',
+      'Allowed types (mirror what the cloud `analyze` command writes):',
+      '  keyword_gap     data: { keyword, ... }       fingerprint = keyword',
+      '  long_tail       data: { phrase, ... }        fingerprint = phrase',
+      '  quick_win       data: { page, issue, ... }   fingerprint = page::issue',
+      '  new_page        data: { target_keyword | title, ... }',
+      '  content_gap     data: { topic, ... }         fingerprint = topic',
+      '  technical_gap   data: { gap, ... }           fingerprint = gap',
+      '  positioning     data: { ...free-form... }    one slot per project',
+      '',
+      'data must include the identifier field above; otherwise the tool returns an error.',
+    ].join('\n'),
+    inputSchema: {
+      project: z.string().describe('Project slug'),
+      type: z.enum(AGENT_INSIGHT_TYPES).describe('Insight type from the allowed set'),
+      data: z.record(z.any()).describe('Insight payload — JSON object. Must include the identifier field for the chosen type.'),
+      agent_name: z.string().optional().describe('Optional provenance tag (e.g. "claude-opus-4-7"). Stored as source="agent:<name>".'),
+    },
+  },
+  async ({ project, type, data, agent_name }) => {
+    try {
+      const db = getDb();
+      const result = insertAgentInsight(db, { project, type, data, agentName: agent_name });
+      if (!result.ok) {
+        return { content: [{ type: 'text', text: `seo-intel ingest error: ${result.error}` }], isError: true };
+      }
+      const payload = {
+        ok: true,
+        project,
+        type,
+        insight_id: result.id,
+        fingerprint: result.fingerprint,
+        deduped: result.deduped,
+        source: result.source,
+        last_seen: new Date(result.last_seen).toISOString(),
+        hint: result.deduped
+          ? 'Insight already existed; last_seen refreshed.'
+          : 'New insight persisted. It will appear in the dashboard ledger and in get_intel(for=audit).',
+      };
+      return {
+        content: [{ type: 'text', text: JSON.stringify(payload, null, 2) }],
+        structuredContent: payload,
+      };
+    } catch (err) {
+      return { content: [{ type: 'text', text: `seo-intel error: ${err.message}` }], isError: true };
+    }
+  }
+);
+
 async function main() {
   const transport = new StdioServerTransport();
   await server.connect(transport);
   // stderr is fine; the host typically surfaces this in its MCP logs panel.
-  console.error(`[seo-intel-mcp] v${VERSION} ready on stdio. Tools: list_projects, get_intel, get_pages, list_keywords, get_headings.`);
+  console.error(`[seo-intel-mcp] v${VERSION} ready on stdio. Tools: list_projects, get_intel, get_pages, list_keywords, get_headings, run_crawl, get_crawl_status, ingest_insight.`);
 }
 
 main().catch(err => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "seo-intel",
-  "version": "1.5.27",
+  "version": "1.5.29",
   "description": "Local Ahrefs-style SEO competitor intelligence. Crawl → SQLite → cloud analysis.",
   "type": "module",
   "license": "SEE LICENSE IN LICENSE",

package/server.js

@@ -4,10 +4,10 @@ import { spawn } from 'child_process';
 import { dirname, join, extname } from 'path';
 import { fileURLToPath } from 'url';
 import { checkForUpdates, getUpdateInfo } from './lib/updater.js';
+import { readProgress, PROGRESS_FILE } from './lib/progress.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PORT = parseInt(process.env.PORT || '3000', 10);
-const PROGRESS_FILE = join(__dirname, '.extraction-progress.json');
 const REPORTS_DIR = join(__dirname, 'reports');
 
 
@@ -100,23 +100,6 @@ const MIME = {
   '.zip': 'application/zip',
 };
 
-// ── Read progress with PID liveness check (mirrors cli.js) ──
-function readProgress() {
-  try {
-    if (!existsSync(PROGRESS_FILE)) return null;
-    const data = JSON.parse(readFileSync(PROGRESS_FILE, 'utf8'));
-    if (data.status === 'running' && data.pid) {
-      try { process.kill(data.pid, 0); } catch (e) {
-        if (e.code === 'ESRCH') {
-          data.status = 'crashed';
-          data.crashed_at = data.updated_at;
-        }
-      }
-    }
-    return data;
-  } catch { return null; }
-}
-
 // ── Parse JSON body from request ──
 function readBody(req) {
   return new Promise((resolve, reject) => {