praxis-sec 1.0.0

package/cli/utils/intel/index.js

@@ -0,0 +1,235 @@
+/**
+ * Intel orchestrator.
+ *
+ * Public API:
+ *   await runUpdate({ onProgress, sources, force })  → MergedReport
+ *   loadMerged()                                     → cached merged threat-intel.json
+ *   isStale(maxAgeMs)                                → boolean
+ *   listSources()                                    → [{ name, tier, description, envKey? }]
+ *
+ * Cache layout under ~/.praxis/:
+ *   intel/<source>.json            per-source TTL'd raw payload
+ *   threat-intel.json              merged feed (replaces the legacy seed cache)
+ *   intel-meta.json                last update summary
+ */
+
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { fileURLToPath } from 'url';
+
+import * as cache from './cache.js';
+import { mergeIntel, serialize } from './merge.js';
+
+import * as osv from './sources/osv.js';
+import * as kev from './sources/kev.js';
+import * as epss from './sources/epss.js';
+import * as nvd from './sources/nvd.js';
+import * as ghsa from './sources/ghsa.js';
+import * as gitleaks from './sources/gitleaks.js';
+import * as snyk from './sources/snyk.js';
+import * as socket from './sources/socket.js';
+import * as gitguardian from './sources/gitguardian.js';
+import * as sonatype from './sources/sonatype.js';
+import * as phylum from './sources/phylum.js';
+import * as threatpack from './sources/threatpack.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const SEED_PATH = path.resolve(__dirname, '..', '..', 'data', 'threat-intel.json');
+
+// Lazy — resolve at call time so HOME overrides (e.g. in tests) take effect.
+const mergedPath = () => path.join(os.homedir(), '.praxis', 'threat-intel.json');
+const metaPath = () => path.join(os.homedir(), '.praxis', 'intel-meta.json');
+
+const ALL_SOURCES = [
+  osv, ghsa, kev, epss, nvd, gitleaks, threatpack,  // free / core
+  snyk, socket, gitguardian, sonatype, phylum,       // optional / paid
+];
+
+export function listSources() {
+  return ALL_SOURCES.map(s => ({
+    name: s.name,
+    tier: s.tier,
+    description: s.description,
+    envKey: s.envKey || null,
+  }));
+}
+
+/**
+ * Run an update across all sources in parallel.
+ *
+ * @param {object} options
+ * @param {(evt: {source, status, message?}) => void} [options.onProgress]
+ * @param {string[]} [options.sources]   limit to a subset
+ * @param {boolean} [options.force]      ignore TTL on per-source caches
+ */
+export async function runUpdate(options = {}) {
+  const { onProgress = () => {}, sources, force = false } = options;
+  const targets = sources
+    ? ALL_SOURCES.filter(s => sources.includes(s.name))
+    : ALL_SOURCES;
+
+  // First pass: parallel fetch of everything except NVD (NVD needs the
+  // CVE list from the others to know what to enrich).
+  const firstPass = targets.filter(s => s.name !== 'nvd');
+  const firstResults = await Promise.all(
+    firstPass.map(s => runOne(s, { force, onProgress }))
+  );
+
+  // Second pass: NVD enrichment, using CVEs we just collected.
+  let nvdResult = null;
+  if (targets.some(s => s.name === 'nvd')) {
+    const cveList = collectCves(firstResults);
+    nvdResult = await runOne(nvd, { force, onProgress, args: { cveList } });
+  }
+
+  const allResults = nvdResult ? [...firstResults, nvdResult] : firstResults;
+
+  // Merge into the unified feed. Start from the existing feed when present
+  // so subset updates (--only) don't wipe data owned by other sources.
+  let seed = loadSeed();
+  try {
+    const existing = loadMerged();
+    if (existing && typeof existing === 'object') seed = existing;
+  } catch { /* corrupt feed — rebuild from bundled seed */ }
+  const merged = mergeIntel(seed, allResults);
+  const out = serialize(merged);
+
+  ensureHomeDir();
+  fs.writeFileSync(mergedPath(), JSON.stringify(out, null, 2));
+
+  const meta = {
+    updatedAt: new Date().toISOString(),
+    version: out.version,
+    sources: out.sources,
+  };
+  fs.writeFileSync(metaPath(), JSON.stringify(meta, null, 2));
+
+  return { merged: out, meta, results: allResults };
+}
+
+async function runOne(source, { force, onProgress, args = {} }) {
+  onProgress({ source: source.name, status: 'start' });
+
+  // Cache hit?
+  const cached = cache.read(source.name);
+  if (!force && cached && cache.isFresh(cached)) {
+    onProgress({ source: source.name, status: 'cached' });
+    return {
+      name: source.name,
+      ok: true,
+      payload: cached.payload,
+      fetchedAt: cached.fetchedAt,
+      fromCache: true,
+      skipped: cached.payload?.skipped === true,
+    };
+  }
+
+  try {
+    const payload = await source.fetchAll(args);
+    if (payload && payload.skipped) {
+      // Optional source declined (env var unset, etc.) — record but don't fail.
+      cache.write(source.name, payload, source.TTL || 60_000);
+      onProgress({ source: source.name, status: 'skipped', message: payload.reason });
+      return {
+        name: source.name,
+        ok: true,
+        skipped: true,
+        payload,
+        fetchedAt: new Date().toISOString(),
+      };
+    }
+    cache.write(source.name, payload, source.TTL || 60_000);
+    onProgress({ source: source.name, status: 'ok' });
+    return {
+      name: source.name,
+      ok: true,
+      payload,
+      fetchedAt: new Date().toISOString(),
+      fromCache: false,
+    };
+  } catch (err) {
+    onProgress({ source: source.name, status: 'error', message: err.message });
+    // Fall back to whatever's in the cache, even if stale, so a transient
+    // outage doesn't blow away our data.
+    if (cached) {
+      return {
+        name: source.name,
+        ok: true,
+        payload: cached.payload,
+        fetchedAt: cached.fetchedAt,
+        fromCache: true,
+        stale: true,
+        error: err.message,
+      };
+    }
+    return {
+      name: source.name,
+      ok: false,
+      error: err.message,
+    };
+  }
+}
+
+function collectCves(results) {
+  const cves = new Set();
+  for (const r of results) {
+    if (!r.ok || r.skipped) continue;
+    const p = r.payload;
+    if (r.name === 'kev') {
+      for (const e of p.entries || []) cves.add(e.cve);
+    } else if (r.name === 'osv') {
+      for (const rec of p.records || []) {
+        for (const c of rec.cves || []) cves.add(c);
+      }
+    } else if (r.name === 'ghsa') {
+      for (const a of p.advisories || []) {
+        if (a.cveId) cves.add(a.cveId);
+      }
+    }
+  }
+  return [...cves];
+}
+
+function loadSeed() {
+  try {
+    return JSON.parse(fs.readFileSync(SEED_PATH, 'utf-8'));
+  } catch {
+    return { version: '1.0.0' };
+  }
+}
+
+function ensureHomeDir() {
+  const dir = path.dirname(mergedPath());
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+}
+
+export function loadMerged() {
+  try {
+    const p = mergedPath();
+    if (fs.existsSync(p)) return JSON.parse(fs.readFileSync(p, 'utf-8'));
+  } catch { /* fall through */ }
+  return null;
+}
+
+export function loadMeta() {
+  try {
+    const p = metaPath();
+    if (fs.existsSync(p)) return JSON.parse(fs.readFileSync(p, 'utf-8'));
+  } catch { /* fall through */ }
+  return null;
+}
+
+export function isStale(maxAgeMs) {
+  const meta = loadMeta();
+  if (!meta || !meta.updatedAt) return true;
+  return Date.now() - new Date(meta.updatedAt).getTime() > maxAgeMs;
+}
+
+export const PATHS = {
+  get MERGED_PATH() { return mergedPath(); },
+  get META_PATH() { return metaPath(); },
+  SEED_PATH,
+};
+export const sources = { osv, ghsa, kev, epss, nvd, gitleaks, snyk, socket, gitguardian, sonatype, phylum };

package/cli/utils/intel/merge.js

@@ -0,0 +1,229 @@
+/**
+ * Merge per-source payloads into the unified threat-intel.json schema
+ * consumed by ThreatIntel and the agents.
+ *
+ * Schema additions on top of the existing seed:
+ *   - cveAdvisories[]   from OSV + GHSA + NVD
+ *   - kevList[]         from CISA KEV
+ *   - epssScores{}      cve → { score, percentile }
+ *   - osvIndex{}        ecosystem/package → [advisory]
+ *   - ghsaIndex{}       ecosystem/package → [advisory]
+ *   - secretRules[]     extra patterns from Gitleaks (+ optional GitGuardian)
+ *   - sources{}         per-source health: { fetchedAt, count, error?, skipped? }
+ */
+
+export function mergeIntel(seedData, results) {
+  // results: array of { name, ok, payload, error, skipped, fetchedAt }
+  const merged = JSON.parse(JSON.stringify(seedData || {}));
+  merged.version = bumpVersion(merged.version || '1.0.0');
+  merged.updated = new Date().toISOString();
+  merged.sources = merged.sources || {};
+
+  // Reset only the collections owned by sources being freshly merged, so a
+  // subset update (--only) preserves data from the other sources.
+  const fresh = new Set(results.filter(r => r.ok && !r.skipped).map(r => r.name));
+  if (fresh.has('kev')) { merged.kevList = []; merged.kevDetails = {}; }
+  if (fresh.has('epss')) { merged.epssScores = {}; }
+  if (fresh.has('nvd')) { merged.nvdDetails = {}; }
+  if (fresh.has('osv')) merged.osvIndex = {};
+  if (fresh.has('ghsa')) merged.ghsaIndex = {};
+  if (fresh.has('gitleaks') || fresh.has('gitguardian')) merged.secretRules = [];
+
+  // Drop stale advisories from sources being refreshed; keep the rest and
+  // rebuild the dedup index from the survivors.
+  merged.cveAdvisories = (merged.cveAdvisories || []).filter(a => !fresh.has(a.source));
+  merged._cveAdvisorySeen = new Set(merged.cveAdvisories.map(a => `${a.cve}@${a.source}`));
+
+  for (const r of results) {
+    merged.sources[r.name] = {
+      ok: r.ok,
+      fetchedAt: r.fetchedAt || null,
+      skipped: r.skipped || false,
+      reason: r.skipped ? r.payload?.reason : undefined,
+      error: r.ok ? null : (r.error || null),
+      stats: r.ok && !r.skipped ? sourceStats(r) : null,
+    };
+
+    if (!r.ok || r.skipped) continue;
+
+    switch (r.name) {
+      case 'osv': mergeOsv(merged, r.payload); break;
+      case 'ghsa': mergeGhsa(merged, r.payload); break;
+      case 'kev': mergeKev(merged, r.payload); break;
+      case 'epss': mergeEpss(merged, r.payload); break;
+      case 'nvd': mergeNvd(merged, r.payload); break;
+      case 'gitleaks': mergeGitleaks(merged, r.payload); break;
+      case 'threatpack': mergeThreatpack(merged, r.payload); break;
+      case 'snyk': mergeSnyk(merged, r.payload); break;
+      case 'socket': mergeSocket(merged, r.payload); break;
+      case 'gitguardian': mergeGitguardian(merged, r.payload); break;
+      case 'sonatype': mergeSonatype(merged, r.payload); break;
+      case 'phylum': mergePhylum(merged, r.payload); break;
+    }
+  }
+
+  return merged;
+}
+
+function bumpVersion(v) {
+  const parts = String(v).split('.').map(Number);
+  while (parts.length < 3) parts.push(0);
+  parts[2] = parts[2] + 1;
+  return parts.join('.');
+}
+
+function sourceStats(r) {
+  const p = r.payload || {};
+  if (Array.isArray(p.records)) return { records: p.records.length };
+  if (Array.isArray(p.entries)) return { entries: p.entries.length };
+  if (Array.isArray(p.advisories)) return { advisories: p.advisories.length };
+  if (Array.isArray(p.detectors)) return { detectors: p.detectors.length };
+  if (Array.isArray(p.rules)) return { rules: p.rules.length };
+  if (Array.isArray(p.issues)) return { issues: p.issues.length };
+  if (typeof p.count === 'number') return { count: p.count };
+  if (p.scores) return { scores: Object.keys(p.scores).length };
+  if (p.details) return { details: Object.keys(p.details).length };
+  return {};
+}
+
+function mergeOsv(merged, payload) {
+  for (const r of payload.records || []) {
+    const key = `${r.ecosystem}/${r.package}`;
+    (merged.osvIndex[key] = merged.osvIndex[key] || []).push(r);
+    for (const cve of r.cves || []) {
+      addCveAdvisory(merged, {
+        cve, source: 'osv', summary: r.summary, severity: r.severity,
+        package: r.package, ecosystem: r.ecosystem, references: r.references,
+      });
+    }
+  }
+}
+
+function mergeGhsa(merged, payload) {
+  for (const adv of payload.advisories || []) {
+    for (const v of adv.vulnerabilities || []) {
+      if (!v.package) continue;
+      const key = `${v.ecosystem}/${v.package}`;
+      (merged.ghsaIndex[key] = merged.ghsaIndex[key] || []).push({
+        ghsaId: adv.ghsaId, cveId: adv.cveId,
+        severity: adv.severity, summary: adv.summary,
+        vulnerable_version_range: v.vulnerable_version_range,
+        first_patched_version: v.first_patched_version,
+      });
+    }
+    if (adv.cveId) {
+      addCveAdvisory(merged, {
+        cve: adv.cveId, source: 'ghsa', summary: adv.summary,
+        severity: adv.severity, references: adv.references, ghsaId: adv.ghsaId,
+      });
+    }
+  }
+}
+
+function mergeKev(merged, payload) {
+  merged.kevList = (payload.entries || []).map(e => e.cve);
+  merged.kevDetails = Object.fromEntries((payload.entries || []).map(e => [e.cve, e]));
+  merged.kevCatalogVersion = payload.catalogVersion;
+}
+
+function mergeEpss(merged, payload) {
+  merged.epssScores = payload.scores || {};
+  merged.epssModelDate = payload.modelDate;
+}
+
+function mergeNvd(merged, payload) {
+  merged.nvdDetails = payload.details || {};
+}
+
+function mergeGitleaks(merged, payload) {
+  for (const r of payload.rules || []) {
+    merged.secretRules.push({
+      id: r.id,
+      description: r.description,
+      regex: r.regex,
+      source: 'gitleaks',
+      severity: 'high',
+    });
+  }
+}
+
+function mergeThreatpack(merged, payload) {
+  // Threat-pack overlay: probe signatures + EAA additions + gateway allowlists.
+  // The corpus loader and agents read this section when it exists.
+  merged.threatPack = {
+    version: payload.version || '0.0.0',
+    fetchedRemote: Boolean(payload.fetchedRemote),
+    sourceUrl: payload.sourceUrl || null,
+    probes: payload.probes || [],
+    eaa: payload.eaa || [],
+    gateways: payload.gateways || [],
+  };
+}
+
+function mergeSnyk(merged, payload) {
+  for (const issue of payload.issues || []) {
+    for (const cve of issue.cves || []) {
+      addCveAdvisory(merged, {
+        cve, source: 'snyk', summary: issue.title,
+        severity: issue.severity,
+      });
+    }
+  }
+}
+
+function mergeSocket(merged, payload) {
+  merged.socketAdvisories = payload.advisories || [];
+}
+
+function mergeGitguardian(merged, payload) {
+  for (const d of payload.detectors || []) {
+    merged.secretRules.push({
+      id: d.id,
+      description: d.summary,
+      family: d.family,
+      source: 'gitguardian',
+      severity: d.severity,
+    });
+  }
+}
+
+function mergeSonatype(merged, payload) {
+  for (const adv of payload.advisories || []) {
+    if (!adv.cve) continue;
+    addCveAdvisory(merged, {
+      cve: adv.cve, source: 'sonatype', summary: adv.title,
+      severity: scoreToSeverity(adv.cvssScore),
+      package: adv.purl, references: adv.reference ? [adv.reference] : [],
+    });
+  }
+}
+
+function mergePhylum(merged, payload) {
+  merged.phylumAdvisories = payload.advisories || [];
+}
+
+function addCveAdvisory(merged, entry) {
+  // Dedup by (cve, source); keep all sources so we can show provenance.
+  const key = `${entry.cve}@${entry.source}`;
+  if (merged._cveAdvisorySeen?.has?.(key)) return;
+  merged._cveAdvisorySeen = merged._cveAdvisorySeen || new Set();
+  merged._cveAdvisorySeen.add(key);
+  merged.cveAdvisories.push(entry);
+}
+
+function scoreToSeverity(score) {
+  if (score == null) return 'medium';
+  if (score >= 9) return 'critical';
+  if (score >= 7) return 'high';
+  if (score >= 4) return 'medium';
+  return 'low';
+}
+
+/**
+ * Strip non-serializable internals before writing to disk.
+ */
+export function serialize(merged) {
+  const out = { ...merged };
+  delete out._cveAdvisorySeen;
+  return out;
+}

package/cli/utils/intel/sources/epss.js

@@ -0,0 +1,54 @@
+/**
+ * EPSS — Exploit Prediction Scoring System (FIRST.org).
+ * https://www.first.org/epss/
+ *
+ * Daily CSV. We download the gzipped current scores and parse into a map
+ * keyed by CVE → { score, percentile }. ~250k CVEs, ~10MB compressed.
+ *
+ * Score is 0..1, the probability the CVE will be exploited in the wild
+ * within 30 days. Used to sort findings by real-world risk.
+ */
+
+import zlib from 'zlib';
+import { promisify } from 'util';
+import { safeFetch } from '../http.js';
+
+const URL = 'https://epss.cyentia.com/epss_scores-current.csv.gz';
+const TTL_MS = 24 * 60 * 60 * 1000; // 24h
+const gunzip = promisify(zlib.gunzip);
+
+export const name = 'epss';
+export const tier = 'core';
+export const description = 'EPSS — exploit-likelihood scores from FIRST.org';
+
+export async function fetchAll() {
+  const res = await safeFetch(URL);
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+
+  const buf = Buffer.from(await res.arrayBuffer());
+  const csv = (await gunzip(buf)).toString('utf-8');
+
+  const scores = {};
+  const lines = csv.split('\n');
+  let modelDate = null;
+
+  for (const line of lines) {
+    if (!line || line.startsWith('#')) {
+      // Header lines look like: #model_version:v2025.03.14
+      const m = line.match(/score_date:([0-9-]+)/);
+      if (m) modelDate = m[1];
+      continue;
+    }
+    if (line.startsWith('cve,')) continue; // column header
+    const [cve, score, percentile] = line.split(',');
+    if (!cve || !score) continue;
+    scores[cve] = {
+      score: parseFloat(score),
+      percentile: parseFloat(percentile),
+    };
+  }
+
+  return { modelDate, count: Object.keys(scores).length, scores };
+}
+
+export const TTL = TTL_MS;

package/cli/utils/intel/sources/ghsa.js

@@ -0,0 +1,81 @@
+/**
+ * GitHub Security Advisories (GHSA).
+ * https://docs.github.com/en/rest/security-advisories/global-advisories
+ *
+ * REST endpoint, paginated, returns published advisories across all
+ * ecosystems. Auth optional (GITHUB_TOKEN raises rate limit from 60 to
+ * 5000 req/h). We pull the latest 1000 advisories — newer than KEV adds,
+ * captures advisories that may not yet be in OSV.
+ */
+
+import { fetchJson, safeFetch } from '../http.js';
+
+const BASE = 'https://api.github.com/advisories';
+const PER_PAGE = 100;
+const MAX_PAGES = 10;
+const TTL_MS = 12 * 60 * 60 * 1000; // 12h
+
+export const name = 'ghsa';
+export const tier = 'core';
+export const description = 'GitHub Security Advisories (global)';
+
+export async function fetchAll() {
+  const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+  const headers = {
+    'Accept': 'application/vnd.github+json',
+    'X-GitHub-Api-Version': '2022-11-28',
+  };
+  if (token) headers.Authorization = `Bearer ${token}`;
+
+  const advisories = [];
+  let url = `${BASE}?per_page=${PER_PAGE}&type=reviewed&sort=published&direction=desc`;
+
+  for (let page = 0; page < MAX_PAGES && url; page++) {
+    const res = await safeFetch(url, { headers });
+    if (!res.ok) {
+      if (page === 0) throw new Error(`HTTP ${res.status} from GHSA`);
+      break;
+    }
+    const data = await res.json();
+    for (const adv of data) {
+      advisories.push(normalize(adv));
+    }
+    url = parseNextLink(res.headers.get('link')) || null;
+  }
+
+  return {
+    tokenUsed: !!token,
+    count: advisories.length,
+    advisories,
+  };
+}
+
+function normalize(adv) {
+  return {
+    ghsaId: adv.ghsa_id,
+    cveId: adv.cve_id,
+    summary: adv.summary,
+    severity: adv.severity, // critical / high / moderate / low
+    published: adv.published_at,
+    updated: adv.updated_at,
+    cwes: (adv.cwes || []).map(c => c.cwe_id),
+    references: (adv.references || []).slice(0, 5).map(r => r.url || r),
+    vulnerabilities: (adv.vulnerabilities || []).map(v => ({
+      ecosystem: v.package?.ecosystem,
+      package: v.package?.name,
+      vulnerable_version_range: v.vulnerable_version_range,
+      first_patched_version: v.first_patched_version,
+    })),
+  };
+}
+
+function parseNextLink(linkHeader) {
+  if (!linkHeader) return null;
+  for (const part of linkHeader.split(',')) {
+    const m = part.match(/<([^>]+)>;\s*rel="next"/);
+    if (m) return m[1];
+  }
+  return null;
+}
+
+export const TTL = TTL_MS;

package/cli/utils/intel/sources/gitguardian.js

@@ -0,0 +1,40 @@
+/**
+ * GitGuardian — paid.
+ * https://api.gitguardian.com/docs
+ *
+ * Requires GITGUARDIAN_API_KEY. Provides ~400 secret-detector definitions
+ * and a /v1/audit/detectors endpoint listing them. We pull detector
+ * definitions to enrich praxis's secret-scanning rule set.
+ */
+
+import { fetchJson } from '../http.js';
+
+const TTL_MS = 7 * 24 * 60 * 60 * 1000;
+
+export const name = 'gitguardian';
+export const tier = 'optional';
+export const description = 'GitGuardian secret detector definitions (requires GITGUARDIAN_API_KEY)';
+export const envKey = 'GITGUARDIAN_API_KEY';
+
+export async function fetchAll() {
+  const key = process.env.GITGUARDIAN_API_KEY;
+  if (!key) return { skipped: true, reason: 'GITGUARDIAN_API_KEY not set' };
+
+  const headers = { Authorization: `Token ${key}` };
+  // GitGuardian's public detector list endpoint
+  const data = await fetchJson('https://api.gitguardian.com/v1/secret_detectors', { headers });
+
+  const detectors = (Array.isArray(data) ? data : data.results || []).map(d => ({
+    id: d.detector_name || d.id,
+    family: d.family,
+    summary: d.detector_group_name || d.name,
+    severity: d.severity || 'high',
+  }));
+
+  return {
+    count: detectors.length,
+    detectors,
+  };
+}
+
+export const TTL = TTL_MS;