ultrastiching-cli 0.0.0-stage → 0.1.1

package/src/marketdb.js

@@ -0,0 +1,482 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, extname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DuckDBInstance } from '@duckdb/node-api';
+import { tableFromIPC } from 'apache-arrow';
+import { buildPayload, endpoint } from './registry.js';
+import { callByName, redactKey } from './http.js';
+import { resolveKey } from './config.js';
+
+export const SCHEMA_VERSION = 1;
+export const MAX_BUFFERED_ROWS = 100000;
+export const MAX_BUFFERED_JSON_BYTES = 64 * 1024 * 1024;
+const IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/;
+let datasetsCache;
+
+export function datasets() {
+  if (!datasetsCache) {
+    const source = new URL('../../registry/marketdb-datasets.json', import.meta.url);
+    const packed = new URL('../registry/marketdb-datasets.json', import.meta.url);
+    let raw;
+    try { raw = readFileSync(fileURLToPath(source), 'utf8'); }
+    catch (error) {
+      if (error.code !== 'ENOENT') throw error;
+      raw = readFileSync(fileURLToPath(packed), 'utf8');
+    }
+    datasetsCache = JSON.parse(raw).datasets;
+  }
+  return datasetsCache;
+}
+
+export function dbPath(explicit, env = process.env, home = homedir()) {
+  const value = explicit ?? env.MERIDIAN_DB ?? '~/.meridian/marketdb.duckdb';
+  if (typeof value !== 'string' || !value.trim()) throw new Error('database path must be a non-empty string');
+  const expanded = value === '~' ? home : value.startsWith('~/') ? join(home, value.slice(2)) : value;
+  return resolve(expanded);
+}
+
+export function canonicalJson(value) {
+  const normalize = (item) => {
+    if (item instanceof Date) return item.toISOString().slice(0, 10);
+    if (Array.isArray(item)) return item.map(normalize);
+    if (item && typeof item === 'object') return Object.fromEntries(Object.keys(item).sort().filter((key) => item[key] !== undefined).map((key) => [key, normalize(item[key])]));
+    if (typeof item === 'number' && !Number.isFinite(item)) throw new TypeError('non-finite number is not JSON serializable');
+    if (typeof item === 'bigint' || item === undefined) throw new TypeError('value is not JSON serializable');
+    return item;
+  };
+  return JSON.stringify(normalize(value));
+}
+
+const sha = (raw) => createHash('sha256').update(raw, 'utf8').digest('hex');
+let lastMicrosecond = 0;
+function now() {
+  lastMicrosecond = Math.max(Date.now() * 1000, lastMicrosecond + 1);
+  const date = new Date(Math.floor(lastMicrosecond / 1000)).toISOString();
+  return `${date.slice(0, 20)}${String(lastMicrosecond % 1000000).padStart(6, '0')}+00:00`;
+}
+function quoted(value) {
+  if (typeof value !== 'string' || !IDENT.test(value)) throw new Error(`unsupported SQL identifier: ${value}`);
+  return `"${value.replaceAll('"', '""')}"`;
+}
+function sqlString(value) {
+  if (value.includes('\0')) throw new Error('NUL is not permitted in an export path');
+  return `'${value.replaceAll("'", "''")}'`;
+}
+function jsonSafe(value) {
+  if (typeof value === 'bigint') return value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER ? Number(value) : value.toString();
+  if (value instanceof Date) return value.toISOString().slice(0, 10);
+  if (Array.isArray(value)) return value.map(jsonSafe);
+  if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, jsonSafe(v)]));
+  return value;
+}
+
+export class UnsupportedArrowValueError extends TypeError {
+  constructor(field, value) {
+    super(`Arrow field ${field} contains int64 ${value}, outside JavaScript's exact integer range`);
+    this.name = 'UnsupportedArrowValueError';
+    this.field = field;
+    this.value = value;
+  }
+  toDiagnostic() { return { type: this.name, field: this.field, value: this.value, message: this.message }; }
+}
+
+function exactArrowValue(value, field) {
+  if (typeof value === 'bigint') {
+    if (value < BigInt(Number.MIN_SAFE_INTEGER) || value > BigInt(Number.MAX_SAFE_INTEGER)) {
+      throw new UnsupportedArrowValueError(field, value.toString());
+    }
+    return Number(value);
+  }
+  if (value instanceof Date) return value.toISOString().slice(0, 10);
+  if (Array.isArray(value)) return value.map((item, index) => exactArrowValue(item, `${field}[${index}]`));
+  if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, exactArrowValue(item, `${field}.${key}`)]));
+  return value;
+}
+
+async function withDb(path, readOnly, fn) {
+  if (!readOnly) mkdirSync(dirname(path), { recursive: true });
+  if (readOnly && !existsSync(path)) throw new Error(`database does not exist: ${path}`);
+  const instance = await DuckDBInstance.create(path, readOnly ? { access_mode: 'READ_ONLY' } : undefined);
+  const con = await instance.connect();
+  try { return await fn(con); }
+  finally { con.closeSync(); instance.closeSync(); }
+}
+
+const rows = async (con, sql, params = []) => jsonSafe((await con.runAndReadAll(sql, params)).getRowsJS());
+const objects = async (con, sql, params = []) => jsonSafe((await con.runAndReadAll(sql, params)).getRowObjectsJS());
+const run = async (con, sql, params = []) => con.run(sql, params);
+
+function catalogTypes(name) {
+  const meta = endpoint(name, 'http');
+  if (!meta) throw new Error(`endpoint missing from registry: ${name}`);
+  const fields = new Map();
+  for (const item of meta.response?.fields ?? []) {
+    quoted(item.name);
+    const kind = String(item.type ?? '').toLowerCase();
+    let dtype = 'VARCHAR';
+    if (['float32', 'float64', 'double', 'number'].includes(kind)) dtype = 'DOUBLE';
+    else if (['int', 'int32', 'int64', 'integer', 'long'].includes(kind)) dtype = 'BIGINT';
+    else if (['bool', 'boolean'].includes(kind)) dtype = 'BOOLEAN';
+    else if (kind === 'date') dtype = 'DATE';
+    const source = ['string', 'float32', 'float64', 'double', 'number', 'int', 'int32', 'int64', 'integer', 'long', 'bool', 'boolean', 'date'].includes(kind) ? 'catalog' : 'unresolved';
+    fields.set(item.name, [dtype, source]);
+  }
+  return fields;
+}
+
+function runtimeType(value) {
+  if (value === null || value === undefined) return null;
+  if (typeof value === 'boolean') return 'BOOLEAN';
+  if (typeof value === 'number' && Number.isFinite(value)) return 'DOUBLE';
+  if (value instanceof Date) return 'DATE';
+  if (typeof value === 'string') return 'VARCHAR';
+  throw new TypeError('nested response fields require explicit catalog typing');
+}
+function compatible(expected, runtime, value) {
+  if (expected === runtime) return true;
+  if (expected === 'BIGINT' && runtime === 'DOUBLE' && Number.isInteger(value)) return true;
+  if (expected === 'DATE' && runtime === 'VARCHAR' && /^\d{4}-\d{2}-\d{2}$/.test(value)) return true;
+  return false;
+}
+
+export class MeridianResultError extends Error {
+  constructor(body, requestId, key) {
+    super('UltraStiching API returned a JSON result error');
+    this.name = 'MeridianResultError';
+    this.body = body;
+    this.requestId = requestId;
+    this.errorCode = typeof body?.error === 'string' ? body.error : body?.error?.code;
+    this.key = key;
+  }
+  toDiagnostic() {
+    return { type: this.name, message: this.message, error: redactKey(this.errorCode, this.key),
+      request_id: redactKey(this.requestId, this.key), raw_body: redactKey(canonicalJson(this.body), this.key) };
+  }
+}
+
+export class MeridianResponseShapeError extends Error {
+  constructor(message, body, requestId, key) {
+    super(message);
+    this.name = 'MeridianResponseShapeError';
+    this.body = body;
+    this.requestId = requestId;
+    this.key = key;
+  }
+  toDiagnostic() {
+    return { type: this.name, message: this.message, request_id: redactKey(this.requestId, this.key),
+      raw_body: redactKey(canonicalJson(this.body), this.key) };
+  }
+}
+
+function decodeResponse(response, key) {
+  if (response.binary) {
+    const table = tableFromIPC(response.data);
+    const data = [];
+    for (const row of table) data.push(exactArrowValue(row.toJSON(), 'row'));
+    return { data, pagination: null };
+  }
+  const body = response.data;
+  if (body && typeof body === 'object' && body.error != null) throw new MeridianResultError(body, response.requestId, key);
+  if (!body || typeof body !== 'object' || Array.isArray(body)) {
+    throw new MeridianResponseShapeError('JSON result is not a successful data_frame', body, response.requestId, key);
+  }
+  if (!Array.isArray(body.data_frame) || !body.data_frame.every((item) => item && typeof item === 'object' && !Array.isArray(item))) {
+    throw new MeridianResponseShapeError('JSON result has no data_frame object list', body, response.requestId, key);
+  }
+  return { data: body.data_frame, pagination: body.pagination && typeof body.pagination === 'object' && !Array.isArray(body.pagination) ? body.pagination : null };
+}
+
+function proof(pagination, request, count) {
+  if (!pagination) return null;
+  const { offset, limit, total } = pagination;
+  if (![offset, limit, total].every(Number.isSafeInteger) || offset < 0 || limit <= 0 || total < 0) return null;
+  if (request.offset !== offset || request.limit !== limit) return null;
+  if (count !== Math.max(0, Math.min(limit, total - offset))) return null;
+  return { offset, limit, total };
+}
+
+function errorDetails(error, key) {
+  const diagnostic = error.toDiagnostic?.() ?? { type: error.name, message: error.message };
+  return redactKey(canonicalJson(diagnostic), key);
+}
+
+export class MarketDB {
+  constructor({ path, env, home, request = callByName, requestOptions = {} } = {}) {
+    this.path = dbPath(path, env, home);
+    this.request = request;
+    this.requestOptions = requestOptions;
+  }
+
+  async init() {
+    await withDb(this.path, false, async (con) => {
+      await run(con, 'CREATE TABLE IF NOT EXISTS _marketdb_meta (schema_version INTEGER PRIMARY KEY, created_at VARCHAR NOT NULL)');
+      const versions = await rows(con, 'SELECT schema_version FROM _marketdb_meta');
+      if (versions.length && (versions.length !== 1 || versions[0][0] !== SCHEMA_VERSION)) throw new Error(`unsupported marketdb schema version: ${JSON.stringify(versions)}`);
+      if (!versions.length) await run(con, 'INSERT INTO _marketdb_meta VALUES (?,?)', [SCHEMA_VERSION, now()]);
+      await run(con, 'CREATE TABLE IF NOT EXISTS _marketdb_columns (dataset VARCHAR NOT NULL, column_name VARCHAR NOT NULL, duckdb_type VARCHAR NOT NULL, source VARCHAR NOT NULL, PRIMARY KEY (dataset, column_name))');
+      await run(con, 'CREATE TABLE IF NOT EXISTS _sync_runs (run_id VARCHAR PRIMARY KEY, dataset VARCHAR NOT NULL, endpoint VARCHAR NOT NULL, params_json VARCHAR NOT NULL, params_sha256 VARCHAR NOT NULL, status VARCHAR NOT NULL, started_at VARCHAR NOT NULL, finished_at VARCHAR, error_json VARCHAR, received_rows BIGINT NOT NULL, written_rows BIGINT NOT NULL, chunk_count INTEGER NOT NULL)');
+      await run(con, 'CREATE TABLE IF NOT EXISTS _sync_chunks (run_id VARCHAR NOT NULL, chunk_index INTEGER NOT NULL, request_json VARCHAR NOT NULL, request_sha256 VARCHAR NOT NULL, status VARCHAR NOT NULL, received_rows BIGINT NOT NULL, completeness_json VARCHAR, error_json VARCHAR, PRIMARY KEY (run_id, chunk_index))');
+    });
+    return { db: this.path, schema_version: SCHEMA_VERSION, datasets: Object.keys(datasets()).sort() };
+  }
+
+  async recordChunk(runId, index, request, status, count, evidence, error) {
+    const raw = canonicalJson(request);
+    await withDb(this.path, false, (con) => run(con, 'INSERT INTO _sync_chunks VALUES (?,?,?,?,?,?,?,?)',
+      [runId, index, raw, sha(raw), status, count, evidence === null ? null : canonicalJson(evidence), error ?? null]));
+  }
+
+  async finish(runId, status, received, written, chunks, error = null) {
+    await withDb(this.path, false, (con) => run(con,
+      'UPDATE _sync_runs SET status=?,finished_at=?,error_json=?,received_rows=?,written_rows=?,chunk_count=? WHERE run_id=?',
+      [status, now(), error, received, written, chunks, runId]));
+  }
+
+  async sync(dataset, overrides = {}) {
+    const spec = datasets()[dataset];
+    if (!spec) throw new Error(`unknown dataset: ${dataset}`);
+    await this.init();
+    const params = { ...spec.kwargs, ...overrides };
+    if (Object.hasOwn(params, 'limit') && (!Number.isSafeInteger(params.limit) || params.limit <= 0)) throw new Error('limit must be a positive integer');
+    if (params.offset !== undefined && params.offset !== 0) throw new Error('complete pagination must start at offset 0');
+    const raw = canonicalJson(params);
+    const runId = randomUUID();
+    await withDb(this.path, false, (con) => run(con, 'INSERT INTO _sync_runs VALUES (?,?,?,?,?,?,?,?,?,?,?,?)',
+      [runId, dataset, spec.endpoint, raw, sha(raw), 'running', now(), null, null, 0, 0, 0]));
+    let count = 0;
+    let bufferedBytes = 0;
+    let page = 0;
+    let total = null;
+    const allRows = [];
+    const pageParams = { ...params };
+    if (Object.hasOwn(pageParams, 'limit')) pageParams.offset = 0;
+    let actualKey = this.requestOptions.key;
+    try {
+      while (true) {
+        const request = { ...pageParams };
+        const index = page++;
+        let fetched;
+        let verified;
+        let pagination;
+        try {
+          const meta = endpoint(spec.endpoint, 'http');
+          if (this.request === callByName) actualKey = resolveKey(actualKey);
+          const response = await this.request(spec.endpoint, buildPayload(meta, request), { ...this.requestOptions, key: actualKey });
+          ({ data: fetched, pagination } = decodeResponse(response, actualKey));
+          count += fetched.length;
+          verified = proof(pagination, request, fetched.length);
+          await this.recordChunk(runId, index, request, verified ? 'complete' : 'incomplete', fetched.length,
+            pagination === null ? null : { server_pagination: pagination, verified });
+        } catch (error) {
+          await this.recordChunk(runId, index, request, 'failed', 0, null, errorDetails(error, actualKey));
+          throw error;
+        }
+        if (!verified) {
+          await this.finish(runId, 'incomplete', count, 0, page);
+          return { table: dataset, endpoint: spec.endpoint, status: 'incomplete', run_id: runId, received_rows: count, written_rows: 0, chunks: page, reason: 'no verified pagination coverage' };
+        }
+        if (total === null) total = verified.total;
+        else if (total !== verified.total) {
+          await this.finish(runId, 'incomplete', count, 0, page);
+          return { table: dataset, endpoint: spec.endpoint, status: 'incomplete', run_id: runId, received_rows: count, written_rows: 0, chunks: page, reason: 'pagination total changed' };
+        }
+        for (const row of fetched) bufferedBytes += Buffer.byteLength(canonicalJson(row), 'utf8');
+        if (total > MAX_BUFFERED_ROWS || count > MAX_BUFFERED_ROWS || bufferedBytes > MAX_BUFFERED_JSON_BYTES) {
+          throw new RangeError("verified result exceeds marketdb's in-memory buffer limit (100000 rows or 64 MiB JSON)");
+        }
+        for (const row of fetched) allRows.push(row);
+        const nextOffset = verified.offset + fetched.length;
+        if (nextOffset >= total) break;
+        pageParams.offset = nextOffset;
+      }
+      const written = await this.writeComplete(dataset, spec, allRows, runId, page);
+      return { table: dataset, endpoint: spec.endpoint, status: 'complete', run_id: runId, received_rows: count, written_rows: written, chunks: page };
+    } catch (error) {
+      await this.finish(runId, 'failed', count, 0, page, errorDetails(error, actualKey));
+      throw error;
+    }
+  }
+
+  async writeComplete(dataset, spec, data, runId, chunks) {
+    const fields = catalogTypes(spec.endpoint);
+    for (const row of data) for (const [name, value] of Object.entries(row)) {
+      quoted(name);
+      const runtime = runtimeType(value);
+      const prior = fields.get(name);
+      if (!prior) fields.set(name, [runtime ?? 'VARCHAR', runtime ? 'runtime' : 'unresolved']);
+      else if (prior[1] === 'unresolved' && runtime) fields.set(name, [runtime, 'runtime']);
+      else if (prior[1] === 'runtime' && runtime && prior[0] !== runtime) throw new TypeError(`conflicting response type for ${dataset}.${name}`);
+      else if (prior[1] === 'catalog' && runtime && !compatible(prior[0], runtime, value)) throw new TypeError(`response type disagrees with catalog for ${dataset}.${name}`);
+    }
+    for (const key of spec.keys) if (!fields.has(key)) throw new Error(`key column missing from response/catalog: ${key}`);
+    const seen = new Set();
+    const prepared = [];
+    for (const row of data) {
+      if (spec.keys.some((key) => row[key] === undefined || row[key] === null)) throw new Error(`response row is missing a key for ${dataset}`);
+      const rowKey = canonicalJson(spec.keys.map((key) => row[key]));
+      if (seen.has(rowKey)) throw new Error(`duplicate key in response for ${dataset}: ${rowKey}`);
+      seen.add(rowKey);
+      prepared.push([rowKey, row]);
+    }
+    return withDb(this.path, false, async (con) => {
+      await run(con, 'BEGIN TRANSACTION');
+      try {
+        const old = await rows(con, 'SELECT column_name,duckdb_type,source FROM _marketdb_columns WHERE dataset=?', [dataset]);
+        const existing = new Map(old.map(([name, dtype, source]) => [name, [dtype, source]]));
+        for (const [name, prior] of existing) {
+          const current = fields.get(name);
+          if (!current) fields.set(name, prior);
+          else if (prior[1] !== 'unresolved' && (prior[0] !== current[0] || prior[1] !== current[1])) throw new TypeError(`stored column type differs for ${dataset}.${name}`);
+        }
+        const table = quoted(dataset);
+        await run(con, `CREATE TABLE IF NOT EXISTS ${table} (_row_key VARCHAR PRIMARY KEY, _raw_json VARCHAR NOT NULL, _updated_at VARCHAR NOT NULL, _run_id VARCHAR NOT NULL)`);
+        for (const [name, [dtype, source]] of fields) {
+          if (name.startsWith('_')) throw new Error(`response field conflicts with reserved column: ${name}`);
+          if (!existing.has(name)) {
+            await run(con, `ALTER TABLE ${table} ADD COLUMN ${quoted(name)} ${dtype}`);
+            await run(con, 'INSERT INTO _marketdb_columns VALUES (?,?,?,?)', [dataset, name, dtype, source]);
+          } else if (existing.get(name)[1] === 'unresolved' && source !== 'unresolved') {
+            await run(con, `ALTER TABLE ${table} ALTER COLUMN ${quoted(name)} SET DATA TYPE ${dtype}`);
+            await run(con, 'UPDATE _marketdb_columns SET duckdb_type=?,source=? WHERE dataset=? AND column_name=?', [dtype, source, dataset, name]);
+          }
+        }
+        const columns = ['_row_key', '_raw_json', '_updated_at', '_run_id', ...fields.keys()];
+        const sql = `INSERT INTO ${table} (${columns.map(quoted).join(',')}) VALUES (${columns.map(() => '?').join(',')})`;
+        for (const [rowKey, row] of prepared) {
+          await run(con, `DELETE FROM ${table} WHERE _row_key=?`, [rowKey]);
+          await run(con, sql, [rowKey, canonicalJson(row), now(), runId, ...[...fields].map(([name, [dtype]]) => {
+            const value = row[name] ?? null;
+            if (value === null) return null;
+            if (dtype === 'VARCHAR' && typeof value !== 'string') throw new TypeError(`expected string response value for ${dataset}.${name}`);
+            return value instanceof Date ? value.toISOString().slice(0, 10) : value;
+          })]);
+        }
+        await run(con, "UPDATE _sync_runs SET status='complete',finished_at=?,received_rows=?,written_rows=?,chunk_count=? WHERE run_id=?", [now(), data.length, prepared.length, chunks, runId]);
+        await run(con, 'COMMIT');
+        return prepared.length;
+      } catch (error) { await run(con, 'ROLLBACK'); throw error; }
+    });
+  }
+
+  async status() {
+    await this.init();
+    return withDb(this.path, true, async (con) => {
+      const tables = [];
+      for (const name of Object.keys(datasets()).sort()) {
+        if ((await rows(con, "SELECT 1 FROM information_schema.tables WHERE table_schema='main' AND table_name=?", [name])).length) {
+          tables.push({ table: name, rows: (await rows(con, `SELECT COUNT(*) FROM ${quoted(name)}`))[0][0] });
+        }
+      }
+      const recent_syncs = await objects(con, 'SELECT run_id,dataset,endpoint,status,received_rows,written_rows,chunk_count FROM _sync_runs ORDER BY started_at DESC LIMIT 20');
+      return { db: this.path, schema_version: SCHEMA_VERSION, tables, recent_syncs };
+    });
+  }
+
+  async validate() {
+    await this.init();
+    return withDb(this.path, true, async (con) => {
+      const issues = [];
+      const completeIds = new Set((await rows(con, "SELECT run_id FROM _sync_runs WHERE status='complete'")).map((item) => item[0]));
+      const latest = new Map();
+      for (const item of await objects(con, 'SELECT run_id,dataset FROM _sync_runs ORDER BY started_at DESC')) {
+        if (!latest.has(item.dataset)) latest.set(item.dataset, item.run_id);
+      }
+      const runs = await objects(con, 'SELECT run_id,dataset,params_json,params_sha256,status,chunk_count FROM _sync_runs');
+      for (const item of runs) {
+        if (sha(item.params_json) !== item.params_sha256) issues.push({ run_id: item.run_id, issue: 'params_hash_mismatch' });
+        else { try { const data = JSON.parse(item.params_json); if (!data || typeof data !== 'object' || Array.isArray(data)) throw Error(); }
+          catch { issues.push({ run_id: item.run_id, issue: 'invalid_params_json' }); } }
+        const chunks = await objects(con, 'SELECT chunk_index,request_json,request_sha256,status,received_rows,completeness_json FROM _sync_chunks WHERE run_id=? ORDER BY chunk_index', [item.run_id]);
+        if (chunks.length !== item.chunk_count || chunks.some((chunk, index) => chunk.chunk_index !== index)) issues.push({ run_id: item.run_id, issue: 'chunk_sequence_mismatch' });
+        let nextOffset = 0;
+        let expectedTotal = null;
+        for (const chunk of chunks) {
+          if (sha(chunk.request_json) !== chunk.request_sha256) issues.push({ run_id: item.run_id, chunk_index: chunk.chunk_index, issue: 'request_hash_mismatch' });
+          let request = null;
+          try { request = JSON.parse(chunk.request_json); }
+          catch { issues.push({ run_id: item.run_id, chunk_index: chunk.chunk_index, issue: 'invalid_request_json' }); }
+          let verified = null;
+          let checked = null;
+          try {
+            const evidence = chunk.completeness_json === null ? null : JSON.parse(chunk.completeness_json);
+            verified = evidence?.verified;
+            checked = request && typeof request === 'object' && !Array.isArray(request)
+              ? proof(evidence?.server_pagination, request, chunk.received_rows) : null;
+          } catch { /* invalid evidence is reported below for complete chunks */ }
+          if (chunk.status === 'complete') {
+            if (!checked || canonicalJson(checked) !== canonicalJson(verified)) {
+              issues.push({ run_id: item.run_id, chunk_index: chunk.chunk_index, issue: 'invalid_completeness_evidence' });
+            } else if (checked.offset !== nextOffset || (expectedTotal !== null && checked.total !== expectedTotal)) {
+              issues.push({ run_id: item.run_id, chunk_index: chunk.chunk_index, issue: 'pagination_gap_or_total_changed' });
+            } else { expectedTotal = checked.total; nextOffset += chunk.received_rows; }
+          }
+        }
+        if (item.status === 'complete' && (!chunks.length || chunks.some((chunk) => chunk.status !== 'complete'))) issues.push({ run_id: item.run_id, issue: 'run_without_complete_chunks' });
+        else if (item.status === 'complete' && expectedTotal !== null && nextOffset !== expectedTotal) issues.push({ run_id: item.run_id, issue: 'incomplete_pagination_coverage' });
+        if (['incomplete', 'failed', 'running'].includes(item.status) && latest.get(item.dataset) === item.run_id) issues.push({ run_id: item.run_id, dataset: item.dataset, issue: item.status });
+      }
+      for (const name of Object.keys(datasets())) {
+        if (!(await rows(con, "SELECT 1 FROM information_schema.tables WHERE table_schema='main' AND table_name=?", [name])).length) continue;
+        const columns = new Map((await rows(con, "SELECT column_name,data_type FROM information_schema.columns WHERE table_schema='main' AND table_name=?", [name])));
+        for (const [column, dtype] of await rows(con, 'SELECT column_name,duckdb_type FROM _marketdb_columns WHERE dataset=?', [name])) {
+          if (columns.get(column) !== dtype) issues.push({ table: name, column, issue: 'column_type_mismatch' });
+        }
+        for (const [rowKey, raw, rowRunId] of await rows(con, `SELECT _row_key,_raw_json,_run_id FROM ${quoted(name)}`)) {
+          try {
+            const payload = JSON.parse(raw);
+            if (canonicalJson(datasets()[name].keys.map((key) => payload[key])) !== rowKey) throw new Error('row key mismatch');
+          } catch { issues.push({ table: name, row_key: rowKey, issue: 'invalid_row_key_or_json' }); }
+          if (!completeIds.has(rowRunId)) issues.push({ table: name, row_key: rowKey, issue: 'row_references_noncomplete_run' });
+        }
+      }
+      return { db: this.path, ok: issues.length === 0, issues };
+    });
+  }
+
+  async repair(dataset) {
+    await this.init();
+    if (dataset !== undefined && !datasets()[dataset]) throw new Error(`unknown dataset: ${dataset}`);
+    const runs = await withDb(this.path, true, (con) => objects(con,
+      'SELECT dataset,params_json,params_sha256,status FROM _sync_runs ORDER BY started_at DESC'));
+    const visited = new Set();
+    const repaired = [];
+    for (const item of runs) {
+      if (visited.has(item.dataset) || (dataset && item.dataset !== dataset)) continue;
+      visited.add(item.dataset);
+      if (!['failed', 'incomplete'].includes(item.status)) continue;
+      if (sha(item.params_json) !== item.params_sha256) throw new Error(`stored request parameters hash mismatch for ${item.dataset}`);
+      let params;
+      try { params = JSON.parse(item.params_json); }
+      catch (cause) { throw new Error(`invalid stored request parameters for ${item.dataset}`, { cause }); }
+      if (!params || typeof params !== 'object' || Array.isArray(params)) throw new Error(`stored request parameters are not an object for ${item.dataset}`);
+      repaired.push(await this.sync(item.dataset, params));
+    }
+    return { repaired };
+  }
+
+  async sql(query, limit) {
+    if (typeof query !== 'string' || !/^\s*(SELECT|WITH)\b/i.test(query)) throw new Error('only SELECT/WITH queries are supported');
+    const stripped = query.trim().replace(/;+$/, '').trim();
+    if (stripped.includes(';')) throw new Error('multiple SQL statements are not supported');
+    if (limit !== undefined && (!Number.isSafeInteger(limit) || limit <= 0)) throw new Error('limit must be a positive integer');
+    return withDb(this.path, true, async (con) => {
+      const reader = await con.runAndReadAll(stripped);
+      const all = reader.getRowsJson();
+      const values = limit ? all.slice(0, limit) : all;
+      return { columns: reader.columnNames(), rows: values, row_count: values.length };
+    });
+  }
+
+  async export(table, path, format) {
+    if (!datasets()[table]) throw new Error(`unknown dataset: ${table}`);
+    if (typeof path !== 'string' || !path) throw new Error('export path is required');
+    const target = dbPath(path, {}, homedir());
+    const fmt = (format ?? (extname(target).slice(1) || 'csv')).toLowerCase();
+    if (!['csv', 'parquet'].includes(fmt)) throw new Error('format must be csv or parquet');
+    mkdirSync(dirname(target), { recursive: true });
+    await withDb(this.path, true, (con) => run(con, `COPY (SELECT * FROM ${quoted(table)}) TO ${sqlString(target)} (FORMAT ${fmt.toUpperCase()})`));
+    return { table, path: target, format: fmt };
+  }
+}

package/src/registry.js

@@ -0,0 +1,91 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+let cached;
+export function loadRegistry() {
+  if (!cached) {
+    // The source checkout reads the shared generated registry. `prepack` copies it
+    // into the package so an installed CLI has the identical snapshot.
+    const source = new URL('../../registry/endpoints.json', import.meta.url);
+    const packed = new URL('../registry/endpoints.json', import.meta.url);
+    let data;
+    try { data = readFileSync(fileURLToPath(source), 'utf8'); }
+    catch (error) {
+      if (error.code !== 'ENOENT') throw error;
+      data = readFileSync(fileURLToPath(packed), 'utf8');
+    }
+    cached = JSON.parse(data);
+  }
+  return cached;
+}
+
+export function endpoint(name, protocol) {
+  return loadRegistry().endpoints.find((item) => item.name === name && (!protocol || item.protocol === protocol));
+}
+
+export function searchEndpoints({ search = '', binary = false } = {}) {
+  const term = search.toLocaleLowerCase();
+  return loadRegistry().endpoints.filter((item) =>
+    (!binary || item.response?.return_format === 'ARROW_IPC') &&
+    (!term || [item.name, item.title, item.description, ...(item.categories ?? [])]
+      .some((value) => String(value).toLocaleLowerCase().includes(term))));
+}
+
+export function buildPayload(meta, values) {
+  if (!meta || !values || typeof values !== 'object' || Array.isArray(values)) {
+    throw new TypeError('endpoint and parameter object are required');
+  }
+  const names = new Set(meta.parameters ?? []);
+  for (const name of Object.keys(values)) {
+    if (!names.has(name)) throw new Error(`${meta.name} has no parameter ${name}; run ultrastiching info ${meta.name}`);
+  }
+  const required = [...names].filter((name) => !(meta.optional_parameters ?? []).includes(name));
+  const missing = required.filter((name) => values[name] === undefined);
+  if (missing.length) throw new Error(`${meta.name} requires: ${missing.join(', ')}`);
+  const retained = new Set(meta.retained_objects ?? []);
+  const walk = (node, path = '') => {
+    if (node && typeof node === 'object' && '$param' in node) {
+      const value = values[node.$param];
+      if (value !== undefined && path.endsWith('.symbols') && !Array.isArray(value)) {
+        return typeof value === 'string' ? value.split(',').map((part) => part.trim()) : [value];
+      }
+      return value;
+    }
+    if (Array.isArray(node)) return node.map((child) => walk(child));
+    if (node && typeof node === 'object') {
+      const result = {};
+      for (const [key, child] of Object.entries(node)) {
+        const value = walk(child, path ? `${path}.${key}` : key);
+        if (value !== undefined) result[key] = value;
+      }
+      if (Object.keys(result).length || retained.has(path)) return result;
+      return undefined;
+    }
+    return node;
+  };
+  return walk(meta.payload) ?? {};
+}
+
+export function parseParameter(raw, field = {}) {
+  if (typeof raw !== 'string') return raw;
+  const value = raw.trim();
+  const type = (field.type ?? '').toLowerCase();
+  if (value.startsWith('{') || value.startsWith('[')) {
+    try { return JSON.parse(value); } catch { throw new Error(`Invalid JSON value for ${field.name}`); }
+  }
+  if (type.includes('boolean') || type === 'bool') {
+    if (value === 'true') return true;
+    if (value === 'false') return false;
+    throw new Error(`${field.name} must be true or false`);
+  }
+  if (type === 'integer' || type === 'int64' || type === 'number' || type === 'float64') {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || (type.startsWith('int') && !Number.isInteger(parsed))) {
+      throw new Error(`${field.name} must be a ${type}`);
+    }
+    return parsed;
+  }
+  if (type === 'array' || (type.endsWith('[]') && !type.includes('|'))) return value.split(',').map((part) => part.trim());
+  if (type.includes('string[]') && value.includes(',')) return value.split(',').map((part) => part.trim());
+  return raw;
+}

package/src/websocket.js

@@ -0,0 +1,72 @@
+import { endpoint, buildPayload } from './registry.js';
+import { normalizeOrigin, resolveKey } from './config.js';
+import { redactKey } from './http.js';
+
+export class MeridianWebSocketError extends Error {
+  constructor(message, { status, requestId, body, key, cause } = {}) {
+    super(message, { cause });
+    this.name = 'MeridianWebSocketError';
+    this.status = status;
+    this.requestId = requestId;
+    this.body = body;
+    this.key = key;
+  }
+  toDiagnostic() {
+    return {
+      type: this.name, message: this.message, status: this.status,
+      request_id: redactKey(this.requestId, this.key), raw_body: redactKey(this.body, this.key),
+      cause: this.cause ? redactKey(String(this.cause), this.key) : undefined,
+    };
+  }
+}
+
+export async function streamNewsflash(values, {
+  key, websocketBaseUrl, count, idleTimeout,
+  onMessage = (line) => process.stdout.write(`${line}\n`),
+  socketFactory,
+} = {}) {
+  const meta = endpoint('ws_newsflash', 'websocket');
+  if (!meta) throw new Error('ws_newsflash is absent from the registry');
+  const subscription = buildPayload(meta, values);
+  const apiKey = resolveKey(key);
+  const origin = websocketBaseUrl === undefined ? meta.origin : normalizeOrigin(websocketBaseUrl, ['ws:', 'wss:']);
+  const max = count === undefined ? Infinity : Number(count);
+  if (!(max > 0 && (max === Infinity || Number.isInteger(max)))) throw new Error('count must be a positive integer');
+  const idleMs = idleTimeout === undefined ? undefined : Number(idleTimeout);
+  if (idleMs !== undefined && (!Number.isFinite(idleMs) || idleMs <= 0)) throw new Error('idle-timeout must be positive milliseconds');
+  const factory = socketFactory ?? ((await import('ws')).default);
+  const socket = new factory(`${origin}${meta.path}`, { headers: { Authorization: `Bearer ${apiKey}` } });
+  return new Promise((resolve, reject) => {
+    let received = 0;
+    let timer;
+    let settled = false;
+    const finish = (error) => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      if (error) reject(error); else resolve({ received });
+    };
+    const resetTimer = () => {
+      if (timer) clearTimeout(timer);
+      if (idleMs) timer = setTimeout(() => socket.close(), idleMs);
+    };
+    socket.on('open', () => { socket.send(JSON.stringify(subscription)); resetTimer(); });
+    socket.on('message', (data) => {
+      resetTimer();
+      received++;
+      try { onMessage(String(data)); }
+      catch (cause) { finish(new MeridianWebSocketError('Could not handle WebSocket message', { cause, key: apiKey })); socket.close(); return; }
+      if (received >= max) socket.close();
+    });
+    socket.on('unexpected-response', (_request, response) => {
+      const chunks = [];
+      response.on('data', (chunk) => chunks.push(chunk));
+      response.on('end', () => finish(new MeridianWebSocketError('WebSocket handshake failed', {
+        status: response.statusCode, requestId: response.headers['x-meridian-request-id'],
+        body: Buffer.concat(chunks).toString('utf8'), key: apiKey,
+      })));
+    });
+    socket.on('error', (cause) => finish(new MeridianWebSocketError('WebSocket transport failed', { cause, key: apiKey })));
+    socket.on('close', () => finish());
+  });
+}