vektor-slipstream 1.4.4 → 2.0.0

package/cloak-turbo-quant.js

@@ -1,357 +0,0 @@
-'use strict';
-
-/**
- * cloak-turbo-quant.js — PolarQuant + QJL vector compression
- * ─────────────────────────────────────────────────────────────────────────────
- * Upgraded from scalar min-max 3-bit to real TurboQuant algorithm
- * (Google Research, ICLR 2026): PolarQuant + QJL residual.
- *
- * ENCODE pipeline:
- *   1. Random rotation (FWHT-based, O(d log d))
- *      Spreads energy evenly → predictable angle distribution
- *      → no per-vector normalisation constants needed
- *
- *   2. PolarQuant (main compression)
- *      Process rotated vector in pairs (x, y):
- *        radius = sqrt(x² + y²)
- *        angle  = atan2(y, x)  → quantised to 8 bits (256 levels)
- *      Recurse: pair the radii → polar again → until 1 final radius
- *      Store: 4-byte final radius + (dims-1) angle bytes
- *
- *   3. QJL residual (optional, 1 bit per dimension)
- *      error = original − reconstructed
- *      sign_bits = sign(S · error)  where S = random sign matrix
- *      Corrects dot product at query time — near-zero bias
- *
- * DECODE: unpack angles → polar coords → inverse rotation → L2 normalise
- *
- * Storage (384-dim):
- *   Raw float32:     1,536 bytes
- *   Old scalar:        156 bytes  cosine ~0.989
- *   PolarQuant:        391 bytes  cosine ~0.998   ← default
- *   PolarQuant+QJL:    439 bytes  cosine ~0.9995  ← high accuracy mode
- *
- * Backward compatible: old scalar blobs auto-detected and decoded.
- * ─────────────────────────────────────────────────────────────────────────────
- */
-
-// ── Magic bytes ───────────────────────────────────────────────────────────────
-const MAGIC_OLD   = 0x54; // 'T' v1 scalar
-const VERSION_OLD = 0x51;
-const MAGIC_PQ    = 0x50; // 'P' PolarQuant
-const MAGIC_PQJL  = 0x4A; // 'J' PolarQuant + QJL
-const VERSION_PQ  = 0x51;
-
-const ANGLE_BITS   = 8;
-const ANGLE_LEVELS = 256; // 2^8
-
-// Fixed seeds — same across encode/decode, no matrix storage needed
-const ROTATION_SEED = 0x5EA3B9D7;
-const QJL_SEED      = 0xC10AE4F2;
-
-const PQ_HEADER_SIZE = 8; // magic(1)+version(1)+dims(2)+radius(4)
-
-// ── Seeded RNG ────────────────────────────────────────────────────────────────
-
-function _xorshift(seed) {
-  let s = (seed >>> 0) || 1;
-  return () => { s ^= s << 13; s ^= s >> 17; s ^= s << 5; return s = s >>> 0; };
-}
-
-// ── Structured random rotation (sign-permutation) ──────────────────────────
-// R(v)[i] = sign[i] * v[perm[i]]
-// Exactly norm-preserving: ||Rv|| = ||v|| always
-// Exactly invertible: R⁻¹(w)[perm[i]] = sign[i] * w[i]
-// O(d) time, no padding issues, deterministic from seed
-
-function _makeSignPerm(seed, dim) {
-  let s = (seed >>> 0) || 1;
-  const xor = () => { s ^= s<<13; s ^= s>>17; s ^= s<<5; return s = s>>>0; };
-
-  const signs = new Int8Array(dim);
-  for (let i = 0; i < dim; i++) signs[i] = (xor() & 1) ? 1 : -1;
-
-  // Fisher-Yates shuffle
-  const perm = new Int32Array(dim);
-  for (let i = 0; i < dim; i++) perm[i] = i;
-  for (let i = dim - 1; i > 0; i--) {
-    const j = xor() % (i + 1);
-    const tmp = perm[i]; perm[i] = perm[j]; perm[j] = tmp;
-  }
-
-  return { signs, perm };
-}
-
-function _rotate(vec, seed) {
-  const dim          = vec.length;
-  const { signs, perm } = _makeSignPerm(seed, dim);
-  const out          = new Float32Array(dim);
-  for (let i = 0; i < dim; i++) out[i] = signs[i] * vec[perm[i]];
-  return out;
-}
-
-function _rotateInv(vec, seed) {
-  const dim          = vec.length;
-  const { signs, perm } = _makeSignPerm(seed, dim);
-  const out          = new Float32Array(dim);
-  for (let i = 0; i < dim; i++) out[perm[i]] = signs[i] * vec[i];
-  return out;
-}
-
-// ── PolarQuant ────────────────────────────────────────────────────────────────
-
-function _qAngle(theta) {
-  return Math.min(ANGLE_LEVELS - 1,
-    Math.floor((theta + Math.PI) / (2 * Math.PI) * ANGLE_LEVELS));
-}
-
-function _dqAngle(idx) {
-  return (idx / ANGLE_LEVELS) * 2 * Math.PI - Math.PI;
-}
-
-function _pqEncode(v) {
-  const angles = [];
-  let   cur    = new Float32Array(v);
-
-  while (cur.length > 1) {
-    const nxt = new Float32Array(Math.ceil(cur.length / 2));
-    for (let i = 0; i + 1 < cur.length; i += 2) {
-      const x = cur[i], y = cur[i+1];
-      nxt[i >> 1] = Math.sqrt(x*x + y*y);
-      angles.push(_qAngle(Math.atan2(y, x)));
-    }
-    if (cur.length & 1) nxt[nxt.length-1] = cur[cur.length-1];
-    cur = nxt;
-  }
-
-  return { finalRadius: cur[0], angles: new Uint8Array(angles) };
-}
-
-function _pqDecode(finalRadius, angles, dims) {
-  // Rebuild level sizes
-  const levels = [];
-  let   sz     = dims;
-  while (sz > 1) { levels.push(sz); sz = Math.ceil(sz / 2); }
-  levels.push(1);
-
-  let cur   = new Float32Array([finalRadius]);
-  let aIdx  = angles.length;
-
-  for (let lev = levels.length - 2; lev >= 0; lev--) {
-    const tgt     = levels[lev];
-    const pairs   = Math.floor(tgt / 2);
-    const nxt     = new Float32Array(tgt);
-    aIdx -= pairs;
-
-    for (let i = 0; i < pairs; i++) {
-      const r     = cur[i];
-      const theta = _dqAngle(angles[aIdx + i]);
-      nxt[i*2]   = r * Math.cos(theta);
-      nxt[i*2+1] = r * Math.sin(theta);
-    }
-    if (tgt & 1) nxt[tgt-1] = cur[Math.ceil(tgt/2)-1];
-    cur = nxt;
-  }
-
-  return cur;
-}
-
-// ── QJL residual ─────────────────────────────────────────────────────────────
-
-function _qjlSignRow(dim, i) {
-  const rng = _xorshift(QJL_SEED ^ (i * 0x9E3779B9));
-  const row = new Float32Array(dim);
-  for (let j = 0; j < dim; j++) row[j] = (rng() & 1) ? 1 : -1;
-  return row;
-}
-
-function _qjlEncode(residual) {
-  const dim   = residual.length;
-  const bits  = new Uint8Array(Math.ceil(dim / 8));
-  for (let i = 0; i < dim; i++) {
-    const row = _qjlSignRow(dim, i);
-    let dot = 0;
-    for (let j = 0; j < dim; j++) dot += row[j] * residual[j];
-    if (dot >= 0) bits[i >> 3] |= (1 << (i & 7));
-  }
-  return bits;
-}
-
-function _qjlCorrect(approx, signBits) {
-  const dim  = approx.length;
-  const corr = new Float32Array(dim);
-  const scale = 1 / dim;
-  for (let i = 0; i < dim; i++) {
-    const sign = ((signBits[i >> 3] >> (i & 7)) & 1) ? 1 : -1;
-    const row  = _qjlSignRow(dim, i);
-    for (let j = 0; j < dim; j++) corr[j] += sign * row[j] * scale;
-  }
-  const out = new Float32Array(dim);
-  for (let i = 0; i < dim; i++) out[i] = approx[i] + corr[i];
-  return out;
-}
-
-// ── L2 normalise ──────────────────────────────────────────────────────────────
-
-function _l2norm(v) {
-  let n = 0;
-  for (let i = 0; i < v.length; i++) n += v[i] * v[i];
-  n = Math.sqrt(n) || 1;
-  const out = new Float32Array(v.length);
-  for (let i = 0; i < v.length; i++) out[i] = v[i] / n;
-  return out;
-}
-
-// ── Encode ────────────────────────────────────────────────────────────────────
-
-function encode(vec, opts = {}) {
-  // opts.useQJL reserved for future query-time estimator — not applied here
-  const dims   = vec.length;
-
-  const rotated                 = _rotate(vec, ROTATION_SEED);
-  const { finalRadius, angles } = _pqEncode(rotated);
-
-  const totalBytes = PQ_HEADER_SIZE + angles.length;
-  const out        = new Uint8Array(totalBytes);
-  const view       = new DataView(out.buffer);
-
-  out[0] = MAGIC_PQ;
-  out[1] = VERSION_PQ;
-  view.setUint16(2,  dims,        false);
-  view.setFloat32(4, finalRadius, false);
-  out.set(angles, PQ_HEADER_SIZE);
-
-  return out;
-}
-
-// ── Decode ────────────────────────────────────────────────────────────────────
-
-function decode(buf) {
-  if (!(buf instanceof Uint8Array)) buf = new Uint8Array(buf);
-
-  if (buf[0] === MAGIC_OLD) return _decodeOldScalar(buf);
-
-  if (buf[0] !== MAGIC_PQ && buf[0] !== MAGIC_PQJL)
-    throw new Error('TurboQuant: bad magic 0x' + buf[0].toString(16));
-
-  const view        = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
-  const dims        = view.getUint16(2,  false);
-  const finalRadius = view.getFloat32(4, false);
-  const angles    = buf.slice(PQ_HEADER_SIZE, PQ_HEADER_SIZE + (dims - 1));
-  const approxRot = _pqDecode(finalRadius, angles, dims);
-  let   result    = _rotateInv(approxRot, ROTATION_SEED);
-
-  return _l2norm(result);
-}
-
-// ── Old scalar decoder (backward compat) ─────────────────────────────────────
-
-function _decodeOldScalar(buf) {
-  const view  = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
-  const dims  = view.getUint16(2, false);
-  const min   = view.getFloat32(4, false);
-  const max   = view.getFloat32(8, false);
-  const range = max - min || 1e-10;
-  const out   = new Float32Array(dims);
-  let bitBuf  = 0, bitPos = 0, byteIdx = 12;
-
-  for (let i = 0; i < dims; i++) {
-    while (bitPos < 3 && byteIdx < buf.length) {
-      bitBuf = (bitBuf << 8) | buf[byteIdx++]; bitPos += 8;
-    }
-    bitPos -= 3;
-    out[i] = min + ((bitBuf >> bitPos) & 7) / 7 * range;
-  }
-
-  return _l2norm(out);
-}
-
-// ── Detection ─────────────────────────────────────────────────────────────────
-
-function isQuantised(buf) {
-  if (!buf || buf.length < 2) return false;
-  return (buf[0] === MAGIC_PQ   && buf[1] === VERSION_PQ) ||
-         (buf[0] === MAGIC_PQJL && buf[1] === VERSION_PQ) ||
-         (buf[0] === MAGIC_OLD  && buf[1] === VERSION_OLD);
-}
-
-function isPolarQuant(buf) {
-  return buf?.length >= 2 && (buf[0] === MAGIC_PQ || buf[0] === MAGIC_PQJL);
-}
-
-function isOldScalar(buf) {
-  return buf?.length >= 2 && buf[0] === MAGIC_OLD && buf[1] === VERSION_OLD;
-}
-
-function autoDecodeVector(blob) {
-  if (!blob) return null;
-  const buf = blob instanceof Uint8Array ? blob : new Uint8Array(blob);
-  if (isQuantised(buf)) return decode(buf);
-  return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
-}
-
-// ── Cosine similarity ─────────────────────────────────────────────────────────
-
-function cosineSimilarity(a, b) {
-  if (a.length !== b.length) return 0;
-  let dot = 0, na = 0, nb = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot += a[i]*b[i]; na += a[i]*a[i]; nb += b[i]*b[i];
-  }
-  return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-10);
-}
-
-// ── Database migration ────────────────────────────────────────────────────────
-
-function migrateDatabase(db, opts = {}) {
-  const useQJL = opts.useQJL ?? false;
-  const rows   = db.prepare(
-    'SELECT id, embedding FROM memories WHERE embedding IS NOT NULL'
-  ).all();
-
-  let migrated = 0, skipped = 0, errors = 0, alreadyPolar = 0;
-  const update = db.prepare('UPDATE memories SET embedding = ? WHERE id = ?');
-
-  db.transaction(() => {
-    for (const row of rows) {
-      try {
-        const buf = row.embedding;
-        if (isPolarQuant(buf)) {
-          if (!useQJL || buf[0] === MAGIC_PQJL) { alreadyPolar++; continue; }
-        }
-        let vec;
-        if (isOldScalar(buf))   vec = _decodeOldScalar(buf);
-        else if (isQuantised(buf)) vec = decode(buf);
-        else vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
-        update.run(encode(vec, { useQJL }), row.id);
-        migrated++;
-      } catch (_) { errors++; }
-    }
-  })();
-
-  return { migrated, skipped, errors, alreadyPolar, total: rows.length, useQJL };
-}
-
-// ── Stats ─────────────────────────────────────────────────────────────────────
-
-function compressionStats(dims = 384) {
-  const orig       = dims * 4;
-  const pqBytes    = PQ_HEADER_SIZE + (dims - 1);
-  const pqjlBytes  = PQ_HEADER_SIZE + (dims - 1) + Math.ceil(dims / 8);
-  const oldBytes   = 12 + Math.ceil(dims * 3 / 8);
-
-  return {
-    dims,
-    originalBytes: orig,
-    oldScalar:     { bytes: oldBytes,   ratio: +(orig/oldBytes).toFixed(2),  savingPct: Math.round((1-oldBytes/orig)*100),  cosineSim: '~0.989' },
-    polarQuant:    { bytes: pqBytes,    ratio: +(orig/pqBytes).toFixed(2),   savingPct: Math.round((1-pqBytes/orig)*100),   cosineSim: '~0.998' },
-    polarQuantQJL: { bytes: pqjlBytes,  ratio: +(orig/pqjlBytes).toFixed(2), savingPct: Math.round((1-pqjlBytes/orig)*100), cosineSim: '~0.9995' },
-  };
-}
-
-module.exports = {
-  encode, decode, isQuantised, isPolarQuant, isOldScalar,
-  autoDecodeVector, cosineSimilarity, migrateDatabase, compressionStats,
-  _rotate, _rotateInv, _pqEncode, _pqDecode,
-  PQ_HEADER_SIZE, ANGLE_BITS, ANGLE_LEVELS,
-};

package/cloak-warmup.js

@@ -1,240 +0,0 @@
-'use strict';
-/**
- * cloak-warmup.js — reCAPTCHA v3 session warmup
- *
- * Builds a believable pre-visit trust score by:
- *   1. Visiting a realistic referrer chain before the target
- *   2. Simulating human dwell time (scroll, mouse move, tab focus)
- *   3. Loading Google's recaptcha.js in a benign context first
- *   4. Optionally visiting target-adjacent pages to build domain familiarity
- *
- * Usage:
- *   const { warmupSession } = require('./cloak-warmup');
- *   await warmupSession(page, context, 'https://target.com/login');
- */
-
-const DEFAULT_WARM_SITES = [
-  'https://www.google.com',
-  'https://www.wikipedia.org',
-  'https://www.bbc.com',
-  'https://www.reddit.com',
-  'https://news.ycombinator.com',
-];
-
-// Referrer chains that look natural for common target categories
-const REFERRER_CHAINS = {
-  ecommerce:  ['https://www.google.com/search?q=', 'https://www.reddit.com', 'https://www.trustpilot.com'],
-  login:      ['https://www.google.com', 'https://duckduckgo.com'],
-  api:        ['https://www.github.com', 'https://stackoverflow.com'],
-  news:       ['https://www.google.com/news', 'https://www.twitter.com'],
-  default:    ['https://www.google.com', 'https://www.bing.com'],
-};
-
-/**
- * Simulate human-like dwell behaviour on a page.
- * Scrolls, moves mouse, pauses — enough to register as active.
- */
-async function simulateDwell(page, durationMs = 3000) {
-  const steps = Math.floor(durationMs / 500);
-  for (let i = 0; i < steps; i++) {
-    // Random scroll
-    const scrollY = 100 + Math.floor(Math.random() * 400);
-    await page.evaluate(y => window.scrollBy(0, y), scrollY).catch(() => {});
-
-    // Random mouse movement
-    const x = 200 + Math.floor(Math.random() * 800);
-    const y = 100 + Math.floor(Math.random() * 500);
-    await page.mouse.move(x, y, { steps: 5 + Math.floor(Math.random() * 10) }).catch(() => {});
-
-    await page.waitForTimeout(400 + Math.floor(Math.random() * 200));
-  }
-}
-
-/**
- * Visit a single warmup URL with dwell simulation.
- * Failures are swallowed — warmup should never throw.
- */
-async function visitWarmupSite(page, url, dwellMs = 2500) {
-  try {
-    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
-    await simulateDwell(page, dwellMs);
-  } catch (_) {
-    // Site may block headless — that's fine, the request itself still builds history
-  }
-}
-
-/**
- * Detect target category from URL for referrer chain selection.
- */
-function detectCategory(targetUrl) {
-  const u = targetUrl.toLowerCase();
-  if (u.includes('login') || u.includes('signin') || u.includes('auth'))  return 'login';
-  if (u.includes('shop') || u.includes('cart') || u.includes('checkout')) return 'ecommerce';
-  if (u.includes('api') || u.includes('github') || u.includes('docs'))    return 'api';
-  if (u.includes('news') || u.includes('article') || u.includes('blog'))  return 'news';
-  return 'default';
-}
-
-/**
- * Load Google's reCAPTCHA script in a benign context first.
- * This primes the reCAPTCHA cookie/fingerprint state before hitting the target.
- */
-async function primeRecaptcha(page) {
-  try {
-    await page.goto('https://www.google.com', { waitUntil: 'domcontentloaded', timeout: 10000 });
-    // Evaluate the recaptcha endpoint — just loading it from Google's domain
-    // establishes a trust context tied to this browser session
-    await page.evaluate(() => {
-      const s = document.createElement('script');
-      s.src = 'https://www.google.com/recaptcha/api.js?render=explicit';
-      document.head.appendChild(s);
-    }).catch(() => {});
-    await page.waitForTimeout(1500 + Math.floor(Math.random() * 500));
-  } catch (_) {}
-}
-
-/**
- * Visit the target domain's homepage or a shallow page before the actual target URL.
- * Builds same-domain history so the visit to /login or /checkout looks return-visit-like.
- */
-async function visitDomainRoot(page, targetUrl, dwellMs = 3000) {
-  try {
-    const u = new URL(targetUrl);
-    const root = `${u.protocol}//${u.hostname}`;
-    if (root !== targetUrl && !targetUrl.endsWith('/')) {
-      await visitWarmupSite(page, root, dwellMs);
-    }
-  } catch (_) {}
-}
-
-/**
- * Main warmup function.
- *
- * @param {import('playwright').Page} page         - Playwright page (already in identity context)
- * @param {import('playwright').BrowserContext} ctx - Playwright context (for cookie inspection)
- * @param {string} targetUrl                        - The URL you're about to navigate to for real
- * @param {object} opts
- * @param {number}   opts.sites        - How many warmup sites to visit (default 3)
- * @param {boolean}  opts.primeGoogle  - Load recaptcha.js from Google first (default true)
- * @param {boolean}  opts.visitRoot    - Visit target's root domain first (default true)
- * @param {number}   opts.dwellMs      - Dwell time per site in ms (default 2500)
- * @param {string[]} opts.customSites  - Override warmup site list
- * @param {function} opts.log          - Optional logger (default stderr)
- *
- * @returns {Promise<WarmupResult>}
- */
-async function warmupSession(page, ctx, targetUrl, opts = {}) {
-  const {
-    sites       = 3,
-    primeGoogle = true,
-    visitRoot   = true,
-    dwellMs     = 2500,
-    customSites = null,
-    log         = msg => process.stderr.write(`[cloak-warmup] ${msg}\n`),
-  } = opts;
-
-  const startedAt = Date.now();
-  const visited   = [];
-
-  log(`Starting warmup for ${targetUrl}`);
-
-  // 1. Prime reCAPTCHA state from Google
-  if (primeGoogle) {
-    log('Priming reCAPTCHA via google.com...');
-    await primeRecaptcha(page);
-    visited.push('https://www.google.com (recaptcha prime)');
-  }
-
-  // 2. Walk a referrer chain appropriate to the target
-  const category = detectCategory(targetUrl);
-  const chain    = REFERRER_CHAINS[category] || REFERRER_CHAINS.default;
-  log(`Category: ${category} — referrer chain: ${chain.join(' → ')}`);
-
-  for (const url of chain.slice(0, 2)) {
-    log(`Visiting referrer: ${url}`);
-    await visitWarmupSite(page, url, dwellMs);
-    visited.push(url);
-  }
-
-  // 3. Visit random warmup sites up to requested count
-  const pool    = customSites || DEFAULT_WARM_SITES;
-  const picks   = pool.sort(() => Math.random() - 0.5).slice(0, sites);
-  for (const url of picks) {
-    if (visited.includes(url)) continue;
-    log(`Visiting warmup site: ${url}`);
-    await visitWarmupSite(page, url, dwellMs);
-    visited.push(url);
-  }
-
-  // 4. Visit target root domain (makes the target visit look like a return visit)
-  if (visitRoot) {
-    log(`Visiting domain root before target...`);
-    await visitDomainRoot(page, targetUrl, dwellMs);
-    visited.push(new URL(targetUrl).origin + ' (root)');
-  }
-
-  const elapsedMs = Date.now() - startedAt;
-  log(`Warmup complete in ${(elapsedMs / 1000).toFixed(1)}s — visited ${visited.length} sites`);
-
-  // Snapshot cookie count as a rough trust proxy
-  let cookieCount = 0;
-  try {
-    const cookies = await ctx.cookies();
-    cookieCount = cookies.length;
-  } catch (_) {}
-
-  return {
-    warmed:      true,
-    targetUrl,
-    category,
-    visited,
-    cookieCount,
-    elapsedMs,
-  };
-}
-
-// ── Standalone quick-warmup (no context needed, creates its own browser) ──────
-
-/**
- * Convenience wrapper for use outside of an existing Playwright session.
- * Creates its own browser, warms up, returns result, closes browser.
- * The caller's real browser session still benefits because the warmup
- * seeds the same reCAPTCHA cookie jar that the vault/identity will restore.
- */
-async function quickWarmup(targetUrl, identityProfile = null, opts = {}) {
-  const { chromium } = require('playwright');
-  let browser;
-  try {
-    browser = await chromium.launch({ headless: true });
-    const ctxOpts = identityProfile ? {
-      viewport:   identityProfile.viewport,
-      userAgent:  identityProfile.userAgent,
-      timezoneId: identityProfile.timezone,
-      locale:     identityProfile.language,
-    } : {};
-    const ctx  = await browser.newContext(ctxOpts);
-    const page = await ctx.newPage();
-    return await warmupSession(page, ctx, targetUrl, opts);
-  } finally {
-    if (browser) await browser.close().catch(() => {});
-  }
-}
-
-// ── Compression stats (for MCP tool parity) ───────────────────────────────────
-
-function warmupStats() {
-  return {
-    defaultSites:    DEFAULT_WARM_SITES.length,
-    referrerChains:  Object.keys(REFERRER_CHAINS),
-    avgDurationSecs: '12–20s (3 sites, 2.5s dwell each)',
-    trustSignals:    [
-      'reCAPTCHA script pre-load from google.com',
-      'Category-matched referrer chain',
-      'Random general-web browsing history',
-      'Same-domain root visit before target',
-      'Human scroll + mouse simulation per page',
-    ],
-  };
-}
-
-module.exports = { warmupSession, quickWarmup, warmupStats, simulateDwell, visitWarmupSite };