vektor-slipstream 1.4.4 → 2.0.0

package/cloak-index.js

@@ -1,261 +0,0 @@
-'use strict';
-
-/**
- * vektor-slipstream/cloak
- * ─────────────────────────────────────────────
- * Re-exports all Cloak tools + new v1.2 modules:
- *
- *   EXISTING (v1.1):
- *     cloak_fetch        — stealth headless browser fetch with cache
- *     cloak_render       — full layout sensor (CSS, fonts, gap analysis)
- *     cloak_diff         — semantic page diff over time
- *     cloak_diff_text    — structural text diff
- *     cloak_passport     — AES-256 session credential vault
- *     tokens_saved       — ROI audit per session
- *
- *   NEW (v1.2):
- *     llms               — llms.txt pre-flight checker
- *     captcha            — CAPTCHA detection + solving pipeline
- *     turboQuant         — 3-bit vector compression for embeddings
- *     CloakIdentity      — persistent browser fingerprint identity
- *
- *   MCP_TOOLS            — all tool definitions for MCP server
- */
-
-// ── v1.1 core (existing) ─────────────────────────────────────────────────────
-const {
-  cloak_fetch,
-  cloak_render,
-  cloak_diff,
-  cloak_diff_text,
-  cloak_passport,
-  tokens_saved,
-  CLOAK_MCP_TOOLS: _CORE_MCP_TOOLS,
-} = require('./cloak-core');   // existing cloak.js renamed to cloak-core.js
-
-// ── v1.2 new modules ─────────────────────────────────────────────────────────
-const llms         = require('./cloak-llms');
-const captcha      = require('./cloak-captcha');
-const turboQuant   = require('./cloak-turbo-quant');
-const { CloakIdentity } = require('./cloak-identity');
-const behaviour        = require('./cloak-behaviour');
-let _patternStore = null; const patternStore = new Proxy({}, { get(_, k) { if (!_patternStore) _patternStore = require('./cloak-pattern-store'); return _patternStore[k]; } });
-let _autoRecorder = null; const autoRecorder = new Proxy({}, { get(_, k) { if (!_autoRecorder) _autoRecorder = require('./cloak-recorder-auto'); return _autoRecorder[k]; } });
-
-// ── Enhanced cloak_fetch with llms.txt pre-flight ────────────────────────────
-/**
- * cloak_fetch_smart: drop-in replacement for cloak_fetch.
- * Checks for llms.txt first — if found, returns structured content
- * without spinning up Playwright (faster, cheaper, cleaner).
- * Falls back to stealth browser if llms.txt not found.
- */
-async function cloak_fetch_smart(url, opts = {}) {
-  if (!opts.skipLlmsCheck) {
-    const llmsResult = await llms.checkLlmsTxt(url);
-    if (llmsResult.found) {
-      const parsed  = llms.parseLlmsTxt(llmsResult.content);
-      const content = llms.extractRelevantContent(parsed, url);
-      const rawLen  = llmsResult.content.length;
-      return {
-        text:        content,
-        tokensSaved: Math.floor((rawLen * 4 - content.length) / 4),
-        fromCache:   llmsResult.fromCache || false,
-        source:      'llms.txt',
-        llmsFriendly: true,
-      };
-    }
-  }
-  // Fall through to stealth browser
-  const result = await cloak_fetch(url, opts);
-  result.source = 'stealth';
-  result.llmsFriendly = false;
-  return result;
-}
-
-// ── New MCP tool definitions ──────────────────────────────────────────────────
-const NEW_MCP_TOOLS = [
-  {
-    name: 'cloak_inject_behaviour',
-    description: 'Inject human-realistic mouse/scroll behaviour into the current browser session. Call before or during cloak_fetch / cloak_identity_use on sites with behavioural fingerprinting (reCAPTCHA v3, Cloudflare, DataDome). Significantly improves trust score.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        url:          { type: 'string',  description: 'URL to open and inject behaviour on' },
-        category:     { type: 'string',  enum: ['reading', 'form', 'shopping', 'login', 'idle'], description: 'Behaviour category to simulate (default: reading)' },
-        patternName:  { type: 'string',  description: 'Specific pattern name to replay (overrides category)' },
-        speedFactor:  { type: 'number',  description: 'Replay speed: 1.0 = real speed, 0.5 = 2x faster (default 1.0)' },
-        synthetic:    { type: 'boolean', description: 'Use procedural synthesis instead of recorded pattern (default false)' },
-        durationMs:   { type: 'integer', description: 'Duration for synthetic mode in ms (default 5000)' },
-      },
-      required: ['url'],
-    },
-  },
-  {
-    name: 'cloak_behaviour_stats',
-    description: 'List available behaviour patterns, categories, and how they work.',
-    input_schema: { type: 'object', properties: {} },
-  },
-  {
-    name: 'cloak_load_pattern',
-    description: 'Load a custom behaviour pattern recorded with the cloak-recorder browser snippet.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        name:    { type: 'string', description: 'Pattern name to register' },
-        pattern: { type: 'string', description: 'JSON string from cloak-recorder-snippet.js' },
-        save:    { type: 'boolean', description: 'Persist pattern to ~/.vektor/behaviour-patterns/ (default true)' },
-      },
-      required: ['name', 'pattern'],
-    },
-  },
-  {
-    name: 'cloak_fetch_smart',
-    description: 'Fetch a URL intelligently — checks llms.txt first (agent-native, no browser), falls back to stealth Playwright browser. Returns compressed clean text. Shows whether site is agent-friendly.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        url:           { type: 'string',  description: 'URL to fetch.' },
-        force:         { type: 'boolean', description: 'Bypass cache.' },
-        skipLlmsCheck: { type: 'boolean', description: 'Skip llms.txt pre-flight and go straight to stealth browser.' },
-      },
-      required: ['url'],
-    },
-  },
-  {
-    name: 'cloak_detect_captcha',
-    description: 'Detect if the current page has a CAPTCHA challenge. Returns type (hcaptcha, recaptcha_v2, recaptcha_v3, turnstile) and sitekey if found.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        url: { type: 'string', description: 'URL to check for CAPTCHA.' },
-      },
-      required: ['url'],
-    },
-  },
-  {
-    name: 'cloak_solve_captcha',
-    description: 'Automatically solve a detected CAPTCHA using vision AI. Supports hCaptcha (Claude/GPT-4o vision), reCAPTCHA v3 session warmup, audio CAPTCHAs (Whisper).',
-    input_schema: {
-      type: 'object',
-      properties: {
-        url:          { type: 'string', description: 'URL with CAPTCHA.' },
-        captchaType:  { type: 'string', enum: ['hcaptcha', 'recaptcha_v2', 'recaptcha_v3', 'turnstile', 'audio'], description: 'Type of CAPTCHA to solve.' },
-        provider:     { type: 'string', enum: ['claude', 'openai', '2captcha'], description: 'Vision model provider.' },
-      },
-      required: ['url', 'captchaType'],
-    },
-  },
-  {
-    name: 'cloak_identity_create',
-    description: 'Create a persistent browser fingerprint identity. Generates consistent UA, WebGL, canvas, fonts, timezone. Use the same identity across sessions to build site trust.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        name: { type: 'string', description: 'Identity name (e.g. "shopping-agent-1").' },
-        seed: { type: 'string', description: 'Optional seed for deterministic fingerprint.' },
-      },
-      required: ['name'],
-    },
-  },
-  {
-    name: 'cloak_identity_use',
-    description: 'Use a saved identity for the next cloak_fetch/cloak_render call. Applies full fingerprint to browser session.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        name: { type: 'string', description: 'Identity name to use.' },
-        url:  { type: 'string', description: 'URL to fetch with this identity.' },
-      },
-      required: ['name', 'url'],
-    },
-  },
-  {
-    name: 'cloak_identity_list',
-    description: 'List all saved browser identities with their age, visit count, and trust summary.',
-    input_schema: { type: 'object', properties: {} },
-  },
-  {
-    name: 'cloak_pattern_stats',
-    description: 'Show the self-improving pattern store stats — tier breakdown, win/loss rates, total sessions.',
-    input_schema: { type: 'object', properties: {} },
-  },
-  {
-    name: 'cloak_pattern_list',
-    description: 'List all patterns in the store with scores and tier.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        tier: { type: 'string', enum: ['elite', 'active', 'probation'], description: 'Filter by tier (optional)' },
-      },
-    },
-  },
-  {
-    name: 'cloak_pattern_prune',
-    description: 'Force a prune pass on the pattern store. Removes stale, low-scoring, and duplicate patterns.',
-    input_schema: { type: 'object', properties: {} },
-  },
-  {
-    name: 'cloak_pattern_seed',
-    description: 'Seed the pattern store with built-in patterns. Only runs if store is empty.',
-    input_schema: { type: 'object', properties: {} },
-  },
-  {
-    name: 'turbo_quant_compress',
-    description: 'Compress the VEKTOR memory database using TurboQuant 3-bit vector quantisation. Reduces embedding storage by ~87% with <2% recall accuracy loss.',
-    input_schema: {
-      type: 'object',
-      properties: {
-        dbPath: { type: 'string', description: 'Path to vektor-slipstream-memory.db. Leave blank for default.' },
-      },
-    },
-  },
-  {
-    name: 'turbo_quant_stats',
-    description: 'Show TurboQuant compression stats — original vs compressed size, ratio, savings percentage.',
-    input_schema: { type: 'object', properties: {} },
-  },
-];
-
-const CLOAK_MCP_TOOLS = [..._CORE_MCP_TOOLS, ...NEW_MCP_TOOLS];
-
-// ── MCP tool handler additions ────────────────────────────────────────────────
-// Add these cases to your runTool() switch in example-claude-mcp.js:
-//
-// case 'cloak_fetch_smart':   return cloak_fetch_smart(input.url, input);
-// case 'cloak_identity_create': {
-//   const id = CloakIdentity.create(input.name, input.seed);
-//   id.save();
-//   return id.summary;
-// }
-// case 'cloak_identity_list':  return CloakIdentity.list().map(n => CloakIdentity.load(n)?.summary);
-// case 'turbo_quant_stats':    return turboQuant.compressionStats();
-// case 'turbo_quant_compress': {
-//   const db = require('better-sqlite3')(input.dbPath || defaultDbPath);
-//   return turboQuant.migrateDatabase(db);
-// }
-
-module.exports = {
-  // v1.1 unchanged exports
-  cloak_fetch,
-  cloak_render,
-  cloak_diff,
-  cloak_diff_text,
-  cloak_passport,
-  tokens_saved,
-
-  // v1.2 new exports
-  cloak_fetch_smart,
-  llms,
-  captcha,
-  turboQuant,
-  CloakIdentity,
-
-  // v1.3 new exports
-  behaviour,
-  patternStore,
-  autoRecorder,
-
-  // MCP
-  CLOAK_MCP_TOOLS,
-  NEW_MCP_TOOLS,
-};

package/cloak-llms.js

@@ -1,163 +0,0 @@
-'use strict';
-
-/**
- * cloak-llms.js
- * Pre-flight llms.txt checker for cloak_fetch.
- * Checks if a site publishes llms.txt (agent-native access).
- * If found, returns clean structured content — no Playwright needed.
- * Falls back to stealth browser if not found or malformed.
- */
-
-const https = require('https');
-const http  = require('http');
-const { URL } = require('url');
-
-const LLMS_TTL_MS  = 24 * 60 * 60 * 1000; // 24hr cache
-const _llmsCache   = new Map(); // origin → { found, content, checkedAt }
-
-/**
- * Check for llms.txt at the origin of a URL.
- * Returns { found: bool, content: string|null, url: string }
- */
-async function checkLlmsTxt(targetUrl) {
-  let origin;
-  try {
-    const u = new URL(targetUrl);
-    origin = u.origin; // e.g. https://example.com
-  } catch {
-    return { found: false, content: null, url: targetUrl };
-  }
-
-  // Cache hit
-  const cached = _llmsCache.get(origin);
-  if (cached && Date.now() - cached.checkedAt < LLMS_TTL_MS) {
-    return { found: cached.found, content: cached.content, url: origin + '/llms.txt', fromCache: true };
-  }
-
-  const llmsUrl = origin + '/llms.txt';
-  try {
-    const content = await _fetchText(llmsUrl, 3000); // 3s timeout — don't block
-    const found   = content && content.length > 20;
-    _llmsCache.set(origin, { found, content: found ? content : null, checkedAt: Date.now() });
-    return { found, content: found ? content : null, url: llmsUrl };
-  } catch {
-    _llmsCache.set(origin, { found: false, content: null, checkedAt: Date.now() });
-    return { found: false, content: null, url: llmsUrl };
-  }
-}
-
-/**
- * Parse llms.txt content into a structured object.
- * Spec: https://llmstxt.org
- * Returns { title, description, sections: [{heading, links: [{url, title, desc}]}] }
- */
-function parseLlmsTxt(content) {
-  const lines    = content.split('\n').map(l => l.trimEnd());
-  const result   = { title: '', description: '', sections: [], raw: content };
-  let   section  = null;
-
-  for (const line of lines) {
-    if (!line.trim()) continue;
-
-    // H1 = page title
-    if (line.startsWith('# ')) {
-      result.title = line.slice(2).trim();
-      continue;
-    }
-
-    // Blockquote = description
-    if (line.startsWith('> ')) {
-      result.description += (result.description ? ' ' : '') + line.slice(2).trim();
-      continue;
-    }
-
-    // H2 = section heading
-    if (line.startsWith('## ')) {
-      section = { heading: line.slice(3).trim(), links: [] };
-      result.sections.push(section);
-      continue;
-    }
-
-    // Markdown link line: - [Title](url): description
-    if (line.startsWith('- [') && section) {
-      const m = line.match(/^- \[([^\]]+)\]\(([^)]+)\)(?::\s*(.*))?$/);
-      if (m) {
-        section.links.push({ title: m[1], url: m[2], desc: m[3] || '' });
-      }
-      continue;
-    }
-
-    // Plain text in a section
-    if (section && line.trim()) {
-      section.links.push({ title: line.trim(), url: null, desc: '' });
-    }
-  }
-
-  return result;
-}
-
-/**
- * Given llms.txt content and a target URL, extract the most relevant
- * section/links for the requested page. Returns a focused text summary.
- */
-function extractRelevantContent(parsed, targetUrl) {
-  if (!parsed.sections.length) return parsed.raw;
-
-  const urlLower = targetUrl.toLowerCase();
-  let best = null, bestScore = -1;
-
-  for (const section of parsed.sections) {
-    // Score section by how many links match the target URL path
-    let score = 0;
-    for (const link of section.links) {
-      if (link.url && urlLower.includes(link.url.toLowerCase().replace(/^https?:\/\/[^/]+/, ''))) {
-        score += 2;
-      }
-      if (link.title && urlLower.includes(link.title.toLowerCase())) score += 1;
-    }
-    if (score > bestScore) { bestScore = score; best = section; }
-  }
-
-  const target = bestScore > 0 ? best : parsed.sections[0];
-  let out = '';
-  if (parsed.title)       out += `# ${parsed.title}\n`;
-  if (parsed.description) out += `${parsed.description}\n\n`;
-  if (target) {
-    out += `## ${target.heading}\n`;
-    for (const link of target.links) {
-      out += link.url
-        ? `- [${link.title}](${link.url})${link.desc ? ': ' + link.desc : ''}\n`
-        : `- ${link.title}\n`;
-    }
-  }
-  return out.trim();
-}
-
-// ── Internal helpers ──────────────────────────────────────────────────────────
-
-function _fetchText(url, timeoutMs = 5000) {
-  return new Promise((resolve, reject) => {
-    const lib      = url.startsWith('https') ? https : http;
-    const timer    = setTimeout(() => reject(new Error('timeout')), timeoutMs);
-    const req      = lib.get(url, {
-      headers: {
-        'User-Agent': 'vektor-slipstream/cloak llms-checker',
-        'Accept':     'text/plain, text/markdown',
-      }
-    }, res => {
-      if (res.statusCode !== 200) {
-        clearTimeout(timer);
-        res.resume();
-        return reject(new Error(`HTTP ${res.statusCode}`));
-      }
-      let body = '';
-      res.setEncoding('utf8');
-      res.on('data', c => { body += c; if (body.length > 50000) res.destroy(); });
-      res.on('end',  () => { clearTimeout(timer); resolve(body); });
-      res.on('error', e => { clearTimeout(timer); reject(e); });
-    });
-    req.on('error', e => { clearTimeout(timer); reject(e); });
-  });
-}
-
-module.exports = { checkLlmsTxt, parseLlmsTxt, extractRelevantContent };