shmakk 1.1.0

package/src/web.js

@@ -0,0 +1,228 @@
+// Web search and URL fetch helpers extracted from agent.js.
+// Uses DuckDuckGo Lite (no API key required) and plain http(s) fetch.
+
+const MAX_FETCH_BYTES = 128 * 1024;
+
+function decodeDdgUrl(url) {
+  try {
+    let raw = String(url || '');
+    // Handle protocol-relative URLs (//duckduckgo.com/...)
+    if (raw.startsWith('//')) raw = 'https:' + raw;
+    // Decode HTML entities that appear in href attribute values
+    raw = raw.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+      .replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+    const u = new URL(raw);
+    const p = u.searchParams;
+    if ((u.hostname === 'duckduckgo.com' || u.hostname.endsWith('.duckduckgo.com')) && p.has('uddg')) {
+      const raw = p.get('uddg');
+      if (raw) {
+        const decoded = decodeURIComponent(raw);
+        if (/^https?:\/\//i.test(decoded)) return decoded;
+      }
+    }
+    return url;
+  } catch { return url; }
+}
+
+function stripTags(html) {
+  return String(html || '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/<[^>]*>/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function parseDdgLite(html, maxResults = 5) {
+  const results = [];
+  const seen = new Set();
+
+  // Primary: old/current lite table result rows
+  const rowRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
+  let row;
+  while ((row = rowRe.exec(html)) && results.length < maxResults) {
+    const block = row[1];
+    const link = /<a[^>]+(?:class="result-link"|rel="nofollow")[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i.exec(block);
+    if (!link) continue;
+    const url = decodeDdgUrl(link[1]);
+    if (!/^https?:\/\//i.test(url) || seen.has(url)) continue;
+    const snippetMatch = /<td[^>]+class="result-snippet"[^>]*>([\s\S]*?)<\/td>/i.exec(block);
+    const title = stripTags(link[2]);
+    if (!title) continue;
+    seen.add(url);
+    results.push({
+      title,
+      url,
+      snippet: snippetMatch ? stripTags(snippetMatch[1]) : '',
+    });
+  }
+
+  // Fallback: generic anchor extraction for changed DDG markup
+  if (results.length < maxResults) {
+    const anchorRe = /<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
+    let a;
+    while ((a = anchorRe.exec(html)) && results.length < maxResults) {
+      const url = decodeDdgUrl(a[1]);
+      if (!/^https?:\/\//i.test(url) || seen.has(url)) continue;
+      if (/duckduckgo\.com\/(?:lite|html|\?|$)/i.test(url)) continue;
+      const title = stripTags(a[2]);
+      if (!title || title.length < 3) continue;
+      seen.add(url);
+      results.push({ title, url, snippet: '' });
+    }
+  }
+
+  return results;
+}
+
+function fetchWithTimeout(url, opts = {}, timeoutMs = 12000) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(new Error('timeout')), timeoutMs);
+  let removeUpstreamAbortListener = null;
+  if (opts.signal) {
+    if (opts.signal.aborted) ctrl.abort(opts.signal.reason);
+    else {
+      const onAbort = () => ctrl.abort(opts.signal.reason);
+      opts.signal.addEventListener('abort', onAbort, { once: true });
+      removeUpstreamAbortListener = () => opts.signal.removeEventListener('abort', onAbort);
+    }
+  }
+  return fetch(url, { ...opts, signal: ctrl.signal }).finally(() => {
+    clearTimeout(timer);
+    if (removeUpstreamAbortListener) removeUpstreamAbortListener();
+  });
+}
+
+async function webSearch(query, maxResults, signal) {
+  const q = String(query || '').trim();
+  if (!q) return { error: 'query required' };
+  const limit = Math.max(1, Math.min(10, Number(maxResults) || 5));
+  const liteUrl = `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(q)}`;
+  const htmlUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`;
+  const headers = { 'user-agent': 'Mozilla/5.0 (compatible; shmakk/0.1; +https://duckduckgo.com)' };
+
+  async function searchOne(url) {
+    const resp = await fetchWithTimeout(url, { signal, headers });
+    const html = await resp.text();
+    if (!resp.ok) return { error: `search failed: HTTP ${resp.status}`, results: [] };
+    return { results: parseDdgLite(html, limit) };
+  }
+
+  try {
+    const lite = await searchOne(liteUrl);
+    if (lite.results.length) return { query: q, results: lite.results, source: 'ddg-lite' };
+
+    const html = await searchOne(htmlUrl);
+    if (html.results.length) return { query: q, results: html.results, source: 'ddg-html-fallback' };
+
+    return { query: q, results: [], source: 'ddg-lite+html', note: lite.error || html.error || 'no results parsed' };
+  } catch (e) {
+    return { error: String(e.message || e) };
+  }
+}
+
+async function fetchUrl(url, signal) {
+  let parsed;
+  try { parsed = new URL(String(url || '')); } catch { return { error: 'invalid URL' }; }
+  if (!/^https?:$/.test(parsed.protocol)) return { error: 'only http(s) URLs are supported' };
+  try {
+    const resp = await fetchWithTimeout(parsed.href, {
+      signal,
+      headers: { 'user-agent': 'shmakk/0.1' },
+    });
+    const text = await resp.text();
+    return {
+      url: parsed.href,
+      status: resp.status,
+      contentType: resp.headers.get('content-type') || '',
+      text: stripTags(text).slice(0, MAX_FETCH_BYTES),
+      truncated: text.length > MAX_FETCH_BYTES,
+    };
+  } catch (e) {
+    return { error: String(e.message || e) };
+  }
+}
+
+// ── JSON fallback action extraction ─────────────────────────────────────────
+
+function stripJsonFence(s) {
+  const t = String(s || '').trim();
+  const m = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(t);
+  return m ? m[1].trim() : t;
+}
+
+function parseFallbackActions(content) {
+  const text = stripJsonFence(content);
+  if (!text) return [];
+
+  let obj = null;
+  try {
+    obj = JSON.parse(text);
+  } catch {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start === -1 || end <= start) return [];
+    try { obj = JSON.parse(text.slice(start, end + 1)); } catch { return []; }
+  }
+
+  const rawActions = Array.isArray(obj?.shmakk_actions) ? obj.shmakk_actions : [];
+  const allowed = new Set(['read_file', 'list_dir', 'web_search', 'fetch_url',
+    'write_file', 'edit_file', 'make_dir', 'delete_file', 'run']);
+  const actions = [];
+  for (const a of rawActions) {
+    const name = a?.tool || a?.name;
+    const args = a?.args && typeof a.args === 'object' ? a.args : {};
+    if (allowed.has(name)) actions.push({ name, args });
+  }
+  return actions;
+}
+
+// ── XML fallback action extraction ──────────────────────────────────────────
+
+function parseXmlFallbackActions(content) {
+  const text = String(content || '');
+  if (!text) return [];
+  const allowed = new Set(['read_file', 'list_dir', 'web_search', 'fetch_url',
+    'write_file', 'edit_file', 'make_dir', 'delete_file', 'run']);
+  const actions = [];
+
+  const tcRe = /<tool_call>([\s\S]*?)<\/tool_call>/gi;
+  let m;
+  while ((m = tcRe.exec(text))) {
+    const block = m[1];
+    const fnMatch = /<function\s*=\s*([a-zA-Z0-9_]+)\s*>([\s\S]*?)<\/function>/i.exec(block);
+    if (!fnMatch) continue;
+    const name = fnMatch[1];
+    if (!allowed.has(name)) continue;
+    const body = fnMatch[2] || '';
+    const args = {};
+    const pRe = /<parameter\s*=\s*([a-zA-Z0-9_]+)\s*>([\s\S]*?)<\/parameter>/gi;
+    let p;
+    while ((p = pRe.exec(body))) {
+      const k = p[1];
+      const raw = (p[2] || '').trim();
+      if (/^(true|false)$/i.test(raw)) args[k] = /^true$/i.test(raw);
+      else if (/^-?\d+(?:\.\d+)?$/.test(raw)) args[k] = Number(raw);
+      else args[k] = raw;
+    }
+    actions.push({ name, args });
+  }
+
+  return actions;
+}
+
+module.exports = {
+  webSearch,
+  fetchUrl,
+  parseDdgLite,
+  stripTags,
+  decodeDdgUrl,
+  fetchWithTimeout,
+  stripJsonFence,
+  parseFallbackActions,
+  parseXmlFallbackActions,
+};

package/src/workspace-index.js

@@ -0,0 +1,213 @@
+const fs = require('fs');
+const path = require('path');
+
+const SKIP_DIRS = new Set(['.git', 'node_modules', '.next', 'dist', 'build', 'coverage']);
+const MAX_FILE_BYTES = 96 * 1024;
+const CODE_EXTS = new Set(['.js', '.cjs', '.mjs', '.ts', '.tsx', '.jsx', '.py', '.go', '.rs']);
+
+function safeRead(p) {
+  try { return fs.readFileSync(p, 'utf8'); } catch { return ''; }
+}
+
+function firstNonEmptyLines(content, limit = 20) {
+  return String(content || '')
+    .split(/\r?\n/)
+    .map((x) => x.trimEnd())
+    .filter((x) => x.trim())
+    .slice(0, limit);
+}
+
+function extractHints(content) {
+  const symbols = [];
+  const imports = [];
+  const exports = [];
+  const lines = String(content || '').split(/\r?\n/).slice(0, 400);
+  for (const l of lines) {
+    const s = l.trim();
+    let m = /^export\s+(?:async\s+)?function\s+([a-zA-Z0-9_]+)/.exec(s)
+      || /^(?:async\s+)?function\s+([a-zA-Z0-9_]+)/.exec(s)
+      || /^class\s+([a-zA-Z0-9_]+)/.exec(s)
+      || /^const\s+([a-zA-Z0-9_]+)\s*=\s*\(/.exec(s);
+    if (m) symbols.push(m[1]);
+    m = /^module\.exports\s*=\s*([a-zA-Z0-9_]+)/.exec(s)
+      || /^export\s+default\s+([a-zA-Z0-9_]+)/.exec(s)
+      || /^export\s*\{\s*([^}]+)\s*\}/.exec(s);
+    if (m) exports.push(m[1]);
+    m = /^import\s+.*?from\s+['"]([^'"]+)['"]/.exec(s)
+      || /^const\s+.*?=\s*require\(['"]([^'"]+)['"]\)/.exec(s);
+    if (m) imports.push(m[1]);
+  }
+  return {
+    symbols: symbols.slice(0, 20),
+    imports: imports.slice(0, 20),
+    exports: exports.slice(0, 20),
+  };
+}
+
+function detectRole(rel) {
+  const base = path.basename(rel).toLowerCase();
+  const dir = path.dirname(rel).toLowerCase();
+  if (base === 'package.json' || /tsconfig|vite\.config|next\.config|dockerfile|readme/.test(base)) return 'config';
+  if (dir.includes('test') || base.includes('.test.') || base.includes('.spec.')) return 'test';
+  if (base === 'index.js' || base === 'main.js' || rel.startsWith('bin/')) return 'entry';
+  if (dir.includes('hooks')) return 'hook';
+  if (dir.includes('services')) return 'service';
+  if (dir.includes('src')) return 'source';
+  return 'file';
+}
+
+function resolveImportTarget(rel, imp, allFiles) {
+  if (!imp || !imp.startsWith('.')) return null;
+  const baseDir = path.dirname(rel);
+  const raw = path.normalize(path.join(baseDir, imp));
+  const candidates = [
+    raw,
+    `${raw}.js`, `${raw}.cjs`, `${raw}.mjs`, `${raw}.ts`, `${raw}.tsx`, `${raw}.jsx`,
+    path.join(raw, 'index.js'), path.join(raw, 'index.ts'), path.join(raw, 'index.tsx'),
+  ];
+  for (const c of candidates) {
+    if (allFiles.has(c)) return c;
+  }
+  return null;
+}
+
+function walkFiles(root, dir = root, out = []) {
+  let ents = [];
+  try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return out; }
+  for (const e of ents) {
+    const abs = path.join(dir, e.name);
+    const rel = path.relative(root, abs);
+    if (!rel) continue;
+    if (e.isDirectory()) {
+      if (SKIP_DIRS.has(e.name)) continue;
+      walkFiles(root, abs, out);
+      continue;
+    }
+    if (!e.isFile()) continue;
+    out.push(rel);
+  }
+  return out;
+}
+
+function indexFilePath(root) {
+  return path.join(root, '.shmakk', 'state', 'index.json');
+}
+
+function loadIndex(root) {
+  const p = indexFilePath(root);
+  try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; }
+}
+
+function saveIndex(root, index) {
+  const p = indexFilePath(root);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  fs.writeFileSync(p, JSON.stringify(index), 'utf8');
+}
+
+function buildOrRefreshIndex(root) {
+  const now = Date.now();
+  const existing = loadIndex(root) || { root, files: {}, updatedAt: 0 };
+  const seen = new Set(walkFiles(root));
+
+  for (const rel of Object.keys(existing.files)) {
+    if (!seen.has(rel)) delete existing.files[rel];
+  }
+
+  for (const rel of seen) {
+    const abs = path.join(root, rel);
+    let st;
+    try { st = fs.statSync(abs); } catch { continue; }
+    const prev = existing.files[rel];
+    const mtimeMs = st.mtimeMs;
+    const size = st.size;
+    if (prev && prev.mtimeMs === mtimeMs && prev.size === size) continue;
+
+    const sample = safeRead(abs).slice(0, MAX_FILE_BYTES);
+    const hints = extractHints(sample);
+    existing.files[rel] = {
+      path: rel,
+      mtimeMs,
+      size,
+      ext: path.extname(rel).toLowerCase(),
+      role: detectRole(rel),
+      symbols: hints.symbols,
+      imports: hints.imports,
+      exports: hints.exports,
+      snippet: firstNonEmptyLines(sample, 12),
+      edges: [],
+    };
+  }
+
+  const allFiles = new Set(Object.keys(existing.files));
+  for (const rel of Object.keys(existing.files)) {
+    const f = existing.files[rel];
+    const edges = [];
+    for (const imp of f.imports || []) {
+      const target = resolveImportTarget(rel, imp, allFiles);
+      if (target) edges.push(target);
+    }
+    f.edges = edges.slice(0, 30);
+  }
+
+  existing.updatedAt = now;
+  saveIndex(root, existing);
+  return existing;
+}
+
+function tokenize(s) {
+  return String(s || '').toLowerCase().split(/[^a-z0-9_./-]+/).filter(Boolean);
+}
+
+function relevantFiles(index, query, limit = 20) {
+  const q = tokenize(query);
+  if (!q.length) return [];
+  const scored = [];
+  for (const f of Object.values(index.files || {})) {
+    const hay = `${f.path} ${(f.symbols || []).join(' ')} ${(f.imports || []).join(' ')}`.toLowerCase();
+    let score = 0;
+    for (const t of q) {
+      if (hay.includes(t)) score += 2;
+      if (f.path.toLowerCase().includes(t)) score += 3;
+    }
+    if (score > 0) scored.push({ file: f.path, score });
+  }
+  scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file));
+  return scored.slice(0, limit).map((x) => x.file);
+}
+
+function relevantSubgraph(index, query, limit = 12, maxHops = 1) {
+  const seeds = relevantFiles(index, query, Math.max(4, Math.min(limit, 8)));
+  const files = index.files || {};
+  const visited = new Set(seeds);
+  const queue = seeds.map((file) => ({ file, hop: 0 }));
+  const out = [];
+
+  while (queue.length && out.length < limit) {
+    const { file, hop } = queue.shift();
+    const node = files[file];
+    if (!node) continue;
+    out.push({
+      path: node.path,
+      role: node.role,
+      symbols: node.symbols || [],
+      imports: node.imports || [],
+      exports: node.exports || [],
+      snippet: node.snippet || [],
+      edges: node.edges || [],
+    });
+    if (hop >= maxHops) continue;
+    for (const next of node.edges || []) {
+      if (visited.has(next)) continue;
+      visited.add(next);
+      queue.push({ file: next, hop: hop + 1 });
+    }
+  }
+
+  return out;
+}
+
+function clearIndex(root) {
+  try { fs.rmSync(indexFilePath(root), { force: true }); } catch {}
+}
+
+module.exports = { buildOrRefreshIndex, clearIndex, relevantFiles, relevantSubgraph };