reasonix 0.4.13 → 0.4.15

package/dist/index.d.ts

@@ -762,6 +762,99 @@ interface FilesystemToolsOptions {
 }
 declare function registerFilesystemTools(registry: ToolRegistry, opts: FilesystemToolsOptions): ToolRegistry;
 
+/**
+ * Built-in web search + fetch tools.
+ *
+ *   - `web_search(query, topK?)` — Mojeek's public search page. No API
+ *     key, no signup. We originally shipped this backed by DuckDuckGo's
+ *     HTML endpoint, but DDG started serving anti-bot interstitials
+ *     (HTTP 202 with a challenge page) for every unauthenticated POST.
+ *     Mojeek runs its own independent index, is bot-friendly, and
+ *     returns parseable HTML.
+ *   - `web_fetch(url)` — HTTP GET + naïve HTML-to-text extraction.
+ *
+ * Both are registered by default on `reasonix chat` / `reasonix code`;
+ * set `search: false` in config (or `REASONIX_SEARCH=off`) to turn
+ * them off. The model decides when to call them based on the query —
+ * no slash command required.
+ */
+
+interface SearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+}
+interface PageContent {
+    url: string;
+    title?: string;
+    text: string;
+    /** True when the extracted text was clipped to fit the cap. */
+    truncated: boolean;
+}
+interface WebFetchOptions {
+    /** Max bytes of extracted text. Defaults to 32_000 to match tool-result cap. */
+    maxChars?: number;
+    /** Timeout in ms. Defaults to 15_000. */
+    timeoutMs?: number;
+    signal?: AbortSignal;
+}
+interface WebSearchOptions {
+    topK?: number;
+    signal?: AbortSignal;
+}
+/**
+ * Search the public web via Mojeek. Returns up to `topK` ranked
+ * results with title, url, snippet.
+ *
+ * Mojeek is an independent index (not a Google/Bing front-end) which
+ * means coverage on niche or very recent topics can be thinner, but
+ * it's reliable from scripts and doesn't gate on cookies or sessions.
+ * If the response has 0 results we distinguish "truly empty" from
+ * "layout changed or blocked" so the caller isn't left guessing.
+ */
+declare function webSearch(query: string, opts?: WebSearchOptions): Promise<SearchResult[]>;
+/**
+ * Extract results from a Mojeek search page.
+ *
+ * Mojeek's stable shape (as of April 2026):
+ *   <a … class="ob" href="URL"> … breadcrumb … </a>
+ *   <h2><a class="title" href="URL">Title</a></h2>
+ *   <p class="s">snippet text …</p>
+ *
+ * We do two tolerant passes — title anchors, then snippet paragraphs —
+ * and pair them positionally. Attribute order inside a tag varies
+ * between versions, so each pass captures the whole element and we
+ * re-extract href / inner text with a second regex. Exported for
+ * unit testing against a fixture.
+ */
+declare function parseMojeekResults(html: string): SearchResult[];
+/**
+ * Download a URL, strip HTML down to readable text, return it. Times
+ * out at 15s, caps extracted text at 32k chars to fit the tool-result
+ * budget.
+ */
+declare function webFetch(url: string, opts?: WebFetchOptions): Promise<PageContent>;
+/**
+ * Strip HTML to readable text. Removes scripts/styles/nav/footer/aside
+ * blocks first, then tags, then collapses whitespace. Not a Readability
+ * clone — purpose-built to keep the extracted text small enough for the
+ * tool-result budget while preserving paragraph breaks.
+ */
+declare function htmlToText(html: string): string;
+interface WebToolsOptions {
+    /** Default top-K for `web_search` when the model doesn't specify. */
+    defaultTopK?: number;
+    /** Byte cap for `web_fetch` extracted text. */
+    maxFetchChars?: number;
+}
+/**
+ * Register `web_search` + `web_fetch` on a ToolRegistry. The model
+ * invokes them automatically when a question needs current info —
+ * no slash command from the user is required.
+ */
+declare function registerWebTools(registry: ToolRegistry, opts?: WebToolsOptions): ToolRegistry;
+declare function formatSearchResults(query: string, results: SearchResult[]): string;
+
 /**
  * Session persistence.
  *
@@ -1745,6 +1838,12 @@ interface ReasonixConfig {
     session?: string | null;
     /** Marks that `reasonix setup` has completed at least once. */
     setupCompleted?: boolean;
+    /**
+     * Whether `web_search` + `web_fetch` tools are registered. Default:
+     * enabled (no key required — backed by DuckDuckGo's public HTML
+     * endpoint). Set to `false` to keep the session offline.
+     */
+    search?: boolean;
 }
 declare function defaultConfigPath(): string;
 declare function readConfig(path?: string): ReasonixConfig;
@@ -1758,6 +1857,6 @@ declare function redactKey(key: string): string;
 
 /** Reasonix — DeepSeek-native agent framework. Library entry point. */
 
-declare const VERSION = "0.4.3";
+declare const VERSION = "0.4.15";
 
-export { AppendOnlyLog, type ApplyResult, type ApplyStatus, type BranchOptions, type BranchProgress, type BranchResult, type BranchSample, type BranchSelector, type BranchSummary, type BridgeOptions, type BridgeResult, CODE_SYSTEM_PROMPT, CacheFirstLoop, type CacheFirstLoopOptions, type CallToolResult, type ChatMessage, type ChatResponse, DEFAULT_MAX_RESULT_CHARS, DeepSeekClient, type DeepSeekClientOptions, type RenderOptions as DiffRenderOptions, type DiffReport, type DiffSide, type EditBlock, type EditSnapshot, type EventRole, type FilesystemToolsOptions, type FlattenDecision, type FlattenOptions, type GetPromptResult, type HarvestOptions, ImmutablePrefix, type ImmutablePrefixOptions, type InitializeResult, type InspectionReport, type JSONSchema, type JsonRpcMessage, type JsonRpcRequest, type JsonRpcResponse, type ListPromptsResult, type ListResourcesResult, type ListToolsResult, type LoopEvent, MCP_PROTOCOL_VERSION, McpClient, type McpClientOptions, type McpContentBlock, type McpProgressHandler, type McpProgressInfo, type McpPrompt, type McpPromptArgument, type McpPromptMessage, type McpPromptResourceBlock, type McpResource, type McpResourceContents, type McpResourceContentsBlob, type McpResourceContentsText, type McpSpec, type McpTool, type McpToolSchema, type McpTransport, type ProgressNotificationParams, type ReadResourceResult, type ReadTranscriptResult, type ReasonixConfig, type ReconfigurableOptions, type RepairReport, type ReplayStats, type RetryInfo, type RetryOptions, type Role, type ScavengeOptions, type ScavengeResult, type SectionResult, type SessionInfo, SessionStats, type SessionSummary, type SseMcpSpec, SseTransport, type SseTransportOptions, type StdioMcpSpec, StdioTransport, type StdioTransportOptions, StormBreaker, type StreamChunk, type ToolCall, type ToolCallContext, ToolCallRepair, type ToolCallRepairOptions, type ToolDefinition, type ToolFunctionSpec, ToolRegistry, type ToolSpec, type TranscriptMeta, type TranscriptRecord, type TruncationRepairResult, type TurnPair, type TurnStats, type TypedPlanState, Usage, VERSION, VolatileScratch, aggregateBranchUsage, analyzeSchema, appendSessionMessage, applyEditBlock, applyEditBlocks, bridgeMcpTools, claudeEquivalentCost, codeSystemPrompt, computeReplayStats, costUsd, defaultConfigPath, defaultSelector, deleteSession, diffTranscripts, emptyPlanState, fetchWithRetry, flattenMcpResult, flattenSchema, formatLoopError, harvest, healLoadedMessages, inputCostUsd, inspectMcpServer, isJsonRpcError, isPlanStateEmpty, isPlausibleKey, listSessions, loadApiKey, loadDotenv, loadSessionMessages, nestArguments, openTranscriptFile, outputCostUsd, parseEditBlocks, parseMcpSpec, parseTranscript, readConfig, readTranscript, recordFromLoopEvent, redactKey, registerFilesystemTools, renderMarkdown as renderDiffMarkdown, renderSummaryTable as renderDiffSummary, repairTruncatedJson, replayFromFile, restoreSnapshots, runBranches, sanitizeName as sanitizeSessionName, saveApiKey, scavengeToolCalls, sessionPath, sessionsDir, similarity, snapshotBeforeEdits, stripHallucinatedToolMarkup, truncateForModel, writeConfig, writeMeta, writeRecord };
+export { AppendOnlyLog, type ApplyResult, type ApplyStatus, type BranchOptions, type BranchProgress, type BranchResult, type BranchSample, type BranchSelector, type BranchSummary, type BridgeOptions, type BridgeResult, CODE_SYSTEM_PROMPT, CacheFirstLoop, type CacheFirstLoopOptions, type CallToolResult, type ChatMessage, type ChatResponse, DEFAULT_MAX_RESULT_CHARS, DeepSeekClient, type DeepSeekClientOptions, type RenderOptions as DiffRenderOptions, type DiffReport, type DiffSide, type EditBlock, type EditSnapshot, type EventRole, type FilesystemToolsOptions, type FlattenDecision, type FlattenOptions, type GetPromptResult, type HarvestOptions, ImmutablePrefix, type ImmutablePrefixOptions, type InitializeResult, type InspectionReport, type JSONSchema, type JsonRpcMessage, type JsonRpcRequest, type JsonRpcResponse, type ListPromptsResult, type ListResourcesResult, type ListToolsResult, type LoopEvent, MCP_PROTOCOL_VERSION, McpClient, type McpClientOptions, type McpContentBlock, type McpProgressHandler, type McpProgressInfo, type McpPrompt, type McpPromptArgument, type McpPromptMessage, type McpPromptResourceBlock, type McpResource, type McpResourceContents, type McpResourceContentsBlob, type McpResourceContentsText, type McpSpec, type McpTool, type McpToolSchema, type McpTransport, type PageContent, type ProgressNotificationParams, type ReadResourceResult, type ReadTranscriptResult, type ReasonixConfig, type ReconfigurableOptions, type RepairReport, type ReplayStats, type RetryInfo, type RetryOptions, type Role, type ScavengeOptions, type ScavengeResult, type SearchResult, type SectionResult, type SessionInfo, SessionStats, type SessionSummary, type SseMcpSpec, SseTransport, type SseTransportOptions, type StdioMcpSpec, StdioTransport, type StdioTransportOptions, StormBreaker, type StreamChunk, type ToolCall, type ToolCallContext, ToolCallRepair, type ToolCallRepairOptions, type ToolDefinition, type ToolFunctionSpec, ToolRegistry, type ToolSpec, type TranscriptMeta, type TranscriptRecord, type TruncationRepairResult, type TurnPair, type TurnStats, type TypedPlanState, Usage, VERSION, VolatileScratch, type WebFetchOptions, type WebSearchOptions, type WebToolsOptions, aggregateBranchUsage, analyzeSchema, appendSessionMessage, applyEditBlock, applyEditBlocks, bridgeMcpTools, claudeEquivalentCost, codeSystemPrompt, computeReplayStats, costUsd, defaultConfigPath, defaultSelector, deleteSession, diffTranscripts, emptyPlanState, fetchWithRetry, flattenMcpResult, flattenSchema, formatLoopError, formatSearchResults, harvest, healLoadedMessages, htmlToText, inputCostUsd, inspectMcpServer, isJsonRpcError, isPlanStateEmpty, isPlausibleKey, listSessions, loadApiKey, loadDotenv, loadSessionMessages, nestArguments, openTranscriptFile, outputCostUsd, parseEditBlocks, parseMcpSpec, parseMojeekResults, parseTranscript, readConfig, readTranscript, recordFromLoopEvent, redactKey, registerFilesystemTools, registerWebTools, renderMarkdown as renderDiffMarkdown, renderSummaryTable as renderDiffSummary, repairTruncatedJson, replayFromFile, restoreSnapshots, runBranches, sanitizeName as sanitizeSessionName, saveApiKey, scavengeToolCalls, sessionPath, sessionsDir, similarity, snapshotBeforeEdits, stripHallucinatedToolMarkup, truncateForModel, webFetch, webSearch, writeConfig, writeMeta, writeRecord };

package/dist/index.js

@@ -2183,6 +2183,187 @@ function lineDiff(a, b) {
   return out;
 }
 
+// src/tools/web.ts
+var DEFAULT_FETCH_MAX_CHARS = 32e3;
+var DEFAULT_FETCH_TIMEOUT_MS = 15e3;
+var DEFAULT_TOPK = 5;
+var USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
+var MOJEEK_ENDPOINT = "https://www.mojeek.com/search";
+async function webSearch(query, opts = {}) {
+  const topK = Math.max(1, Math.min(10, opts.topK ?? DEFAULT_TOPK));
+  const resp = await fetch(`${MOJEEK_ENDPOINT}?q=${encodeURIComponent(query)}`, {
+    headers: {
+      "User-Agent": USER_AGENT,
+      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9",
+      "Accept-Language": "en-US,en;q=0.9"
+    },
+    signal: opts.signal,
+    redirect: "follow"
+  });
+  if (!resp.ok) throw new Error(`web_search ${resp.status}`);
+  const html = await resp.text();
+  const results = parseMojeekResults(html).slice(0, topK);
+  if (results.length === 0) {
+    if (/no results found|did not match any documents/i.test(html)) return [];
+    if (/captcha|verify you are human|access denied|forbidden/i.test(html)) {
+      throw new Error("web_search: Mojeek anti-bot page \u2014 rate-limited or blocked");
+    }
+    throw new Error(
+      `web_search: 0 results but response doesn't look like a real empty page (${html.length} chars, first 120: ${html.slice(0, 120).replace(/\s+/g, " ")})`
+    );
+  }
+  return results;
+}
+function parseMojeekResults(html) {
+  const titles = [];
+  const titleAnchorRe = /<a\b[^>]*\bclass="title"[^>]*>[\s\S]*?<\/a>/g;
+  let m;
+  while (true) {
+    m = titleAnchorRe.exec(html);
+    if (m === null) break;
+    titles.push(m[0]);
+  }
+  const snippets = [];
+  const snippetRe = /<p\b[^>]*\bclass="s"[^>]*>([\s\S]*?)<\/p>/g;
+  while (true) {
+    m = snippetRe.exec(html);
+    if (m === null) break;
+    snippets.push(m[1] ?? "");
+  }
+  const hrefRe = /href="([^"]+)"/;
+  const innerRe = /<a\b[^>]*>([\s\S]*?)<\/a>/;
+  const results = [];
+  for (let i = 0; i < titles.length; i++) {
+    const anchor = titles[i];
+    const hrefMatch = anchor.match(hrefRe);
+    const innerMatch = anchor.match(innerRe);
+    if (!hrefMatch?.[1]) continue;
+    results.push({
+      title: decodeHtmlEntities(stripHtml(innerMatch?.[1] ?? "")).trim(),
+      url: hrefMatch[1],
+      snippet: decodeHtmlEntities(stripHtml(snippets[i] ?? "")).replace(/\s+/g, " ").trim()
+    });
+  }
+  return results;
+}
+async function webFetch(url, opts = {}) {
+  const maxChars = opts.maxChars ?? DEFAULT_FETCH_MAX_CHARS;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
+  const ctl = new AbortController();
+  const timer = setTimeout(() => ctl.abort(), timeoutMs);
+  const cancel = () => ctl.abort();
+  opts.signal?.addEventListener("abort", cancel, { once: true });
+  let resp;
+  try {
+    resp = await fetch(url, {
+      headers: { "User-Agent": USER_AGENT, Accept: "text/html,text/plain,*/*" },
+      signal: ctl.signal,
+      redirect: "follow"
+    });
+  } finally {
+    clearTimeout(timer);
+    opts.signal?.removeEventListener("abort", cancel);
+  }
+  if (!resp.ok) throw new Error(`web_fetch ${resp.status} for ${url}`);
+  const contentType = resp.headers.get("content-type") ?? "";
+  const raw = await resp.text();
+  const title = extractTitle(raw);
+  const text = contentType.includes("text/html") ? htmlToText(raw) : raw;
+  const truncated = text.length > maxChars;
+  const finalText = truncated ? `${text.slice(0, maxChars)}
+
+[\u2026 truncated ${text.length - maxChars} chars \u2026]` : text;
+  return { url, title, text: finalText, truncated };
+}
+function htmlToText(html) {
+  let s = html;
+  s = s.replace(/<script[\s\S]*?<\/script>/gi, "");
+  s = s.replace(/<style[\s\S]*?<\/style>/gi, "");
+  s = s.replace(/<noscript[\s\S]*?<\/noscript>/gi, "");
+  s = s.replace(/<nav[\s\S]*?<\/nav>/gi, "");
+  s = s.replace(/<footer[\s\S]*?<\/footer>/gi, "");
+  s = s.replace(/<aside[\s\S]*?<\/aside>/gi, "");
+  s = s.replace(/<svg[\s\S]*?<\/svg>/gi, "");
+  s = s.replace(/<\/?(p|div|br|h[1-6]|li|tr|section|article)\b[^>]*>/gi, "\n");
+  s = s.replace(/<[^>]+>/g, "");
+  s = decodeHtmlEntities(s);
+  s = s.replace(/[ \t]+/g, " ");
+  s = s.replace(/\n[ \t]+/g, "\n");
+  s = s.replace(/\n{3,}/g, "\n\n");
+  return s.trim();
+}
+function stripHtml(s) {
+  return s.replace(/<[^>]+>/g, "");
+}
+function decodeHtmlEntities(s) {
+  return s.replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+}
+function extractTitle(html) {
+  const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  if (!m?.[1]) return void 0;
+  return m[1].replace(/\s+/g, " ").trim() || void 0;
+}
+function registerWebTools(registry, opts = {}) {
+  const defaultTopK = opts.defaultTopK ?? DEFAULT_TOPK;
+  const maxFetchChars = opts.maxFetchChars ?? DEFAULT_FETCH_MAX_CHARS;
+  registry.register({
+    name: "web_search",
+    description: "Search the public web. Returns ranked results with title, url, and snippet. Use this when the question needs information more current than your training data, when you're unsure of a factual detail, or when the user asks about a specific webpage/library/release you haven't seen.",
+    parameters: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "Natural-language search query." },
+        topK: {
+          type: "integer",
+          description: `Number of results to return (1..10). Default ${defaultTopK}.`
+        }
+      },
+      required: ["query"]
+    },
+    fn: async (args, ctx) => {
+      const results = await webSearch(args.query, {
+        topK: args.topK ?? defaultTopK,
+        signal: ctx?.signal
+      });
+      return formatSearchResults(args.query, results);
+    }
+  });
+  registry.register({
+    name: "web_fetch",
+    description: "Download a URL and return its visible text content (HTML pages get scripts/styles/nav stripped). Truncated at the tool-result cap. Use after web_search when a snippet isn't enough.",
+    parameters: {
+      type: "object",
+      properties: {
+        url: { type: "string", description: "Absolute http:// or https:// URL." }
+      },
+      required: ["url"]
+    },
+    fn: async (args, ctx) => {
+      if (!/^https?:\/\//i.test(args.url)) {
+        throw new Error("web_fetch: url must start with http:// or https://");
+      }
+      const page = await webFetch(args.url, { maxChars: maxFetchChars, signal: ctx?.signal });
+      const header = page.title ? `${page.title}
+${page.url}` : page.url;
+      return `${header}
+
+${page.text}`;
+    }
+  });
+  return registry;
+}
+function formatSearchResults(query, results) {
+  const lines = [`query: ${query}`, `
+results (${results.length}):`];
+  results.forEach((r, i) => {
+    lines.push(`
+${i + 1}. ${r.title}`);
+    lines.push(`   ${r.url}`);
+    if (r.snippet) lines.push(`   ${r.snippet}`);
+  });
+  return lines.join("\n");
+}
+
 // src/env.ts
 import { readFileSync as readFileSync2 } from "fs";
 import { resolve as resolve2 } from "path";
@@ -3556,7 +3737,7 @@ function redactKey(key) {
 }
 
 // src/index.ts
-var VERSION = "0.4.3";
+var VERSION = "0.4.15";
 export {
   AppendOnlyLog,
   CODE_SYSTEM_PROMPT,
@@ -3594,8 +3775,10 @@ export {
   flattenMcpResult,
   flattenSchema,
   formatLoopError,
+  formatSearchResults,
   harvest,
   healLoadedMessages,
+  htmlToText,
   inputCostUsd,
   inspectMcpServer,
   isJsonRpcError,
@@ -3610,12 +3793,14 @@ export {
   outputCostUsd,
   parseEditBlocks,
   parseMcpSpec,
+  parseMojeekResults,
   parseTranscript,
   readConfig,
   readTranscript,
   recordFromLoopEvent,
   redactKey,
   registerFilesystemTools,
+  registerWebTools,
   renderMarkdown as renderDiffMarkdown,
   renderSummaryTable as renderDiffSummary,
   repairTruncatedJson,
@@ -3631,6 +3816,8 @@ export {
   snapshotBeforeEdits,
   stripHallucinatedToolMarkup,
   truncateForModel,
+  webFetch,
+  webSearch,
   writeConfig,
   writeMeta,
   writeRecord