rird 1.0.200

package/src/tool/webfetch.ts

@@ -0,0 +1,198 @@
+import z from "zod"
+import { Tool } from "./tool"
+import TurndownService from "turndown"
+import DESCRIPTION from "./webfetch.txt"
+import { Config } from "../config/config"
+import { Permission } from "../permission"
+import { checkDomain, logBlockedAttempt } from "../security"
+
+const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB
+const DEFAULT_TIMEOUT = 30 * 1000 // 30 seconds
+const MAX_TIMEOUT = 120 * 1000 // 2 minutes
+
+export const WebFetchTool = Tool.define("webfetch", {
+  description: DESCRIPTION,
+  parameters: z.object({
+    url: z.string().describe("The URL to fetch content from"),
+    format: z
+      .enum(["text", "markdown", "html"])
+      .describe("The format to return the content in (text, markdown, or html)"),
+    timeout: z.number().describe("Optional timeout in seconds (max 120)").optional(),
+  }),
+  async execute(params, ctx) {
+    // Validate URL
+    if (!params.url.startsWith("http://") && !params.url.startsWith("https://")) {
+      throw new Error("URL must start with http:// or https://")
+    }
+
+    // Security guardrail: check for blocked domains (banking, crypto, government)
+    const domainCheck = checkDomain(params.url)
+    if (domainCheck.blocked) {
+      logBlockedAttempt(domainCheck, {
+        url: params.url,
+        timestamp: new Date(),
+      })
+      throw new Error(`URL blocked by security guardrails: ${domainCheck.reason}`)
+    }
+
+    const cfg = await Config.get()
+    if (cfg.permission?.webfetch === "ask")
+      await Permission.ask({
+        type: "webfetch",
+        sessionID: ctx.sessionID,
+        messageID: ctx.messageID,
+        callID: ctx.callID,
+        title: "Fetch content from: " + params.url,
+        metadata: {
+          url: params.url,
+          format: params.format,
+          timeout: params.timeout,
+        },
+      })
+
+    const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT)
+
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), timeout)
+
+    // Build Accept header based on requested format with q parameters for fallbacks
+    let acceptHeader = "*/*"
+    switch (params.format) {
+      case "markdown":
+        acceptHeader = "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1"
+        break
+      case "text":
+        acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1"
+        break
+      case "html":
+        acceptHeader = "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1"
+        break
+      default:
+        acceptHeader =
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
+    }
+
+    const response = await fetch(params.url, {
+      signal: AbortSignal.any([controller.signal, ctx.abort]),
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        Accept: acceptHeader,
+        "Accept-Language": "en-US,en;q=0.9",
+      },
+    })
+
+    clearTimeout(timeoutId)
+
+    if (!response.ok) {
+      throw new Error(`Request failed with status code: ${response.status}`)
+    }
+
+    // Check content length
+    const contentLength = response.headers.get("content-length")
+    if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+      throw new Error("Response too large (exceeds 5MB limit)")
+    }
+
+    const arrayBuffer = await response.arrayBuffer()
+    if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
+      throw new Error("Response too large (exceeds 5MB limit)")
+    }
+
+    const content = new TextDecoder().decode(arrayBuffer)
+    const contentType = response.headers.get("content-type") || ""
+
+    const title = `${params.url} (${contentType})`
+
+    // Handle content based on requested format and actual content type
+    switch (params.format) {
+      case "markdown":
+        if (contentType.includes("text/html")) {
+          const markdown = convertHTMLToMarkdown(content)
+          return {
+            output: markdown,
+            title,
+            metadata: {},
+          }
+        }
+        return {
+          output: content,
+          title,
+          metadata: {},
+        }
+
+      case "text":
+        if (contentType.includes("text/html")) {
+          const text = await extractTextFromHTML(content)
+          return {
+            output: text,
+            title,
+            metadata: {},
+          }
+        }
+        return {
+          output: content,
+          title,
+          metadata: {},
+        }
+
+      case "html":
+        return {
+          output: content,
+          title,
+          metadata: {},
+        }
+
+      default:
+        return {
+          output: content,
+          title,
+          metadata: {},
+        }
+    }
+  },
+})
+
+async function extractTextFromHTML(html: string) {
+  let text = ""
+  let skipContent = false
+
+  const rewriter = new HTMLRewriter()
+    .on("script, style, noscript, iframe, object, embed", {
+      element() {
+        skipContent = true
+      },
+      text() {
+        // Skip text content inside these elements
+      },
+    })
+    .on("*", {
+      element(element) {
+        // Reset skip flag when entering other elements
+        if (!["script", "style", "noscript", "iframe", "object", "embed"].includes(element.tagName)) {
+          skipContent = false
+        }
+      },
+      text(input) {
+        if (!skipContent) {
+          text += input.text
+        }
+      },
+    })
+    .transform(new Response(html))
+
+  await rewriter.text()
+  return text.trim()
+}
+
+function convertHTMLToMarkdown(html: string): string {
+  const turndownService = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+  })
+  turndownService.remove(["script", "style", "meta", "link"])
+  return turndownService.turndown(html)
+}

package/src/tool/webfetch.txt

@@ -0,0 +1,13 @@
+- Fetches content from a specified URL
+- Takes a URL and a prompt as input
+- Fetches the URL content, converts HTML to markdown
+- Returns the model's response about the content
+- Use this tool when you need to retrieve and analyze web content
+
+Usage notes:
+  - IMPORTANT: if another tool is present that offers better web fetching capabilities, is more targeted to the task, or has fewer restrictions, prefer using that tool instead of this one.
+  - The URL must be a fully-formed valid URL
+  - HTTP URLs will be automatically upgraded to HTTPS
+  - The prompt should describe what information you want to extract from the page
+  - This tool is read-only and does not modify any files
+  - Results may be summarized if the content is very large

package/src/tool/websearch.ts

@@ -0,0 +1,180 @@
+import z from "zod"
+import { Tool } from "./tool"
+import DESCRIPTION from "./websearch.txt"
+import { Config } from "../config/config"
+import { Permission } from "../permission"
+
+const API_CONFIG = {
+  BASE_URL: "https://mcp.exa.ai",
+  ENDPOINTS: {
+    SEARCH: "/mcp",
+  },
+  DEFAULT_NUM_RESULTS: 8,
+} as const
+
+interface McpSearchRequest {
+  jsonrpc: string
+  id: number
+  method: string
+  params: {
+    name: string
+    arguments: {
+      query: string
+      numResults?: number
+      livecrawl?: "fallback" | "preferred"
+      type?: "auto" | "fast" | "deep"
+      contextMaxCharacters?: number
+    }
+  }
+}
+
+interface McpSearchResponse {
+  jsonrpc: string
+  result: {
+    content: Array<{
+      type: string
+      text: string
+    }>
+  }
+}
+
+export const WebSearchTool = Tool.define("websearch", {
+  description: DESCRIPTION,
+  parameters: z.object({
+    query: z.string().describe("Websearch query"),
+    numResults: z.number().optional().describe("Number of search results to return (default: 8)"),
+    livecrawl: z
+      .enum(["fallback", "preferred"])
+      .optional()
+      .describe(
+        "Live crawl mode - 'fallback': use live crawling as backup if cached content unavailable, 'preferred': prioritize live crawling (default: 'fallback')",
+      ),
+    type: z
+      .enum(["auto", "fast", "deep"])
+      .optional()
+      .describe("Search type - 'auto': balanced search (default), 'fast': quick results, 'deep': comprehensive search"),
+    contextMaxCharacters: z
+      .number()
+      .optional()
+      .describe("Maximum characters for context string optimized for LLMs (default: 10000)"),
+  }),
+  async execute(params, ctx) {
+    const cfg = await Config.get()
+    if (cfg.permission?.webfetch === "ask")
+      await Permission.ask({
+        type: "websearch",
+        sessionID: ctx.sessionID,
+        messageID: ctx.messageID,
+        callID: ctx.callID,
+        title: "Search web for: " + params.query,
+        metadata: {
+          query: params.query,
+          numResults: params.numResults,
+          livecrawl: params.livecrawl,
+          type: params.type,
+          contextMaxCharacters: params.contextMaxCharacters,
+        },
+      })
+
+    const searchRequest: McpSearchRequest = {
+      jsonrpc: "2.0",
+      id: 1,
+      method: "tools/call",
+      params: {
+        name: "web_search_exa",
+        arguments: {
+          query: params.query,
+          type: params.type || "auto",
+          numResults: params.numResults || API_CONFIG.DEFAULT_NUM_RESULTS,
+          livecrawl: params.livecrawl || "fallback",
+          contextMaxCharacters: params.contextMaxCharacters,
+        },
+      },
+    }
+
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), 25000)
+
+    try {
+      const headers: Record<string, string> = {
+        accept: "application/json, text/event-stream",
+        "content-type": "application/json",
+      }
+
+      const response = await fetch(`${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.SEARCH}`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(searchRequest),
+        signal: AbortSignal.any([controller.signal, ctx.abort]),
+      })
+
+      clearTimeout(timeoutId)
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        throw new Error(`Search error (${response.status}): ${errorText}`)
+      }
+
+      const responseText = await response.text()
+
+      // Parse SSE response
+      const lines = responseText.split("\n")
+      for (const line of lines) {
+        if (line.startsWith("data: ")) {
+          const data: McpSearchResponse = JSON.parse(line.substring(6))
+          if (data.result && data.result.content && data.result.content.length > 0) {
+            return {
+              output: data.result.content[0].text,
+              title: `RIRD Search: ${params.query}`,
+              metadata: { source: "rird-web" },
+            }
+          }
+        }
+      }
+
+      return {
+        output: "No results found in RIRD index. Please try a different query.",
+        title: `RIRD Search: ${params.query}`,
+        metadata: { source: "rird-web" },
+      }
+    } catch (error) {
+      clearTimeout(timeoutId)
+
+      // Fallback to local RIRD Index (Option B)
+      try {
+        console.log("External lookup unavailable, attempting internal lookup...")
+        const searxUrl = "http://localhost:8080/search"
+        const searxParams = new URLSearchParams({
+          q: params.query,
+          format: "json",
+          safesearch: "0"
+        })
+        
+        const searxResponse = await fetch(`${searxUrl}?${searxParams.toString()}`)
+        if (searxResponse.ok) {
+          const data = await searxResponse.json()
+          if (data.results && data.results.length > 0) {
+            // Format results to match expected output
+            const formattedResults = data.results.map((r: any) => 
+              `[${r.title}](${r.url})\n${r.content}`
+            ).join("\n\n")
+            
+            return {
+              output: formattedResults,
+              title: `RIRD Search: ${params.query}`,
+              metadata: { source: "rird-local" }
+            }
+          }
+        }
+      } catch (fallbackError) {
+        console.error("RIRD index fallback failed:", fallbackError)
+      }
+
+      if (error instanceof Error && error.name === "AbortError") {
+        throw new Error("Search request timed out")
+      }
+
+      throw error
+    }
+  },
+})

package/src/tool/websearch.txt

@@ -0,0 +1,11 @@
+- Search the web using Exa AI - performs real-time web searches and can scrape content from specific URLs
+- Provides up-to-date information for current events and recent data
+- Supports configurable result counts and returns the content from the most relevant websites
+- Use this tool for accessing information beyond knowledge cutoff
+- Searches are performed automatically within a single API call
+
+Usage notes:
+  - Supports live crawling modes: 'fallback' (backup if cached unavailable) or 'preferred' (prioritize live crawling)
+  - Search types: 'auto' (balanced), 'fast' (quick results), 'deep' (comprehensive search)
+  - Configurable context length for optimal LLM integration
+  - Domain filtering and advanced search options available

package/src/tool/write.ts

@@ -0,0 +1,110 @@
+import z from "zod"
+import * as path from "path"
+import { Tool } from "./tool"
+import { LSP } from "../lsp"
+import { Permission } from "../permission"
+import DESCRIPTION from "./write.txt"
+import { Bus } from "../bus"
+import { File } from "../file"
+import { FileTime } from "../file/time"
+import { Filesystem } from "../util/filesystem"
+import { Instance } from "../project/instance"
+import { Agent } from "../agent/agent"
+
+const MAX_DIAGNOSTICS_PER_FILE = 20
+const MAX_PROJECT_DIAGNOSTICS_FILES = 5
+
+export const WriteTool = Tool.define("write", {
+  description: DESCRIPTION,
+  parameters: z.object({
+    content: z.string().describe("The content to write to the file"),
+    filePath: z.string().describe("The absolute path to the file to write (must be absolute, not relative)"),
+  }),
+  async execute(params, ctx) {
+    const agent = await Agent.get(ctx.agent)
+
+    const filepath = path.isAbsolute(params.filePath) ? params.filePath : path.join(Instance.directory, params.filePath)
+    if (!Filesystem.contains(Instance.directory, filepath)) {
+      const parentDir = path.dirname(filepath)
+      if (agent.permission.external_directory === "ask") {
+        await Permission.ask({
+          type: "external_directory",
+          pattern: [parentDir, path.join(parentDir, "*")],
+          sessionID: ctx.sessionID,
+          messageID: ctx.messageID,
+          callID: ctx.callID,
+          title: `Write file outside working directory: ${filepath}`,
+          metadata: {
+            filepath,
+            parentDir,
+          },
+        })
+      } else if (agent.permission.external_directory === "deny") {
+        throw new Permission.RejectedError(
+          ctx.sessionID,
+          "external_directory",
+          ctx.callID,
+          {
+            filepath: filepath,
+            parentDir,
+          },
+          `File ${filepath} is not in the current working directory`,
+        )
+      }
+    }
+
+    const file = Bun.file(filepath)
+    const exists = await file.exists()
+    if (exists) await FileTime.assert(ctx.sessionID, filepath)
+
+    if (agent.permission.edit === "ask")
+      await Permission.ask({
+        type: "write",
+        sessionID: ctx.sessionID,
+        messageID: ctx.messageID,
+        callID: ctx.callID,
+        title: exists ? "Overwrite this file: " + filepath : "Create new file: " + filepath,
+        metadata: {
+          filePath: filepath,
+          content: params.content,
+          exists,
+        },
+      })
+
+    await Bun.write(filepath, params.content)
+    await Bus.publish(File.Event.Edited, {
+      file: filepath,
+    })
+    FileTime.read(ctx.sessionID, filepath)
+
+    let output = ""
+    await LSP.touchFile(filepath, true)
+    const diagnostics = await LSP.diagnostics()
+    const normalizedFilepath = Filesystem.normalizePath(filepath)
+    let projectDiagnosticsCount = 0
+    for (const [file, issues] of Object.entries(diagnostics)) {
+      if (issues.length === 0) continue
+      const sorted = issues.toSorted((a, b) => (a.severity ?? 4) - (b.severity ?? 4))
+      const limited = sorted.slice(0, MAX_DIAGNOSTICS_PER_FILE)
+      const suffix =
+        issues.length > MAX_DIAGNOSTICS_PER_FILE ? `\n... and ${issues.length - MAX_DIAGNOSTICS_PER_FILE} more` : ""
+      if (file === normalizedFilepath) {
+        output += `\nThis file has errors, please fix\n<file_diagnostics>\n${limited.map(LSP.Diagnostic.pretty).join("\n")}${suffix}\n</file_diagnostics>\n`
+        continue
+      }
+      if (projectDiagnosticsCount >= MAX_PROJECT_DIAGNOSTICS_FILES) continue
+      projectDiagnosticsCount++
+      output += `\n<project_diagnostics>\n${file}\n${limited.map(LSP.Diagnostic.pretty).join("\n")}${suffix}\n</project_diagnostics>\n`
+    }
+
+    return {
+      title: path.relative(Instance.worktree, filepath),
+      metadata: {
+        diagnostics,
+        filepath,
+        exists: exists,
+      },
+      output,
+    }
+  },
+})

package/src/tool/write.txt

@@ -0,0 +1,8 @@
+Writes a file to the local filesystem.
+
+Usage:
+- This tool will overwrite the existing file if there is one at the provided path.
+- If this is an existing file, you MUST use the Read tool first to read the file's contents. This tool will fail if you did not read the file first.
+- ALWAYS prefer editing existing files in the codebase. NEVER write new files unless explicitly required.
+- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
+- Only use emojis if the user explicitly requests it. Avoid writing emojis to files unless asked.

package/src/util/archive.ts

@@ -0,0 +1,16 @@
+import { $ } from "bun"
+import path from "path"
+
+export namespace Archive {
+  export async function extractZip(zipPath: string, destDir: string) {
+    if (process.platform === "win32") {
+      const winZipPath = path.resolve(zipPath)
+      const winDestDir = path.resolve(destDir)
+      // $global:ProgressPreference suppresses PowerShell's blue progress bar popup
+      const cmd = `$global:ProgressPreference = 'SilentlyContinue'; Expand-Archive -Path '${winZipPath}' -DestinationPath '${winDestDir}' -Force`
+      await $`powershell -NoProfile -NonInteractive -Command ${cmd}`.quiet()
+    } else {
+      await $`unzip -o -q ${zipPath} -d ${destDir}`.quiet()
+    }
+  }
+}

package/src/util/color.ts

@@ -0,0 +1,19 @@
+export namespace Color {
+  export function isValidHex(hex?: string): hex is string {
+    if (!hex) return false
+    return /^#[0-9a-fA-F]{6}$/.test(hex)
+  }
+
+  export function hexToRgb(hex: string): { r: number; g: number; b: number } {
+    const r = parseInt(hex.slice(1, 3), 16)
+    const g = parseInt(hex.slice(3, 5), 16)
+    const b = parseInt(hex.slice(5, 7), 16)
+    return { r, g, b }
+  }
+
+  export function hexToAnsiBold(hex?: string): string | undefined {
+    if (!isValidHex(hex)) return undefined
+    const { r, g, b } = hexToRgb(hex)
+    return `\x1b[38;2;${r};${g};${b}m\x1b[1m`
+  }
+}

package/src/util/context.ts

@@ -0,0 +1,25 @@
+import { AsyncLocalStorage } from "async_hooks"
+
+export namespace Context {
+  export class NotFound extends Error {
+    constructor(public override readonly name: string) {
+      super(`No context found for ${name}`)
+    }
+  }
+
+  export function create<T>(name: string) {
+    const storage = new AsyncLocalStorage<T>()
+    return {
+      use() {
+        const result = storage.getStore()
+        if (!result) {
+          throw new NotFound(name)
+        }
+        return result
+      },
+      provide<R>(value: T, fn: () => R) {
+        return storage.run(value, fn)
+      },
+    }
+  }
+}

package/src/util/defer.ts

@@ -0,0 +1,12 @@
+export function defer<T extends () => void | Promise<void>>(
+  fn: T,
+): T extends () => Promise<void> ? { [Symbol.asyncDispose]: () => Promise<void> } : { [Symbol.dispose]: () => void } {
+  return {
+    [Symbol.dispose]() {
+      fn()
+    },
+    [Symbol.asyncDispose]() {
+      return Promise.resolve(fn())
+    },
+  } as any
+}

package/src/util/eventloop.ts

@@ -0,0 +1,20 @@
+import { Log } from "./log"
+
+export namespace EventLoop {
+  export async function wait() {
+    return new Promise<void>((resolve) => {
+      const check = () => {
+        const active = [...(process as any)._getActiveHandles(), ...(process as any)._getActiveRequests()]
+        Log.Default.info("eventloop", {
+          active,
+        })
+        if ((process as any)._getActiveHandles().length === 0 && (process as any)._getActiveRequests().length === 0) {
+          resolve()
+        } else {
+          setImmediate(check)
+        }
+      }
+      check()
+    })
+  }
+}