saeeol 1.2.1 → 1.2.3

package/src/tool/search/warpgrep.ts

@@ -0,0 +1,107 @@
+import { Effect, Schema } from "effect"
+import * as Tool from "../core/tool"
+import { WarpGrepClient } from "@morphllm/morphsdk/tools/warp-grep/client" 
+import { Telemetry } from "@saeeol/telemetry" 
+import { Instance } from "../../project/instance"
+import { Bus } from "../../bus"
+import { TuiEvent } from "../../cli/cmd/tui/event"
+import DESCRIPTION from "./warpgrep.txt"
+
+// FREE_PERIOD_TODO: Remove SAEEOL_WARPGREP_PROXY_URL constant and the proxy
+// fallback below. After the free period ends, require MORPH_API_KEY and
+// return an error when it is missing.
+const SAEEOL_WARPGREP_PROXY_URL = "https://api.saeeol.ai/api/gateway"
+
+const Parameters = Schema.Struct({
+  query: Schema.String.annotate({
+    description: "Search query describing what code you are looking for. Be specific and descriptive for best results.", 
+  }),
+})
+
+export const CodebaseSearchTool = Tool.define(
+  "codebase_search",
+  Effect.gen(function* () {
+    return {
+      description: DESCRIPTION,
+      parameters: Parameters,
+      execute: (params: Schema.Schema.Type<typeof Parameters>, ctx: Tool.Context) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "codebase_search",
+            patterns: [params.query],
+            always: ["*"],
+            metadata: { query: params.query },
+          })
+          Telemetry.trackToolUsed("codebase_search", ctx.sessionID) 
+
+          const apiKey = process.env["MORPH_API_KEY"]
+
+          // FREE_PERIOD_TODO: Remove proxy fallback — require apiKey, error if missing:
+          //   if (!apiKey) return { title: ..., output: "Set MORPH_API_KEY to use codebase search.", metadata: {} }
+          const client = new WarpGrepClient({
+            morphApiKey: apiKey ?? "saeeol-free",
+            ...(apiKey ? {} : { morphApiUrl: SAEEOL_WARPGREP_PROXY_URL }),
+            timeout: 60_000,
+          })
+
+          const result = yield* Effect.promise(() =>
+            client.execute({
+              searchTerm: params.query,
+              repoRoot: Instance.directory,
+            }),
+          )
+
+          if (!result.success || !result.contexts?.length) {
+            // FREE_PERIOD_TODO: When the proxy stops serving free requests, errors
+            // from the proxy (401/402/429) will surface here. The message below
+            // tells the user exactly what to do.
+            const isAuthOrRateLimit =
+              result.error && /401|402|429|rate.limit|free.period|unauthorized/i.test(result.error) 
+            const apiKeyMsg =
+              "Codebase search unavailable: free period ended. Set MORPH_API_KEY to continue. Get your key at https://www.morphllm.com/"
+            if (isAuthOrRateLimit) {
+              yield* Effect.promise(() =>
+                Bus.publish(TuiEvent.ToastShow, {
+                  title: "Codebase Search Unavailable",
+                  message: "Free period has ended. Set MORPH_API_KEY to continue. Get your key at morphllm.com",
+                  variant: "error",
+                  duration: 10000,
+                }).catch(() => {}),
+              )
+            }
+            return {
+              title: `Codebase Search: ${params.query}`,
+              output: isAuthOrRateLimit ? apiKeyMsg : (result.error ?? "No relevant code found."),
+              metadata: { count: 0 },
+            }
+          }
+
+          const MAX_OUTPUT_CHARS = 45_000
+          const fullOutput = result.contexts.map((c) => `### ${c.file}\n\`\`\`\n${c.content}\n\`\`\``).join("\n\n") 
+
+          let output: string
+          if (fullOutput.length > MAX_OUTPUT_CHARS) {
+            const summary = result.contexts
+              .map((c) => {
+                const lineInfo = !c.lines
+                  ? ""
+                  : c.lines === "*"
+                    ? " (full file)"
+                    : ` (lines ${c.lines.map((r) => r.join("-")).join(", ")})`
+                return `- ${c.file}${lineInfo}`
+              })
+              .join("\n")
+            output = `Results too large to show inline. Showing file paths and line ranges. Use Read tool to view specific files.\n\n${summary}`
+          } else {
+            output = fullOutput
+          }
+
+          return {
+            title: `Codebase Search: ${params.query}`,
+            output,
+            metadata: { count: result.contexts.length },
+          }
+        }).pipe(Effect.orDie),
+    }
+  }),
+)

package/src/tool/search/warpgrep.txt

@@ -0,0 +1,10 @@
+- Searches the codebase using a natural language search string
+- Takes a natural language search string as input: a question about something you want to understand, or a description of what you're looking for
+- This is not pattern matching or semantic search — it is an AI-powered search agent that performs intelligent multi-step searches across the repository, returning only the most relevant spans
+- Use this when you're looking for a feature or implementation, or trying to understand how something works in a large codebase but are unclear about the exact patterns to search for
+
+Usage notes:
+  - This does NOT accept regex or keyword dumps — use natural language
+  - Best practice is using this at the start of long codebase explorations when you haven't yet mapped out the relevant areas
+  - Firing two or three concurrent codebase_search calls at different angles helps go through a large codebase faster
+  - This tool is read-only and does not modify any files

package/src/tool/search/webfetch.ts

@@ -0,0 +1,202 @@
+import { Effect, Schema } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import * as Tool from "../core/tool"
+import TurndownService from "turndown"
+import DESCRIPTION from "./webfetch.txt"
+import { isImageAttachment } from "@/util/media"
+import { normalizeUrls } from "@/saeeol/util/url" 
+
+const MAX_RESPONSE_SIZE = 5 * 1024 * 1024 // 5MB
+const DEFAULT_TIMEOUT = 30 * 1000 // 30 seconds
+const MAX_TIMEOUT = 120 * 1000 // 2 minutes
+
+export const Parameters = Schema.Struct({
+  url: Schema.String.annotate({ description: "The URL to fetch content from" }),
+  format: Schema.Literals(["text", "markdown", "html"])
+    .pipe(Schema.optional, Schema.withDecodingDefault(Effect.succeed("markdown" as const)))
+    .annotate({
+      description: "The format to return the content in (text, markdown, or html). Defaults to markdown.",
+    }),
+  timeout: Schema.optional(Schema.Number).annotate({ description: "Optional timeout in seconds (max 120)" }),
+})
+
+export const WebFetchTool = Tool.define(
+  "webfetch",
+  Effect.gen(function* () {
+    const http = yield* HttpClient.HttpClient
+    const httpOk = HttpClient.filterStatusOk(http)
+
+    return {
+      description: DESCRIPTION,
+      parameters: Parameters,
+      execute: (params: Schema.Schema.Type<typeof Parameters>, ctx: Tool.Context) =>
+        Effect.gen(function* () {
+          if (!params.url.startsWith("http://") && !params.url.startsWith("https://")) {
+            throw new Error("URL must start with http:// or https://")
+          }
+
+          const url = normalizeUrls(params.url) 
+
+          yield* ctx.ask({
+            permission: "webfetch",
+            patterns: [url], 
+            always: ["*"],
+            metadata: {
+              url, 
+              format: params.format,
+              timeout: params.timeout,
+            },
+          })
+
+          const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT)
+
+          // Build Accept header based on requested format with q parameters for fallbacks
+          let acceptHeader = "*/*"
+          switch (params.format) {
+            case "markdown":
+              acceptHeader = "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1"
+              break
+            case "text":
+              acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1"
+              break
+            case "html":
+              acceptHeader =
+                "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1"
+              break
+            default:
+              acceptHeader =
+                "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
+          }
+          const headers = {
+            "User-Agent":
+              "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
+            Accept: acceptHeader,
+            "Accept-Language": "en-US,en;q=0.9",
+          }
+
+          const request = HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers)) 
+
+          // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch)
+          const response = yield* httpOk.execute(request).pipe(
+            Effect.catchIf(
+              (err) =>
+                err.reason._tag === "StatusCodeError" &&
+                err.reason.response.status === 403 &&
+                err.reason.response.headers["cf-mitigated"] === "challenge",
+              () =>
+                httpOk.execute(
+                  HttpClientRequest.get(url).pipe( 
+                    HttpClientRequest.setHeaders({ ...headers, "User-Agent": "saeeol" }), 
+                  ),
+                ),
+            ),
+            Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }),
+          )
+
+          // Check content length
+          const contentLength = response.headers["content-length"]
+          if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+            throw new Error("Response too large (exceeds 5MB limit)")
+          }
+
+          const arrayBuffer = yield* response.arrayBuffer
+          if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
+            throw new Error("Response too large (exceeds 5MB limit)")
+          }
+
+          const contentType = response.headers["content-type"] || ""
+          const mime = contentType.split(";")[0]?.trim().toLowerCase() || ""
+          const title = `${url} (${contentType})` 
+
+          if (isImageAttachment(mime)) {
+            const base64Content = Buffer.from(arrayBuffer).toString("base64")
+            return {
+              title,
+              output: "Image fetched successfully",
+              metadata: {},
+              attachments: [
+                {
+                  type: "file" as const,
+                  mime,
+                  url: `data:${mime};base64,${base64Content}`,
+                },
+              ],
+            }
+          }
+
+          const content = new TextDecoder().decode(arrayBuffer)
+
+          // Handle content based on requested format and actual content type
+          switch (params.format) {
+            case "markdown":
+              if (contentType.includes("text/html")) {
+                const markdown = convertHTMLToMarkdown(content)
+                return {
+                  output: markdown,
+                  title,
+                  metadata: {},
+                }
+              }
+              return { output: content, title, metadata: {} }
+
+            case "text":
+              if (contentType.includes("text/html")) {
+                const text = yield* Effect.promise(() => extractTextFromHTML(content))
+                return { output: text, title, metadata: {} }
+              }
+              return { output: content, title, metadata: {} }
+
+            case "html":
+              return { output: content, title, metadata: {} }
+
+            default:
+              return { output: content, title, metadata: {} }
+          }
+        }).pipe(Effect.orDie),
+    }
+  }),
+)
+
+async function extractTextFromHTML(html: string) {
+  let text = ""
+  let skipContent = false
+
+  const rewriter = new HTMLRewriter()
+    .on("script, style, noscript, iframe, object, embed", {
+      element() {
+        skipContent = true
+      },
+      text() {
+        // Skip text content inside these elements
+      },
+    })
+    .on("*", {
+      element(element) {
+        // Reset skip flag when entering other elements
+        if (!["script", "style", "noscript", "iframe", "object", "embed"].includes(element.tagName)) {
+          skipContent = false
+        }
+      },
+      text(input) {
+        if (!skipContent) {
+          text += input.text
+        }
+      },
+    })
+    .transform(new Response(html))
+
+  await rewriter.text()
+  return text.trim()
+}
+
+function convertHTMLToMarkdown(html: string): string {
+  const turndownService = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+  })
+  turndownService.remove(["script", "style", "meta", "link"])
+  return turndownService.turndown(html)
+}

package/src/tool/search/webfetch.txt

@@ -0,0 +1,13 @@
+- Fetches content from a specified URL
+- Takes a URL and optional format as input
+- Fetches the URL content, converts to requested format (markdown by default)
+- Returns the content in the specified format
+- Use this tool when you need to retrieve and analyze web content
+
+Usage notes:
+  - IMPORTANT: if another tool is present that offers better web fetching capabilities, is more targeted to the task, or has fewer restrictions, prefer using that tool instead of this one.
+  - The URL must be a fully-formed valid URL
+  - HTTP URLs will be automatically upgraded to HTTPS
+  - Format options: "markdown" (default), "text", or "html"
+  - This tool is read-only and does not modify any files
+  - Results may be summarized if the content is very large

package/src/tool/search/websearch.ts

@@ -0,0 +1,71 @@
+import { Effect, Schema } from "effect"
+import { HttpClient } from "effect/unstable/http"
+import * as Tool from "../core/tool"
+import * as McpExa from "../integration/mcp-exa"
+import DESCRIPTION from "./websearch.txt"
+
+export const Parameters = Schema.Struct({
+  query: Schema.String.annotate({ description: "Websearch query" }),
+  numResults: Schema.optional(Schema.Number).annotate({
+    description: "Number of search results to return (default: 8)",
+  }),
+  livecrawl: Schema.optional(Schema.Literals(["fallback", "preferred"])).annotate({
+    description:
+      "Live crawl mode - 'fallback': use live crawling as backup if cached content unavailable, 'preferred': prioritize live crawling (default: 'fallback')",
+  }),
+  type: Schema.optional(Schema.Literals(["auto", "fast", "deep"])).annotate({
+    description: "Search type - 'auto': balanced search (default), 'fast': quick results, 'deep': comprehensive search",
+  }),
+  contextMaxCharacters: Schema.optional(Schema.Number).annotate({
+    description: "Maximum characters for context string optimized for LLMs (default: 10000)",
+  }),
+})
+
+export const WebSearchTool = Tool.define(
+  "websearch",
+  Effect.gen(function* () {
+    const http = yield* HttpClient.HttpClient
+
+    return {
+      get description() {
+        return DESCRIPTION.replace("{{year}}", new Date().getFullYear().toString())
+      },
+      parameters: Parameters,
+      execute: (params: Schema.Schema.Type<typeof Parameters>, ctx: Tool.Context) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "websearch",
+            patterns: [params.query],
+            always: ["*"],
+            metadata: {
+              query: params.query,
+              numResults: params.numResults,
+              livecrawl: params.livecrawl,
+              type: params.type,
+              contextMaxCharacters: params.contextMaxCharacters,
+            },
+          })
+
+          const result = yield* McpExa.call(
+            http,
+            "web_search_exa",
+            McpExa.SearchArgs,
+            {
+              query: params.query,
+              type: params.type || "auto",
+              numResults: params.numResults || 8,
+              livecrawl: params.livecrawl || "fallback",
+              contextMaxCharacters: params.contextMaxCharacters,
+            },
+            "25 seconds",
+          )
+
+          return {
+            output: result ?? "No search results found. Please try a different query.",
+            title: `Web search: ${params.query}`,
+            metadata: {},
+          }
+        }).pipe(Effect.orDie),
+    }
+  }),
+)

package/src/tool/search/websearch.txt

@@ -0,0 +1,14 @@
+- Search the web using Exa AI - performs real-time web searches and can scrape content from specific URLs
+- Provides up-to-date information for current events and recent data
+- Supports configurable result counts and returns the content from the most relevant websites
+- Use this tool for accessing information beyond knowledge cutoff
+- Searches are performed automatically within a single API call
+
+Usage notes:
+  - Supports live crawling modes: 'fallback' (backup if cached unavailable) or 'preferred' (prioritize live crawling)
+  - Search types: 'auto' (balanced), 'fast' (quick results), 'deep' (comprehensive search)
+  - Configurable context length for optimal LLM integration
+  - Domain filtering and advanced search options available
+
+The current year is {{year}}. You MUST use this year when searching for recent information or current events
+- Example: If the current year is 2026 and the user asks for "latest AI news", search for "AI news 2026", NOT "AI news 2025"

package/src/tool/skill.ts

@@ -1,91 +1 @@
-import path from "path"
-import { pathToFileURL } from "url"
-import { Effect, Schema } from "effect"
-import * as Stream from "effect/Stream"
-import { Ripgrep } from "../file/ripgrep"
-import { Skill } from "../skill"
-import * as Tool from "./tool"
-import DESCRIPTION from "./skill.txt"
-
-export const Parameters = Schema.Struct({
-  name: Schema.String.annotate({ description: "The name of the skill from available_skills" }),
-})
-
-export const SkillTool = Tool.define(
-  "skill",
-  Effect.gen(function* () {
-    const skill = yield* Skill.Service
-    const rg = yield* Ripgrep.Service
-
-    return {
-      description: DESCRIPTION,
-      parameters: Parameters,
-      execute: (params: Schema.Schema.Type<typeof Parameters>, ctx: Tool.Context) =>
-        Effect.gen(function* () {
-          const info = yield* skill.get(params.name)
-          if (!info) {
-            const all = yield* skill.all()
-            const available = all.map((item) => item.name).join(", ")
-            throw new Error(`Skill "${params.name}" not found. Available skills: ${available || "none"}`)
-          }
-
-          yield* ctx.ask({
-            permission: "skill",
-            patterns: [params.name],
-            always: [params.name],
-            metadata: {},
-          })
-          if (info.location === Skill.BUILTIN_LOCATION) {
-            return {
-              title: `Loaded skill: ${info.name}`,
-              output: [
-                `<skill_content name="${info.name}">`,
-                `# Skill: ${info.name}`,
-                "",
-                info.content.trim(),
-                "</skill_content>",
-              ].join("\n"),
-              metadata: {
-                name: info.name,
-                dir: Skill.BUILTIN_LOCATION,
-              },
-            }
-          }
-
-          const dir = path.dirname(info.location)
-          const base = pathToFileURL(dir).href
-          const limit = 10
-          const files = yield* rg.files({ cwd: dir, follow: false, hidden: true, signal: ctx.abort }).pipe(
-            Stream.filter((file) => !file.includes("SKILL.md")),
-            Stream.map((file) => path.resolve(dir, file)),
-            Stream.take(limit),
-            Stream.runCollect,
-            Effect.map((chunk) => [...chunk].map((file) => `<file>${file}</file>`).join("\n")),
-          )
-
-          return {
-            title: `Loaded skill: ${info.name}`,
-            output: [
-              `<skill_content name="${info.name}">`,
-              `# Skill: ${info.name}`,
-              "",
-              info.content.trim(),
-              "",
-              `Base directory for this skill: ${base}`,
-              "Relative paths in this skill (e.g., scripts/, reference/) are relative to this base directory.",
-              "Note: file list is sampled.",
-              "",
-              "<skill_files>",
-              files,
-              "</skill_files>",
-              "</skill_content>",
-            ].join("\n"),
-            metadata: {
-              name: info.name,
-              dir,
-            },
-          }
-        }).pipe(Effect.orDie),
-    }
-  }),
-)
+export * from "./workflow/skill"